diff --git a/.agents/skills/audit-copilot-workarounds/SKILL.md b/.agents/skills/audit-copilot-workarounds/SKILL.md index 0cc2029cc7..4c0fa17549 100644 --- a/.agents/skills/audit-copilot-workarounds/SKILL.md +++ b/.agents/skills/audit-copilot-workarounds/SKILL.md @@ -16,23 +16,24 @@ Copilot upstream. The provider code and the reference URLs beside each workaround are the inventory; there is no separate documentation list to reconcile. -1. Start from - `packages/provider-copilot/src/interceptors/{openai-chat-completions,anthropic-messages,openai-responses}/index.ts` - and `packages/provider-copilot/src/defaults.ts`. Follow every registered - interceptor and default-enabled shim to its implementation and tests. +1. Start from `packages/provider-copilot/src/pipelines.ts`, + `chat-pipelines.ts`, and `defaults.ts`. Follow every registered stage and + default-enabled shim to its implementation under `src/stages/` and its + mirrored tests under `__tests__/stages/`. 2. Sweep the rest of `packages/provider-copilot/src` for non-pricing reference URLs and for vendor constants, thresholds, timeouts, retries, and pinned wire values that require a reference but may be missing one. Pricing citations belong to `fetching-models-pricing`; everything else remains in this audit. -3. Include provider-level request/result shaping and catalog shaping even when - they are not interceptors. In particular, inspect `provider.ts`, +3. Include request/result shaping and catalog shaping. Inspect `provider.ts`, `fetch-models.ts`, `known-models.ts`, `model-selection.ts`, and - `merge-claude-variants.ts` together with their imports and tests. -4. Include the authentication fingerprint and management/data-plane behavior in - `auth.ts`, plus OpenAI Responses item identity and replay handling rooted at - `interceptors/openai-responses/item-id-membrane.ts`. Follow adjacent carrier and - compaction modules rather than assuming the interceptor registry contains the - whole workaround. + `merge-claude-variants.ts` together with their imports and tests. Exercise + data-plane behavior through the provider's operation pipelines rather than + its catalog-only instance. +4. Include the authentication fingerprint and control/data-plane behavior in + `auth.ts` and `control-http.ts`, plus OpenAI Responses item identity and replay + handling rooted at `stages/openai-responses/item-id-membrane.ts` and + `opaque-item-id.ts`. Follow adjacent carrier and compaction modules; the + operation assemblies are entrypoints, not a complete workaround inventory. 5. Record each item's owning module, reference URLs, affected source and target APIs, models, account scope, default flag state, tests, and exit condition: delete an obsolete workaround, refresh pinned mimicry, or retain it with diff --git a/AGENTS.md b/AGENTS.md index e2426d055a..9ea2a610f7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,7 +62,6 @@ | Package | `packages/dump` | Records pipeline runs and publishes diagnostics. | | Package | `packages/gateway` | Composes gateway services. | | Package | `packages/http` | Provides HTTP transport primitives. | -| Package | `packages/interceptor` | Intercepts gateway traffic. | | Package | `packages/pipeline` | Runs stages over a fact record. | | Package | `packages/platform` | Defines portable runtime contracts. | | Package | `packages/protocols` | Defines protocol contracts. | diff --git a/README.md b/README.md index 8e159795f2..c56d74462b 100644 --- a/README.md +++ b/README.md @@ -206,13 +206,13 @@ stage throws. Client JSON serialization runs inside settlement and publishes UTF while retaining the canonical content for inspection. Streaming quantities settle at the deferred exit. Explicit protocol and transport errors take precedence over the settlement's generic failure marker. -Pipeline owns the stage-event format and object-space encoding. `@floway-dev/dump` owns -run recording, stream references, completed NDJSON artifacts and metadata publication. -Gateway supplies storage/broker ports and applies request admission, model attribution, -HTTP byte measurement and retention policy. Closing waits for owned readers and deferred -outcomes before storing the completed record and publishing its metadata. LogStream -infrastructure is available on both platforms; live recording and its business readers -remain outside these three PRs. +`@floway-dev/dump` owns portable run recording, stream recording, metadata contracts, +broker framing and wire conversion. Pipeline encodes stage facts, logs and protocol frames +into one object space. Gateway owns attribution, admission, SQL storage and HTTP measurement. +Closing waits for owned readers and deferred outcomes before storing the completed NDJSON +record and publishing its metadata. The Collected view selects the recorded client stream and uses its +`stream.end` event to establish recording completion. LogStream infrastructure is available +on both platforms; live recording and its business readers remain outside these three PRs. Pipeline endpoints are assembled from individual stage modules. Each stage owns its request and response contract; a large stage keeps local helpers in the same diff --git a/apps/web/__tests__/components/requests/body-rendering_test.ts b/apps/web/__tests__/components/requests/body-rendering_test.ts deleted file mode 100644 index 1fa698998d..0000000000 --- a/apps/web/__tests__/components/requests/body-rendering_test.ts +++ /dev/null @@ -1,149 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import { contentTypeOf, EMPTY_BODY, renderBody } from '../../../src/components/requests/body-render'; - -// The dump carries bodies as they went over the wire, so a base64 fixture is -// built from bytes rather than from a string literal. -const base64 = (bytes: string) => btoa(bytes); - -const withoutTypedArrayBase64 = (run: () => T): T => { - const fromBase64 = Object.getOwnPropertyDescriptor(Uint8Array, 'fromBase64'); - const toBase64 = Object.getOwnPropertyDescriptor(Uint8Array.prototype, 'toBase64'); - Object.defineProperty(Uint8Array, 'fromBase64', { configurable: true, value: undefined }); - Object.defineProperty(Uint8Array.prototype, 'toBase64', { configurable: true, value: undefined }); - try { - return run(); - } finally { - if (fromBase64) Object.defineProperty(Uint8Array, 'fromBase64', fromBase64); - else Reflect.deleteProperty(Uint8Array, 'fromBase64'); - if (toBase64) Object.defineProperty(Uint8Array.prototype, 'toBase64', toBase64); - else Reflect.deleteProperty(Uint8Array.prototype, 'toBase64'); - } -}; - -describe('content type lookup', () => { - it('finds the header however it was capitalized, and says nothing when it is absent', () => { - expect(contentTypeOf([['Content-Type', 'application/json']])).toBe('application/json'); - expect(contentTypeOf([['content-length', '12']])).toBe(''); - }); -}); - -describe('body rendering', () => { - it('shows nothing for a body that carries no data', () => { - expect(renderBody({ data: '', encoding: 'utf8' }, 'application/json')).toEqual(EMPTY_BODY); - }); - - it('pretty-prints JSON and says that is what it is', () => { - const rendered = renderBody({ data: '{"model":"gpt-5","stream":true}', encoding: 'utf8' }, 'application/json'); - - expect(rendered.isJson).toBe(true); - expect(rendered.text).toBe('{\n "model": "gpt-5",\n "stream": true\n}'); - expect(rendered.copyText).toBe(rendered.text); - }); - - it('leaves a body that is not JSON exactly as it arrived', () => { - const rendered = renderBody({ data: 'data: [DONE]\n\n', encoding: 'utf8' }, 'text/event-stream'); - - expect(rendered.isJson).toBe(false); - expect(rendered.text).toBe('data: [DONE]\n\n'); - expect(rendered.decodeError).toBeNull(); - }); - - it('decodes a base64 body before reading it as JSON', () => { - const rendered = renderBody({ data: base64('{"ok":true}'), encoding: 'base64' }, 'application/json'); - - expect(rendered.isJson).toBe(true); - expect(rendered.text).toBe('{\n "ok": true\n}'); - }); - - // A body that is not text at all keeps its base64 on screen rather than - // becoming replacement characters, and says why. - it('reports a body it cannot decode instead of showing mojibake', () => { - const data = base64('ÿþý'); - - const rendered = renderBody({ data, encoding: 'base64' }, 'application/octet-stream'); - - expect(rendered.decodeError).toBeTruthy(); - expect(rendered.text).toBe(data); - expect(rendered.copyText).toBe(data); - expect(rendered.isJson).toBe(false); - }); -}); - -describe('multipart body rendering', () => { - const boundary = 'floway-boundary'; - const bytes = (...values: number[]) => String.fromCharCode(...values); - const part = (headers: string, body: string) => `${headers}\r\n\r\n${body}`; - const multipart = (parts: string[]) => - [`--${boundary}`, parts.join(`\r\n--${boundary}\r\n`), `--${boundary}--`, ''].join('\r\n'); - - const jsonPart = part('content-disposition: form-data; name="model"\r\ncontent-type: Application/Vnd.Floway+JSON; charset=utf-8', '{"id":1}'); - const textWire = multipart([jsonPart, part('content-disposition: form-data; name="note"', 'hello')]); - const render = (wire: string, contentType = `multipart/form-data; boundary=${boundary}`) => - renderBody({ data: base64(wire), encoding: 'base64' }, contentType); - - it('shows every textual part as text rather than as one base64 blob', () => { - const rendered = render(textWire); - - expect(rendered.text).toContain('{"id":1}'); - expect(rendered.text).toContain('hello'); - expect(rendered.text).toContain(`--${boundary}--`); - expect(rendered.isJson).toBe(false); - }); - - // Bytes no textual decoding could survive, so a render that reached the - // placeholder cannot have come from the plain-body fallback. - const image = bytes(0x00, 0x01, 0xFF); - const binaryWire = multipart([jsonPart, part('content-disposition: form-data; name="file"\r\ncontent-type: image/png', image)]); - - it('stands in for a part that is not text', () => { - const rendered = render(binaryWire); - - expect(rendered.text).toContain('{"id":1}'); - expect(rendered.text).toContain('[binary, 3 bytes, content-type=image/png]'); - expect(rendered.text).toContain(base64(image)); - expect(rendered.text).not.toContain(image); - }); - - it('renders binary parts directly through the legacy-browser fallback', () => { - const rendered = withoutTypedArrayBase64(() => render(binaryWire)); - - expect(rendered.text).toContain('[binary, 3 bytes, content-type=image/png]'); - expect(rendered.text).toContain(base64(image)); - }); - - it('copies the body as it arrived rather than as it is shown', () => { - const data = base64(textWire); - - expect(renderBody({ data, encoding: 'base64' }, `multipart/form-data; boundary=${boundary}`).copyText).toBe(data); - expect(renderBody({ data, encoding: 'base64' }, `multipart/form-data; boundary=${boundary}`).text).not.toBe(data); - }); - - it('reads the boundary out of a quoted parameter too', () => { - expect(render(binaryWire, `multipart/form-data; boundary="${boundary}"`).text).toContain('[binary, 3 bytes'); - }); - - it('falls back to the plain body when the content type names no boundary', () => { - const rendered = render(textWire, 'multipart/form-data'); - - expect(rendered.text).toBe(textWire); - expect(rendered.text).not.toContain('[binary,'); - }); - - // A PNG signature leads with 0x89, inside the 0x80-0x9F range where the - // windows-1252 decoder the Encoding Standard resolves latin1 to - // (https://encoding.spec.whatwg.org/#names-and-labels) is not byte-transparent. - // Reading the wire through any text encoding loses those bytes and abandons - // the whole multipart render, so every PNG upload arrives as a raw blob. - const png = bytes(0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A); - const pngWire = multipart([jsonPart, part('content-disposition: form-data; name="file"\r\ncontent-type: image/png', png)]); - - it('round-trips a PNG signature rather than losing it to the encoding', () => { - const rendered = render(pngWire); - - expect(rendered.text).toContain('{"id":1}'); - expect(rendered.text).toContain('[binary, 8 bytes, content-type=image/png]'); - expect(rendered.text).toContain(base64(png)); - expect(rendered.decodeError).toBeNull(); - }); -}); diff --git a/apps/web/__tests__/components/requests/detail_test.tsx b/apps/web/__tests__/components/requests/detail_test.tsx index 96180a2b3a..c488e5e128 100644 --- a/apps/web/__tests__/components/requests/detail_test.tsx +++ b/apps/web/__tests__/components/requests/detail_test.tsx @@ -1,30 +1,30 @@ import { fireEvent, screen } from '@testing-library/react'; -import type { ReactNode } from 'react'; import { describe, expect, it, vi } from 'vitest'; import { RequestDetailPanel } from '../../../src/components/requests/detail'; import { renderInApp } from '../../render'; -import type { DumpEdgeRecord, DumpRunRecord } from '@floway-dev/gateway/dump-types'; +import type { DumpRecord } from '@floway-dev/dump/types'; +import { encodeRun, toNdjson } from '@floway-dev/pipeline'; -vi.mock('../../../src/components/ui/body-editor', () => ({ default: ({ text, toolbarStart }: { text: string; toolbarStart?: ReactNode }) => <>{toolbarStart}
{text}
})); +vi.mock('../../../src/components/ui/body-editor', () => ({ default: ({ text }: { text: string }) => 
{text}
})); -const record: DumpEdgeRecord = { - shape: 'edge', +const record: DumpRecord = { meta: { id: 'detail', method: 'POST', path: '/v1/chat/completions', startedAt: 0, completedAt: 1, status: 200, upstream: null, model: 'm', inputTokens: null, outputTokens: null, requestBytes: 0, responseBytes: 0, durationMs: 1, error: null }, - request: { method: 'POST', path: '/v1/chat/completions', headers: [], body: { encoding: 'utf8', data: '{"client":"large request"}' } }, - response: { status: 200, headers: [], body: { type: 'stream', events: [] } }, - capture: { exchanges: [{ upstreamId: 'u', request: { url: 'https://upstream.test', method: 'POST', headers: [], body: { encoding: 'utf8', data: '{"upstream":"translated request"}' } }, response: { status: 200, headers: [], body: { encoding: 'utf8', data: 'data: {broken\n' }, complete: false, error: null }, error: null }], response: { body: { encoding: 'utf8', data: 'data: downstream\n' }, complete: true, error: null } }, + events: toNdjson(encodeRun([ + { type: 'stage.entered', stageId: 1, name: 'serve', parentStageId: null, facts: {} }, + { type: 'stage.leaved', stageId: 1, facts: { 'response.http.status': 200 } }, + ])), }; it('opens stage facts and offers searchable events with timing and export controls', async () => { - const run: DumpRunRecord = { - shape: 'run', meta: { ...record.meta, ttftMs: 125 }, - events: '{"type":"stage.entered","stageId":1,"name":"serve","parentStageId":null,"facts":{"serve.model":"m"}}\n', + const run: DumpRecord = { + meta: { ...record.meta, ttftMs: 125 }, + events: toNdjson(encodeRun([{ type: 'stage.entered', stageId: 1, name: 'serve', parentStageId: null, facts: { 'serve.model': 'm' } }])), }; - renderInApp(); + renderInApp(); expect(screen.getByRole('treeitem', { name: 'serve #1' })).toBeTruthy(); expect((await screen.findByTestId('body-content')).textContent).toContain('serve.model'); - fireEvent.click(screen.getByRole('combobox', { name: 'Run' })); + fireEvent.click(screen.getByRole('combobox', { name: 'Response body view' })); fireEvent.click(screen.getByRole('option', { name: 'Events (1)' })); expect(screen.getByRole('button', { name: '#1 stage.entered serve' })).toBeTruthy(); expect(screen.getByRole('button', { name: 'Search events' })).toBeTruthy(); @@ -34,64 +34,43 @@ it('opens stage facts and offers searchable events with timing and export contro }); describe('request detail navigation', () => { - it('opens each body directly and preserves malformed raw response text', async () => { - renderInApp(); - expect((await screen.findByTestId('body-content')).textContent).toContain('parsed response'); - expect(screen.queryByText('large request')).toBeNull(); - fireEvent.click(screen.getByRole('combobox', { name: 'Request details' })); - fireEvent.click(screen.getByRole('option', { name: 'Client request' })); - expect((await screen.findByTestId('body-content')).textContent).toContain('large request'); - fireEvent.click(screen.getByRole('combobox', { name: 'Request details' })); - fireEvent.click(screen.getByRole('option', { name: 'Upstream response' })); - expect((await screen.findByTestId('body-content')).textContent).toContain('data: {broken\n'); - fireEvent.click(screen.getByRole('button', { name: 'Issues (1)' })); - expect(screen.getByText('Capture ended before EOF. These are the bytes received so far.')).toBeTruthy(); - fireEvent.click(screen.getByRole('button', { name: 'Dismiss' })); - fireEvent.click(screen.getByRole('combobox', { name: 'Request details' })); - fireEvent.click(screen.getByRole('option', { name: 'Upstream request' })); - expect((await screen.findByTestId('body-content')).textContent).toContain('translated request'); - fireEvent.click(screen.getByRole('combobox', { name: 'Request details' })); - fireEvent.click(screen.getByRole('option', { name: 'Client response' })); + it('shows run facts, searches events, and opens the collected client response', async () => { + renderInApp(); + expect(screen.getByRole('treeitem', { name: 'serve #1' })).toBeTruthy(); + fireEvent.click(screen.getByRole('combobox', { name: 'Response body view' })); + fireEvent.click(screen.getByRole('option', { name: 'Events (2)' })); + expect(screen.getByRole('button', { name: '#1 stage.entered serve' })).toBeTruthy(); + fireEvent.click(screen.getByRole('button', { name: 'Search events' })); + fireEvent.change(screen.getByRole('textbox', { name: 'Search events' }), { target: { value: 'response.http.status' } }); + expect(screen.getByText('1 / 2')).toBeTruthy(); fireEvent.click(screen.getByRole('combobox', { name: 'Response body view' })); - fireEvent.click(screen.getByRole('option', { name: 'Raw bytes' })); - expect((await screen.findByTestId('body-content')).textContent).toBe('data: downstream\n'); + fireEvent.click(screen.getByRole('option', { name: 'Collected' })); + expect((await screen.findByTestId('body-content')).textContent).toContain('parsed response'); }); -}); -it('keeps legacy upstream streams readable without a raw exchange', async () => { - const legacy = { ...record, capture: undefined, response: { ...record.response, upstream: { status: 200, headers: [], body: { type: 'stream' as const, events: [] } } } }; - renderInApp(); - fireEvent.click(screen.getByRole('combobox', { name: 'Request details' })); - fireEvent.click(screen.getByRole('option', { name: 'Upstream response' })); - expect((await screen.findByTestId('body-content')).textContent).toContain('legacy output'); -}); + it('warns about an incomplete recording while leaving its partial collected response accessible', async () => { + renderInApp(); + expect(screen.getByText('The recorded client stream is incomplete. The collected response may be partial.')).toBeTruthy(); + fireEvent.click(screen.getByRole('combobox', { name: 'Response body view' })); + fireEvent.click(screen.getByRole('option', { name: 'Collected' })); + expect((await screen.findByTestId('body-content')).textContent).toContain('partial response'); + }); -it('switches upstream attempts without showing the last attempt’s payload', async () => { - const multiple = { - ...record, capture: { - ...record.capture!, exchanges: [ - { ...record.capture!.exchanges[0]!, request: { ...record.capture!.exchanges[0]!.request, body: { encoding: 'utf8' as const, data: 'first attempt' } } }, - { ...record.capture!.exchanges[0]!, request: { ...record.capture!.exchanges[0]!.request, body: { encoding: 'utf8' as const, data: 'second attempt' } } }, - ], - }, - }; - renderInApp(); - fireEvent.click(screen.getByRole('combobox', { name: 'Request details' })); - fireEvent.click(screen.getByRole('option', { name: 'Upstream request' })); - expect((await screen.findByTestId('body-content')).textContent).toContain('second attempt'); - fireEvent.click(screen.getByRole('combobox', { name: 'Upstream calls' })); - fireEvent.click(screen.getAllByRole('option')[0]!); - expect((await screen.findByTestId('body-content')).textContent).toContain('first attempt'); -}); + it('masks credential headers in rendered facts', async () => { + const secret = 'Bearer private-request-token-1234567890'; + const events = [ + { type: 'object', fromObjectId: 1, nodes: [[['authorization', secret]]] }, + { type: 'stage.entered', stageId: 1, name: 'serve', parentStageId: null, facts: { 'ingress.http.headers': { $: 1 } } }, + ].map(event => `${JSON.stringify(event)}\n`).join(''); + const view = renderInApp(); + await screen.findByTestId('body-content'); + expect(view.container.textContent).not.toContain(secret); + expect(view.container.textContent).toContain('authorization'); + expect(view.container.textContent).toContain('••••'); + }); -it('masks credential headers in rendered run facts', () => { - const secret = 'Bearer private-request-token-1234567890'; - const events = [ - { type: 'object', fromObjectId: 1, nodes: [[['authorization', secret]]] }, - { type: 'stage.entered', stageId: 1, name: 'serve', parentStageId: null, facts: { 'ingress.http.headers': { $: 1 } } }, - ].map(event => `${JSON.stringify(event)}\n`).join(''); - const view = renderInApp(); - expect(view.container.textContent).not.toContain(secret); - expect(view.container.textContent).toContain('authorization'); - expect(view.container.textContent).toContain('••••'); + it('distinguishes an empty run from a search with no matches', () => { + renderInApp(); + expect(screen.getByText('This run recorded no events.')).toBeTruthy(); + }); }); diff --git a/apps/web/__tests__/components/requests/events_test.tsx b/apps/web/__tests__/components/requests/events_test.tsx index 1d10630f87..4de2082ea6 100644 --- a/apps/web/__tests__/components/requests/events_test.tsx +++ b/apps/web/__tests__/components/requests/events_test.tsx @@ -1,22 +1,23 @@ import { fireEvent, screen, within } from '@testing-library/react'; import { describe, expect, it } from 'vitest'; -import { EventList } from '../../../src/components/requests/events'; +import { RenderedEventList, type RenderedEvent } from '../../../src/components/requests/events'; import { renderInApp } from '../../render'; -import type { DumpStreamEvent } from '@floway-dev/gateway/dump-types'; -const events: DumpStreamEvent[] = Array.from({ length: 120 }, (_, i) => ({ - ts: i * 15, - frame: { +const events: RenderedEvent[] = Array.from({ length: 120 }, (_, i) => ({ + timestamp: i * 15, + event: null, + parseError: null, + text: JSON.stringify({ type: 'event', event: { id: 'demo', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { content: i === 99 ? 'needle ' : `chunk ${i + 1}` }, finish_reason: null }], }, - }, + }, null, 2), })); -events.push({ ts: 1800, frame: { type: 'done' } }); +events.push({ timestamp: 1800, event: 'done', parseError: null, text: JSON.stringify({ type: 'done' }, null, 2) }); -const renderEvents = () => renderInApp(View selector} />); +const renderEvents = () => renderInApp( event.text).join('\n')} toolbarStart={View selector} />); describe('per-event view', () => { it('renders separate highlighted JSON blocks and collapses each event independently', () => { @@ -41,8 +42,8 @@ describe('per-event view', () => { expect(view.container.querySelector('code')?.textContent).toContain(''); fireEvent.change(screen.getByRole('textbox', { name: 'Search events' }), { target: { value: 'unmatched' } }); expect(screen.getByText('No matching events')).toBeTruthy(); - fireEvent.change(screen.getByRole('textbox', { name: 'Search events' }), { target: { value: '[DONE]' } }); - expect(screen.getByRole('button', { name: '#121 [DONE]' })).toBeTruthy(); + fireEvent.change(screen.getByRole('textbox', { name: 'Search events' }), { target: { value: 'done' } }); + expect(screen.getByRole('button', { name: '#121 done' })).toBeTruthy(); expect(screen.queryByText('Invalid JSON')).toBeNull(); }); }); diff --git a/apps/web/__tests__/components/requests/export_test.ts b/apps/web/__tests__/components/requests/export_test.ts index 34a8671b13..09d22d8bb8 100644 --- a/apps/web/__tests__/components/requests/export_test.ts +++ b/apps/web/__tests__/components/requests/export_test.ts @@ -2,34 +2,22 @@ import { describe, expect, it } from 'vitest'; import { exportRecords } from '../../../src/components/requests/export'; import { redactHeaderValue } from '../../../src/components/requests/header-redact'; -import type { DumpEdgeRecord } from '@floway-dev/gateway/dump-types'; +import type { DumpRecord } from '@floway-dev/dump/types'; const SECRET = 'Bearer client-secret-token-1234567890abcd'; const MASK = redactHeaderValue(SECRET); -const record = (id: string): DumpEdgeRecord => ({ - shape: 'edge', +const record = (id: string): DumpRecord => ({ meta: { id, method: 'POST', path: '/v1/chat/completions', startedAt: 0, completedAt: 1000, status: 200, upstream: null, model: 'm', inputTokens: null, outputTokens: null, requestBytes: 0, responseBytes: 0, durationMs: 1000, error: null }, - request: { method: 'POST', path: '/v1/chat/completions', headers: [['authorization', SECRET], ['content-type', 'application/json']], body: { encoding: 'utf8', data: '中文 request' } }, - response: { status: 200, headers: [['set-cookie', SECRET]], body: { type: 'none' }, upstream: { status: 200, headers: [['api-key', SECRET]], body: { type: 'none' } } }, - capture: { exchanges: [{ upstreamId: 'u', request: { url: 'https://upstream.test', method: 'POST', headers: [['x-api-key', SECRET]], body: { encoding: 'utf8', data: '{"upstream":"translated request"}' } }, response: { status: 200, headers: [['x-goog-api-key', SECRET]], body: { encoding: 'utf8', data: 'data: {broken\n' }, complete: true, error: null }, error: null }], response: { body: { encoding: 'base64', data: '/4A=' }, complete: false, error: 'socket reset' } }, + events: [ + { type: 'object', fromObjectId: 1, nodes: [[['authorization', SECRET], ['content-type', 'application/json']]] }, + { type: 'stage.leaved', stageId: 1, facts: { 'response.http.headers': { $: 1 }, 'response.text': '中文 response' } }, + ].map(event => JSON.stringify(event)).join('\n'), }); -const redactedRecord = (id: string): DumpEdgeRecord => { +const redactedRecord = (id: string): DumpRecord => { const source = record(id); - return { - ...source, - request: { ...source.request, headers: [['authorization', MASK], ['content-type', 'application/json']] }, - response: { ...source.response, headers: [['set-cookie', MASK]], upstream: { ...source.response.upstream!, headers: [['api-key', MASK]] } }, - capture: { - ...source.capture!, - exchanges: [{ - ...source.capture!.exchanges[0]!, - request: { ...source.capture!.exchanges[0]!.request, headers: [['x-api-key', MASK]] }, - response: { ...source.capture!.exchanges[0]!.response!, headers: [['x-goog-api-key', MASK]] }, - }], - }, - }; + return { ...source, events: `${source.events.replace(SECRET, MASK)}\n` }; }; describe('request export', () => { @@ -43,7 +31,7 @@ describe('request export', () => { expect(new TextDecoder().decode(new Uint8Array(await exportRecords([record('one')], true).arrayBuffer()))).not.toContain(SECRET); }); - it('preserves raw bytes, completeness and errors in combined JSON', async () => { + it('preserves facts and protocol records in combined JSON', async () => { const records = [record('one'), record('two')]; expect(JSON.parse(await exportRecords(records, false).text())).toEqual({ format: 'floway-request-dump', version: 1, records: [redactedRecord('one'), redactedRecord('two')] }); }); diff --git a/apps/web/__tests__/components/requests/run-redact_test.ts b/apps/web/__tests__/components/requests/run-redact_test.ts index ead440d782..a98eeae7e8 100644 --- a/apps/web/__tests__/components/requests/run-redact_test.ts +++ b/apps/web/__tests__/components/requests/run-redact_test.ts @@ -7,8 +7,8 @@ it('redacts shared header string definitions while preserving unrelated payloads const secret = 'Bearer shared-token'.repeat(100); const events = [ { type: 'object', fromObjectId: 1, nodes: [[{ $: 2 }, { $: 4 }], ['authorization', { $: 3 }], secret, ['content-type', 'application/json']] }, - { type: 'stage.entered', stageId: 1, name: 'entry', parentStageId: null, facts: { 'ingress.http.headers': { $: 1 }, 'request.payload': { $: 5 } } }, { type: 'object', fromObjectId: 5, nodes: [{ text: 'payload remains exact' }] }, + { type: 'stage.entered', stageId: 1, name: 'entry', parentStageId: null, facts: { 'ingress.http.headers': { $: 1 }, 'request.payload': { $: 5 } } }, { type: 'stage.leaved', stageId: 1, facts: { 'response.http.headers': { $: 1 } } }, ]; const source = events.map(event => `${JSON.stringify(event)}\n`).join(''); diff --git a/apps/web/__tests__/components/requests/run-stages_test.tsx b/apps/web/__tests__/components/requests/run-stages_test.tsx index 9812d978f4..6f773d964f 100644 --- a/apps/web/__tests__/components/requests/run-stages_test.tsx +++ b/apps/web/__tests__/components/requests/run-stages_test.tsx @@ -68,4 +68,25 @@ describe('run stages', () => { expect(screen.queryByText(/upstream content/)).toBeNull(); }); + it('shows and copies the recorded Anthropic citation source without wire projection', async () => { + const writeText = vi.fn(async (_text: string) => {}); + Object.defineProperty(navigator, 'clipboard', { configurable: true, value: { writeText } }); + const citation = { type: 'search_result_location', source: 'https://citation.test/anthropic', title: 'Reference', search_result_index: 0, start_block_index: 0, end_block_index: 1 }; + const frame = { type: 'event' as const, event: { type: 'content_block_delta', index: 0, delta: { type: 'citations_delta', citation } } }; + const events = toNdjson(encodeRun([ + { type: 'stage.entered', stageId: 1, name: 'emit', parentStageId: null, facts: {} }, + { type: 'stage.leaved', stageId: 1, facts: { 'response.chat.clientFrames': streamFact(1) } }, + { type: 'stream.frame', streamId: 1, frames: [frame] }, + { type: 'stream.end', streamId: 1 }, + ])); + renderInApp(); + await screen.findByTestId('state'); + fireEvent.click(screen.getByRole('combobox', { name: 'Stage view' })); + fireEvent.click(screen.getByRole('option', { name: 'Protocol frames' })); + expect(screen.getByText(/https:\/\/citation.test\/anthropic/)).toBeTruthy(); + fireEvent.click(screen.getAllByRole('button', { name: 'Copy' })[0]!); + await waitFor(() => expect(writeText).toHaveBeenCalledTimes(1)); + expect(JSON.parse(writeText.mock.calls[0]![0])).toEqual(frame); + }); + }); diff --git a/apps/web/__tests__/components/requests/run-stream_test.ts b/apps/web/__tests__/components/requests/run-stream_test.ts new file mode 100644 index 0000000000..7d95ad3998 --- /dev/null +++ b/apps/web/__tests__/components/requests/run-stream_test.ts @@ -0,0 +1,63 @@ +import { expect, test } from 'vitest'; + +import { clientStreamOf } from '../../../src/components/requests/run-stream'; +import { encodeRun, streamFact, toNdjson } from '@floway-dev/pipeline'; + +for (const collected of [true, false]) { + for (const ended of [true, false]) { + test(`selects client frames and their own completion (${collected ? 'collected' : 'streamed'}, ${ended ? 'ended' : 'interrupted'})`, () => { + const selected = { type: 'stage.leaved' as const, stageId: 1, facts: { 'response.chat.clientFrames': streamFact(3) } }; + const events = toNdjson(encodeRun([ + { type: 'stage.entered', stageId: 1, name: 'emit', parentStageId: null, facts: {} }, + { type: 'stream.frame', streamId: 1, frames: [{ type: 'event', event: { text: 'upstream' } }] }, + { type: 'stream.end', streamId: 1 }, + ...collected ? [] : [selected], + { type: 'stream.frame', streamId: 3, frames: [{ type: 'event', event: { text: 'client' } }, { type: 'done' }] }, + ...ended ? [{ type: 'stream.end' as const, streamId: 3 }] : [], + { type: 'stream.frame', streamId: 2, frames: [{ type: 'event', event: { text: 'intermediate' } }] }, + { type: 'stream.end', streamId: 2 }, + ...collected ? [selected] : [], + ])); + expect(clientStreamOf(events)).toEqual({ + frames: [ + { type: 'event', event: { text: 'client' } }, + { type: 'done' }, + ], + ended, + }); + }); + } +} + +test('keeps a selected empty stream and distinguishes it from no client stream', () => { + expect(clientStreamOf(toNdjson(encodeRun([ + { type: 'stage.entered', stageId: 1, name: 'emit', parentStageId: null, facts: {} }, + { type: 'stage.leaved', stageId: 1, facts: { 'response.chat.clientFrames': streamFact(1) } }, + { type: 'stream.end', streamId: 1 }, + ])))).toEqual({ frames: [], ended: true }); + expect(clientStreamOf(toNdjson(encodeRun([ + { type: 'stage.entered', stageId: 1, name: 'emit', parentStageId: null, facts: {} }, + { type: 'stage.leaved', stageId: 1, facts: { 'response.chat.clientFrames': null } }, + ])))).toBeNull(); +}); + +test('selects the rendered classic Completions stream and its own end proof', () => { + const events = toNdjson(encodeRun([ + { type: 'stage.entered', stageId: 1, name: 'emitOpenAICompletions', parentStageId: null, facts: {} }, + { type: 'stream.frame', streamId: 1, frames: [{ type: 'event', event: { choices: [{ text: 'upstream' }] } }] }, + { type: 'stream.frame', streamId: 2, frames: [{ type: 'event', event: { id: 'completion-1', object: 'text_completion', created: 1, model: 'm', choices: [{ index: 0, text: 'client', finish_reason: 'stop', logprobs: null }] } }, { type: 'done' }] }, + { type: 'stream.end', streamId: 2 }, + { type: 'stage.leaved', stageId: 1, facts: { 'response.openaiCompletions.rendered': streamFact(2) } }, + ])); + expect(clientStreamOf(events)).toMatchObject({ frames: [{ event: { choices: [{ text: 'client' }] } }, { type: 'done' }], ended: true }); +}); + +test('does not select a JSON Completions extension that resembles a decoded stream reference', () => { + const events = toNdjson(encodeRun([ + { type: 'stage.entered', stageId: 1, name: 'emitOpenAICompletions', parentStageId: null, facts: {} }, + { type: 'stream.frame', streamId: 7, frames: [{ type: 'event', event: { text: 'unselected' } }] }, + { type: 'stream.end', streamId: 7 }, + { type: 'stage.leaved', stageId: 1, facts: { 'response.openaiCompletions.rendered': { stream: 7, $stream: 7 } } }, + ])); + expect(clientStreamOf(events)).toBeNull(); +}); diff --git a/apps/web/__tests__/components/requests/stream-render_test.ts b/apps/web/__tests__/components/requests/stream-render_test.ts index 64071b8c97..ad64b59fa1 100644 --- a/apps/web/__tests__/components/requests/stream-render_test.ts +++ b/apps/web/__tests__/components/requests/stream-render_test.ts @@ -1,39 +1,64 @@ -import { describe, expect, it } from 'vitest'; +import { expect, test } from 'vitest'; -import { collectKindFromTargetApi, streamEndedCleanly } from '../../../src/components/requests/stream-render'; -import type { DumpStreamEvent } from '@floway-dev/gateway/dump-types'; +import { clientStreamOf } from '../../../src/components/requests/run-stream'; +import { collectStream } from '../../../src/components/requests/stream-render'; +import { encodeRun, streamFact, toNdjson, type Event } from '@floway-dev/pipeline'; +import { anthropicMessagesEventToSsePayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -const event = (frame: DumpStreamEvent['frame']): DumpStreamEvent => ({ frame, ts: 1 }); +const recorded = (frames: readonly ProtocolFrame[], ended: boolean) => { + const events: Event[] = [ + { type: 'stage.entered', stageId: 1, name: 'emit', parentStageId: null, facts: {} }, + { type: 'stage.leaved', stageId: 1, facts: { 'response.chat.clientFrames': streamFact(1) } }, + { type: 'stream.frame', streamId: 1, frames }, + ...ended ? [{ type: 'stream.end' as const, streamId: 1 }] : [], + ]; + return clientStreamOf(toNdjson(encodeRun(events)))!; +}; -describe('captured stream completion', () => { - it('recognizes the protocol done frame', () => { - expect(streamEndedCleanly([ - event({ type: 'event', event: { value: 'partial' } }), - event({ type: 'done' }), - ])).toBe(true); - }); +const chunk = eventFrame({ id: 'chat-1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { role: 'assistant', content: 'hello' }, finish_reason: 'stop' }] }); - it('marks a recording with no done frame as incomplete', () => { - expect(streamEndedCleanly([ - event({ type: 'event', event: { value: 'partial' } }), - ])).toBe(false); - }); +test('collects normal Chat Completions EOF without inventing a done frame', async () => { + const stream = recorded([chunk], true); + const collected = await collectStream('openai-chat-completions', stream); + expect(collected).toMatchObject({ result: { choices: [{ message: { content: 'hello' } }] }, error: null, truncated: false }); + expect(stream.frames).toHaveLength(1); + expect(stream.frames.map(frame => frame.type)).toEqual(['event']); }); -// The upstream (pre-translation) view dispatches its serializer by the TARGET -// protocol the inner attempt spoke, not by the client path (which names the -// source protocol). `collectKindFromTargetApi` mirrors `detectCollectKind` -// for that purpose; null on native turns (no targetApi) and unknown values. -describe('collectKindFromTargetApi', () => { - it('maps each ChatTargetApi to its CollectKind', () => { - expect(collectKindFromTargetApi('anthropicMessages')).toBe('anthropic-messages'); - expect(collectKindFromTargetApi('openaiResponses')).toBe('openai-responses'); - expect(collectKindFromTargetApi('openaiChatCompletions')).toBe('openai-chat-completions'); +for (const terminal of [false, true]) { + test(`marks an interrupted recording incomplete even with ${terminal ? 'a terminal frame' : 'partial output'}`, async () => { + const stream = recorded([chunk, ...terminal ? [{ type: 'done' as const }] : []], false); + expect(await collectStream('openai-chat-completions', stream)).toMatchObject({ error: null, truncated: true }); }); +} - it('returns null for absent or unknown targetApi (native turns)', () => { - expect(collectKindFromTargetApi(null)).toBeNull(); - expect(collectKindFromTargetApi(undefined)).toBeNull(); - expect(collectKindFromTargetApi('unknown')).toBeNull(); - }); +test('separates an ended protocol error from a truncated recording', async () => { + const stream = recorded([eventFrame({ type: 'error', error: { type: 'api_error', message: 'upstream failed' } })], true); + expect(await collectStream('anthropic-messages', stream)).toMatchObject({ result: null, error: expect.stringContaining('upstream failed'), truncated: false }); +}); + +test('collects final Anthropic citation wire fields into the canonical result', async () => { + const citation = { type: 'search_result_location' as const, url: 'https://example.com/source', title: 'Reference', search_result_index: 0, start_block_index: 0, end_block_index: 1, cited_text: 'evidence' }; + const events: AnthropicMessagesStreamEvent[] = [ + { type: 'message_start', message: { id: 'msg-1', type: 'message', role: 'assistant', content: [], model: 'm', stop_reason: null, stop_sequence: null, usage: { input_tokens: 1, output_tokens: 0 } } }, + { type: 'content_block_start', index: 0, content_block: { type: 'text', text: 'answer', citations: [citation] } }, + { type: 'content_block_delta', index: 0, delta: { type: 'citations_delta', citation } }, + { type: 'content_block_stop', index: 0 }, + { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 1 } }, + { type: 'message_stop' }, + ]; + const wire = events.map(event => eventFrame(anthropicMessagesEventToSsePayload(event))); + const stream = recorded(wire, true); + expect(await collectStream('anthropic-messages', stream)).toMatchObject({ result: { content: [{ type: 'text', text: 'answer', citations: [citation, citation] }] }, error: null, truncated: false }); +}); + +test('collects the classic Completions stream published by its rendered fact', async () => { + const stream = clientStreamOf(toNdjson(encodeRun([ + { type: 'stage.entered', stageId: 1, name: 'emitOpenAICompletions', parentStageId: null, facts: {} }, + { type: 'stream.frame', streamId: 4, frames: [eventFrame({ id: 'completion-1', object: 'text_completion', created: 1, model: 'm', choices: [{ index: 0, text: 'classic answer', finish_reason: 'stop', logprobs: null }] }), { type: 'done' }] }, + { type: 'stream.end', streamId: 4 }, + { type: 'stage.leaved', stageId: 1, facts: { 'response.openaiCompletions.rendered': streamFact(4) } }, + ])))!; + expect(await collectStream('openai-completions', stream)).toMatchObject({ result: { choices: [{ text: 'classic answer' }] }, error: null, truncated: false }); }); diff --git a/apps/web/__tests__/components/requests/ttft_test.tsx b/apps/web/__tests__/components/requests/ttft_test.tsx index 2cc4831271..e40e0b131f 100644 --- a/apps/web/__tests__/components/requests/ttft_test.tsx +++ b/apps/web/__tests__/components/requests/ttft_test.tsx @@ -7,8 +7,7 @@ import type { ApiKey } from '../../../src/api/types'; import { RequestDetailPanel } from '../../../src/components/requests/detail'; import { RequestListPanel } from '../../../src/components/requests/list'; import { renderInApp } from '../../render'; -import type { DumpMetadata } from '@floway-dev/dump/types'; -import type { DumpEdgeRecord } from '@floway-dev/gateway/dump-types'; +import type { DumpMetadata, DumpRecord } from '@floway-dev/dump/types'; vi.mock('../../../src/components/ui/body-editor', () => ({ default: ({ text, toolbarStart }: { text: string; toolbarStart?: ReactNode }) => ( @@ -46,15 +45,9 @@ const makeRecordMeta = (id: string, ttftMs: number | null): DumpMetadata => ({ upstream: null, }); -const makeFullRecord = (id: string, ttftMs: number | null): DumpEdgeRecord => ({ - shape: 'edge', +const makeFullRecord = (id: string, ttftMs: number | null): DumpRecord => ({ meta: makeRecordMeta(id, ttftMs), - request: { method: 'POST', path: '/v1/chat/completions', headers: [], body: { encoding: 'utf8', data: '{}' } }, - response: { status: 200, headers: [], body: { type: 'stream', events: [] } }, - capture: { - exchanges: [], - response: { body: { encoding: 'utf8', data: '' }, complete: true, error: null }, - }, + events: '', }); describe('requests TTFT UI presentation', () => { @@ -92,7 +85,6 @@ describe('requests TTFT UI presentation', () => { recordId="detail-streamed" error={null} collected={null} - upstreamCollected={null} retainLastRecord={false} />, ); @@ -107,7 +99,6 @@ describe('requests TTFT UI presentation', () => { recordId="detail-nonstreamed" error={null} collected={null} - upstreamCollected={null} retainLastRecord={false} />, ); diff --git a/apps/web/__tests__/lib/source-mapped-stack-status_test.tsx b/apps/web/__tests__/lib/source-mapped-stack-status_test.tsx index 6285974354..f2d444d0bd 100644 --- a/apps/web/__tests__/lib/source-mapped-stack-status_test.tsx +++ b/apps/web/__tests__/lib/source-mapped-stack-status_test.tsx @@ -1,4 +1,4 @@ -import { act, screen } from '@testing-library/react'; +import { act, screen, waitFor } from '@testing-library/react'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { useSourceMappedStack } from '../../src/lib/source-mapped-stack'; @@ -60,8 +60,10 @@ describe('what the error page is told about its trace', () => { it('replaces the trace once the maps land', async () => { vi.stubEnv('DEV', false); renderInApp(); - await settle(); - expect(read()).toEqual({ status: 'settled', stack: 'Error: boom\n at handler (/src/first.ts:1:1)' }); + // Waited for rather than drained once: restoring a frame fetches the script and then its + // map, and reading each body is a real stream read rather than a microtask, so how many + // turns the chain takes is the machine's business and not this test's. + await waitFor(() => expect(read()).toEqual({ status: 'settled', stack: 'Error: boom\n at handler (/src/first.ts:1:1)' })); }); it('keeps the minified trace and says so when the maps cannot be read', async () => { @@ -69,8 +71,7 @@ describe('what the error page is told about its trace', () => { vi.stubGlobal('fetch', vi.fn(() => Promise.resolve(new Response('', { status: 503 })))); vi.spyOn(console, 'error').mockImplementation(() => {}); renderInApp(); - await settle(); - expect(read()).toEqual({ status: 'failed', stack: RAW }); + await waitFor(() => expect(read()).toEqual({ status: 'failed', stack: RAW })); }); it('goes back to waiting when a second failure replaces the first', async () => { @@ -102,9 +103,11 @@ describe('what the error page is told about its trace', () => { const { rerender } = renderInApp(); const second = `Error: later\n at other (${SCRIPT}:1:1)`; rerender(); - await settle(); - expect(read()).toEqual({ status: 'settled', stack: 'Error: later\n at other (/src/first.ts:1:1)' }); + await waitFor(() => expect(read()).toEqual({ status: 'settled', stack: 'Error: later\n at other (/src/first.ts:1:1)' })); + // The superseded restoration finishes here, and what it must not do is write. A settle is + // the right instrument for that one: the assertion is that nothing changed, so draining + // everything the release queued is exactly the window it could have changed something in. await act(async () => { releaseFirst(new Response('const a=1;\n//# sourceMappingURL=chunk.js.map')); }); diff --git a/apps/web/__tests__/routes/dashboard-monitor-requests_test.tsx b/apps/web/__tests__/routes/dashboard-monitor-requests_test.tsx index c5cfb1bcc0..fbd0d35ed5 100644 --- a/apps/web/__tests__/routes/dashboard-monitor-requests_test.tsx +++ b/apps/web/__tests__/routes/dashboard-monitor-requests_test.tsx @@ -11,7 +11,6 @@ import { renderInApp } from '../render'; // all, which the page reports differently. const loaderData = { collected: null, - upstreamCollected: null, error: 'HTTP 500', keys: [], record: null, diff --git a/apps/web/src/components/requests/batch-export.tsx b/apps/web/src/components/requests/batch-export.tsx index 3adafbc5b4..6b84fea37e 100644 --- a/apps/web/src/components/requests/batch-export.tsx +++ b/apps/web/src/components/requests/batch-export.tsx @@ -6,7 +6,7 @@ import { fluentComponents } from '../../fluent'; import { useTranslation } from '../../i18n/translation'; import { Checkbox } from '../ui/fluent-form-controls'; import { OutcomeMessageBar } from '../ui/outcome-message-bar'; -import type { DumpRecord } from '@floway-dev/gateway/dump-types'; +import type { DumpRecord } from '@floway-dev/dump/types'; const { Button, Menu, MenuItem, MenuList, MenuPopover, MenuTrigger } = fluentComponents; diff --git a/apps/web/src/components/requests/body-render.ts b/apps/web/src/components/requests/body-render.ts index d81c7f4a78..3a1e0b451b 100644 --- a/apps/web/src/components/requests/body-render.ts +++ b/apps/web/src/components/requests/body-render.ts @@ -1,7 +1,5 @@ -import { decodeWebBase64, decodeWebBase64BinaryString, encodeBase64BinaryString } from '../../lib/base-encoding'; -import { errorMessage } from '../../lib/error-message'; -import type { DumpBody } from '@floway-dev/gateway/dump-types'; -import { isTextualMediaType, parseMediaType } from '@floway-dev/protocols/common'; +// What a code block renders: the text, what a copy button copies, and the failure to name if +// the text could not be produced. export interface RenderedBody { text: string; @@ -9,75 +7,3 @@ export interface RenderedBody { decodeError: string | null; isJson: boolean; } - -export const EMPTY_BODY: RenderedBody = { text: '', copyText: '', decodeError: null, isJson: false }; - -export const contentTypeOf = (headers: Array<[string, string]>): string => { - return headers.find(([name]) => name.toLowerCase() === 'content-type')?.[1] ?? ''; -}; - -export const renderBody = (body: DumpBody, contentType: string): RenderedBody => { - if (!body.data) return EMPTY_BODY; - if (parseMediaType(contentType)?.type === 'multipart' && body.encoding === 'base64') { - const multipart = renderMultipart(body.data, contentType); - if (multipart !== null) { - return { text: multipart, copyText: body.data, decodeError: null, isJson: false }; - } - } - - let text = body.data; - if (body.encoding === 'base64') { - try { - const bytes = decodeWebBase64(body.data); - text = new TextDecoder('utf-8', { fatal: true }).decode(bytes); - } catch (error) { - return { - text: body.data, - copyText: body.data, - decodeError: errorMessage(error), - isJson: false, - }; - } - } - - try { - const pretty = JSON.stringify(JSON.parse(text) as unknown, null, 2); - return { text: pretty, copyText: pretty, decodeError: null, isJson: true }; - } catch { - return { text, copyText: text, decodeError: null, isJson: false }; - } -}; - -const renderMultipart = (base64: string, contentType: string): string | null => { - const boundaryMatch = /;\s*boundary=(?:"([^"]+)"|([^;\s]+))/i.exec(contentType); - const boundary = boundaryMatch?.[1] ?? boundaryMatch?.[2]; - if (!boundary) return null; - - try { - // The wire stays a binary string with one code unit per byte, so slicing it - // and encoding a part back through the byte codec round-trips. Decoding it - // through a text encoding first does not: the Encoding Standard resolves - // latin1 to windows-1252 (https://encoding.spec.whatwg.org/#names-and-labels), - // which maps 0x80-0x9F -- a PNG signature leads with 0x89 -- to code points - // above 255. - const wire = decodeWebBase64BinaryString(base64); - const chunks = wire.split(`--${boundary}`); - if (chunks.length < 3) return null; - - const rendered = chunks.slice(1, -1).map(chunk => { - const part = chunk.replace(/^\r\n/, '').replace(/\r\n$/, ''); - const separator = part.indexOf('\r\n\r\n'); - if (separator < 0) throw new Error('Malformed multipart body'); - const headers = part.slice(0, separator); - const data = part.slice(separator + 4); - const type = /^content-type:\s*(.+)$/im.exec(headers)?.[1]?.trim() ?? ''; - const textual = !type || isTextualMediaType(type); - if (textual) return `${headers}\r\n\r\n${new TextDecoder().decode(Uint8Array.from(data, c => c.charCodeAt(0)))}`; - const encoded = encodeBase64BinaryString(data).replace(/.{76}(?=.)/g, '$&\n'); - return `${headers}\r\n\r\n[binary, ${data.length} bytes, content-type=${type}]\r\n${encoded}`; - }); - return `--${boundary}\r\n${rendered.join(`\r\n--${boundary}\r\n`)}\r\n--${boundary}--\r\n`; - } catch { - return null; - } -}; diff --git a/apps/web/src/components/requests/detail.tsx b/apps/web/src/components/requests/detail.tsx index 9e48c4a8e9..9f64a48dcf 100644 --- a/apps/web/src/components/requests/detail.tsx +++ b/apps/web/src/components/requests/detail.tsx @@ -1,164 +1,42 @@ -import { ArrowDownloadRegular, EyeOffRegular, EyeRegular, FlashRegular, InfoRegular, TimerRegular, WarningRegular } from '@fluentui/react-icons'; +import { ArrowDownloadRegular, FlashRegular, TimerRegular } from '@fluentui/react-icons'; import { lazy, Suspense, useMemo, useState } from 'react'; -import { contentTypeOf, renderBody } from './body-render'; -import { EventList, RenderedEventList } from './events'; +import { RenderedEventList } from './events'; import { downloadRecords } from './export'; import { errorLabel, requestSeverity } from './format'; -import { isSensitiveHeader, redactHeaderValue } from './header-redact'; import { redactRunHeaders } from './run-redact'; import { renderRunEvents } from './run-render'; import { RunStages } from './run-stages'; -import { collectKindFromTargetApi, detectCollectKind, type CollectedStream } from './stream-render'; +import type { CollectedStream } from './stream-render'; import { fluentComponents } from '../../fluent'; import { useTranslation } from '../../i18n/translation'; import { formatDuration } from '../../lib/format-duration'; -import { DialogShell } from '../ui/dialog-shell'; import { EmptyStateLine } from '../ui/empty-state'; import { Dropdown } from '../ui/fluent-form-controls'; -import { HttpMethodBadge, HttpStatusBadge } from '../ui/http-badge'; +import { HttpStatusBadge } from '../ui/http-badge'; import { OutcomeMessageBar } from '../ui/outcome-message-bar'; import { PANEL_BAND_CLASS } from '../ui/panel'; import { TooltipIconButton } from '../ui/tooltip-icon-button'; -import { copyOutcomeIcon, useCopyLabel, useCopyToClipboard } from '../ui/use-copy-to-clipboard'; -import type { DumpMetadata } from '@floway-dev/dump/types'; -import type { DumpRecord, DumpResponseBody, DumpRunRecord, DumpEdgeRecord } from '@floway-dev/gateway/dump-types'; +import type { DumpMetadata, DumpRecord } from '@floway-dev/dump/types'; const BodyEditor = lazy(() => import('../ui/body-editor')); -const { Button, DialogActions, DialogTitle, Option, Spinner, Text, Tooltip } = fluentComponents; +const { Option, Spinner, Text, Tooltip } = fluentComponents; -type Source = 'request' | 'upstreamRequest' | 'upstreamResponse' | 'response'; - -function CopyButton({ text }: { text: string }) { - const { t } = useTranslation(); - const { copy, outcomeFor } = useCopyToClipboard(); - const copyLabel = useCopyLabel(); - return copy(text)} />; -} - -function HeaderTable({ headers }: { headers: Array<[string, string]> }) { - const { t } = useTranslation(); - const [revealed, setRevealed] = useState>(new Set()); - return - {headers.map(([name, value], index) => - - - )} -
{name} - {isSensitiveHeader(name) && !revealed.has(index) ? redactHeaderValue(value) : value} - {isSensitiveHeader(name) && : } - label={revealed.has(index) ? t('dashboard.requests.hideValue') : t('dashboard.requests.revealValue')} - onClick={() => setRevealed(current => { const next = new Set(current); if (next.has(index)) next.delete(index); else next.add(index); return next; })} - />} -
; -} - -export function RequestDetailPanel({ collected, upstreamCollected, error, record, recordId, retainLastRecord }: { +export function RequestDetailPanel({ collected, error, record, recordId, retainLastRecord }: { collected: CollectedStream | null; - upstreamCollected: CollectedStream | null; error: string | null; record: DumpRecord | null; recordId: string | null; retainLastRecord: boolean; }) { - const [shown, setShown] = useState({ collected, upstreamCollected, error, record, recordId }); - const incoming = retainLastRecord && recordId === null ? shown : { collected, upstreamCollected, error, record, recordId }; + const [shown, setShown] = useState({ collected, error, record, recordId }); + const incoming = retainLastRecord && recordId === null ? shown : { collected, error, record, recordId }; if (shown.record !== incoming.record || shown.error !== incoming.error || shown.recordId !== incoming.recordId) setShown(incoming); const { t } = useTranslation(); if (!shown.recordId) return {t('dashboard.requests.selectPrompt')}; if (shown.error) return {shown.error}; if (!shown.record) return null; - if (shown.record.shape === 'run') return ; - return ; -} - -function RecordDetail({ record, collected, upstreamCollected }: { record: DumpEdgeRecord; collected: CollectedStream | null; upstreamCollected: CollectedStream | null }) { - const { t } = useTranslation(); - const [source, setSource] = useState('response'); - const [view, setView] = useState('collected'); - const [detailsOpen, setDetailsOpen] = useState(false); - const exchanges = record.capture?.exchanges ?? []; - const [index, setIndex] = useState(Math.max(0, exchanges.length - 1)); - const upstream = source === 'upstreamRequest' || source === 'upstreamResponse'; - const request = source === 'request' || source === 'upstreamRequest'; - const exchange = upstream ? exchanges[index] : undefined; - const legacy = record.response.upstream; - const headers = useMemo(() => upstream ? (request ? exchange?.request.headers : exchange?.response?.headers ?? legacy?.headers) ?? [] - : request ? record.request.headers : record.response.headers, [exchange, legacy, record, request, upstream]); - const status = upstream ? exchange?.response?.status ?? legacy?.status ?? null : record.response.status; - const endpoint = upstream ? exchange?.request.url : record.request.path; - const method = upstream ? exchange?.request.method : record.request.method; - const raw = request ? undefined : upstream ? exchange?.response ?? undefined : record.capture?.response; - const result = request ? null : upstream ? (!exchange || index === exchanges.length - 1 ? upstreamCollected : null) : collected; - const kind = upstream ? collectKindFromTargetApi(record.meta.targetApi) : detectCollectKind(record.meta.path); - const body = useMemo(() => upstream - ? request ? exchange ? { type: 'bytes', body: exchange.request.body } : { type: 'none' } - : (!exchange || index === exchanges.length - 1) && legacy ? legacy.body : raw ? { type: 'bytes', body: raw.body } : { type: 'none' } - : request ? { type: 'bytes', body: record.request.body } : record.response.body, [upstream, request, exchange, index, exchanges.length, legacy, raw, record]); - const displayed = useMemo(() => { - if (view === 'raw' && raw) return { text: raw.body.data, isJson: false, decodeError: null }; - if (body.type === 'bytes') return renderBody(body.body, contentTypeOf(headers)); - return { text: result?.result == null ? '' : JSON.stringify(result.result, null, 2), isJson: true, decodeError: null }; - }, [body, headers, raw, result, view]); - const diagnostics = [...new Set([ - errorLabel(record.meta.error), exchange?.error, raw?.error, result?.error, - raw && !raw.complete ? t('dashboard.requests.partialCapture') : null, - result?.truncated && !result.error ? t('dashboard.requests.truncatedStream') : null, - displayed.decodeError ? t('dashboard.requests.decodeError', { error: displayed.decodeError }) : null, - ].filter((value): value is string => Boolean(value)))]; - const labels: Record = { - request: t('dashboard.requests.clientRequest'), upstreamRequest: t('dashboard.requests.upstreamRequest'), - upstreamResponse: t('dashboard.requests.upstreamResponse'), response: t('dashboard.requests.clientResponse'), - }; - const viewLabels: Record = { - collected: t(body.type === 'stream' ? 'dashboard.requests.collected' : 'dashboard.requests.formattedBody'), events: t('dashboard.requests.events', { count: body.type === 'stream' ? body.events.length : 0 }), - raw: raw?.body.encoding === 'base64' ? t('dashboard.requests.base64') : t('dashboard.requests.raw'), - }; - const chooseSource = (value: string | undefined) => { - if (value === undefined) return; - setSource(value as Source); - setView('collected'); - }; - const toolbar =
- {upstream && exchanges.length > 1 && { setIndex(Number(data.optionValue)); setView('collected'); }}> - {exchanges.map((item, i) => )} - } - {(body.type === 'stream' || raw) ? { if (data.optionValue) setView(data.optionValue); }}> - - {body.type === 'stream' && } - {raw && } - : {t(request ? 'dashboard.requests.requestBody' : 'dashboard.requests.responseBody')}} -
; - return
-
- chooseSource(data.optionValue)}> - {Object.entries(labels).map(([value, label]) => )} - - {status ?? t('dashboard.requests.noStatus')} - - - } label={t('dashboard.requests.exportRecord')} onClick={() => downloadRecords([record])} /> -
-
- {view === 'events' && body.type === 'stream' - ? - : }>} -
- setDetailsOpen(data.open)} title={{labels[source]}} actions={}> -
{endpoint ?? record.request.path}
- {diagnostics.map((message, i) => {message})} - {upstream && !exchange && !legacy && {t('dashboard.requests.noUpstreamCapture')}} -
{t('dashboard.requests.headers', { count: String(headers.length) })} `${name}: ${value}`).join('\n')} />
- - -
; + return ; } function RecordTiming({ meta }: { meta: DumpMetadata }) { @@ -179,7 +57,7 @@ function RecordTiming({ meta }: { meta: DumpMetadata }) { ; } -function RunRecordDetail({ record }: { record: DumpRunRecord }) { +function RunRecordDetail({ record, collected }: { record: DumpRecord; collected: CollectedStream | null }) { const { t } = useTranslation(); const [view, setView] = useState('stages'); const redacted = useMemo(() => redactRunHeaders(record.events), [record.events]); @@ -187,20 +65,25 @@ function RunRecordDetail({ record }: { record: DumpRunRecord }) { const events = useMemo(() => view === 'events' ? renderRunEvents(redacted).map(event => ({ event: `${event.type} ${event.subject ?? ''}`.trim(), text: event.text, parseError: event.parseError, })) : [], [redacted, view]); - const failure = errorLabel(record.meta.error); + const failure = errorLabel(record.meta.error) ?? collected?.error; + const label = view === 'stages' ? t('dashboard.requests.stages') : view === 'events' ? t('dashboard.requests.events', { count }) : t('dashboard.requests.collected'); return
- setView(data.optionValue!)}> + setView(data.optionValue!)}> + {collected?.result != null && } {record.meta.status ?? t('dashboard.requests.noStatus')} } label={t('dashboard.requests.exportRecord')} onClick={() => downloadRecords([record])} />
{failure && {failure}} + {collected?.truncated && {t('dashboard.requests.incompleteRecording')}}
- {view === 'stages' ? : {t('dashboard.requests.events', { count })}} emptyText={t('dashboard.requests.noRunEvents')} />} + {view === 'stages' ? : view === 'events' + ? {t('dashboard.requests.events', { count })}} emptyText={t('dashboard.requests.noRunEvents')} /> + : }>}
; } diff --git a/apps/web/src/components/requests/events.tsx b/apps/web/src/components/requests/events.tsx index fd54317fed..c4fc4d8d97 100644 --- a/apps/web/src/components/requests/events.tsx +++ b/apps/web/src/components/requests/events.tsx @@ -2,7 +2,6 @@ import { ChevronDownRegular, ChevronRightRegular, DismissRegular, MoreHorizontal import { useCallback, useMemo, useRef, useState, type ReactNode } from 'react'; import { List, useDynamicRowHeight, type ListImperativeAPI, type RowComponentProps } from 'react-window'; -import { renderStreamEvents, streamEventsCopyText, type CollectKind, type RenderedStreamEvent } from './stream-render'; import { fluentComponents } from '../../fluent'; import { useTranslation } from '../../i18n/translation'; import { CodeBlock } from '../ui/code-block'; @@ -12,11 +11,15 @@ import { PANEL_BAND_CLASS } from '../ui/panel'; import { useScrollAreaHost } from '../ui/scroll-area'; import { TooltipIconButton } from '../ui/tooltip-icon-button'; import { copyOutcomeIcon, useCopyLabel, useCopyToClipboard } from '../ui/use-copy-to-clipboard'; -import type { DumpStreamEvent } from '@floway-dev/gateway/dump-types'; const { Button, Menu, MenuItem, MenuItemCheckbox, MenuList, MenuPopover, MenuTrigger, Text, mergeClasses } = fluentComponents; -export type RenderedEvent = Omit & { timestamp?: number }; +export interface RenderedEvent { + event: string | null; + text: string; + parseError: string | null; + timestamp?: number; +} type IndexedEvent = RenderedEvent & { index: number }; interface EventRowProps { events: IndexedEvent[]; @@ -38,7 +41,7 @@ function EventRow({ index, style, ariaAttributes, events, collapsed, toggle, wra wrap={wrap} copyOutcome={outcomeFor()} onCopy={() => copy(event.text)} - language={event.parseError || event.text === '[DONE]' ? 'plain' : 'json'} + language={event.parseError ? 'plain' : 'json'} header={
{event.timestamp !== undefined && +{event.timestamp.toFixed(event.timestamp < 1 ? 3 : 0)}ms} @@ -48,12 +51,6 @@ function EventRow({ index, style, ariaAttributes, events, collapsed, toggle, wra
; } -export function EventList({ events, kind, toolbarStart }: { events: DumpStreamEvent[]; kind: CollectKind | null; toolbarStart: ReactNode }) { - const rendered = useMemo(() => renderStreamEvents(kind, events), [events, kind]); - const copyText = useMemo(() => streamEventsCopyText(kind, events), [events, kind]); - return ; -} - export function RenderedEventList({ events, copyText, toolbarStart, emptyText }: { events: RenderedEvent[]; copyText: string; toolbarStart: ReactNode; emptyText?: string }) { const { t } = useTranslation(); const [query, setQuery] = useState(''); diff --git a/apps/web/src/components/requests/export.ts b/apps/web/src/components/requests/export.ts index ec61f64aa5..044884f03a 100644 --- a/apps/web/src/components/requests/export.ts +++ b/apps/web/src/components/requests/export.ts @@ -1,34 +1,7 @@ -import { isSensitiveHeader, redactHeaderValue } from './header-redact'; import { redactRunHeaders } from './run-redact'; -import type { DumpRecord } from '@floway-dev/gateway/dump-types'; +import type { DumpRecord } from '@floway-dev/dump/types'; -// The dump API serves headers verbatim so the dashboard's reveal affordance -// keeps working, but an exported file leaves the operator's machine, so every -// credential header is masked to its redacted form before serialization. -const redactHeaders = (headers: Array<[string, string]>): Array<[string, string]> => - headers.map(([name, value]) => isSensitiveHeader(name) ? [name, redactHeaderValue(value)] as [string, string] : [name, value]); - -const redactRecord = (record: DumpRecord): DumpRecord => { - if (record.shape === 'run') return { ...record, events: redactRunHeaders(record.events) }; - const capture = record.capture === undefined ? undefined : { - ...record.capture, - exchanges: record.capture.exchanges.map(exchange => ({ - ...exchange, - request: { ...exchange.request, headers: redactHeaders(exchange.request.headers) }, - response: exchange.response === null ? null : { ...exchange.response, headers: redactHeaders(exchange.response.headers) }, - })), - }; - return { - ...record, - request: { ...record.request, headers: redactHeaders(record.request.headers) }, - response: { - ...record.response, - headers: redactHeaders(record.response.headers), - ...(record.response.upstream !== undefined ? { upstream: { ...record.response.upstream, headers: redactHeaders(record.response.upstream.headers) } } : {}), - }, - ...(capture === undefined ? {} : { capture }), - }; -}; +const redactRecord = (record: DumpRecord): DumpRecord => ({ ...record, events: redactRunHeaders(record.events) }); const serializeRecord = (record: DumpRecord): string => JSON.stringify({ format: 'floway-request-dump', version: 1, record: redactRecord(record) }, null, 2); diff --git a/apps/web/src/components/requests/run-stream.ts b/apps/web/src/components/requests/run-stream.ts new file mode 100644 index 0000000000..d20a1d7f98 --- /dev/null +++ b/apps/web/src/components/requests/run-stream.ts @@ -0,0 +1,34 @@ +import { createRunReader, type DumpEvent } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; + +export interface RecordedClientStream { + readonly frames: readonly ProtocolFrame[]; + readonly ended: boolean; +} + +// Collected responses consume frames before the edge publishes its identifying fact; +// streaming responses consume them afterwards. Resolve the complete record before selecting. +export const clientStreamOf = (ndjson: string): RecordedClientStream | null => { + const read = createRunReader(); + const streams = new Map[]; ended: boolean }>(); + let client: number | null = null; + for (const line of ndjson.split('\n')) { + if (line.length === 0) continue; + const event = JSON.parse(line) as DumpEvent; + const decoded = read(event); + if ((event.type === 'stage.entered' || event.type === 'stage.leaved') && event.facts !== undefined) { + for (const key of ['response.chat.clientFrames', 'response.openaiCompletions.rendered']) { + if (!(key in event.facts)) continue; + const selected = event.facts[key]; + client = selected !== null && typeof selected === 'object' && '$stream' in selected ? selected.$stream as number : null; + } + } + if (event.type === 'stream.frame' || event.type === 'stream.end') { + const stream = streams.get(event.streamId) ?? { frames: [], ended: false }; + if (event.type === 'stream.end') stream.ended = true; + else for (const frame of decoded?.frames ?? []) stream.frames.push(frame as ProtocolFrame); + streams.set(event.streamId, stream); + } + } + return client === null ? null : streams.get(client) ?? { frames: [], ended: false }; +}; diff --git a/apps/web/src/components/requests/stream-render.ts b/apps/web/src/components/requests/stream-render.ts index 3e538fe688..a88f4f8e5f 100644 --- a/apps/web/src/components/requests/stream-render.ts +++ b/apps/web/src/components/requests/stream-render.ts @@ -1,28 +1,17 @@ +import type { RecordedClientStream } from './run-stream'; import { errorMessage } from '../../lib/error-message'; -import type { DumpStreamEvent } from '@floway-dev/gateway/dump-types'; -import { - collectAnthropicMessagesProtocolEventsToResult, - anthropicMessagesProtocolFrameToSSEFrame, -} from '@floway-dev/protocols/anthropic-messages'; -import type { ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; +import { collectAnthropicMessagesProtocolEventsToResult } from '@floway-dev/protocols/anthropic-messages'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; import { collectGeminiGenerateContentProtocolEventsToResult, - geminiGenerateContentProtocolFrameToSSEFrame, type GeminiGenerateContentStreamEvent, } from '@floway-dev/protocols/gemini-generate-content'; +import { collectOpenAIChatCompletionsProtocolEventsToResult } from '@floway-dev/protocols/openai-chat-completions'; import { - openaiChatCompletionsProtocolFrameToSSEFrame, - collectOpenAIChatCompletionsProtocolEventsToResult, -} from '@floway-dev/protocols/openai-chat-completions'; -import { - openaiCompletionsProtocolFrameToSSEFrame, reassembleOpenAICompletionsEvents, type OpenAICompletionsStreamEvent, } from '@floway-dev/protocols/openai-completions'; -import { - collectOpenAIResponsesProtocolEventsToResult, - openaiResponsesProtocolFrameToSSEFrame, -} from '@floway-dev/protocols/openai-responses'; +import { collectOpenAIResponsesProtocolEventsToResult } from '@floway-dev/protocols/openai-responses'; export type CollectKind = 'openai-completions' | 'openai-chat-completions' | 'anthropic-messages' | 'openai-responses' | 'gemini-generate-content'; @@ -32,13 +21,6 @@ export interface CollectedStream { truncated: boolean; } -export interface RenderedStreamEvent { - event: string | null; - text: string; - parseError: string | null; - timestamp: number; -} - export const detectCollectKind = (path: string): CollectKind | null => { if (path.includes('/messages')) return 'anthropic-messages'; if (path.includes('/responses')) return 'openai-responses'; @@ -48,85 +30,33 @@ export const detectCollectKind = (path: string): CollectKind | null => { return null; }; -// Mirror of `detectCollectKind` for the target protocol of a translated turn. -// The downstream view derives its kind from `meta.path` (the client path = -// source protocol); the upstream view cannot, because the client path names -// the source, not the target. `meta.targetApi` is stamped at capture time by -// `traverseTranslation` and names the target protocol the inner attempt spoke. -export const collectKindFromTargetApi = (api: string | null | undefined): CollectKind | null => { - if (api === 'anthropicMessages') return 'anthropic-messages'; - if (api === 'openaiResponses') return 'openai-responses'; - if (api === 'openaiChatCompletions') return 'openai-chat-completions'; - return null; -}; - -export const streamEndedCleanly = (events: DumpStreamEvent[]): boolean => - events.at(-1)?.frame.type === 'done'; - -const complete = (result: unknown, events: DumpStreamEvent[]): CollectedStream => - ({ result, error: null, truncated: !streamEndedCleanly(events) }); - -export const collectStream = async (kind: CollectKind, events: DumpStreamEvent[]): Promise => { +export const collectStream = async (kind: CollectKind, { frames, ended }: RecordedClientStream): Promise => { + const complete = (result: unknown): CollectedStream => ({ result, error: null, truncated: !ended }); try { switch (kind) { case 'openai-chat-completions': - return complete(await collectOpenAIChatCompletionsProtocolEventsToResult(frames(events) as never), events); + return complete(await collectOpenAIChatCompletionsProtocolEventsToResult(asIterable(frames) as never)); case 'anthropic-messages': - return complete(await collectAnthropicMessagesProtocolEventsToResult(frames(events) as never), events); + return complete(await collectAnthropicMessagesProtocolEventsToResult(asIterable(frames) as never)); case 'openai-responses': - return complete(await collectOpenAIResponsesProtocolEventsToResult(frames(events) as never), events); + return complete(await collectOpenAIResponsesProtocolEventsToResult(asIterable(frames) as never)); case 'gemini-generate-content': - return complete(await collectGeminiGenerateContentProtocolEventsToResult(frames(events) as AsyncIterable>), events); + return complete(await collectGeminiGenerateContentProtocolEventsToResult(asIterable(frames) as AsyncIterable>)); case 'openai-completions': { const stream = (async function* () { - for (const { frame } of events) { + for (const frame of frames) { const typed = frame as ProtocolFrame; if (typed.type === 'event') yield typed.event; } })(); - return complete(await reassembleOpenAICompletionsEvents(stream), events); + return complete(await reassembleOpenAICompletionsEvents(stream)); } } } catch (error) { - return { result: null, error: errorMessage(error), truncated: true }; + return { result: null, error: errorMessage(error), truncated: !ended }; } }; -export const renderStreamEvents = (kind: CollectKind | null, events: DumpStreamEvent[]): RenderedStreamEvent[] => { - return events.map(({ frame, ts }) => { - const sse = frameToSse(kind, frame); - if (!sse) return { event: null, text: '', parseError: null, timestamp: ts }; - if (frame.type === 'done') return { event: sse.event ?? '[DONE]', text: sse.data, parseError: null, timestamp: ts }; - try { - return { event: sse.event ?? null, text: JSON.stringify(JSON.parse(sse.data) as unknown, null, 2), parseError: null, timestamp: ts }; - } catch (error) { - return { event: sse.event ?? null, text: sse.data, parseError: errorMessage(error), timestamp: ts }; - } - }); -}; - -export const streamEventsCopyText = (kind: CollectKind | null, events: DumpStreamEvent[]): string => { - return events.map(({ frame }) => { - const sse = frameToSse(kind, frame); - return sse ? `${sse.event ? `event: ${sse.event}\n` : ''}data: ${sse.data}\n` : ''; - }).filter(Boolean).join('\n'); -}; - -async function* frames(events: DumpStreamEvent[]) { - for (const event of events) yield event.frame; +async function* asIterable(frames: readonly ProtocolFrame[]) { + yield* frames; } - -const frameToSse = (kind: CollectKind | null, frame: ProtocolFrame): SseFrame | null => { - try { - switch (kind) { - case 'openai-chat-completions': return openaiChatCompletionsProtocolFrameToSSEFrame(frame as never, { includeUsageChunk: true }); - case 'openai-completions': return openaiCompletionsProtocolFrameToSSEFrame(frame as never); - case 'anthropic-messages': return anthropicMessagesProtocolFrameToSSEFrame(frame as never); - case 'openai-responses': return openaiResponsesProtocolFrameToSSEFrame(frame as never); - case 'gemini-generate-content': return geminiGenerateContentProtocolFrameToSSEFrame(frame as never); - default: return null; - } - } catch (error) { - return { type: 'sse', event: 'serialize_error', data: errorMessage(error) }; - } -}; diff --git a/apps/web/src/i18n/locales/en.ts b/apps/web/src/i18n/locales/en.ts index d77347db60..058a33ccfe 100644 --- a/apps/web/src/i18n/locales/en.ts +++ b/apps/web/src/i18n/locales/en.ts @@ -1052,8 +1052,6 @@ const en = { noKeys: 'No API key has request retention enabled', noKeysDescription: 'Enable request dump retention on an API key to start capturing requests.', apiKeysLink: 'API Keys', - requestBody: 'Request body', - run: 'Run', stages: 'Stages', requestFacts: 'Request facts', responseFacts: 'Response facts', @@ -1071,17 +1069,12 @@ const en = { stageView: 'Stage view', descent: 'Descent', noRunEvents: 'This run recorded no events.', - responseBody: 'Response body', emptyBody: 'Empty body', - truncatedStream: 'No terminal marker was recorded. The assembled content is shown; completeness is unknown.', noStatus: 'No status', - revealValue: 'Reveal value', - hideValue: 'Hide value', - decodeError: 'Could not decode this body ({{error}}); showing raw base64.', streamView: 'Response body view', noMatches: 'No requests match these filters', - formattedBody: 'Formatted body', collected: 'Collected', + incompleteRecording: 'The recorded client stream is incomplete. The collected response may be partial.', streamEvent: 'Event', eventParseError: 'Invalid JSON', findEvents: 'Search events', @@ -1090,19 +1083,6 @@ const en = { expandEvents: 'Expand all events', noEventMatches: 'No matching events', events: 'Events ({{count, number}})', - headers: 'Headers ({{count}})', - raw: 'Raw bytes', - base64: 'Raw bytes (Base64)', - partialCapture: 'Capture ended before EOF. These are the bytes received so far.', - noUpstreamCapture: 'No HTTP upstream capture is available for this record.', - upstreamCall: 'Upstream calls', - callNumber: 'Call {{number}} of {{count}}', - upstreamRequest: 'Upstream request', - upstreamResponse: 'Upstream response', - metadata: 'Details', - diagnostics: 'Issues ({{count}})', - clientRequest: 'Client request', - clientResponse: 'Client response', exportRecord: 'Export record', search: 'Search ID, model, path, upstream, status or error', failuresOnly: 'Failures only', diff --git a/apps/web/src/i18n/locales/zh-Hans.ts b/apps/web/src/i18n/locales/zh-Hans.ts index 9461e455ca..df935b347f 100644 --- a/apps/web/src/i18n/locales/zh-Hans.ts +++ b/apps/web/src/i18n/locales/zh-Hans.ts @@ -1004,8 +1004,6 @@ const zhHansCN = { noKeys: '没有 API 密钥启用请求保留', noKeysDescription: '在 API 密钥上启用请求转储保留后，即可开始捕获请求。', apiKeysLink: 'API 密钥', - requestBody: '请求体', - run: '运行过程', stages: 'Stages', requestFacts: '请求 facts', responseFacts: '响应 facts', @@ -1023,17 +1021,12 @@ const zhHansCN = { stageView: 'Stage 视图', descent: '分支', noRunEvents: '这次运行没有记录到任何事件。', - responseBody: '响应体', emptyBody: '响应体为空', - truncatedStream: '记录中没有结束标记。下面仍展示已解析内容，完整性未知。', noStatus: '无状态码', - revealValue: '显示值', - hideValue: '隐藏值', - decodeError: '无法解码此内容（{{error}}），以下显示原始 base64。', streamView: '响应体视图', noMatches: '没有符合筛选条件的请求', - formattedBody: '格式化正文', collected: '聚合结果', + incompleteRecording: '记录的客户端流不完整，聚合结果可能只包含部分响应。', streamEvent: '事件', eventParseError: 'JSON 解析失败', findEvents: '搜索事件', @@ -1042,19 +1035,6 @@ const zhHansCN = { expandEvents: '展开全部事件', noEventMatches: '没有匹配的事件', events: '事件（{{count, number}}）', - headers: '请求头 / 响应头（{{count}}）', - raw: '原始内容', - base64: '原始字节（Base64）', - partialCapture: '采集在 EOF 前结束。这里保留了目前收到的字节。', - noUpstreamCapture: '这条记录没有可用的上游 HTTP 采集。', - upstreamCall: '上游交互', - callNumber: '第 {{number}} / {{count}} 次调用', - upstreamRequest: '上游请求', - upstreamResponse: '上游响应', - metadata: '详情', - diagnostics: '诊断（{{count}}）', - clientRequest: '客户端请求', - clientResponse: '客户端响应', exportRecord: '导出记录', search: '搜索 ID、模型、路径、上游、状态或错误', failuresOnly: '仅看失败', diff --git a/apps/web/src/routes/dashboard-monitor-requests.tsx b/apps/web/src/routes/dashboard-monitor-requests.tsx index 8366870cbd..75061d5a62 100644 --- a/apps/web/src/routes/dashboard-monitor-requests.tsx +++ b/apps/web/src/routes/dashboard-monitor-requests.tsx @@ -10,7 +10,8 @@ import type { ApiKey } from '../api/types'; import { RequestDetailPanel } from '../components/requests/detail'; import { refreshRequestKeys } from '../components/requests/key-refresh'; import { RequestListPanel } from '../components/requests/list'; -import { collectKindFromTargetApi, collectStream, detectCollectKind, type CollectedStream } from '../components/requests/stream-render'; +import { clientStreamOf } from '../components/requests/run-stream'; +import { collectStream, detectCollectKind, type CollectedStream } from '../components/requests/stream-render'; import { useDumpSubscription } from '../components/requests/use-dump-subscription'; import { DashboardPageHeader } from '../components/ui/dashboard-page-header'; import { EmptyState, EmptyStateLine } from '../components/ui/empty-state'; @@ -25,8 +26,7 @@ import { fluentComponents } from '../fluent'; import { dashboardWorkspaceHandle } from '../lib/dashboard-route-handle'; import { useEntryRewrite } from '../lib/page-navigation'; import { useMediaQuery } from '../lib/use-media-query'; -import type { DumpMetadata } from '@floway-dev/dump/types'; -import type { DumpRecord } from '@floway-dev/gateway/dump-types'; +import type { DumpMetadata, DumpRecord } from '@floway-dev/dump/types'; export const handle = dashboardWorkspaceHandle; @@ -37,7 +37,6 @@ const { Button, DrawerBody, DrawerHeader, DrawerHeaderTitle, OverlayDrawer } = f // key they may already have. interface LoaderData { collected: CollectedStream | null; - upstreamCollected: CollectedStream | null; error: string | null; keys: ApiKey[] | null; record: DumpRecord | null; @@ -58,7 +57,7 @@ export async function clientLoader({ request }: Route.ClientLoaderArgs): Promise : keys.some(key => key.id === requestedKeyId) ? requestedKeyId : keys[0]?.id ?? null; const recordId = url.searchParams.get('record'); if (!selectedKeyId) { - return { collected: null, upstreamCollected: null, error: keysResult.error?.message ?? null, keys, record: null, recordError: null, records: [], recordsError: null, selectedKeyId }; + return { collected: null, error: keysResult.error?.message ?? null, keys, record: null, recordError: null, records: [], recordsError: null, selectedKeyId }; } const [recordsResult, recordResult] = await Promise.all([ callApi(() => api.api.dump.keys[':keyId'].records.$get({ param: { keyId: selectedKeyId }, query: { limit: '100', q: url.searchParams.get('q') ?? '', failures: url.searchParams.get('failures') === 'true' ? 'true' : 'false' } })), @@ -68,19 +67,10 @@ export async function clientLoader({ request }: Route.ClientLoaderArgs): Promise ]); const record = recordResult?.data ?? null; const collectKind = record ? detectCollectKind(record.meta.path) : null; - // Only an edge-shaped record carries a captured frame log to collect; a run - // records its stream inside its own events and the detail panel reads it there. - const streamEvents = record?.shape === 'edge' && record.response.body.type === 'stream' ? record.response.body.events : []; - const collected = collectKind && streamEvents.length ? await collectStream(collectKind, streamEvents) : null; - // The pre-translation upstream view: dispatched by `meta.targetApi` (the - // target protocol) rather than `meta.path` (the source protocol). Only - // translated turns carry an upstream body. - const upstreamCollectKind = record?.meta.targetApi ? collectKindFromTargetApi(record.meta.targetApi) : null; - const upstreamStreamEvents = record?.shape === 'edge' && record.response.upstream?.body.type === 'stream' ? record.response.upstream.body.events : []; - const upstreamCollected = upstreamCollectKind && upstreamStreamEvents.length ? await collectStream(upstreamCollectKind, upstreamStreamEvents) : null; + const stream = record ? clientStreamOf(record.events) : null; + const collected = collectKind && stream ? await collectStream(collectKind, stream) : null; return { collected, - upstreamCollected, error: keysResult.error?.message ?? null, keys, record, @@ -216,14 +206,14 @@ export default function DashboardMonitorRequests({ loaderData }: Route.Component
- +
: (
- + baseFakeMeta({ id, completedAt, startedAt: completedAt - 1 }); +// A record with an empty stream: these cases are about the list and the row, not about what a +// run put in it. const fakeRecord = (id: string, completedAt: number): StoredDumpRecord => - baseFakeRecord({ id, completedAt, startedAt: completedAt - 1 }); + fakeRunRecord([], { id, completedAt, startedAt: completedAt - 1 }); test('GET /api/dump/keys/:keyId/records lists newest-first', async () => { const { repo, apiKey } = await setupAppTest(); @@ -81,14 +81,12 @@ test('GET /api/dump/keys/:keyId/records/:recordId returns the rehydrated record' assertEquals(body.meta.id, '01HZZ0000000000000000000XX'); }); -// One list, both shapes — the metadata is common, so the dashboard's list says -// nothing about which mechanism served a turn — and a detail fetch hands each -// one back as what it is. -test('GET /api/dump/keys/:keyId/records serves a run record beside an edge one', async () => { +// The list is metadata and the detail is the run's own event stream. Both come from the same +// row, which is what lets the dashboard list a turn without opening it. +test('GET /api/dump/keys/:keyId/records serves the run a record holds', async () => { const { repo, apiKey } = await setupAppTest(); await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); const stubs = installDumpStubs(initDumpStore, initDumpBroker); - stubs.seed(apiKey.id, fakeRecord('01HZZ0000000000000000EDGE', 1000)); stubs.seed(apiKey.id, fakeRunRecord( [ { type: 'stage.entered', stageId: 1, name: 'serve', parentStageId: null, facts: { 'request.payload': { model: 'm' } } }, @@ -100,20 +98,14 @@ test('GET /api/dump/keys/:keyId/records serves a run record beside an edge one', const listed = await requestApp(`/api/dump/keys/${apiKey.id}/records`, { headers: { 'x-api-key': apiKey.key } }); assertEquals(listed.status, 200); const { records } = await listed.json() as { records: DumpMetadata[] }; - assertEquals(records.map(meta => meta.id), ['01HZZ00000000000000000RUN', '01HZZ0000000000000000EDGE']); + assertEquals(records.map(meta => meta.id), ['01HZZ00000000000000000RUN']); const run = await requestApp(`/api/dump/keys/${apiKey.id}/records/01HZZ00000000000000000RUN`, { headers: { 'x-api-key': apiKey.key } }); const runBody = await run.json() as DumpRecord; - if (runBody.shape !== 'run') throw new Error(`expected the run shape, got ${runBody.shape}`); // The object space rides with the events that reference it, so the `object` // event carrying the payload arrives before the entry that points at it. assertEquals(runBody.events.trimEnd().split('\n').map(line => (JSON.parse(line) as { type: string }).type), ['object', 'stage.entered', 'stage.leaved']); - - const edge = await requestApp(`/api/dump/keys/${apiKey.id}/records/01HZZ0000000000000000EDGE`, { headers: { 'x-api-key': apiKey.key } }); - const edgeBody = await edge.json() as DumpRecord; - if (edgeBody.shape !== 'edge') throw new Error(`expected the edge shape, got ${edgeBody.shape}`); - assertEquals(edgeBody.request.method, 'POST'); }); test('GET /api/dump/keys/:keyId/records/:recordId 404s on unknown id', async () => { diff --git a/packages/gateway/__tests__/data-plane/chat-rules_test.ts b/packages/gateway/__tests__/data-plane/chat-rules_test.ts new file mode 100644 index 0000000000..4fdd8768ca --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat-rules_test.ts @@ -0,0 +1,1198 @@ +// The chat rules. Two properties matter and neither is about the rewrites themselves — what +// each rule does to a payload is settled by the protocol it speaks. What matters here is that +// a stage returns a record instead of assigning to a context, and that a rewrite which changes +// nothing changes nothing by identity. A rule that speaks about the response direction has a +// third: what it read on the way down has to reach the way back, which is the only thing its +// closure is for. + +import { describe, expect, it } from 'vitest'; + +import { applyRoleCompatibilityToAnthropicMessages } from '../../src/data-plane/chat/anthropic-messages/apply-role-compatibility.ts'; +import { disableReasoningOnForcedToolChoiceForAnthropicMessages } from '../../src/data-plane/chat/anthropic-messages/disable-reasoning-on-forced-tool-choice.ts'; +import { normalizeEmptyToolsForAnthropicMessages } from '../../src/data-plane/chat/anthropic-messages/normalize-empty-tools-tool-choice.ts'; +import { stripBillingAttributionFromAnthropicMessages } from '../../src/data-plane/chat/anthropic-messages/strip-billing-attribution.ts'; +import { stripSafetySettingsFromGeminiGenerateContent } from '../../src/data-plane/chat/gemini-generate-content/strip-safety-settings.ts'; +import { stripUnsupportedPartFieldsFromGeminiGenerateContent } from '../../src/data-plane/chat/gemini-generate-content/strip-unsupported-part-fields.ts'; +import { stripUnsupportedToolsFromGeminiGenerateContent } from '../../src/data-plane/chat/gemini-generate-content/strip-unsupported-tools.ts'; +import { suppressThoughtPartsFromGeminiGenerateContent } from '../../src/data-plane/chat/gemini-generate-content/suppress-thought-parts.ts'; +import { applyRoleCompatibilityToOpenAIChatCompletions } from '../../src/data-plane/chat/openai-chat-completions/apply-role-compatibility.ts'; +import { disableReasoningOnForcedToolChoiceForOpenAIChatCompletions } from '../../src/data-plane/chat/openai-chat-completions/disable-reasoning-on-forced-tool-choice.ts'; +import { includeUsageStreamOptionsForOpenAIChatCompletions } from '../../src/data-plane/chat/openai-chat-completions/include-usage-stream-options.ts'; +import { normalizeEmptyToolsForOpenAIChatCompletions } from '../../src/data-plane/chat/openai-chat-completions/normalize-empty-tools-tool-choice.ts'; +import { normalizeExclusiveCachedTokensForOpenAIChatCompletions } from '../../src/data-plane/chat/openai-chat-completions/normalize-exclusive-cached-tokens.ts'; +import { normalizeUsageForOpenAIChatCompletions } from '../../src/data-plane/chat/openai-chat-completions/normalize-usage.ts'; +import { stripPromptCacheKeyForOpenAIChatCompletions } from '../../src/data-plane/chat/openai-chat-completions/strip-prompt-cache-key.ts'; +import { vendorDeepSeekNormalizeForOpenAIChatCompletions } from '../../src/data-plane/chat/openai-chat-completions/vendor-deep-seek-normalize.ts'; +import { vendorKimiNormalizeForOpenAIChatCompletions } from '../../src/data-plane/chat/openai-chat-completions/vendor-kimi-normalize.ts'; +import { vendorQwenNormalizeForOpenAIChatCompletions } from '../../src/data-plane/chat/openai-chat-completions/vendor-qwen-normalize.ts'; +import { applyRoleCompatibilityToOpenAIResponses } from '../../src/data-plane/chat/openai-responses/apply-role-compatibility.ts'; +import { disableReasoningOnForcedToolChoiceForOpenAIResponses } from '../../src/data-plane/chat/openai-responses/disable-reasoning-on-forced-tool-choice.ts'; +import { normalizeEmptyToolsForOpenAIResponses } from '../../src/data-plane/chat/openai-responses/normalize-empty-tools-tool-choice.ts'; +import { normalizeExclusiveCachedTokensForOpenAIResponses } from '../../src/data-plane/chat/openai-responses/normalize-exclusive-cached-tokens.ts'; +import { stripPromptCacheKeyForOpenAIResponses } from '../../src/data-plane/chat/openai-responses/strip-prompt-cache-key.ts'; +import { vendorDeepSeekNormalizeForOpenAIResponses } from '../../src/data-plane/chat/openai-responses/vendor-deep-seek-normalize.ts'; +import { vendorQwenNormalizeForOpenAIResponses } from '../../src/data-plane/chat/openai-responses/vendor-qwen-normalize.ts'; +import type { AttemptSelector } from '../../src/data-plane/pipeline/facts.ts'; +import { compose, defineStage, move, run } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; +import type { GeminiGenerateContentPayload, GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; +import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; +import type { OpenAIResponsesInputItem, OpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; + +/** What an attempt is, to a stage that only reads flags. The selector carries them as data, + * so nothing here needs the live candidate the resolver kept. */ +const attemptWith = (...flags: string[]): AttemptSelector => ({ candidateId: 0, upstreamId: 'u', modelId: 'm', flags }); + +/** + * Runs a family's whole rule array, in the order its chain runs it, between stages + * that declare what its real neighbours declare: an edge that needs the answer, the headers + * and the billed set on the way up, and an ending that answers with all three. + * + * `compose` refuses an array whose declarations do not line up, so an array that runs here is + * one that family's `pipeline.ts` can hold — which is the whole of what these harnesses check + * beyond the rewrites themselves. + */ +const runChain = async ( + responseKey: string, + stages: readonly unknown[], + facts: Record, + answer: unknown = { kind: 'value' as const, body: null }, +): Promise<{ readonly down: Record; readonly up: Record }> => { + let down: Record = {}; + const edge = defineStage< + Record, + Record, + Record, + Record + >({ + name: 'edge', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { + needs: [responseKey, 'response.http.headers', 'response.usage.billable'], + consumes: [], + provides: [], + }, + }, + execute: async (received, next) => await next(received), + }); + const ending = defineStage, Record>({ + name: 'ending', + return: { provides: [responseKey, 'response.http.headers', 'response.usage.billable'] }, + execute: async received => { + down = received; + return move({ ...received, [responseKey]: answer, 'response.http.headers': [], 'response.usage.billable': [] }); + }, + }); + const { facts: up } = await run(compose('chain', [edge, ...(stages as never[]), ending]), move(facts) as never, {}); + return { down, up: up as Record }; +}; + +/** One stage is the same harness with an array of one. */ +const runStage = async ( + stage: unknown, + responseKey: string, + facts: Record, + answer: unknown = { kind: 'value' as const, body: null }, +): Promise<{ readonly down: Record; readonly up: Record }> => + await runChain(responseKey, [stage], facts, answer); + +/** An answer in the shape a wire hands up: frames, read once. */ +const streamOf = (frames: readonly ProtocolFrame[]) => ({ + kind: 'stream' as const, + frames: { [Symbol.asyncIterator]: () => (async function* () { yield* frames; })() }, +}); + +const drain = async (answer: unknown): Promise[]> => { + const collected: ProtocolFrame[] = []; + for await (const frame of (answer as { frames: AsyncIterable> }).frames) collected.push(frame); + return collected; +}; + +const openaiChatCompletionsPayload = ( + messages: OpenAIChatCompletionsPayload['messages'], + rest: Partial = {}, +): OpenAIChatCompletionsPayload => ({ model: 'm', messages, ...rest }) as OpenAIChatCompletionsPayload; + +const anthropicMessagesPayload = (messages: unknown[], rest: Record = {}): AnthropicMessagesPayload => + ({ model: 'm', max_tokens: 16, messages, ...rest }) as unknown as AnthropicMessagesPayload; + +const openaiResponsesPayload = (input: unknown[], rest: Record = {}): OpenAIResponsesPayload => + ({ model: 'm', input, ...rest }) as unknown as OpenAIResponsesPayload; + +describe('the OpenAI Chat Completions rules', () => { + it('rewrites the roles its flags name, in the settled order', async () => { + const { down } = await runStage(applyRoleCompatibilityToOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([ + { role: 'system', content: 'lead' }, + { role: 'user', content: 'hi' }, + { role: 'system', content: 'mid' }, + ]), + 'route.attempt': attemptWith('rewrite-system-to-developer'), + }); + const after = down['request.chat.openaiChatCompletions'] as OpenAIChatCompletionsPayload; + expect(after.messages.map(message => message.role)).toEqual(['developer', 'user', 'developer']); + }); + + it('turns only a mid-conversation system message into a user one', async () => { + const { down } = await runStage(applyRoleCompatibilityToOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([ + { role: 'system', content: 'lead' }, + { role: 'system', content: 'still lead' }, + { role: 'user', content: 'hi' }, + { role: 'system', content: 'mid' }, + ]), + 'route.attempt': attemptWith('rewrite-mid-conv-system-to-user'), + }); + const after = down['request.chat.openaiChatCompletions'] as OpenAIChatCompletionsPayload; + expect(after.messages.map(message => message.role)).toEqual(['system', 'system', 'user', 'user']); + }); + + // The convention the whole exact-recording story rests on. A stage whose flags do not + // fire must hand the payload back as the *same object*, or every layer costs a copy of + // the conversation and the dump stops being affordable. + it('hands the same payload back by identity when no flag fires', async () => { + const original = openaiChatCompletionsPayload([{ role: 'system', content: 'lead' }]); + const { down } = await runStage(applyRoleCompatibilityToOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': original, + 'route.attempt': attemptWith(), + }); + expect(down['request.chat.openaiChatCompletions']).toBe(original); + }); + + it('hands the same payload back when a flag fires but changes nothing', async () => { + const original = openaiChatCompletionsPayload([{ role: 'user', content: 'hi' }]); + const { down } = await runStage(applyRoleCompatibilityToOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': original, + 'route.attempt': attemptWith('rewrite-system-to-developer'), + }); + expect(down['request.chat.openaiChatCompletions']).toBe(original); + }); + + it('shares every message it did not rewrite', async () => { + const messages = Array.from({ length: 49 }, (_, index) => ( + index === 0 ? { role: 'system' as const, content: 'lead' } : { role: 'user' as const, content: `turn ${index}` } + )); + const original = openaiChatCompletionsPayload(messages); + const { down } = await runStage(applyRoleCompatibilityToOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': original, + 'route.attempt': attemptWith('rewrite-system-to-developer'), + }); + const after = down['request.chat.openaiChatCompletions'] as OpenAIChatCompletionsPayload; + const shared = after.messages.filter((message, index) => message === messages[index]).length; + expect(shared).toBe(48); // every message but the one that changed + }); + + it('disables reasoning only when a tool choice is forced', async () => { + const { down: forced } = await runStage(disableReasoningOnForcedToolChoiceForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([], { tool_choice: 'required' }), + 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), + }); + expect((forced['request.chat.openaiChatCompletions'] as OpenAIChatCompletionsPayload).reasoning_effort).toBe('none'); + + const free = openaiChatCompletionsPayload([], { tool_choice: 'auto' }); + const { down: untouched } = await runStage(disableReasoningOnForcedToolChoiceForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': free, + 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), + }); + expect(untouched['request.chat.openaiChatCompletions']).toBe(free); + }); + + it('removes the prompt cache key, and the key itself rather than its value', async () => { + const { down } = await runStage(stripPromptCacheKeyForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([], { prompt_cache_key: 'k' } as Partial), + 'route.attempt': attemptWith('strip-prompt-cache-key'), + }); + const after = down['request.chat.openaiChatCompletions'] as OpenAIChatCompletionsPayload; + // `undefined` is not a removal: the key has to be gone, not present and empty. + expect('prompt_cache_key' in after).toBe(false); + }); + + it('leaves a payload that never had the key alone, by identity', async () => { + const original = openaiChatCompletionsPayload([]); + const { down } = await runStage(stripPromptCacheKeyForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': original, + 'route.attempt': attemptWith('strip-prompt-cache-key'), + }); + expect(down['request.chat.openaiChatCompletions']).toBe(original); + }); + + // The usage chunk is what every stream here is billed from, so what the client asked for and + // what the upstream is asked for differ — and this is the rule that makes them differ. + it('asks the upstream for the usage chunk when the client sent no stream options', async () => { + const { down } = await runStage(includeUsageStreamOptionsForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([]), + }); + expect((down['request.chat.openaiChatCompletions'] as OpenAIChatCompletionsPayload).stream_options).toEqual({ include_usage: true }); + }); + + it('asks for it over a client that asked for it to be left out', async () => { + const { down } = await runStage(includeUsageStreamOptionsForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([], { stream_options: { include_usage: false } }), + }); + expect((down['request.chat.openaiChatCompletions'] as OpenAIChatCompletionsPayload).stream_options).toEqual({ include_usage: true }); + }); + + // Everything else on `stream_options` is the client's own. An upstream that reads a key our + // typed surface does not name still gets it. + it('keeps the client-s other stream options while it forces the one it wants', async () => { + const { down } = await runStage(includeUsageStreamOptionsForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([], { + stream_options: { extra: 'keep-me', include_usage: false } as unknown as OpenAIChatCompletionsPayload['stream_options'], + }), + }); + expect((down['request.chat.openaiChatCompletions'] as OpenAIChatCompletionsPayload).stream_options) + .toEqual({ extra: 'keep-me', include_usage: true }); + }); + + it('leaves a payload that already asked for it alone, by identity', async () => { + const original = openaiChatCompletionsPayload([], { stream_options: { include_usage: true } }); + const { down } = await runStage(includeUsageStreamOptionsForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': original, + }); + expect(down['request.chat.openaiChatCompletions']).toBe(original); + }); + + // The spec puts the final usage on a carrier chunk of its own; an upstream that hangs it off + // the last delta is split in two so that everything downstream reads the one shape. + it('moves usage off the chunk that carried the last delta and onto a carrier of its own', async () => { + const { up } = await runStage(normalizeUsageForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', {}, + streamOf([eventFrame({ + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', + choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], + usage: { prompt_tokens: 100, completion_tokens: 20, total_tokens: 120 }, + } as unknown as OpenAIChatCompletionsStreamEvent)])); + const frames = await drain(up['response.chat.openaiChatCompletions']); + + expect(frames).toHaveLength(2); + const delta = openaiChatCompletionsEventOf(frames[0]!); + expect(delta.choices).toEqual([{ index: 0, delta: {}, finish_reason: 'stop' }]); + expect('usage' in delta).toBe(false); + const carrier = openaiChatCompletionsEventOf(frames[1]!); + // The carrier is the same chunk otherwise: an id and a model a client can correlate. + expect(carrier).toMatchObject({ id: 'c1', model: 'm', choices: [] }); + expect(carrier.usage).toEqual({ prompt_tokens: 100, completion_tokens: 20, total_tokens: 120 }); + }); + + it('hands a chunk that is already a carrier, and one with no usage at all, back by identity', async () => { + const carrier = eventFrame({ + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [], + usage: { prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, prompt_tokens_details: { cached_tokens: 60 } }, + } as unknown as OpenAIChatCompletionsStreamEvent); + const content = eventFrame({ + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', + choices: [{ index: 0, delta: { content: 'hi' }, finish_reason: null }], + } as unknown as OpenAIChatCompletionsStreamEvent); + const done = doneFrame(); + + const { up } = await runStage(normalizeUsageForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', {}, + streamOf([carrier, content, done])); + const frames = await drain(up['response.chat.openaiChatCompletions']); + + expect(frames).toEqual([carrier, content, done]); + }); + + // Naming a vendor's cache fields is the vendor stage's job, and it has already run by the + // time the answer reaches this one. A chunk still carrying them is one no vendor claimed. + it('leaves a vendor-s own cache fields where it found them', async () => { + const { up } = await runStage(normalizeUsageForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', {}, + streamOf([openaiChatCompletionsUsageChunk({ + prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, prompt_cache_hit_tokens: 70, cached_tokens: 25, + })])); + const frames = await drain(up['response.chat.openaiChatCompletions']); + + expect(openaiChatCompletionsUsageOf(frames[0]!)).toEqual({ + prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, prompt_cache_hit_tokens: 70, cached_tokens: 25, + }); + }); + + // Verbatim from a Charm Hyper kimi-k3 turn: 479 + 13312 + 373 = 14164, so `prompt_tokens` + // counts only what the cache did not serve, and `total_tokens` witnesses it whatever anyone + // flagged. + it('folds the cache buckets into the input total when the totals witness it', async () => { + const { up } = await runStage(normalizeExclusiveCachedTokensForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'route.attempt': attemptWith(), + }, streamOf([openaiChatCompletionsUsageChunk({ + prompt_tokens: 479, completion_tokens: 373, total_tokens: 14164, prompt_tokens_details: { cached_tokens: 13312 }, + })])); + const frames = await drain(up['response.chat.openaiChatCompletions']); + + expect(openaiChatCompletionsUsageOf(frames[0]!)).toMatchObject({ + prompt_tokens: 13791, + // The total already counted the real input, and now agrees with it. + total_tokens: 14164, + prompt_tokens_details: { cached_tokens: 13312 }, + }); + }); + + // The flag settles the responses whose totals witness nothing, which makes it a declaration + // input rather than the gate — and it arrives on the attempt this stage read on the way + // down, so this is also what proves the closure carries it to the way back. + it('folds on the flag alone when the chunk states no total, cache writes included', async () => { + const { up } = await runStage(normalizeExclusiveCachedTokensForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'route.attempt': attemptWith('usage-exclusive-cached-tokens'), + }, streamOf([openaiChatCompletionsUsageChunk({ + prompt_tokens: 100, completion_tokens: 10, prompt_tokens_details: { cached_tokens: 120, cache_creation_input_tokens: 80 }, + })])); + const frames = await drain(up['response.chat.openaiChatCompletions']); + + expect(openaiChatCompletionsUsageOf(frames[0]!)).toMatchObject({ prompt_tokens: 300 }); + }); + + it('hands an inclusive chunk, and one with no cache counts, back by identity', async () => { + const inclusive = openaiChatCompletionsUsageChunk({ + prompt_tokens: 1000, completion_tokens: 50, total_tokens: 1050, prompt_tokens_details: { cached_tokens: 400 }, + }); + const uncached = openaiChatCompletionsUsageChunk({ prompt_tokens: 40, completion_tokens: 1, total_tokens: 41 }); + + const { up } = await runStage(normalizeExclusiveCachedTokensForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'route.attempt': attemptWith('usage-exclusive-cached-tokens'), + }, streamOf([uncached])); + expect(await drain(up['response.chat.openaiChatCompletions'])).toEqual([uncached]); + + const { up: unflagged } = await runStage(normalizeExclusiveCachedTokensForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'route.attempt': attemptWith(), + }, streamOf([inclusive])); + expect(await drain(unflagged['response.chat.openaiChatCompletions'])).toEqual([inclusive]); + }); + + // Both contradictions name the flag and the upstream, because the remedy is a setting an + // operator makes on that upstream. Neither may pass silently: one over-charges the input by + // the whole cached prefix, the other underflows it. + it('raises when the flag claims exclusive and the totals say inclusive', async () => { + const { up } = await runStage(normalizeExclusiveCachedTokensForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'route.attempt': attemptWith('usage-exclusive-cached-tokens'), + }, streamOf([openaiChatCompletionsUsageChunk({ + prompt_tokens: 1000, completion_tokens: 50, total_tokens: 1050, prompt_tokens_details: { cached_tokens: 400 }, + })])); + + await expect(drain(up['response.chat.openaiChatCompletions'])) + .rejects.toThrow(/usage-exclusive-cached-tokens is enabled/); + }); + + it('raises naming the flag when the cache counts underflow with no verdict', async () => { + const { up } = await runStage(normalizeExclusiveCachedTokensForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'route.attempt': attemptWith(), + }, streamOf([openaiChatCompletionsUsageChunk({ + prompt_tokens: 479, completion_tokens: 373, prompt_tokens_details: { cached_tokens: 13312 }, + })])); + + await expect(drain(up['response.chat.openaiChatCompletions'])) + .rejects.toThrow(/enable usage-exclusive-cached-tokens/); + }); + + it('leaves an answer that is not a stream alone, by identity', async () => { + const answer = { kind: 'value' as const, body: null }; + const { up } = await runStage(normalizeExclusiveCachedTokensForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'route.attempt': attemptWith(), + }, answer); + expect(up['response.chat.openaiChatCompletions']).toBe(answer); + }); +}); + +/** A `choices: []` carrier holding nothing but the counts under test — the shape the spec puts + * a stream's final usage on. */ +const openaiChatCompletionsUsageChunk = (usage: Record): ProtocolFrame => + eventFrame({ + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [], usage, + } as unknown as OpenAIChatCompletionsStreamEvent); + +const openaiChatCompletionsEventOf = (frame: ProtocolFrame): Record => + (frame as unknown as { event: Record }).event; + +const openaiChatCompletionsUsageOf = (frame: ProtocolFrame): Record => + openaiChatCompletionsEventOf(frame).usage as Record; + +const deepSeekDelta = (delta: Record): ProtocolFrame => + eventFrame({ + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', + choices: [{ index: 0, delta, finish_reason: null }], + } as unknown as OpenAIChatCompletionsStreamEvent); + +const deltaOf = (frame: ProtocolFrame): Record => + (openaiChatCompletionsEventOf(frame).choices as { delta: Record }[])[0]!.delta; + +/** An assistant turn as the gateway holds it: the canonical reasoning fields, and a tool call + * the replay of a multi-turn loop has to keep. */ +const assistantReplay = (reasoning: Record): OpenAIChatCompletionsPayload['messages'] => [ + { role: 'user', content: 'first turn' }, + { + role: 'assistant', content: null, ...reasoning, + tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'lookup', arguments: '{}' } }], + }, + { role: 'tool', tool_call_id: 'call_1', content: 'result' }, +] as unknown as OpenAIChatCompletionsPayload['messages']; + +const assistantOf = (payload: unknown): Record => + (payload as OpenAIChatCompletionsPayload).messages[1] as unknown as Record; + +describe('the OpenAI Chat Completions vendor dialects', () => { + it('projects the canonical assistant reasoning onto the one field DeepSeek reads', async () => { + const { down } = await runStage(vendorDeepSeekNormalizeForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload(assistantReplay({ + reasoning_text: 'let me check the docs', + reasoning_opaque: 'opaque-blob', + reasoning_items: [{ type: 'reasoning', summary: [] }], + })), + 'route.attempt': attemptWith('vendor-deepseek'), + }); + const assistant = assistantOf(down['request.chat.openaiChatCompletions']); + + expect(assistant.reasoning_content).toBe('let me check the docs'); + // The other three are not DeepSeek's, and `reasoning_opaque` in particular is the + // canonical cross-turn signature it does not accept. + expect('reasoning_text' in assistant).toBe(false); + expect('reasoning_opaque' in assistant).toBe(false); + expect('reasoning_items' in assistant).toBe(false); + expect(assistant.tool_calls).toHaveLength(1); + }); + + it('writes that field from the summary items when the scalar is the thing that is missing', async () => { + const { down } = await runStage(vendorDeepSeekNormalizeForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload(assistantReplay({ + reasoning_items: [{ + type: 'reasoning', id: 'rs_1', + summary: [{ type: 'summary_text', text: 'step one. ' }, { type: 'summary_text', text: 'step two.' }], + }], + })), + 'route.attempt': attemptWith('vendor-deepseek'), + }); + + expect(assistantOf(down['request.chat.openaiChatCompletions']).reasoning_content).toBe('step one. step two.'); + }); + + it('strips the fields it cannot project even when there is nothing to project', async () => { + const { down } = await runStage(vendorDeepSeekNormalizeForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([ + { role: 'user', content: 'first turn' }, + { role: 'assistant', content: 'answer', reasoning_items: [{ type: 'reasoning' }], reasoning_opaque: 'opaque-chain' }, + ] as unknown as OpenAIChatCompletionsPayload['messages']), + 'route.attempt': attemptWith('vendor-deepseek'), + }); + const assistant = assistantOf(down['request.chat.openaiChatCompletions']); + + expect('reasoning_content' in assistant).toBe(false); + expect('reasoning_items' in assistant).toBe(false); + expect('reasoning_opaque' in assistant).toBe(false); + expect(assistant.content).toBe('answer'); + }); + + it('puts the reasoning sentinel into the shape DeepSeek reads', async () => { + const { down } = await runStage(vendorDeepSeekNormalizeForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([], { reasoning_effort: 'none' }), + 'route.attempt': attemptWith('vendor-deepseek'), + }); + const after = down['request.chat.openaiChatCompletions'] as Record; + + expect('reasoning_effort' in after).toBe(false); + expect(after.thinking).toEqual({ type: 'disabled' }); + }); + + it('puts the reasoning sentinel into the shape Qwen reads', async () => { + const { down } = await runStage(vendorQwenNormalizeForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([], { reasoning_effort: 'none' }), + 'route.attempt': attemptWith('vendor-qwen'), + }); + const after = down['request.chat.openaiChatCompletions'] as Record; + + expect('reasoning_effort' in after).toBe(false); + expect(after.enable_thinking).toBe(false); + }); + + // Only the sentinel means "no reasoning". A real effort is the client asking for reasoning, + // and DeepSeek and Qwen both take it in the canonical field. + it('leaves a payload that asked for real reasoning alone, by identity', async () => { + const original = openaiChatCompletionsPayload([], { reasoning_effort: 'high' }); + for (const stage of [vendorDeepSeekNormalizeForOpenAIChatCompletions, vendorQwenNormalizeForOpenAIChatCompletions]) { + const { down } = await runStage(stage, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': original, + 'route.attempt': attemptWith('vendor-deepseek', 'vendor-qwen'), + }); + expect(down['request.chat.openaiChatCompletions']).toBe(original); + } + }); + + // DeepSeek's structured output takes `json_object` and nothing else, so the schema goes + // rather than the request being refused for carrying it. + it('downgrades a JSON schema to the JSON object DeepSeek supports', async () => { + const { down } = await runStage(vendorDeepSeekNormalizeForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([], { + response_format: { + type: 'json_schema', + json_schema: { name: 'messages_response', strict: true, schema: { type: 'object' } }, + }, + }), + 'route.attempt': attemptWith('vendor-deepseek'), + }); + + expect((down['request.chat.openaiChatCompletions'] as OpenAIChatCompletionsPayload).response_format).toEqual({ type: 'json_object' }); + }); + + it('leaves a payload already asking for a JSON object alone, by identity', async () => { + const original = openaiChatCompletionsPayload([], { response_format: { type: 'json_object' } }); + const { down } = await runStage(vendorDeepSeekNormalizeForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': original, + 'route.attempt': attemptWith('vendor-deepseek'), + }); + expect(down['request.chat.openaiChatCompletions']).toBe(original); + }); + + it('renames DeepSeek-s reasoning deltas to the canonical field', async () => { + const { up } = await runStage(vendorDeepSeekNormalizeForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([]), + 'route.attempt': attemptWith('vendor-deepseek'), + }, streamOf([ + deepSeekDelta({ role: 'assistant' }), + deepSeekDelta({ reasoning_content: 'thinking...' }), + deepSeekDelta({ content: 'answer' }), + doneFrame(), + ])); + const frames = await drain(up['response.chat.openaiChatCompletions']); + + expect(deltaOf(frames[1]!).reasoning_text).toBe('thinking...'); + expect('reasoning_content' in deltaOf(frames[1]!)).toBe(false); + // The deltas either side of it are not this rule's business. + expect(deltaOf(frames[0]!)).toEqual({ role: 'assistant' }); + expect(deltaOf(frames[2]!)).toEqual({ content: 'answer' }); + }); + + it('rewrites DeepSeek-s cache-hit count into the field the rules above it read', async () => { + const { up } = await runStage(vendorDeepSeekNormalizeForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([]), + 'route.attempt': attemptWith('vendor-deepseek'), + }, streamOf([openaiChatCompletionsUsageChunk({ + prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, + prompt_cache_hit_tokens: 70, prompt_cache_miss_tokens: 30, + prompt_tokens_details: { upstream_metric: 'preserved' }, + })])); + const frames = await drain(up['response.chat.openaiChatCompletions']); + const usage = openaiChatCompletionsUsageOf(frames[0]!); + + expect(usage.prompt_tokens).toBe(100); + expect(usage.prompt_tokens_details).toEqual({ upstream_metric: 'preserved', cached_tokens: 70 }); + // The miss count is what is left of the input, which `prompt_tokens` already says. + expect('prompt_cache_hit_tokens' in usage).toBe(false); + expect('prompt_cache_miss_tokens' in usage).toBe(false); + }); + + it('rewrites Kimi-s flat cached count into that same field', async () => { + const { up } = await runStage(vendorKimiNormalizeForOpenAIChatCompletions, 'response.chat.openaiChatCompletions', { + 'route.attempt': attemptWith('vendor-kimi'), + }, streamOf([openaiChatCompletionsUsageChunk({ + prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, cached_tokens: 50, + prompt_tokens_details: { upstream_metric: 'preserved' }, + })])); + const frames = await drain(up['response.chat.openaiChatCompletions']); + const usage = openaiChatCompletionsUsageOf(frames[0]!); + + expect(usage.prompt_tokens_details).toEqual({ upstream_metric: 'preserved', cached_tokens: 50 }); + expect('cached_tokens' in usage).toBe(false); + }); + + // An upstream that answers with an array where the details object should be is not carrying + // details, so what goes on the rewritten block is the one count that was read — not the + // array's own indices spread across it. + it('replaces a details field that is not an object rather than spreading it', async () => { + for (const [stage, flag, vendorField] of [ + [vendorDeepSeekNormalizeForOpenAIChatCompletions, 'vendor-deepseek', 'prompt_cache_hit_tokens'], + [vendorKimiNormalizeForOpenAIChatCompletions, 'vendor-kimi', 'cached_tokens'], + ] as const) { + const { up } = await runStage(stage, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([]), + 'route.attempt': attemptWith(flag), + }, streamOf([openaiChatCompletionsUsageChunk({ + prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, + [vendorField]: 70, prompt_tokens_details: [{ cached_tokens: 1 }], + })])); + const frames = await drain(up['response.chat.openaiChatCompletions']); + + expect(openaiChatCompletionsUsageOf(frames[0]!).prompt_tokens_details).toEqual({ cached_tokens: 70 }); + } + }); + + // A flag is data on an upstream model. A candidate that does not carry one is an upstream + // whose wire is already canonical, and nothing here may touch it. + it('stands down on a candidate that carries none of the vendor flags', async () => { + const payload = openaiChatCompletionsPayload([], { reasoning_effort: 'none' }); + const usage = openaiChatCompletionsUsageChunk({ + prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, prompt_cache_hit_tokens: 70, cached_tokens: 50, + }); + for (const stage of [ + vendorDeepSeekNormalizeForOpenAIChatCompletions, + vendorQwenNormalizeForOpenAIChatCompletions, + vendorKimiNormalizeForOpenAIChatCompletions, + ]) { + const { down, up } = await runStage(stage, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': payload, + 'route.attempt': attemptWith(), + }, streamOf([usage])); + + expect(down['request.chat.openaiChatCompletions']).toBe(payload); + expect(await drain(up['response.chat.openaiChatCompletions'])).toEqual([usage]); + } + }); + + // A stream that breaks is the upstream's failure and belongs to whoever is reading it; a + // rewrite that swallowed it would present a broken turn as a whole one. + it('lets an upstream stream failure through as it was thrown', async () => { + const failure = new Error('upstream stream failed'); + for (const [stage, flag] of [ + [vendorDeepSeekNormalizeForOpenAIChatCompletions, 'vendor-deepseek'], + [vendorKimiNormalizeForOpenAIChatCompletions, 'vendor-kimi'], + ] as const) { + const { up } = await runStage(stage, 'response.chat.openaiChatCompletions', { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([]), + 'route.attempt': attemptWith(flag), + }, { kind: 'stream' as const, frames: { [Symbol.asyncIterator]: () => (async function* () { throw failure; })() } }); + + await expect(drain(up['response.chat.openaiChatCompletions'])).rejects.toBe(failure); + } + }); +}); + +describe('the Anthropic Messages rules', () => { + it('turns every inline system message into a user one', async () => { + const { down } = await runStage(applyRoleCompatibilityToAnthropicMessages, 'response.chat.anthropicMessages', { + 'request.chat.anthropicMessages': anthropicMessagesPayload([ + { role: 'system', content: 'lead' }, + { role: 'user', content: 'hi' }, + { role: 'system', content: 'mid' }, + ]), + 'route.attempt': attemptWith('rewrite-mid-conv-system-to-user'), + }); + const after = down['request.chat.anthropicMessages'] as AnthropicMessagesPayload; + // The top-level `system` field is the only first-position slot on this protocol, so even + // the leading one is past the leading run. + expect(after.messages.map(message => message.role)).toEqual(['user', 'user', 'user']); + }); + + it('hands the same conversation back by identity when the flag does not fire', async () => { + const original = anthropicMessagesPayload([{ role: 'system', content: 'lead' }]); + const { down } = await runStage(applyRoleCompatibilityToAnthropicMessages, 'response.chat.anthropicMessages', { + 'request.chat.anthropicMessages': original, + 'route.attempt': attemptWith(), + }); + expect(down['request.chat.anthropicMessages']).toBe(original); + }); + + it('hands the same conversation back when the flag fires but there is no system message', async () => { + const original = anthropicMessagesPayload([{ role: 'user', content: 'hi' }]); + const { down } = await runStage(applyRoleCompatibilityToAnthropicMessages, 'response.chat.anthropicMessages', { + 'request.chat.anthropicMessages': original, + 'route.attempt': attemptWith('rewrite-mid-conv-system-to-user'), + }); + expect(down['request.chat.anthropicMessages']).toBe(original); + }); + + it('disables thinking on a forced tool choice and leaves the structured-output format', async () => { + const { down } = await runStage(disableReasoningOnForcedToolChoiceForAnthropicMessages, 'response.chat.anthropicMessages', { + 'request.chat.anthropicMessages': anthropicMessagesPayload([], { + tool_choice: { type: 'any' }, + output_config: { effort: 'high', format: { type: 'json_schema', schema: {} } }, + }), + 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), + }); + const after = down['request.chat.anthropicMessages'] as AnthropicMessagesPayload; + expect(after.thinking).toEqual({ type: 'disabled' }); + // Forced tool choice composes fine with structured output; it is thinking it does not + // compose with, so only the effort goes. + expect(after.output_config).toEqual({ format: { type: 'json_schema', schema: {} } }); + }); + + it('drops an output_config that held nothing but the effort', async () => { + const { down } = await runStage(disableReasoningOnForcedToolChoiceForAnthropicMessages, 'response.chat.anthropicMessages', { + 'request.chat.anthropicMessages': anthropicMessagesPayload([], { + tool_choice: { type: 'tool', name: 'f' }, + output_config: { effort: 'high' }, + }), + 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), + }); + expect('output_config' in (down['request.chat.anthropicMessages'] as AnthropicMessagesPayload)).toBe(false); + }); + + it('leaves a free tool choice alone, by identity', async () => { + const free = anthropicMessagesPayload([], { tool_choice: { type: 'auto' } }); + const { down } = await runStage(disableReasoningOnForcedToolChoiceForAnthropicMessages, 'response.chat.anthropicMessages', { + 'request.chat.anthropicMessages': free, + 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), + }); + expect(down['request.chat.anthropicMessages']).toBe(free); + }); + + it('scrubs the billing header line and the per-turn hash out of the system prompt', async () => { + const { down } = await runStage(stripBillingAttributionFromAnthropicMessages, 'response.chat.anthropicMessages', { + 'request.chat.anthropicMessages': anthropicMessagesPayload([], { + system: 'x-anthropic-billing-header: cch=abcdef12\nreal instructions\ncch=beef01;', + }), + 'route.attempt': attemptWith('strip-billing-attribution'), + }); + expect((down['request.chat.anthropicMessages'] as AnthropicMessagesPayload).system).toBe('real instructions'); + }); + + it('drops a system block the scrub emptied, and the field when nothing is left', async () => { + const { down } = await runStage(stripBillingAttributionFromAnthropicMessages, 'response.chat.anthropicMessages', { + 'request.chat.anthropicMessages': anthropicMessagesPayload([], { + system: [{ type: 'text', text: 'x-anthropic-billing-header: cch=abcdef12' }], + }), + 'route.attempt': attemptWith('strip-billing-attribution'), + }); + // An empty system prompt is not a system prompt: the field goes rather than riding on + // empty. + expect('system' in (down['request.chat.anthropicMessages'] as AnthropicMessagesPayload)).toBe(false); + }); + + it('leaves a system prompt carrying no attribution block alone, by identity', async () => { + const original = anthropicMessagesPayload([], { system: [{ type: 'text', text: 'real instructions' }] }); + const { down } = await runStage(stripBillingAttributionFromAnthropicMessages, 'response.chat.anthropicMessages', { + 'request.chat.anthropicMessages': original, + 'route.attempt': attemptWith('strip-billing-attribution'), + }); + expect(down['request.chat.anthropicMessages']).toBe(original); + }); +}); + +describe('the Gemini generateContent rules', () => { + it('drops the part fields no translation carries, and a part they were all of', async () => { + const { down } = await runStage(stripUnsupportedPartFieldsFromGeminiGenerateContent, 'response.chat.geminiGenerateContent', { + 'request.chat.geminiGenerateContent': { + contents: [{ + role: 'user', + parts: [ + { text: 'hi', fileData: { mimeType: 'text/plain', fileUri: 'u' } }, + { fileData: { mimeType: 'text/plain', fileUri: 'u' } }, + ], + }], + systemInstruction: { parts: [{ text: 'sys', codeExecutionResult: {} }] }, + } satisfies GeminiGenerateContentPayload, + }); + const after = down['request.chat.geminiGenerateContent'] as GeminiGenerateContentPayload; + expect(after.contents?.[0]?.parts).toEqual([{ text: 'hi' }]); + expect(after.systemInstruction?.parts).toEqual([{ text: 'sys' }]); + }); + + it('hands a payload carrying no unsupported part field back by identity', async () => { + const original: GeminiGenerateContentPayload = { contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }; + const { down } = await runStage(stripUnsupportedPartFieldsFromGeminiGenerateContent, 'response.chat.geminiGenerateContent', { + 'request.chat.geminiGenerateContent': original, + }); + expect(down['request.chat.geminiGenerateContent']).toBe(original); + }); + + it('keeps a tool group only for its function declarations', async () => { + const { down } = await runStage(stripUnsupportedToolsFromGeminiGenerateContent, 'response.chat.geminiGenerateContent', { + 'request.chat.geminiGenerateContent': { + tools: [{ functionDeclarations: [{ name: 'f' }], googleSearch: {} }, { codeExecution: {} }], + } satisfies GeminiGenerateContentPayload, + }); + expect((down['request.chat.geminiGenerateContent'] as GeminiGenerateContentPayload).tools).toEqual([{ functionDeclarations: [{ name: 'f' }] }]); + }); + + it('removes the tool list itself when no group declared a function', async () => { + const { down } = await runStage(stripUnsupportedToolsFromGeminiGenerateContent, 'response.chat.geminiGenerateContent', { + 'request.chat.geminiGenerateContent': { tools: [{ googleSearch: {} }] } satisfies GeminiGenerateContentPayload, + }); + // An empty tool list is a different request from no tools at all. + expect('tools' in (down['request.chat.geminiGenerateContent'] as GeminiGenerateContentPayload)).toBe(false); + }); + + it('hands a payload whose groups only declare functions back by identity', async () => { + const original: GeminiGenerateContentPayload = { tools: [{ functionDeclarations: [{ name: 'f' }] }] }; + const { down } = await runStage(stripUnsupportedToolsFromGeminiGenerateContent, 'response.chat.geminiGenerateContent', { + 'request.chat.geminiGenerateContent': original, + }); + expect(down['request.chat.geminiGenerateContent']).toBe(original); + }); + + it('removes the safety settings, and leaves a payload that carries none alone', async () => { + const { down } = await runStage(stripSafetySettingsFromGeminiGenerateContent, 'response.chat.geminiGenerateContent', { + 'request.chat.geminiGenerateContent': { safetySettings: [{ category: 'c', threshold: 't' }] } satisfies GeminiGenerateContentPayload, + }); + expect('safetySettings' in (down['request.chat.geminiGenerateContent'] as GeminiGenerateContentPayload)).toBe(false); + + const original: GeminiGenerateContentPayload = { contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }; + const { down: untouched } = await runStage(stripSafetySettingsFromGeminiGenerateContent, 'response.chat.geminiGenerateContent', { + 'request.chat.geminiGenerateContent': original, + }); + expect(untouched['request.chat.geminiGenerateContent']).toBe(original); + }); + + it('hides thought parts from a caller who did not opt in', async () => { + const answer = streamOf([ + eventFrame({ candidates: [{ index: 0, content: { parts: [{ text: 'pondering', thought: true }] } }] }), + eventFrame({ + candidates: [{ + index: 0, + content: { parts: [{ text: 'more pondering', thought: true }, { text: 'the answer' }] }, + finishReason: 'STOP', + }], + }), + ]); + const { up } = await runStage(suppressThoughtPartsFromGeminiGenerateContent, 'response.chat.geminiGenerateContent', { + 'request.chat.geminiGenerateContent': {} satisfies GeminiGenerateContentPayload, + }, answer); + const frames = await drain(up['response.chat.geminiGenerateContent']); + // The first event was pure thought and had not finished, so it is nothing at all and + // never reaches the client; the second keeps only what the caller asked for. + expect(frames).toHaveLength(1); + const event = (frames[0] as { event: Extract }).event; + expect(event.candidates?.[0]?.content.parts).toEqual([{ text: 'the answer' }]); + }); + + // The opt-in is something the client sent, so it is read on the way down and used on the + // way back — the closure carrying it across is the only mechanism this proves. + it('leaves the stream alone when the caller asked to see thoughts', async () => { + const answer = streamOf([ + eventFrame({ candidates: [{ index: 0, content: { parts: [{ text: 'pondering', thought: true }] }, finishReason: 'STOP' }] }), + ]); + const { up } = await runStage(suppressThoughtPartsFromGeminiGenerateContent, 'response.chat.geminiGenerateContent', { + 'request.chat.geminiGenerateContent': { generationConfig: { thinkingConfig: { includeThoughts: true } } } satisfies GeminiGenerateContentPayload, + }, answer); + expect(up['response.chat.geminiGenerateContent']).toBe(answer); + }); + + it('leaves an answer that is not a stream alone, by identity', async () => { + const answer = { kind: 'value' as const, body: null }; + const { up } = await runStage(suppressThoughtPartsFromGeminiGenerateContent, 'response.chat.geminiGenerateContent', { + 'request.chat.geminiGenerateContent': {} satisfies GeminiGenerateContentPayload, + }, answer); + expect(up['response.chat.geminiGenerateContent']).toBe(answer); + }); +}); + +describe('the OpenAI Responses rules', () => { + const rolesOf = (down: Record): unknown[] => + ((down['request.chat.openaiResponses'] as OpenAIResponsesPayload).input as OpenAIResponsesInputItem[]) + .map(item => (item as { role?: unknown }).role); + + it('rewrites the roles its flags name, in the settled order', async () => { + const { down } = await runStage(applyRoleCompatibilityToOpenAIResponses, 'response.chat.openaiResponses', { + 'request.chat.openaiResponses': openaiResponsesPayload([ + { type: 'message', role: 'system', content: 'lead' }, + { type: 'message', role: 'user', content: 'hi' }, + { type: 'message', role: 'system', content: 'mid' }, + ]), + 'route.attempt': attemptWith('rewrite-system-to-developer'), + }); + expect(rolesOf(down)).toEqual(['developer', 'user', 'developer']); + }); + + // The ordering the rule depends on: an item with no role of its own still ends the leading + // system run, which is what makes the system message after it mid-conversation. + it('lets an item carrying no role cross the leading system run', async () => { + const { down } = await runStage(applyRoleCompatibilityToOpenAIResponses, 'response.chat.openaiResponses', { + 'request.chat.openaiResponses': openaiResponsesPayload([ + { type: 'message', role: 'system', content: 'lead' }, + { type: 'reasoning', id: 'rs_1', summary: [] }, + { type: 'message', role: 'system', content: 'after' }, + ]), + 'route.attempt': attemptWith('rewrite-mid-conv-system-to-user'), + }); + expect(rolesOf(down)).toEqual(['system', undefined, 'user']); + }); + + it('hands the same payload back by identity when no flag fires', async () => { + const original = openaiResponsesPayload([{ type: 'message', role: 'system', content: 'lead' }]); + const { down } = await runStage(applyRoleCompatibilityToOpenAIResponses, 'response.chat.openaiResponses', { + 'request.chat.openaiResponses': original, + 'route.attempt': attemptWith(), + }); + expect(down['request.chat.openaiResponses']).toBe(original); + }); + + it('disables reasoning on a required tool choice and on a named one', async () => { + for (const toolChoice of ['required', { type: 'function', name: 'f' }]) { + const { down } = await runStage(disableReasoningOnForcedToolChoiceForOpenAIResponses, 'response.chat.openaiResponses', { + 'request.chat.openaiResponses': openaiResponsesPayload([], { tool_choice: toolChoice, reasoning: { effort: 'high', summary: 'auto' } }), + 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), + }); + // The object is replaced rather than merged: a summary has nothing to summarize once + // reasoning is off. + expect((down['request.chat.openaiResponses'] as OpenAIResponsesPayload).reasoning).toEqual({ effort: 'none' }); + } + }); + + it('leaves an automatic tool choice alone, by identity', async () => { + const free = openaiResponsesPayload([], { tool_choice: 'auto' }); + const { down } = await runStage(disableReasoningOnForcedToolChoiceForOpenAIResponses, 'response.chat.openaiResponses', { + 'request.chat.openaiResponses': free, + 'route.attempt': attemptWith('disable-reasoning-on-forced-tool-choice'), + }); + expect(down['request.chat.openaiResponses']).toBe(free); + }); + + it('removes the prompt cache key, and leaves a payload that never had it alone', async () => { + const { down } = await runStage(stripPromptCacheKeyForOpenAIResponses, 'response.chat.openaiResponses', { + 'request.chat.openaiResponses': openaiResponsesPayload([], { prompt_cache_key: 'k' }), + 'route.attempt': attemptWith('strip-prompt-cache-key'), + }); + expect('prompt_cache_key' in (down['request.chat.openaiResponses'] as OpenAIResponsesPayload)).toBe(false); + + const original = openaiResponsesPayload([]); + const { down: untouched } = await runStage(stripPromptCacheKeyForOpenAIResponses, 'response.chat.openaiResponses', { + 'request.chat.openaiResponses': original, + 'route.attempt': attemptWith('strip-prompt-cache-key'), + }); + expect(untouched['request.chat.openaiResponses']).toBe(original); + }); + + it('folds the cache buckets into the input total when the totals witness it', async () => { + // 100 + 50 + 10 = 160 reaches the stated total and 100 + 10 does not, so the upstream + // reports the buckets outside the input count whatever anyone flagged. + const answer = streamOf([usageEvent({ input_tokens: 100, output_tokens: 10, total_tokens: 160, input_tokens_details: { cached_tokens: 50 } })]); + const { up } = await runStage(normalizeExclusiveCachedTokensForOpenAIResponses, 'response.chat.openaiResponses', { + 'route.attempt': attemptWith(), + }, answer); + const frames = await drain(up['response.chat.openaiResponses']); + expect(usageOf(frames[0]!)).toMatchObject({ input_tokens: 150, output_tokens: 10 }); + }); + + // The flag settles the responses whose totals witness nothing, which makes it a declaration + // input rather than the gate — and it arrives on the attempt this stage read on the way + // down, so this is also what proves the closure carries it to the way back. + it('folds on the flag alone when the response states no total', async () => { + const answer = streamOf([usageEvent({ input_tokens: 100, output_tokens: 10, input_tokens_details: { cached_tokens: 50, cache_write_tokens: 5 } })]); + const { up } = await runStage(normalizeExclusiveCachedTokensForOpenAIResponses, 'response.chat.openaiResponses', { + 'route.attempt': attemptWith('usage-exclusive-cached-tokens'), + }, answer); + const frames = await drain(up['response.chat.openaiResponses']); + expect(usageOf(frames[0]!)).toMatchObject({ input_tokens: 155 }); + }); + + it('hands a frame back by identity when the totals say the buckets are already inside', async () => { + const frame = usageEvent({ input_tokens: 100, output_tokens: 10, total_tokens: 110, input_tokens_details: { cached_tokens: 50 } }); + const { up } = await runStage(normalizeExclusiveCachedTokensForOpenAIResponses, 'response.chat.openaiResponses', { + 'route.attempt': attemptWith(), + }, streamOf([frame])); + const frames = await drain(up['response.chat.openaiResponses']); + expect(frames[0]).toBe(frame); + }); + + it('leaves an answer that is not a stream alone, by identity', async () => { + const answer = { kind: 'value' as const, body: null }; + const { up } = await runStage(normalizeExclusiveCachedTokensForOpenAIResponses, 'response.chat.openaiResponses', { + 'route.attempt': attemptWith(), + }, answer); + expect(up['response.chat.openaiResponses']).toBe(answer); + }); + + it('puts the reasoning sentinel into the shape DeepSeek reads', async () => { + const { down } = await runStage(vendorDeepSeekNormalizeForOpenAIResponses, 'response.chat.openaiResponses', { + 'request.chat.openaiResponses': openaiResponsesPayload([], { reasoning: { effort: 'none' } }), + 'route.attempt': attemptWith('vendor-deepseek'), + }); + const after = down['request.chat.openaiResponses'] as Record; + expect('reasoning' in after).toBe(false); + expect(after.thinking).toEqual({ type: 'disabled' }); + }); + + it('puts the reasoning sentinel into the shape Qwen reads', async () => { + const { down } = await runStage(vendorQwenNormalizeForOpenAIResponses, 'response.chat.openaiResponses', { + 'request.chat.openaiResponses': openaiResponsesPayload([], { reasoning: { effort: 'none' } }), + 'route.attempt': attemptWith('vendor-qwen'), + }); + const after = down['request.chat.openaiResponses'] as Record; + expect('reasoning' in after).toBe(false); + expect(after.enable_thinking).toBe(false); + }); + + it('leaves the sentinel alone for an upstream that is not Qwen', async () => { + // The dialect is the flag's, not the payload's: the same sentinel means something else on + // the next upstream, and a rewrite that fired without the flag would put a Qwen field on a + // wire that has never heard of it. + const original = openaiResponsesPayload([], { reasoning: { effort: 'none' } }); + const { down } = await runStage(vendorQwenNormalizeForOpenAIResponses, 'response.chat.openaiResponses', { + 'request.chat.openaiResponses': original, + 'route.attempt': attemptWith(), + }); + expect(down['request.chat.openaiResponses']).toBe(original); + }); + + it('leaves a payload that asked for real reasoning alone, by identity', async () => { + const original = openaiResponsesPayload([], { reasoning: { effort: 'high' } }); + for (const stage of [vendorDeepSeekNormalizeForOpenAIResponses, vendorQwenNormalizeForOpenAIResponses]) { + const { down } = await runStage(stage, 'response.chat.openaiResponses', { + 'request.chat.openaiResponses': original, + 'route.attempt': attemptWith('vendor-deepseek', 'vendor-qwen'), + }); + expect(down['request.chat.openaiResponses']).toBe(original); + } + }); +}); + +/** A lifecycle envelope carrying nothing but the counts under test. Every event that carries + * a response resource repeats the whole resource, which is why the rewrite is per event. */ +const usageEvent = (usage: Record): ProtocolFrame => + eventFrame({ type: 'response.completed', sequence_number: 0, response: { usage } } as unknown as OpenAIResponsesStreamEvent); + +const usageOf = (frame: ProtocolFrame): Record => + (frame as { event: { response: { usage: Record } } }).event.response.usage; + +// Each array below is the order a family's chain runs these rules in, and running it is what +// says the order is a property of the array rather than of any one stage: assembly refuses +// declarations that do not line up, and a rewrite that had to see another rewrite's output +// only sees it from where it sits. +// +// It is not that family's *whole* chain. Rules a wire owns run in the wire's own chain, below +// the stage that picks it, so what an array states is the order of what sits above the fork. +describe('a family\'s rule array, in the order its chain runs it', () => { + it('assembles and runs the Anthropic Messages array', async () => { + const { down } = await runChain('response.chat.anthropicMessages', [ + stripBillingAttributionFromAnthropicMessages, + disableReasoningOnForcedToolChoiceForAnthropicMessages, + applyRoleCompatibilityToAnthropicMessages, + ], { + 'request.chat.anthropicMessages': anthropicMessagesPayload([{ role: 'system', content: 'inline' }], { + system: 'x-anthropic-billing-header: cch=abcdef12\nreal instructions', + tool_choice: { type: 'any' }, + }), + 'route.attempt': attemptWith( + 'strip-billing-attribution', + 'disable-reasoning-on-forced-tool-choice', + 'rewrite-mid-conv-system-to-user', + ), + }); + const after = down['request.chat.anthropicMessages'] as AnthropicMessagesPayload; + expect(after.system).toBe('real instructions'); + expect(after.thinking).toEqual({ type: 'disabled' }); + expect(after.messages.map(message => message.role)).toEqual(['user']); + }); + + it('assembles and runs the Gemini generateContent array', async () => { + const answer = streamOf([ + eventFrame({ candidates: [{ index: 0, content: { parts: [{ text: 'pondering', thought: true }, { text: 'the answer' }] }, finishReason: 'STOP' }] }), + ]); + const { down, up } = await runChain('response.chat.geminiGenerateContent', [ + stripUnsupportedPartFieldsFromGeminiGenerateContent, + stripUnsupportedToolsFromGeminiGenerateContent, + stripSafetySettingsFromGeminiGenerateContent, + suppressThoughtPartsFromGeminiGenerateContent, + ], { + 'request.chat.geminiGenerateContent': { + contents: [{ role: 'user', parts: [{ text: 'hi', fileData: { mimeType: 'text/plain', fileUri: 'u' } }] }], + tools: [{ googleSearch: {} }], + safetySettings: [{ category: 'c', threshold: 't' }], + } satisfies GeminiGenerateContentPayload, + }, answer); + const after = down['request.chat.geminiGenerateContent'] as GeminiGenerateContentPayload; + expect(after.contents?.[0]?.parts).toEqual([{ text: 'hi' }]); + expect('tools' in after).toBe(false); + expect('safetySettings' in after).toBe(false); + const frames = await drain(up['response.chat.geminiGenerateContent']); + const event = (frames[0] as { event: Extract }).event; + expect(event.candidates?.[0]?.content.parts).toEqual([{ text: 'the answer' }]); + }); + + // The one ordering between stages rather than inside one: the sentinel is the gateway's + // canonical form, and a vendor normalizer can only put it on the wire in the vendor's shape + // because it runs after the stage that wrote it. + it('assembles and runs the OpenAI Responses array, the vendor normalizers last', async () => { + const answer = streamOf([usageEvent({ input_tokens: 100, output_tokens: 10, total_tokens: 160, input_tokens_details: { cached_tokens: 50 } })]); + const { down, up } = await runChain('response.chat.openaiResponses', [ + disableReasoningOnForcedToolChoiceForOpenAIResponses, + applyRoleCompatibilityToOpenAIResponses, + stripPromptCacheKeyForOpenAIResponses, + normalizeExclusiveCachedTokensForOpenAIResponses, + vendorDeepSeekNormalizeForOpenAIResponses, + vendorQwenNormalizeForOpenAIResponses, + ], { + 'request.chat.openaiResponses': openaiResponsesPayload([ + { type: 'message', role: 'system', content: 'lead' }, + { type: 'message', role: 'user', content: 'hi' }, + { type: 'message', role: 'system', content: 'mid' }, + ], { tool_choice: 'required', prompt_cache_key: 'k' }), + 'route.attempt': attemptWith( + 'disable-reasoning-on-forced-tool-choice', + 'rewrite-mid-conv-system-to-user', + 'strip-prompt-cache-key', + 'vendor-deepseek', + ), + }, answer); + const after = down['request.chat.openaiResponses'] as Record; + expect(rolesOfPayload(after)).toEqual(['system', 'user', 'user']); + expect('prompt_cache_key' in after).toBe(false); + // The sentinel this chain wrote reached DeepSeek's normalizer and left in DeepSeek's shape. + expect('reasoning' in after).toBe(false); + expect(after.thinking).toEqual({ type: 'disabled' }); + const frames = await drain(up['response.chat.openaiResponses']); + expect(usageOf(frames[0]!)).toMatchObject({ input_tokens: 150 }); + }); + + // The OpenAI Chat Completions wire's own array, which is not one family's: every source protocol + // that reaches an upstream over this endpoint runs it. What the order states is one thing in + // each direction. Going down, the usage chunk is asked for before any vendor can rewrite the + // body. Coming back, the vendor dialect has the first say — so the fold reads a cache count + // under OpenAI's name rather than DeepSeek's, and the carrier split reads the folded total. + it('assembles and runs the OpenAI Chat Completions wire, the vendor dialects innermost', async () => { + const answer = streamOf([eventFrame({ + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', + choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], + // DeepSeek's own names, and the exclusive convention: 479 + 13312 + 373 = 14164. + usage: { + prompt_tokens: 479, completion_tokens: 373, total_tokens: 14164, + prompt_cache_hit_tokens: 13312, prompt_cache_miss_tokens: 479, + }, + } as unknown as OpenAIChatCompletionsStreamEvent)]); + const { down, up } = await runChain('response.chat.openaiChatCompletions', [ + includeUsageStreamOptionsForOpenAIChatCompletions, + normalizeUsageForOpenAIChatCompletions, + normalizeEmptyToolsForOpenAIChatCompletions, + applyRoleCompatibilityToOpenAIChatCompletions, + normalizeExclusiveCachedTokensForOpenAIChatCompletions, + vendorDeepSeekNormalizeForOpenAIChatCompletions, + vendorQwenNormalizeForOpenAIChatCompletions, + vendorKimiNormalizeForOpenAIChatCompletions, + ], { + 'request.chat.openaiChatCompletions': openaiChatCompletionsPayload([ + { role: 'system', content: 'lead' }, + { role: 'user', content: 'hi' }, + { role: 'system', content: 'mid' }, + ], { reasoning_effort: 'none' }), + 'route.attempt': attemptWith('rewrite-mid-conv-system-to-user', 'vendor-deepseek'), + }, answer); + + const sent = down['request.chat.openaiChatCompletions'] as Record; + expect(sent.stream_options).toEqual({ include_usage: true }); + expect((sent as unknown as OpenAIChatCompletionsPayload).messages.map(message => message.role)).toEqual(['system', 'user', 'user']); + // The sentinel this chain carried reached DeepSeek's dialect and left in DeepSeek's shape. + expect('reasoning_effort' in sent).toBe(false); + expect(sent.thinking).toEqual({ type: 'disabled' }); + + const frames = await drain(up['response.chat.openaiChatCompletions']); + // Three rules ran on the way back, in the order that makes each one's input the one it + // expects: the vendor rename, then the fold, then the split onto a carrier. + expect(frames).toHaveLength(2); + expect('usage' in openaiChatCompletionsEventOf(frames[0]!)).toBe(false); + expect(openaiChatCompletionsUsageOf(frames[1]!)).toMatchObject({ + prompt_tokens: 13791, + prompt_tokens_details: { cached_tokens: 13312 }, + }); + }); +}); + +const rolesOfPayload = (payload: Record): unknown[] => + (payload.input as OpenAIResponsesInputItem[]).map(item => (item as { role?: unknown }).role); + +describe('empty-tools choice normalization', () => { + it.each([ + ['openaiChatCompletions', normalizeEmptyToolsForOpenAIChatCompletions, 'none'], + ['openaiResponses', normalizeEmptyToolsForOpenAIResponses, 'none'], + ['anthropicMessages', normalizeEmptyToolsForAnthropicMessages, { type: 'none' }], + ] as const)('normalizes explicit empty %s tools without changing the input', async (protocol, stage, none) => { + const key = `request.chat.${protocol}`; + const payload = { model: 'm', messages: [], input: [], max_tokens: 1, tools: [], tool_choice: protocol === 'anthropicMessages' ? { type: 'auto' } : 'auto' }; + const { down } = await runChain(`response.chat.${protocol}`, [stage], { [key]: payload, 'route.attempt': attemptWith('empty-tools-tool-choice-none') }); + expect((down[key] as { tool_choice: unknown }).tool_choice).toEqual(none); + expect(payload.tool_choice).toEqual(protocol === 'anthropicMessages' ? { type: 'auto' } : 'auto'); + const disabled = await runChain(`response.chat.${protocol}`, [stage], { [key]: payload, 'route.attempt': attemptWith() }); + expect(disabled.down[key]).toBe(payload); + const missing = { ...payload, tools: undefined }; + const absent = await runChain(`response.chat.${protocol}`, [stage], { [key]: missing, 'route.attempt': attemptWith('empty-tools-tool-choice-none') }); + expect(absent.down[key]).toBe(missing); + }); + it('retains Responses choices when deferred inventories still supply tools', async () => { + const payload = { model: 'm', input: [{ type: 'additional_tools', tools: [{ type: 'function', name: 'f' }] }], tools: [], tool_choice: 'auto' }; + const { down } = await runChain('response.chat.openaiResponses', [normalizeEmptyToolsForOpenAIResponses], { 'request.chat.openaiResponses': payload, 'route.attempt': attemptWith('empty-tools-tool-choice-none') }); + expect(down['request.chat.openaiResponses']).toBe(payload); + }); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts new file mode 100644 index 0000000000..b4f5662a10 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-count-tokens_test.ts @@ -0,0 +1,353 @@ +// The Anthropic Messages count-tokens chain, run. What is written down here is what only running it +// can say: that a measurement is taken on the body generation would have sent — the payload +// affinity materialized, the alias' own rules, the three request rules and the web-search +// rewrite — that a refusal is answered in the upstream's own words after every candidate has +// been tried, and that measuring bills nothing. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { anthropicMessagesCountTokensPipeline } from '../../../src/data-plane/chat/anthropic-messages/count-tokens.ts'; +import { createCandidateRegistry } from '../../../src/data-plane/pipeline/candidates.ts'; +import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; +import { initRepo } from '../../../src/repo/index.ts'; +import { chatFixtureHttpServices, stubChatProviderPipelines } from '../../test-utils/chat-provider-pipelines.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { move, run } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import type { AliasRules, ModelEndpoints } from '@floway-dev/protocols/common'; +import { type FlagId, type AnthropicMessagesUpstreamCallOptions, type ModelCandidate } from '@floway-dev/provider'; +import { type StubChatProviderCall, type ProviderCallResult } from '@floway-dev/test-utils'; +import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; + +vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ + ...(await importOriginal()), + enumerateModelCandidates: vi.fn(), +})); + +vi.mock('../../../src/data-plane/chat/anthropic-messages/affinity/ingress.ts', async importOriginal => ({ + ...(await importOriginal()), + analyzeAnthropicMessagesAffinity: async () => ({ + requiredTargets: [], + evaluateCandidate: () => ({ kind: 'accepted', degrades: false, preferred: true, materialize: () => affinityPayload }), + }), +})); + +let live: readonly ModelCandidate[] = []; + +type CountTokens = StubChatProviderCall<'anthropicMessagesCountTokens'>; + +const candidate = ( + callAnthropicMessagesCountTokens: CountTokens, + overrides: { + upstreamId?: string; + endpoints?: ModelEndpoints; + enabledFlags?: ReadonlySet; + rules?: AliasRules; + } = {}, +): ModelCandidate => { + const upstreamId = overrides.upstreamId ?? 'up_a'; + const endpoints = overrides.endpoints ?? { anthropicMessages: {} }; + const enabledFlags = overrides.enabledFlags ?? new Set(); + return { + provider: { + upstreamId, kind: 'claude-code', name: upstreamId, + // Wide enough that a header reaching the provider proves the ending let it through, + // rather than proving only that the allowlist did. + inboundHeaderAllowlist: [/^(anthropic-beta|x-trace)$/], + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + pipelines: stubChatProviderPipelines({ callAnthropicMessagesCountTokens: callAnthropicMessagesCountTokens as never }), + instance: stubProvider(), + }, + model: stubInternalModel( + { + id: 'claude-model', + endpoints, + providerModels: { [upstreamId]: stubProviderModel({ id: 'claude-model', endpoints, enabledFlags }) }, + }, + upstreamId, + ), + ...(overrides.rules === undefined ? {} : { rules: overrides.rules }), + } as unknown as ModelCandidate; +}; + +const resolves = (candidates: readonly ModelCandidate[], sawModel = true): void => { + live = candidates; + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel, failedUpstreams: [] } as never); +}; + +const counted = (input: number): ProviderCallResult => ({ + response: Response.json({ input_tokens: input }), + modelKey: 'claude-model-key', +}); + +const payload = { model: 'claude-model', max_tokens: 64, messages: [{ role: 'user', content: 'hi' }] } as unknown as AnthropicMessagesPayload; + +/** What affinity materialized for the candidate about to be dialled. It differs from the + * payload the client sent, which is the point: carried state is rewritten per candidate. */ +let affinityPayload: AnthropicMessagesPayload = payload; + +const count = async ( + request: AnthropicMessagesPayload = payload, + headers: readonly (readonly [string, string])[] = [], +) => { + const gateway = mockChatGatewayCtx({ wantsStream: false }); + return await run( + anthropicMessagesCountTokensPipeline(), + move({ + 'ingress.http.headers': headers, + 'ingress.chat.sourceProtocol': 'anthropicMessages', + 'request.chat.anthropicMessages': request, + 'serve.model': request.model, + }) as never, + { + gateway, + background: () => {}, + ...chatFixtureHttpServices(gateway), + ...createCandidateRegistry(), + selectAffinity: () => { throw new Error('a measurement pins nothing; it must not select affinity'); }, + resolveAttempt: (selector: { readonly upstreamId: string }) => { + const found = live.find(c => c.provider.upstreamId === selector.upstreamId); + if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); + return found; + }, + } as never, + ); +}; + +beforeEach(() => { + vi.mocked(enumerateModelCandidates).mockReset(); + affinityPayload = payload; + initRepo({ + apiKeys: { update: async () => {} }, + usage: { record: async () => {} }, + performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, + } as never); +}); + +describe('the messages count-tokens chain', () => { + // The same statement the generate chain makes: what is measured is the body affinity + // materialized for this candidate, stamped with the id that candidate actually serves, + // with an alias' own rules overlaid on top of it. + it('measures the payload this attempt is owed, under the id the upstream resolved', async () => { + let sent: Record | undefined; + let seenModel: { readonly id: string } | undefined; + affinityPayload = { ...payload, messages: [{ role: 'user', content: 'rewritten' }] } as AnthropicMessagesPayload; + resolves([candidate(async (model, body) => { + seenModel = model as { readonly id: string }; + sent = body as Record; + return counted(11); + }, { rules: { reasoning: { effort: 'high' } } })]); + + const { facts } = await count(); + + expect(facts['response.http.status']).toBe(200); + expect(facts['response.chat.anthropicMessages.rendered']).toEqual({ input_tokens: 11 }); + expect(seenModel?.id).toBe('claude-model'); + expect(sent).toMatchObject({ messages: [{ role: 'user', content: 'rewritten' }] }); + // The id the client addressed does not travel: the provider re-stamps what it resolved. + expect(sent).not.toHaveProperty('model'); + // An alias' rules reach the measured body, so a count and the generation it precedes + // are taken on the same request. + expect(sent).toMatchObject({ output_config: { effort: 'high' } }); + }); + + // Counting observes the same gateway-level rewrites generation does, in the order the + // chain applies them, so a client is told what the request it is about to send costs. + it('applies the three request rules generation applies', async () => { + let sent: Record | undefined; + affinityPayload = { + ...payload, + system: 'x-anthropic-billing-header: token\ncch=deadbeef1234;\nbase rules', + messages: [ + { role: 'system', content: 'inline rules' }, + { role: 'user', content: 'hello' }, + ], + thinking: { type: 'enabled', budget_tokens: 1024 }, + output_config: { effort: 'high' }, + tool_choice: { type: 'tool', name: 'lookup' }, + } as unknown as AnthropicMessagesPayload; + resolves([candidate(async (_model, body) => { sent = body as Record; return counted(9); }, { + enabledFlags: new Set([ + 'strip-billing-attribution', + 'disable-reasoning-on-forced-tool-choice', + 'rewrite-mid-conv-system-to-user', + ]), + })]); + + await count(); + + expect(sent).toEqual({ + max_tokens: 64, + system: 'base rules', + messages: [ + { role: 'user', content: 'inline rules' }, + { role: 'user', content: 'hello' }, + ], + thinking: { type: 'disabled' }, + tool_choice: { type: 'tool', name: 'lookup' }, + }); + }); + + // Anthropic's native hosted tool never reaches an upstream as itself — the shim sends an + // ordinary client tool — so a count taken on the client's own body would measure a request + // nobody is sent. + it('measures the client-tool shape the web-search shim would actually send', async () => { + let sent: AnthropicMessagesPayload | undefined; + const asked = { ...payload, tools: [{ type: 'web_search_20260209', max_uses: 3 }] } as unknown as AnthropicMessagesPayload; + affinityPayload = asked; + resolves([candidate(async (_model, body) => { sent = body as AnthropicMessagesPayload; return counted(13); }, { + enabledFlags: new Set(['anthropic-messages-web-search-shim']), + })]); + + await count(asked); + + const tool = sent?.tools?.[0] as { name?: string; type?: string; input_schema?: unknown } | undefined; + expect(tool?.name).toBe('web_search'); + expect(tool).not.toHaveProperty('type'); + expect(tool?.input_schema).toEqual({ + type: 'object', + properties: { query: { type: 'string', description: 'Search query' } }, + required: ['query'], + }); + }); + + // A tool definition this protocol rejects is an answer the chain already holds, so no + // upstream is asked what an unsendable request would cost. + it('refuses a native web-search definition the protocol rejects without dialling', async () => { + const asked = { + ...payload, + tools: [ + { type: 'web_search_20260209' }, + { type: 'web_search_20250305' }, + ], + } as unknown as AnthropicMessagesPayload; + affinityPayload = asked; + let dialled = 0; + resolves([candidate(async () => { dialled += 1; return counted(0); }, { + enabledFlags: new Set(['anthropic-messages-web-search-shim']), + })]); + + const { facts } = await count(asked); + + expect(dialled).toBe(0); + expect(facts['response.http.status']).toBe(400); + expect(facts['response.chat.anthropicMessages.rendered']).toMatchObject({ + type: 'error', + error: { type: 'invalid_request_error', message: 'Only one native web search tool definition is supported per request.' }, + }); + }); + + // The flag is the whole of the gate here: counting has only the native wire, so the + // structural half of the shim's condition — a translated target — can never hold. + it('leaves the native tool alone on a candidate the shim is off for', async () => { + let sent: AnthropicMessagesPayload | undefined; + const asked = { ...payload, tools: [{ type: 'web_search_20260209', max_uses: 3 }] } as unknown as AnthropicMessagesPayload; + affinityPayload = asked; + resolves([candidate(async (_model, body) => { sent = body as AnthropicMessagesPayload; return counted(13); })]); + + await count(asked); + + expect(sent?.tools).toEqual([{ type: 'web_search_20260209', max_uses: 3 }]); + }); + + // Anthropic beta flags have a typed path of their own precisely so no provider's header + // allowlist can admit them, and a measurement asks the same question generation does. + it('hands the beta flags over on their own path and not as a header', async () => { + let seen: AnthropicMessagesUpstreamCallOptions | undefined; + resolves([candidate(async (_model, _body, _signal, opts) => { seen = opts; return counted(5); })]); + + await count(payload, [ + ['anthropic-beta', 'context-1m-2025-08-07, advanced-tool-use-2025-11-20'], + ['x-trace', 'abc'], + ]); + + expect(seen?.anthropicBeta).toEqual(['context-1m-2025-08-07', 'advanced-tool-use-2025-11-20']); + expect(seen?.headers.get('anthropic-beta')).toBeNull(); + expect(seen?.headers.get('x-trace')).toBe('abc'); + }); + + // A measurement one upstream refused is a measurement another may answer, which is what + // the fork is for — and the surviving refusal keeps the upstream's own status and words. + it('fails a refused measurement over to the next candidate', async () => { + const tried: string[] = []; + resolves([ + candidate(async () => { + tried.push('busy'); + return { response: new Response('unavailable', { status: 503 }), modelKey: 'k' }; + }, { upstreamId: 'up_busy' }), + candidate(async () => { tried.push('free'); return counted(42); }, { upstreamId: 'up_free' }), + ]); + + const { facts } = await count(); + + expect(tried).toEqual(['busy', 'free']); + expect(facts['response.http.status']).toBe(200); + expect(facts['response.chat.anthropicMessages.rendered']).toEqual({ input_tokens: 42 }); + }); + + it('answers the last refusal in the upstream-s own status and words', async () => { + resolves([candidate(async () => ({ + response: new Response(JSON.stringify({ type: 'error', error: { type: 'rate_limit_error', message: 'slow down' } }), { + status: 429, headers: { 'content-type': 'application/json' }, + }), + modelKey: 'k', + }))]); + + const { facts } = await count(); + + expect(facts['response.http.status']).toBe(429); + expect(facts['response.chat.anthropicMessages.rendered']).toEqual({ + type: 'error', + error: { type: 'rate_limit_error', message: 'slow down' }, + }); + }); + + // A refused connection is an outcome the fork has to be able to see, not a fault that ends + // the run. + it('fails a dial that never connected over to the next candidate', async () => { + const tried: string[] = []; + resolves([ + candidate(async () => { tried.push('dead'); return { type: 'transportFailure' as const, error: new Error('ECONNREFUSED') }; }, { upstreamId: 'up_dead' }), + candidate(async () => { tried.push('alive'); return counted(7); }, { upstreamId: 'up_alive' }), + ]); + + const { facts } = await count(); + + expect(tried).toEqual(['dead', 'alive']); + expect(facts['response.chat.anthropicMessages.rendered']).toEqual({ input_tokens: 7 }); + }); + + // Only a native Anthropic Messages endpoint measures — no translation carries the question — so a + // candidate that would serve generation over a translated wire cannot serve this, and the + // refusal names the endpoint the client actually addressed. + it('refuses a candidate no wire can measure on, naming the count endpoint', async () => { + resolves([candidate(async () => counted(0), { endpoints: { openaiChatCompletions: {} } })]); + + const { facts } = await count(); + + expect(facts['response.http.status']).toBe(400); + expect(facts['response.chat.anthropicMessages.rendered']).toMatchObject({ + error: { + type: 'invalid_request_error', + message: 'Model claude-model does not support the /messages/count_tokens endpoint.', + }, + }); + }); + + // Measuring is not generating: no upstream charged for the answer, so the run has nothing + // to bill and says so rather than naming an entity that reported zero. + it('bills nothing for a measurement', async () => { + resolves([candidate(async () => counted(3))]); + + const { facts } = await count(); + + expect((facts as Record)['response.usage.billable']).toEqual([]); + }); +}); + +it('preserves the accepted native counting HTTP status', async () => { + resolves([candidate(async () => ({ response: new Response(JSON.stringify({ input_tokens: 3 }), { status: 201, headers: { 'content-type': 'application/json' } }), modelKey: 'counted-model' }))]); + const { facts, drain } = await count(); + expect(facts['response.http.status']).toBe(201); + await drain(); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts new file mode 100644 index 0000000000..cc519662d3 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages-pipeline_test.ts @@ -0,0 +1,405 @@ +// The Anthropic Messages chain, run. Assembly is checked where every family's is; what is written +// down here is what only running it can say — that the edge writes Anthropic's own named +// SSE events when the client asked to stream and reassembles them into one message when it +// did not, that it hands the turn's own state back for the client to carry, that a refusal +// keeps the upstream's own status and words, that a dial nobody answered is a value the fork +// can move past, that the beta flags travel on their typed path rather than as a header, and +// that what the two usage-bearing events add up to is billed in the metrics settlement reads. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { anthropicMessagesServePipeline } from '../../../src/data-plane/chat/anthropic-messages/pipeline.ts'; +import { createCandidateRegistry } from '../../../src/data-plane/pipeline/candidates.ts'; +import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; +import { initRepo } from '../../../src/repo/index.ts'; +import { chatFixtureHttpServices, stubChatProviderPipelines } from '../../test-utils/chat-provider-pipelines.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { move, run } from '@floway-dev/pipeline'; +import { ANTHROPIC_MESSAGES_MISSING_TERMINAL_MESSAGE, type AnthropicMessagesPayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import type { SseFrame } from '@floway-dev/protocols/common'; +import { directFetcher, type AnthropicMessagesUpstreamCallOptions, type ModelCandidate } from '@floway-dev/provider'; +import { type StubChatProviderCall, type ProviderStreamResult } from '@floway-dev/test-utils'; +import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; + +vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ + ...(await importOriginal()), + enumerateModelCandidates: vi.fn(), +})); + +vi.mock('../../../src/data-plane/chat/anthropic-messages/affinity/ingress.ts', async importOriginal => ({ + ...(await importOriginal()), + analyzeAnthropicMessagesAffinity: async () => ({ + requiredTargets: [], + evaluateCandidate: () => ({ kind: 'accepted', degrades: false, preferred: true, materialize: () => { asked += 1; return affinityPayload; } }), + }), +})); + +let live: readonly ModelCandidate[] = []; + +type CallAnthropicMessages = StubChatProviderCall<'anthropicMessages'>; + +const candidate = (callAnthropicMessages: CallAnthropicMessages, upstreamId = 'up_a'): ModelCandidate => { + const endpoints = { anthropicMessages: {} }; + return { + provider: { + upstreamId, kind: 'claude-code', name: upstreamId, + // Wide enough that a header reaching the provider proves the stage let it through, + // rather than proving only that the allowlist did. + inboundHeaderAllowlist: [/^(anthropic-beta|x-trace)$/], + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + pipelines: stubChatProviderPipelines({ callAnthropicMessages: callAnthropicMessages as never }), + instance: stubProvider(), + }, + model: stubInternalModel( + { id: 'claude-model', endpoints, providerModels: { [upstreamId]: stubProviderModel({ id: 'claude-model', endpoints }) } }, + upstreamId, + ), + fetcher: directFetcher, + } as unknown as ModelCandidate; +}; + +const resolves = (candidates: readonly ModelCandidate[]): void => { + live = candidates; + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] } as never); +}; + +// One well-formed turn: Anthropic states input accounting on `message_start` and output +// accounting on `message_delta`, so a stream that says what it cost says it in two places. +const messageStart = { + type: 'message_start', + message: { + id: 'msg_1', type: 'message', role: 'assistant', content: [], model: 'claude-model', + stop_reason: null, stop_sequence: null, + usage: { input_tokens: 5, output_tokens: 0, cache_read_input_tokens: 2 }, + }, +}; +const blockStart = { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }; +const textDelta = (text: string) => ({ type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text } }); +const blockStop = { type: 'content_block_stop', index: 0 }; +const messageDelta = { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 7 } }; +const messageStop = { type: 'message_stop' }; + +const turn = (...text: readonly string[]): readonly unknown[] => + [messageStart, blockStart, ...text.map(textDelta), blockStop, messageDelta, messageStop]; + +/** The same turn with its terminal event cut off, which is what an upstream that died + * mid-answer leaves behind. */ +const truncatedTurn = (...text: readonly string[]): readonly unknown[] => + turn(...text).filter(event => event !== messageStop); + +const stream = (events: readonly unknown[]): ProviderStreamResult => ({ + ok: true, + modelKey: 'claude-model-key', + headers: new Headers({ 'x-request-id': 'req-1', 'content-length': '99' }), + events: (async function* () { + for (const event of events) yield { type: 'event' as const, event: event as AnthropicMessagesStreamEvent }; + yield { type: 'done' as const }; + })(), +}); + +const payload = { model: 'claude-model', max_tokens: 64, messages: [{ role: 'user', content: 'hi' }] } as unknown as AnthropicMessagesPayload; + +/** What affinity materialized for the candidate about to be dialled. It differs from the + * payload the client sent, which is the point: carried state is rewritten per candidate. */ +let affinityPayload: AnthropicMessagesPayload = payload; + +/** How many times the chain asked the resolver for that payload. One stage owns the reading; + * anything below it reads the record. */ +let asked = 0; + +const serve = async (wantsStream: boolean, headers: readonly (readonly [string, string])[] = []) => + await serveWith(mockChatGatewayCtx({ wantsStream }), wantsStream, headers); + +const serveWith = async ( + gateway: ReturnType, + wantsStream: boolean, + headers: readonly (readonly [string, string])[] = [], +) => await run( + anthropicMessagesServePipeline(), + move({ + 'ingress.http.headers': headers, + 'ingress.chat.sourceProtocol': 'anthropicMessages', + 'ingress.chat.anthropicMessages.wantsStream': wantsStream, + 'request.chat.anthropicMessages': payload, + 'serve.model': 'claude-model', + }) as never, + { + gateway, + background: () => {}, + ...chatFixtureHttpServices(gateway), + ...createCandidateRegistry(), + // Wired where the app wires it: the carrier the edge writes is addressed to whatever the + // chain named here. + selectAffinity: (selected: ModelCandidate) => { gateway.affinity.select(selected); }, + resolveAttempt: (selector: { readonly upstreamId: string }) => { + const found = live.find(c => c.provider.upstreamId === selector.upstreamId); + if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); + return found; + }, + } as never, +); + +const collect = async (rendered: unknown): Promise => { + const frames: SseFrame[] = []; + for await (const frame of rendered as AsyncIterable) frames.push(frame); + return frames; +}; + +// The carrier the edge writes back is a redacted_thinking block holding this turn's own +// state. Anthropic Messages has real block boundaries, so it takes content block 0 of its own: its +// start and stop are the two extra frames, and the turn's own blocks answer behind it. + +/** The turn's own frames, with the carrier block's dropped. */ +const withoutCarrier = (frames: readonly SseFrame[]): readonly SseFrame[] => { + const carrierBlocks = new Set(); + return frames.filter(frame => { + const event = JSON.parse(frame.data) as { + readonly type: string; + readonly index?: number; + readonly content_block?: { readonly type: string }; + }; + if (event.index === undefined) return true; + if (event.type === 'content_block_start' && event.content_block?.type === 'redacted_thinking') { + carrierBlocks.add(event.index); + } + return !carrierBlocks.has(event.index); + }); +}; + +/** The same events, at the block index the carrier leaves them at. */ +const behindCarrier = (events: readonly unknown[]): readonly unknown[] => + events.map(event => { + const block = event as { readonly index?: number }; + return block.index === undefined ? event : { ...block, index: block.index + 1 }; + }); + +beforeEach(() => { + vi.mocked(enumerateModelCandidates).mockReset(); + affinityPayload = payload; + asked = 0; + initRepo({ + apiKeys: { update: async () => {} }, + usage: { record: async () => {} }, + performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, + } as never); +}); + +describe('the messages chain', () => { + // Affinity rewrites client-carried state — a thinking signature only its issuer can read — + // for the upstream that will see it, so the body that goes out is the one it materialized + // rather than the one the client sent. It enters the record below the fork because + // everything between there and the dial rewrites the request as a fact, and one stage owns + // that reading: an ending that asked the resolver again would send a payload no stage in + // the chain had touched. + it('puts the payload this attempt is owed into the record, and sends that', async () => { + let sent: Record | undefined; + affinityPayload = { ...payload, messages: [{ role: 'user', content: 'rewritten' }] } as AnthropicMessagesPayload; + resolves([candidate(async (_model, body) => { + sent = body as Record; + return stream(turn('hi')); + })]); + + const { facts } = await serve(false); + + // The exit contract names what the run answers with, and the request the chain sent is + // not that — but the record carries every key in flight, so what was sent is still there + // to read. + expect((facts as Record)['request.chat.anthropicMessages']).toBe(affinityPayload); + expect(asked).toBe(1); + // The id the client addressed does not travel: the provider re-stamps what it resolved. + expect(sent).toMatchObject({ messages: [{ role: 'user', content: 'rewritten' }] }); + expect(sent).not.toHaveProperty('model'); + }); + + // Anthropic's clients dispatch on the SSE event name, not on the payload alone, so the + // name each frame carries is part of the answer. + it('writes the frames out under their own event names when the client asked to stream', async () => { + resolves([candidate(async () => stream(turn('he', 'llo')))]); + + const { facts } = await serve(true); + const frames = withoutCarrier(await collect(facts['response.chat.anthropicMessages.rendered'])); + + expect(facts['response.http.status']).toBe(200); + expect(frames.map(frame => frame.event)).toEqual([ + 'message_start', 'content_block_start', 'content_block_delta', 'content_block_delta', + 'content_block_stop', 'message_delta', 'message_stop', + ]); + expect(frames.map(frame => frame.data)).toEqual(behindCarrier(turn('he', 'llo')).map(event => JSON.stringify(event))); + }); + + // The other half of affinity: the resolver reads client-carried state on the way down, and + // this is what the client is given to carry back. A turn that handed back nothing would + // leave the follow-up quoting it with no way to name the upstream that issued it. + it('hands the turn-s own state back on a block of its own', async () => { + resolves([candidate(async () => stream(turn('hi')))]); + const gateway = mockChatGatewayCtx({ wantsStream: true }); + + const { facts } = await serveWith(gateway, true); + const frames = await collect(facts['response.chat.anthropicMessages.rendered']); + const carrier = frames + .map(frame => JSON.parse(frame.data) as { readonly type: string; readonly content_block?: { readonly type: string; readonly data: string } }) + .filter(event => event.type === 'content_block_start' && event.content_block?.type === 'redacted_thinking'); + + expect(carrier).toHaveLength(1); + // It names the upstream that answered, sealed under this run's own secret — which is what + // lets the next turn be pinned to it. + expect(await gateway.affinity.codec.unwrap(carrier[0]!.content_block!.data, 'anthropic-messages.redacted_thinking.data')).toMatchObject({ + kind: 'owned', + affinity: { upstreamId: 'up_a', modelId: 'claude-model' }, + }); + }); + + // The upstream speaks SSE whatever the client asked for, so a client that did not ask to + // stream is answered from the same frames — folded here rather than read a second time. + it('reassembles the frames into one message when the client did not', async () => { + resolves([candidate(async () => stream(turn('he', 'llo')))]); + + const { facts } = await serve(false); + const rendered = facts['response.chat.anthropicMessages.rendered'] as { + readonly type: string; + readonly content: readonly { readonly type: string; readonly text?: string }[]; + readonly stop_reason: string; + readonly usage: { readonly input_tokens: number; readonly output_tokens: number }; + }; + + expect(facts['response.http.status']).toBe(200); + expect(rendered.type).toBe('message'); + expect(rendered.content.filter(block => block.type !== 'redacted_thinking').map(block => block.text)).toEqual(['hello']); + expect(rendered.stop_reason).toBe('end_turn'); + expect(rendered.usage).toMatchObject({ input_tokens: 5, output_tokens: 7 }); + }); + + // Content-length would misdescribe a body this gateway serialized itself; a vendor trace + // is what a client and an operator both need to correlate a turn. + it('forwards the upstream headers a client may see and drops the ones it may not', async () => { + resolves([candidate(async () => stream(turn('hi')))]); + + const { facts } = await serve(true); + + expect(Object.fromEntries(facts['response.http.headers'])).toMatchObject({ 'x-request-id': 'req-1' }); + expect(Object.fromEntries(facts['response.http.headers'])).not.toHaveProperty('content-length'); + }); + + it('answers an upstream refusal with its own status and words', async () => { + resolves([candidate(async () => ({ + ok: false, + modelKey: 'claude-model-key', + response: new Response(JSON.stringify({ type: 'error', error: { type: 'rate_limit_error', message: 'slow down' } }), { + status: 429, headers: { 'content-type': 'application/json' }, + }), + }))]); + + const { facts } = await serve(false); + + expect(facts['response.http.status']).toBe(429); + expect(facts['response.chat.anthropicMessages.rendered']).toEqual({ + type: 'error', + error: { type: 'rate_limit_error', message: 'slow down' }, + }); + }); + + // A refused connection is an outcome the fork has to be able to see, not a fault that ends + // the run — so the second candidate is tried and its answer is the one served. + it('fails a dial that never connected over to the next candidate', async () => { + const tried: string[] = []; + resolves([ + candidate(async () => { tried.push('dead'); return { type: 'transportFailure' as const, error: new Error('ECONNREFUSED') }; }, 'up_dead'), + candidate(async () => { tried.push('alive'); return stream(turn('hi')); }, 'up_alive'), + ]); + + const { facts } = await serve(false); + + expect(tried).toEqual(['dead', 'alive']); + expect(facts['response.http.status']).toBe(200); + expect(facts['response.chat.anthropicMessages.rendered']).toMatchObject({ type: 'message' }); + }); + + // Anthropic beta flags have a typed path of their own precisely so no provider's header + // allowlist can admit them and no other source protocol can leak them in — so the field is + // read off the client's request and does not travel beside itself. + it('hands the beta flags over on their own path and not as a header', async () => { + let seen: AnthropicMessagesUpstreamCallOptions | undefined; + resolves([candidate(async (_model, _body, _signal, opts) => { + seen = opts; + return stream(turn('hi')); + })]); + + await serve(false, [ + ['anthropic-beta', 'oauth-2025-04-20,fine-grained-tool-streaming-2025-05-14'], + ['x-trace', 'abc'], + ]); + + expect(seen?.anthropicBeta).toEqual(['oauth-2025-04-20', 'fine-grained-tool-streaming-2025-05-14']); + expect(seen?.headers.get('anthropic-beta')).toBeNull(); + expect(seen?.headers.get('x-trace')).toBe('abc'); + }); + + // The reading is per billing metric, which is not the shape Anthropic reports in: input + // accounting arrives on `message_start` and output accounting on `message_delta`, and what + // settlement writes is what the two add up to after the client's own stream drove them out. + it('bills what the upstream metered across both of its usage events', async () => { + resolves([candidate(async () => stream(turn('hi')))]); + + const { facts } = await serve(true); + + // Before the drain: called, and reported nothing — the numbers arrive with the last + // frame, which is after this run has answered. + expect(facts['response.usage.billable']).toEqual([ + { identity: { model: 'claude-model', upstream: 'up_a', modelKey: 'claude-model-key', pricing: null }, quantities: {} }, + ]); + + await collect(facts['response.chat.anthropicMessages.rendered']); + const outcome = await facts['response.chat.anthropicMessages.streamedUsage']!; + const billable = outcome.billable; + + expect(billable[0]).toMatchObject({ + identity: { model: 'claude-model', upstream: 'up_a', modelKey: 'claude-model-key', pricing: null }, + quantities: { input_tokens: '5', input_cache_read_tokens: '2', output_tokens: '7' }, + }); + // A rate can depend on the tier and on how much input there was; both are selector + // coordinates, so they travel as pricing facts rather than as quantities. + expect(billable[0]!.pricingFacts).toMatchObject({ inputTokens: 7 }); + }); + + // Time to first token is measured where the token is — `message_start` and + // `content_block_start` are the envelope around the answer, not the answer, and a run that + // never stamps it is recorded as having produced nothing to time. + it('stamps the first frame that carried a generated token', async () => { + resolves([candidate(async () => stream(turn('hi')))]); + const gateway = mockChatGatewayCtx({ wantsStream: true }); + + const { facts } = await serveWith(gateway, true); + expect(gateway.attempt.timing.firstOutputTokenAt).toBeNull(); + await collect(facts['response.chat.anthropicMessages.rendered']); + + expect(gateway.attempt.timing.firstOutputTokenAt).toBeTypeOf('number'); + }); + + // A turn is over at `message_stop`, so what an upstream writes after it is not part of the + // answer and the client is not shown it. Reading stops there, which also closes the + // upstream rather than leaving the client's stream open behind a connection nobody reads. + it('stops reading at the terminal event', async () => { + resolves([candidate(async () => stream([...turn('hi'), textDelta(' and more')]))]); + + const { facts } = await serve(true); + const frames = withoutCarrier(await collect(facts['response.chat.anthropicMessages.rendered'])); + + expect(frames.map(frame => frame.event)).toEqual([ + 'message_start', 'content_block_start', 'content_block_delta', + 'content_block_stop', 'message_delta', 'message_stop', + ]); + }); + + // A stream that ran out before `message_stop` is a turn nobody can answer from: the message + // was never stopped and never failed, so serving what did arrive would report a truncated + // answer as a whole one. + it('fails a stream that ended without its terminal event', async () => { + resolves([candidate(async () => stream(truncatedTurn('hi')))]); + + const { facts } = await serve(true); + + const frames = await collect(facts['response.chat.anthropicMessages.rendered']); + expect(frames.at(-1)?.event).toBe('error'); + expect(JSON.parse(frames.at(-1)!.data)).toMatchObject({ type: 'error', error: { message: ANTHROPIC_MESSAGES_MISSING_TERMINAL_MESSAGE } }); + }); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/attempt_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/attempt_test.ts deleted file mode 100644 index f809fa0d70..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/attempt_test.ts +++ /dev/null @@ -1,529 +0,0 @@ -import { test, vi } from 'vitest'; - -import { anthropicMessagesAttempt } from '../../../../src/data-plane/chat/anthropic-messages/attempt.ts'; -import { initRepo } from '../../../../src/repo/index.ts'; -import { InMemoryRepo } from '../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import type { AnthropicMessagesClientTool, AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import { doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import type { OpenAIResponsesPayload, OpenAIResponsesResult } from '@floway-dev/protocols/openai-responses'; -import { type AnthropicMessagesUpstreamCallOptions, type ModelCandidate, directFetcher, type ProviderCallResult, type ProviderOpenAIResponsesResult, type ProviderStreamResult, type OpenAIResponsesAction, type UpstreamCallOptions } from '@floway-dev/provider'; -import type { FlagId } from '@floway-dev/provider/flags'; -import { assertEquals, assertExists, stubProvider, stubInternalModel, stubProviderModel } from '@floway-dev/test-utils'; - -const API_KEY_ID = 'key_messages_attempt_test'; - -const makeGatewayCtx = () => mockChatGatewayCtx({ apiKeyId: API_KEY_ID, wantsStream: true }); - -const makePayload = (overrides: Partial = {}): AnthropicMessagesPayload => ({ - model: 'test-model', - max_tokens: 32, - messages: [{ role: 'user', content: 'hello' }], - ...overrides, -}); - -const makeAnthropicMessagesEvents = (): readonly AnthropicMessagesStreamEvent[] => [ - { - type: 'message_start', - message: { - id: 'msg_1', - type: 'message', - role: 'assistant', - content: [], - model: 'test-model', - stop_reason: null, - stop_sequence: null, - usage: { input_tokens: 4, output_tokens: 0 }, - }, - }, - { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, - { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'hi' } }, - { type: 'content_block_stop', index: 0 }, - { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 1 } }, - { type: 'message_stop' }, -]; - -const makeProtocolFrames = async function* (events: readonly TEvent[]): AsyncGenerator> { - for (const event of events) yield eventFrame(event); - yield doneFrame(); -}; - -const makeCandidate = (overrides: { - upstream?: string; - endpoints?: ModelEndpoints; - callAnthropicMessages?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: AnthropicMessagesUpstreamCallOptions) => Promise>; - callOpenAIResponses?: (model: unknown, body: unknown, action: OpenAIResponsesAction, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise; - callOpenAIChatCompletions?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise>; - callAnthropicMessagesCountTokens?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: AnthropicMessagesUpstreamCallOptions) => Promise; - enabledFlags?: ReadonlySet; -} = {}): ModelCandidate => { - const upstream = overrides.upstream ?? 'up_test'; - const endpoints = overrides.endpoints ?? { openaiChatCompletions: {}, openaiResponses: {}, anthropicMessages: {} }; - const provider = stubProvider({ - callAnthropicMessages: overrides.callAnthropicMessages, - callOpenAIResponses: overrides.callOpenAIResponses, - callOpenAIChatCompletions: overrides.callOpenAIChatCompletions, - callAnthropicMessagesCountTokens: overrides.callAnthropicMessagesCountTokens, - }); - return { - provider: { - upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: provider, - }, - model: stubInternalModel({ - endpoints, - providerModels: { - [upstream]: stubProviderModel({ endpoints, enabledFlags: new Set(overrides.enabledFlags ?? []) }), - }, - }, upstream), - fetcher: directFetcher, - }; -}; - -const collectEvents = async (events: AsyncIterable>): Promise => { - const out: TEvent[] = []; - for await (const frame of events) { - if (frame.type === 'event') out.push(frame.event); - } - return out; -}; - -const installRepo = (): InMemoryRepo => { - const repo = new InMemoryRepo(); - initRepo(repo); - return repo; -}; - -test('generate native messages target calls provider.callAnthropicMessages with no rewrite', async () => { - installRepo(); - const callAnthropicMessages = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeAnthropicMessagesEvents()), modelKey: 'k', headers: new Headers(), - })); - const result = await anthropicMessagesAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callAnthropicMessages }), - headers: new Headers(), - anthropicBeta: [], - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callAnthropicMessages.mock.calls.length, 1); -}); - -test('generate carries anthropic-beta through the Anthropic Messages boundary outside the provider allowlist', async () => { - installRepo(); - let upstreamHeaders: Headers | undefined; - let upstreamAnthropicBeta: readonly string[] | undefined; - const callAnthropicMessages = vi.fn(async (_model, _body, _signal, opts): Promise> => { - upstreamHeaders = opts?.headers; - upstreamAnthropicBeta = opts?.anthropicBeta; - return { ok: true, events: makeProtocolFrames(makeAnthropicMessagesEvents()), modelKey: 'k', headers: new Headers() }; - }); - - const result = await anthropicMessagesAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callAnthropicMessages }), - headers: new Headers({ - 'anthropic-beta': 'must-not-enter-ordinary-headers', - 'x-unlisted': 'discard', - }), - anthropicBeta: ['context-1m-2025-08-07', 'advanced-tool-use-2025-11-20'], - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertExists(upstreamHeaders); - assertEquals(Object.fromEntries(upstreamHeaders), {}); - assertEquals(upstreamAnthropicBeta, ['context-1m-2025-08-07', 'advanced-tool-use-2025-11-20']); -}); - -test('generate translate-to-responses branch routes through openaiResponsesAttempt', async () => { - installRepo(); - let upstreamHeaders: Headers | undefined; - const respResp: OpenAIResponsesResult = { - id: 'resp_x', object: 'response', model: 'test-model', status: 'completed', - output: [{ - type: 'message', id: 'msg_resp', role: 'assistant', status: 'completed', - content: [{ type: 'output_text', text: 'hi', annotations: [] }], - }], - output_text: 'hi', error: null, incomplete_details: null, - }; - const callOpenAIResponses = vi.fn(async (_model, _body, _action, _signal, opts): Promise => { - upstreamHeaders = opts?.headers; - return { - action: 'generate', ok: true, - events: makeProtocolFrames([{ type: 'response.completed', sequence_number: 0, response: respResp }]), - modelKey: 'k', - headers: new Headers(), - }; - }); - const result = await anthropicMessagesAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callOpenAIResponses, endpoints: { openaiResponses: {} } }), - headers: new Headers({ 'anthropic-beta': 'must-not-enter-ordinary-headers' }), - anthropicBeta: ['context-1m-2025-08-07'], - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callOpenAIResponses.mock.calls.length, 1); - assertEquals(upstreamHeaders?.has('anthropic-beta'), false); -}); - -test('generate does not carry Anthropic Messages beta metadata through translation to OpenAI Chat Completions', async () => { - installRepo(); - let upstreamHeaders: Headers | undefined; - const callOpenAIChatCompletions = vi.fn(async (_model, _body, _signal, opts): Promise> => { - upstreamHeaders = opts?.headers; - return { - ok: true, - events: makeProtocolFrames([{ - id: 'chatcmpl_1', object: 'chat.completion.chunk', created: 1, model: 'test-model', - choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], - }]), - modelKey: 'k', - headers: new Headers(), - }; - }); - - const result = await anthropicMessagesAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callOpenAIChatCompletions, endpoints: { openaiChatCompletions: {} } }), - headers: new Headers({ 'anthropic-beta': 'must-not-enter-ordinary-headers' }), - anthropicBeta: ['context-1m-2025-08-07'], - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(upstreamHeaders?.has('anthropic-beta'), false); -}); - -test('generate lets the target system-to-developer rewrite take precedence over the source system-to-user rewrite', async () => { - installRepo(); - const observedBodies: Omit[] = []; - const callOpenAIResponses = vi.fn(async (_model, body): Promise => { - observedBodies.push(body as Omit); - return { - action: 'generate', - ok: true, - events: makeProtocolFrames([{ - type: 'response.completed', - sequence_number: 0, - response: { - id: 'resp_x', - object: 'response', - model: 'test-model', - status: 'completed', - output: [], - output_text: '', - error: null, - incomplete_details: null, - }, - }]), - modelKey: 'k', - headers: new Headers(), - }; - }); - - const result = await anthropicMessagesAttempt.generate({ - payload: makePayload({ - messages: [ - { role: 'user', content: 'hello' }, - { role: 'system', content: 'inline instructions' }, - ], - }), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ - callOpenAIResponses, - endpoints: { openaiResponses: {} }, - enabledFlags: new Set([ - 'rewrite-mid-conv-system-to-user', - 'rewrite-system-to-developer', - ]), - }), - headers: new Headers(), - anthropicBeta: [], - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callOpenAIResponses.mock.calls.length, 1); - const observedBody = observedBodies[0]; - if (!observedBody) throw new Error('expected observed OpenAI Responses body'); - const input = observedBody.input; - if (!Array.isArray(input)) throw new Error('expected OpenAI Responses input array'); - assertEquals(input[0], { type: 'message', role: 'user', content: 'hello' }); - assertEquals(input[1], { type: 'message', role: 'developer', content: 'inline instructions' }); -}); - -test('generate translate-to-responses branch rewrites a multi-block system prefix to developer', async () => { - installRepo(); - const observedBodies: Omit[] = []; - const callOpenAIResponses = vi.fn(async (_model, body): Promise => { - observedBodies.push(body as Omit); - return { - action: 'generate', - ok: true, - events: makeProtocolFrames([{ - type: 'response.completed', - sequence_number: 0, - response: { - id: 'resp_x', - object: 'response', - model: 'test-model', - status: 'completed', - output: [], - output_text: '', - error: null, - incomplete_details: null, - }, - }]), - modelKey: 'k', - headers: new Headers(), - }; - }); - - const result = await anthropicMessagesAttempt.generate({ - payload: makePayload({ - system: [{ type: 'text', text: 'base A' }, { type: 'text', text: 'base B' }], - messages: [ - { role: 'user', content: 'hello' }, - { role: 'system', content: 'inline instructions' }, - ], - }), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ - callOpenAIResponses, - endpoints: { openaiResponses: {} }, - enabledFlags: new Set(['rewrite-system-to-developer']), - }), - headers: new Headers(), - anthropicBeta: [], - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callOpenAIResponses.mock.calls.length, 1); - const observedBody = observedBodies[0]; - if (!observedBody) throw new Error('expected observed OpenAI Responses body'); - const input = observedBody.input; - if (!Array.isArray(input)) throw new Error('expected OpenAI Responses input array'); - assertEquals(input[0], { - type: 'message', - role: 'developer', - content: [{ type: 'input_text', text: 'base A' }, { type: 'input_text', text: 'base B' }], - }); - assertEquals(input[1], { type: 'message', role: 'user', content: 'hello' }); - assertEquals(input[2], { type: 'message', role: 'developer', content: 'inline instructions' }); -}); - -test('countTokens proxies the upstream response as a plain result', async () => { - installRepo(); - const callAnthropicMessagesCountTokens = vi.fn(async (): Promise => ({ - response: new Response(JSON.stringify({ input_tokens: 7 }), { status: 200, headers: new Headers({ 'content-type': 'application/json' }) }), - modelKey: 'k', - })); - - const result = await anthropicMessagesAttempt.countTokens({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callAnthropicMessagesCountTokens }), - headers: new Headers(), - anthropicBeta: [], - }); - - assertEquals(result.type, 'plain'); - if (result.type !== 'plain') throw new Error('unreachable'); - assertEquals(result.status, 200); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assertEquals(body.input_tokens, 7); - assertEquals(callAnthropicMessagesCountTokens.mock.calls.length, 1); -}); - -test('countTokens carries anthropic-beta through the Anthropic Messages boundary outside the provider allowlist', async () => { - installRepo(); - let upstreamHeaders: Headers | undefined; - let upstreamAnthropicBeta: readonly string[] | undefined; - const callAnthropicMessagesCountTokens = vi.fn(async (_model, _body, _signal, opts): Promise => { - upstreamHeaders = opts?.headers; - upstreamAnthropicBeta = opts?.anthropicBeta; - return { response: Response.json({ input_tokens: 7 }), modelKey: 'k' }; - }); - - await anthropicMessagesAttempt.countTokens({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callAnthropicMessagesCountTokens }), - headers: new Headers({ - 'anthropic-beta': 'must-not-enter-ordinary-headers', - 'x-unlisted': 'discard', - }), - anthropicBeta: ['context-1m-2025-08-07'], - }); - - assertExists(upstreamHeaders); - assertEquals(Object.fromEntries(upstreamHeaders), {}); - assertEquals(upstreamAnthropicBeta, ['context-1m-2025-08-07']); -}); - -test('countTokens applies generation request transforms before provider dispatch', async () => { - installRepo(); - const observedBodies: Array> = []; - const callAnthropicMessagesCountTokens = vi.fn(async (_model, body): Promise => { - observedBodies.push(body as Omit); - return { response: Response.json({ input_tokens: 9 }), modelKey: 'k' }; - }); - - const result = await anthropicMessagesAttempt.countTokens({ - payload: makePayload({ - system: 'x-anthropic-billing-header: token\ncch=deadbeef1234;\nbase rules', - messages: [ - { role: 'system', content: 'inline rules' }, - { role: 'user', content: 'hello' }, - ], - thinking: { type: 'enabled', budget_tokens: 1024 }, - output_config: { effort: 'high' }, - tool_choice: { type: 'tool', name: 'lookup' }, - }), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ - callAnthropicMessagesCountTokens, - enabledFlags: new Set([ - 'strip-billing-attribution', - 'disable-reasoning-on-forced-tool-choice', - 'rewrite-mid-conv-system-to-user', - ]), - }), - headers: new Headers(), - anthropicBeta: [], - }); - - assertEquals(result.type, 'plain'); - assertEquals(observedBodies, [{ - max_tokens: 32, - system: 'base rules', - messages: [ - { role: 'user', content: 'inline rules' }, - { role: 'user', content: 'hello' }, - ], - thinking: { type: 'disabled' }, - tool_choice: { type: 'tool', name: 'lookup' }, - }]); -}); - -test('countTokens prepares the generation web-search request shape', async () => { - installRepo(); - const observedBodies: Array> = []; - const callAnthropicMessagesCountTokens = vi.fn(async (_model, body): Promise => { - observedBodies.push(body as Omit); - return { response: Response.json({ input_tokens: 11 }), modelKey: 'k' }; - }); - - const result = await anthropicMessagesAttempt.countTokens({ - payload: makePayload({ tools: [{ type: 'web_search_20260209', max_uses: 3 }] }), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ - callAnthropicMessagesCountTokens, - enabledFlags: new Set(['anthropic-messages-web-search-shim']), - }), - headers: new Headers(), - anthropicBeta: [], - }); - - assertEquals(result.type, 'plain'); - const tool = observedBodies[0]?.tools?.[0] as AnthropicMessagesClientTool | undefined; - if (tool === undefined) throw new Error('expected rewritten web-search tool'); - assertEquals(tool.name, 'web_search'); - assertEquals('type' in tool, false); - assertEquals(tool.input_schema, { - type: 'object', - properties: { query: { type: 'string', description: 'Search query' } }, - required: ['query'], - }); -}); - -test('countTokens refuses a non-anthropic-messages candidate', async () => { - installRepo(); - let thrown: unknown = null; - try { - await anthropicMessagesAttempt.countTokens({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ endpoints: { openaiResponses: {} } }), - headers: new Headers(), - anthropicBeta: [], - }); - } catch (error) { - thrown = error; - } - if (!(thrown instanceof Error)) throw new Error('expected an Error to be thrown'); - assertEquals(thrown.message.includes('chatTargetPicker.pick'), true); -}); - -test('generate attaches the performance context to the result', async () => { - installRepo(); - const ctx = mockChatGatewayCtx({ - apiKeyId: API_KEY_ID, - wantsStream: true, - runtimeLocation: 'SJC', - }); - const callAnthropicMessages = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeAnthropicMessagesEvents()), modelKey: 'gpt-test', headers: new Headers(), - })); - - const result = await anthropicMessagesAttempt.generate({ - payload: makePayload(), - ctx, - candidate: makeCandidate({ upstream: 'up_perf', callAnthropicMessages }), - headers: new Headers(), - anthropicBeta: [], - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - // result.performance carries the full dimension set that respond.ts will - // use to record telemetry once the stream settles. - assertExists(result.performance); - assertEquals(result.performance.keyId, API_KEY_ID); - assertEquals(result.performance.model, 'test-model'); - assertEquals(result.performance.upstream, 'up_perf'); - assertEquals(result.performance.runtimeLocation, 'SJC'); - - await collectEvents(result.events); -}); - -test('generate propagates upstream response headers onto the EventResult so respond can forward them', async () => { - installRepo(); - const upstreamHeaders = new Headers({ - 'anthropic-ratelimit-unified-status': 'allowed', - 'request-id': 'req_messages_xyz', - }); - const callAnthropicMessages = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeAnthropicMessagesEvents()), modelKey: 'k', headers: upstreamHeaders, - })); - const result = await anthropicMessagesAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callAnthropicMessages }), - headers: new Headers(), - anthropicBeta: [], - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - assertEquals(result.headers?.get('anthropic-ratelimit-unified-status'), 'allowed'); - assertEquals(result.headers?.get('request-id'), 'req_messages_xyz'); - await collectEvents(result.events); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/claude-code-probe_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/claude-code-probe_test.ts new file mode 100644 index 0000000000..ec6587acd9 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/claude-code-probe_test.ts @@ -0,0 +1,115 @@ +import { test } from 'vitest'; + +import { isClaudeCodeProbe, probeFrames } from '../../../../src/data-plane/chat/anthropic-messages/claude-code-probe.ts'; +import { collectAnthropicMessagesProtocolEventsToResult, type AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import { assert, assertEquals } from '@floway-dev/test-utils'; + +// The CLI emits `claude-cli/ (external, …)`, where the +// entrypoint substitutes for `cli` and up to three optional segments append — +// which is why the predicate anchors on the `claude-cli/` prefix only. +// This is the default entrypoint form. +const PROBE_USER_AGENT = 'claude-cli/2.1.226 (external, cli)'; + +// The shape of the 2.1.226 `/model` validation probe: one user turn holding +// one ephemeral text block, `max_tokens: 1`, no tools. The real body also +// carries the CLI's system-prompt array and a `betas` list, neither of which +// the predicate looks at. +const probePayload = (overrides: Partial = {}): AnthropicMessagesPayload => ({ + model: 'test-model', + max_tokens: 1, + system: [{ type: 'text', text: "You are Claude Code, Anthropic's official CLI for Claude." }], + messages: [{ role: 'user', content: [{ type: 'text', text: 'Hi', cache_control: { type: 'ephemeral' } }] }], + metadata: { user_id: 'user_0_account__session_0' }, + ...overrides, +}); + +/** What the stage that answers this asks: is the turn in front of it the CLI's own probe? + * Everything below is that predicate, over the bodies the CLI has been observed to send. */ +const invocation = (payload: AnthropicMessagesPayload, userAgent: string | null = PROBE_USER_AGENT): boolean => + isClaudeCodeProbe(payload, new Headers(userAgent === null ? {} : { 'user-agent': userAgent })); + +const assertForwarded = (answered: boolean, message: string) => { assertEquals(answered, false, message); }; + +const assertAnswered = (answered: boolean, message: string) => { assertEquals(answered, true, message); }; + +test('answers the /model validation probe without dialing the upstream', async () => { + assertEquals(invocation(probePayload()), true); + + const message = await collectAnthropicMessagesProtocolEventsToResult(probeFrames('test-model')); + assertEquals(message.type, 'message'); + assertEquals(message.role, 'assistant'); + assertEquals(message.model, 'test-model'); + assertEquals(message.content, []); + assertEquals(message.stop_reason, 'max_tokens'); + assertEquals(message.usage.input_tokens, 0); + assertEquals(message.usage.output_tokens, 0); + assert(message.id.startsWith('msg_')); +}); + +test('answers every fixed probe prompt, in block and bare-string form', () => { + for (const prompt of ['Hi', 'test']) { + assertAnswered(invocation(probePayload({ messages: [{ role: 'user', content: prompt }] })), `bare string: ${prompt}`); + assertAnswered(invocation(probePayload({ messages: [{ role: 'user', content: [{ type: 'text', text: prompt }] }] })), `text block: ${prompt}`); + } +}); + +test('forwards the probe prompts the gateway cannot answer truthfully', () => { + // `quota` reads rate-limit response headers a synthesized turn cannot carry; + // `.` only ever reaches a Bedrock / Vertex / Mantle base URL; `hello` is a + // third-party report's reconstruction that no observed build sends. + for (const prompt of ['quota', '.', 'hello']) { + assertForwarded(invocation(probePayload({ messages: [{ role: 'user', content: prompt }] })), `prompt: ${prompt}`); + } +}); + +test('forwards a probe prompt whose casing the CLI has never been observed to send', () => { + assertForwarded(invocation(probePayload({ messages: [{ role: 'user', content: 'HI' }] })), 'uppercased'); +}); + +test('forwards a one-token request from a client that is not Claude Code', () => { + assertForwarded(invocation(probePayload(), 'python-httpx/0.28.1'), 'other client'); + assertForwarded(invocation(probePayload(), null), 'no user-agent'); +}); + +test('forwards a Claude Code turn whose output cap is not one token', () => { + assertForwarded(invocation(probePayload({ max_tokens: 32_000 })), 'max_tokens 32000'); +}); + +test('forwards a Claude Code turn that carries the session tools', () => { + assertForwarded(invocation(probePayload({ + tools: [{ name: 'Bash', input_schema: { type: 'object' } }], + })), 'tools present'); +}); + +test('forwards a conversation whose sole turn is not one of the fixed prompts', () => { + assertForwarded(invocation(probePayload({ + messages: [{ role: 'user', content: 'Hi, can you explain this file?' }], + })), 'real question'); +}); + +test('forwards a multi-turn conversation that happens to end on a probe prompt', () => { + assertForwarded(invocation(probePayload({ + messages: [ + { role: 'user', content: 'Hi' }, + { role: 'assistant', content: 'Hello!' }, + { role: 'user', content: 'Hi' }, + ], + })), 'three turns'); +}); + +test('forwards a sole turn that carries more than one block', () => { + assertForwarded(invocation(probePayload({ + messages: [{ role: 'user', content: [{ type: 'text', text: 'Hi' }, { type: 'text', text: 'and explain this file' }] }], + })), 'two text blocks'); +}); + +test('forwards a sole turn that is not a user turn', () => { + assertForwarded(invocation(probePayload({ messages: [{ role: 'assistant', content: 'Hi' }] })), 'assistant role'); + assertForwarded(invocation(probePayload({ messages: [{ role: 'system', content: 'Hi' }] })), 'system role'); +}); + +test('forwards a turn whose sole block is not text', () => { + assertForwarded(invocation(probePayload({ + messages: [{ role: 'user', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'AA==' } }] }], + })), 'image block'); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/errors_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/errors_test.ts index bd739593e1..f13c9af0ce 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/errors_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/errors_test.ts @@ -1,37 +1,42 @@ +// The Anthropic error envelope, at the function that writes it. +// +// Key order is load-bearing: a client comparing this gateway's refusal against +// Anthropic-direct's compares bytes, so `type`, then `error`, then a top-level `request_id`. + import { test } from 'vitest'; -import { translatorInputErrorResult } from '../../../../src/data-plane/chat/anthropic-messages/errors.ts'; -import type { ApiErrorResult } from '@floway-dev/provider'; +import { renderAnthropicMessagesError } from '../../../../src/data-plane/chat/anthropic-messages/errors.ts'; import { assert, assertEquals } from '@floway-dev/test-utils'; -import { TranslatorInputError } from '@floway-dev/translate'; - -const apiErrorOf = (result: ReturnType): ApiErrorResult => result as ApiErrorResult; -const bodyOf = (result: ReturnType): Record => - JSON.parse(new TextDecoder().decode(apiErrorOf(result).body)); -test('translatorInputErrorResult renders an Anthropic 400 invalid_request_error envelope with top-level request_id', () => { - const result = translatorInputErrorResult( - new TranslatorInputError("Invalid 'image_url' content part in system or developer message. Only 'text' content parts are supported in system messages on this model."), +test('renders an Anthropic invalid_request_error envelope with a top-level request_id', () => { + const body = renderAnthropicMessagesError( + 400, + "Invalid 'image_url' content part in system or developer message. Only 'text' content parts are supported in system messages on this model.", ); - const apiError = apiErrorOf(result); - assertEquals(apiError.type, 'api-error'); - assertEquals(apiError.source, 'gateway'); - assertEquals(apiError.status, 400); - assertEquals(apiError.headers.get('content-type'), 'application/json'); - - const body = bodyOf(result); assertEquals(body.type, 'error'); assertEquals(body.error, { type: 'invalid_request_error', message: "Invalid 'image_url' content part in system or developer message. Only 'text' content parts are supported in system messages on this model.", }); - assert(typeof body.request_id === 'string' && /^req_[A-Za-z0-9]{24}$/.test(body.request_id), `request_id ${String(body.request_id)} must match Anthropic-shape req_<24 opaque chars>`); + assert( + typeof body.request_id === 'string' && /^req_[A-Za-z0-9]{24}$/.test(body.request_id), + `request_id ${String(body.request_id)} must match Anthropic-shape req_<24 opaque chars>`, + ); +}); + +test('preserves Anthropic key order: type, error, request_id', () => { + const raw = JSON.stringify(renderAnthropicMessagesError(400, 'whatever')); + assert( + /^\{"type":"error","error":\{"type":"invalid_request_error","message":"whatever"\},"request_id":"req_[A-Za-z0-9]{24}"\}$/.test(raw), + `body ${raw} must match Anthropic-direct byte shape`, + ); }); -test('translatorInputErrorResult preserves Anthropic key order: type, error, request_id', () => { - const result = translatorInputErrorResult(new TranslatorInputError('whatever')); - const raw = new TextDecoder().decode(apiErrorOf(result).body); - // Key-order is load-bearing for byte-faithfulness with Anthropic-direct. - assert(/^\{"type":"error","error":\{"type":"invalid_request_error","message":"whatever"\},"request_id":"req_[A-Za-z0-9]{24}"\}$/.test(raw), `body ${raw} must match Anthropic-direct byte shape`); +test('names the condition each status carries in this protocol', () => { + // The status a refusal was made with is what selects the type name, so the mapping is what + // makes a gateway-made refusal read like the upstream's own. + assertEquals((renderAnthropicMessagesError(429, 'slow down').error as { type: string }).type, 'rate_limit_error'); + assertEquals((renderAnthropicMessagesError(529, 'overloaded').error as { type: string }).type, 'overloaded_error'); + assertEquals((renderAnthropicMessagesError(502, 'bad gateway').error as { type: string }).type, 'api_error'); }); diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts index 4fb932c576..525c681f07 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts @@ -6,10 +6,11 @@ import { initRepo } from '../../../../src/repo/index.ts'; import type { ApiKey, User } from '../../../../src/repo/types.ts'; import { InMemoryRepo } from '../../../repo/memory.ts'; import { flushBackground } from '../../../test-utils/background-tracker.ts'; +import { stubChatProviderPipelines } from '../../../test-utils/chat-provider-pipelines.ts'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; -import { type ModelCandidate, directFetcher, type ProviderCallResult, type ProviderStreamResult, type UpstreamCallOptions } from '@floway-dev/provider'; -import { assert, assertEquals, stubProvider, stubInternalModel } from '@floway-dev/test-utils'; +import { type ModelCandidate, directFetcher, type UpstreamCallOptions } from '@floway-dev/provider'; +import { assert, assertEquals, stubProvider, stubInternalModel, type ProviderCallResult, type ProviderStreamResult } from '@floway-dev/test-utils'; const candidatesQueue: { readonly candidates: readonly ModelCandidate[]; readonly sawModel: boolean; readonly failedUpstreams: readonly string[] }[] = []; vi.mock('../../../../src/data-plane/providers/resolution.ts', async importOriginal => { @@ -108,14 +109,15 @@ const makeCandidate = (overrides: { callAnthropicMessagesCountTokens?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise; } = {}): ModelCandidate => { const upstream = overrides.upstream ?? 'up_test'; - const provider = stubProvider({ + const calls = { callAnthropicMessages: overrides.callAnthropicMessages, callAnthropicMessagesCountTokens: overrides.callAnthropicMessagesCountTokens, - }); + }; + const provider = stubProvider(); return { provider: { upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: provider, + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: stubChatProviderPipelines(calls), instance: provider, }, model: stubInternalModel(overrides.endpoints ? { endpoints: overrides.endpoints } : {}, upstream), fetcher: directFetcher, @@ -189,13 +191,8 @@ test('POST /v1/messages answers the Claude Code model-validation probe without c assertEquals(body.usage.output_tokens, 0); assertEquals(callAnthropicMessages.mock.calls.length, 0); - // The turn is recorded as served, at zero cost, and contributes no latency - // sample — there was no upstream call to measure. await flushBackground(); - const usage = await repo.usage.listAll(); - assertEquals(usage.length, 1); - assertEquals(usage[0]?.requests, 1); - assertEquals(usage[0]?.metrics, []); + assertEquals(await repo.usage.listAll(), []); assertEquals(await repo.performance.listAll(), []); }); @@ -220,7 +217,7 @@ test('POST /v1/messages rejects body anthropic_beta with a 400 before routing', }); test('POST /v1/messages/count_tokens proxies the upstream measurement body', async () => { - installRepo(); + const repo = installRepo(); const callAnthropicMessagesCountTokens = vi.fn(async (): Promise => ({ response: new Response(JSON.stringify({ input_tokens: 99 }), { status: 200, headers: new Headers({ 'content-type': 'application/json' }) }), modelKey: 'k', @@ -237,6 +234,31 @@ test('POST /v1/messages/count_tokens proxies the upstream measurement body', asy const body = await response.json() as { input_tokens: number }; assertEquals(body.input_tokens, 99); assertEquals(callAnthropicMessagesCountTokens.mock.calls.length, 1); + + await flushBackground(); + assertEquals(await repo.usage.listAll(), []); + assertEquals(await repo.performance.listAll(), []); +}); + +test('POST /v1/messages/count_tokens preserves a body read failure without charging the native count call', async () => { + const repo = installRepo(); + const fault = new Error('native count body read failed'); + const callAnthropicMessagesCountTokens = vi.fn(async (): Promise => ({ + response: new Response(new ReadableStream({ pull(controller) { controller.error(fault); } }), { headers: { 'content-type': 'application/json' } }), + modelKey: 'count-model', + })); + queueCandidates([makeCandidate({ callAnthropicMessagesCountTokens })]); + const response = await makeApp().request('/v1/messages/count_tokens', { + method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ model: 'test-model', max_tokens: 32, messages: [{ role: 'user', content: 'hello' }] }), + }); + assertEquals(callAnthropicMessagesCountTokens.mock.calls.length, 1); + assertEquals(response.status, 500); + const body = await response.text(); + assert(body.includes(fault.message)); + await flushBackground(); + assertEquals(await repo.usage.listAll(), []); + assertEquals(await repo.performance.listAll(), []); }); test('POST /v1/messages forwards upstream response headers end-to-end (streaming) and strips hop-by-hop / cookies', async () => { @@ -296,10 +318,10 @@ test('POST /v1/messages forwards upstream response headers end-to-end (non-strea assertEquals(response.headers.get('cf-ray'), 'cf_ray_e2e'); }); -test('POST /v1/messages renders the Anthropic-shaped model-unsupported 400 when no candidate matches the messages-generate picker', async () => { +test('POST /v1/messages renders the Anthropic-shaped model-unsupported 400 when no candidate matches the anthropic-messages-generate picker', async () => { installRepo(); - // Queue a chat-kind candidate whose endpoints expose only `openaiCompletions` — - // anthropicMessagesGenerateTarget (messages > responses > openai-chat-completions) rejects + // Queue a chat-kind candidate whose endpoints expose only `completions` — + // anthropicMessagesGenerateTarget (messages > responses > chat-completions) rejects // it, leaving zero viable candidates, and with sawModel=true the serve // renders model-unsupported as a 400. queueCandidates([makeCandidate({ endpoints: { openaiCompletions: {} } })]); diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe_test.ts deleted file mode 100644 index 72ff2aabf5..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe_test.ts +++ /dev/null @@ -1,148 +0,0 @@ -import { test, vi } from 'vitest'; - -import { answerClaudeCodeProbe } from '../../../../../src/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe.ts'; -import type { AnthropicMessagesInvocation } from '../../../../../src/data-plane/chat/anthropic-messages/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { collectAnthropicMessagesProtocolEventsToResult, type AnthropicMessagesPayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { type ExecuteResult, eventResult } from '@floway-dev/provider'; -import { assert, assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -// The CLI emits `claude-cli/ (external, …)`, where the -// entrypoint substitutes for `cli` and up to three optional segments append — -// which is why the predicate anchors on the `claude-cli/` prefix only. -// This is the default entrypoint form. -const PROBE_USER_AGENT = 'claude-cli/2.1.226 (external, cli)'; - -// The shape of the 2.1.226 `/model` validation probe: one user turn holding -// one ephemeral text block, `max_tokens: 1`, no tools. The real body also -// carries the CLI's system-prompt array and a `betas` list, neither of which -// the predicate looks at. -const probePayload = (overrides: Partial = {}): AnthropicMessagesPayload => ({ - model: 'test-model', - max_tokens: 1, - system: [{ type: 'text', text: "You are Claude Code, Anthropic's official CLI for Claude." }], - messages: [{ role: 'user', content: [{ type: 'text', text: 'Hi', cache_control: { type: 'ephemeral' } }] }], - metadata: { user_id: 'user_0_account__session_0' }, - ...overrides, -}); - -const invocation = (payload: AnthropicMessagesPayload, userAgent: string | null = PROBE_USER_AGENT): AnthropicMessagesInvocation => ({ - payload, - candidate: stubModelCandidate({ model: { endpoints: { openaiResponses: {} } } }), - targetApi: 'openaiResponses', - headers: new Headers(userAgent === null ? {} : { 'user-agent': userAgent }), -}); - -const passthrough = async (): Promise>> => - eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity); - -const runProbe = async (input: AnthropicMessagesInvocation) => { - const run = vi.fn(passthrough); - const result = await answerClaudeCodeProbe(input, stubCtx, run); - return { result, run }; -}; - -const assertForwarded = async (input: AnthropicMessagesInvocation, message: string) => { - const { run } = await runProbe(input); - assertEquals(run.mock.calls.length, 1, message); -}; - -const assertAnswered = async (input: AnthropicMessagesInvocation, message: string) => { - const { run } = await runProbe(input); - assertEquals(run.mock.calls.length, 0, message); -}; - -test('answers the /model validation probe without dialing the upstream', async () => { - const { result, run } = await runProbe(invocation(probePayload())); - - assertEquals(run.mock.calls.length, 0); - assert(result.type === 'events'); - const message = await collectAnthropicMessagesProtocolEventsToResult(result.events); - assertEquals(message.type, 'message'); - assertEquals(message.role, 'assistant'); - assertEquals(message.model, 'test-model'); - assertEquals(message.content, []); - assertEquals(message.stop_reason, 'max_tokens'); - assertEquals(message.usage.input_tokens, 0); - assertEquals(message.usage.output_tokens, 0); - assert(message.id.startsWith('msg_')); -}); - -test('reports no performance context so the turn contributes no latency sample', async () => { - const { result } = await runProbe(invocation(probePayload())); - - assert(result.type === 'events'); - assertEquals(result.performance, undefined); - assertEquals(result.finalMetadata, undefined); -}); - -test('answers every fixed probe prompt, in block and bare-string form', async () => { - for (const prompt of ['Hi', 'test']) { - await assertAnswered(invocation(probePayload({ messages: [{ role: 'user', content: prompt }] })), `bare string: ${prompt}`); - await assertAnswered(invocation(probePayload({ messages: [{ role: 'user', content: [{ type: 'text', text: prompt }] }] })), `text block: ${prompt}`); - } -}); - -test('forwards the probe prompts the gateway cannot answer truthfully', async () => { - // `quota` reads rate-limit response headers a synthesized turn cannot carry; - // `.` only ever reaches a Bedrock / Vertex / Mantle base URL; `hello` is a - // third-party report's reconstruction that no observed build sends. - for (const prompt of ['quota', '.', 'hello']) { - await assertForwarded(invocation(probePayload({ messages: [{ role: 'user', content: prompt }] })), `prompt: ${prompt}`); - } -}); - -test('forwards a probe prompt whose casing the CLI has never been observed to send', async () => { - await assertForwarded(invocation(probePayload({ messages: [{ role: 'user', content: 'HI' }] })), 'uppercased'); -}); - -test('forwards a one-token request from a client that is not Claude Code', async () => { - await assertForwarded(invocation(probePayload(), 'python-httpx/0.28.1'), 'other client'); - await assertForwarded(invocation(probePayload(), null), 'no user-agent'); -}); - -test('forwards a Claude Code turn whose output cap is not one token', async () => { - await assertForwarded(invocation(probePayload({ max_tokens: 32_000 })), 'max_tokens 32000'); -}); - -test('forwards a Claude Code turn that carries the session tools', async () => { - await assertForwarded(invocation(probePayload({ - tools: [{ name: 'Bash', input_schema: { type: 'object' } }], - })), 'tools present'); -}); - -test('forwards a conversation whose sole turn is not one of the fixed prompts', async () => { - await assertForwarded(invocation(probePayload({ - messages: [{ role: 'user', content: 'Hi, can you explain this file?' }], - })), 'real question'); -}); - -test('forwards a multi-turn conversation that happens to end on a probe prompt', async () => { - await assertForwarded(invocation(probePayload({ - messages: [ - { role: 'user', content: 'Hi' }, - { role: 'assistant', content: 'Hello!' }, - { role: 'user', content: 'Hi' }, - ], - })), 'three turns'); -}); - -test('forwards a sole turn that carries more than one block', async () => { - await assertForwarded(invocation(probePayload({ - messages: [{ role: 'user', content: [{ type: 'text', text: 'Hi' }, { type: 'text', text: 'and explain this file' }] }], - })), 'two text blocks'); -}); - -test('forwards a sole turn that is not a user turn', async () => { - await assertForwarded(invocation(probePayload({ messages: [{ role: 'assistant', content: 'Hi' }] })), 'assistant role'); - await assertForwarded(invocation(probePayload({ messages: [{ role: 'system', content: 'Hi' }] })), 'system role'); -}); - -test('forwards a turn whose sole block is not text', async () => { - await assertForwarded(invocation(probePayload({ - messages: [{ role: 'user', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'AA==' } }] }], - })), 'image block'); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/apply-role-compatibility_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/apply-role-compatibility_test.ts deleted file mode 100644 index a1d9af1a3e..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/apply-role-compatibility_test.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { test } from 'vitest'; - -import { withRoleCompatibilityApplied } from '../../../../../src/data-plane/chat/anthropic-messages/interceptors/apply-role-compatibility.ts'; -import type { AnthropicMessagesInvocation } from '../../../../../src/data-plane/chat/anthropic-messages/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { AnthropicMessagesMessage, AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { type ExecuteResult, eventResult, type FlagId } from '@floway-dev/provider'; -import { assert, assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const gatewayCtx = mockChatGatewayCtx(); -const okEvents = (): Promise>> => - Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); - -const applyRoles = async ( - messages: AnthropicMessagesMessage[], - enabledFlags: ReadonlySet, - targetApi: AnthropicMessagesInvocation['targetApi'] = 'anthropicMessages', -): Promise => { - const payload: AnthropicMessagesPayload = { model: 'test-model', max_tokens: 1, messages }; - const invocation: AnthropicMessagesInvocation = { - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi, - headers: new Headers(), - }; - await withRoleCompatibilityApplied(invocation, gatewayCtx, okEvents); - return invocation.payload.messages; -}; - -test('leaves roles unchanged without the flag or at a translated target', async () => { - const messages: AnthropicMessagesMessage[] = [{ role: 'system', content: 'inline rules' }]; - assertEquals(await applyRoles(messages, new Set()), messages); - assertEquals( - await applyRoles(messages, new Set(['rewrite-mid-conv-system-to-user']), 'openaiResponses'), - messages, - ); -}); - -test('rewrites every inline system message and preserves content', async () => { - const content = [{ type: 'text' as const, text: 'inline rules' }]; - assertEquals( - await applyRoles( - [ - { role: 'system', content: 'first rules' }, - { role: 'user', content: 'hello' }, - { role: 'system', content }, - ], - new Set(['rewrite-mid-conv-system-to-user']), - ), - [ - { role: 'user', content: 'first rules' }, - { role: 'user', content: 'hello' }, - { role: 'user', content }, - ], - ); - const result = await applyRoles( - [{ role: 'system', content }], - new Set(['rewrite-mid-conv-system-to-user']), - ); - assert(result[0]?.content === content); -}); - -test('handles empty input and leaves non-system messages unchanged', async () => { - assertEquals(await applyRoles([], new Set(['rewrite-mid-conv-system-to-user'])), []); - const messages: AnthropicMessagesMessage[] = [ - { role: 'user', content: 'hello' }, - { role: 'assistant', content: 'hi' }, - ]; - assertEquals(await applyRoles(messages, new Set(['rewrite-mid-conv-system-to-user'])), messages); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/disable-reasoning-on-forced-tool-choice_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/disable-reasoning-on-forced-tool-choice_test.ts deleted file mode 100644 index 833799e0ce..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/disable-reasoning-on-forced-tool-choice_test.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { test } from 'vitest'; - -import { withReasoningDisabledOnForcedToolChoice } from '../../../../../src/data-plane/chat/anthropic-messages/interceptors/disable-reasoning-on-forced-tool-choice.ts'; -import type { AnthropicMessagesInvocation } from '../../../../../src/data-plane/chat/anthropic-messages/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { type ExecuteResult, eventResult, type FlagId } from '@floway-dev/provider'; -import { stubModelCandidate, testTelemetryModelIdentity, assertEquals } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = (): Promise>> => - Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); - -const invocation = ( - payload: AnthropicMessagesPayload, - enabledFlags: ReadonlySet = new Set(['disable-reasoning-on-forced-tool-choice']), -): AnthropicMessagesInvocation => ({ - payload, - candidate: stubModelCandidate({ - model: { endpoints: { anthropicMessages: {} } }, - enabledFlags, - }), - targetApi: 'anthropicMessages', - headers: new Headers(), -}); - -test('messages forced tool_choice disables thinking and strips output_config', async () => { - const input = invocation({ - model: 'm', - messages: [], - max_tokens: 1, - thinking: { type: 'enabled', budget_tokens: 1024 }, - output_config: { effort: 'high' }, - tool_choice: { type: 'tool', name: 'x' }, - }); - - await withReasoningDisabledOnForcedToolChoice(input, stubCtx, okEvents); - - assertEquals(input.payload.thinking, { type: 'disabled' }); - assertEquals(input.payload.output_config, undefined); -}); - -test('messages any tool_choice also disables thinking', async () => { - const input = invocation({ - model: 'm', - messages: [], - max_tokens: 1, - thinking: { type: 'enabled', budget_tokens: 1024 }, - tool_choice: { type: 'any' }, - }); - - await withReasoningDisabledOnForcedToolChoice(input, stubCtx, okEvents); - - assertEquals(input.payload.thinking, { type: 'disabled' }); -}); - -test('messages non-forced tool_choice leaves reasoning untouched', async () => { - for (const type of ['auto', 'none'] as const) { - const input = invocation({ - model: 'm', - messages: [], - max_tokens: 1, - thinking: { type: 'enabled', budget_tokens: 1024 }, - tool_choice: { type }, - }); - - await withReasoningDisabledOnForcedToolChoice(input, stubCtx, okEvents); - - assertEquals(input.payload.thinking, { - type: 'enabled', - budget_tokens: 1024, - }); - } -}); - -test('messages forced tool_choice preserves structured-output format while stripping reasoning effort', async () => { - const schema = { type: 'object', properties: { x: { type: 'string' } }, required: ['x'], additionalProperties: false }; - const input = invocation({ - model: 'm', - messages: [], - max_tokens: 1, - thinking: { type: 'enabled', budget_tokens: 1024 }, - output_config: { effort: 'high', format: { type: 'json_schema', schema } }, - tool_choice: { type: 'tool', name: 'x' }, - }); - - await withReasoningDisabledOnForcedToolChoice(input, stubCtx, okEvents); - - assertEquals(input.payload.thinking, { type: 'disabled' }); - assertEquals(input.payload.output_config, { format: { type: 'json_schema', schema } }); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/normalize-empty-tools-tool-choice_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/normalize-empty-tools-tool-choice_test.ts deleted file mode 100644 index 101d869e86..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/normalize-empty-tools-tool-choice_test.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { test } from 'vitest'; - -import { withEmptyToolsToolChoiceNormalized } from '../../../../../src/data-plane/chat/anthropic-messages/interceptors/normalize-empty-tools-tool-choice.ts'; -import type { AnthropicMessagesInvocation } from '../../../../../src/data-plane/chat/anthropic-messages/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; -import { type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); -const okResponse = () => Promise.resolve(new Response()); - -const invocation = ( - payload: AnthropicMessagesPayload, - enabledFlags: ReadonlySet = new Set(['empty-tools-tool-choice-none']), -): AnthropicMessagesInvocation => ({ - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi: 'anthropicMessages', - headers: new Headers(), -}); - -const base = { model: 'm', messages: [], max_tokens: 1 }; - -test('messages empty tools rewrites tool_choice to native none', async () => { - const input = invocation({ ...base, tools: [], tool_choice: { type: 'auto' } }); - - await withEmptyToolsToolChoiceNormalized(input, stubCtx, okResponse); - - assertEquals(input.payload.tools, []); - assertEquals(input.payload.tool_choice, { type: 'none' }); -}); - -test('messages rewrite requires the flag and an explicit empty array', async () => { - const disabled = invocation({ ...base, tools: [], tool_choice: { type: 'auto' } }, new Set()); - const missing = invocation({ ...base, tool_choice: { type: 'auto' } }); - const present = invocation({ - ...base, - tools: [{ name: 'x', description: 'x', input_schema: { type: 'object' } }], - tool_choice: { type: 'auto' }, - }); - - await withEmptyToolsToolChoiceNormalized(disabled, stubCtx, okResponse); - await withEmptyToolsToolChoiceNormalized(missing, stubCtx, okResponse); - await withEmptyToolsToolChoiceNormalized(present, stubCtx, okResponse); - - assertEquals(disabled.payload.tool_choice, { type: 'auto' }); - assertEquals(missing.payload.tool_choice, { type: 'auto' }); - assertEquals(present.payload.tool_choice, { type: 'auto' }); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/strip-billing-attribution_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/strip-billing-attribution_test.ts deleted file mode 100644 index 9a5965138d..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/strip-billing-attribution_test.ts +++ /dev/null @@ -1,154 +0,0 @@ -import { test } from 'vitest'; - -import { stripBillingAttribution } from '../../../../../src/data-plane/chat/anthropic-messages/interceptors/strip-billing-attribution.ts'; -import type { AnthropicMessagesInvocation } from '../../../../../src/data-plane/chat/anthropic-messages/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { type ExecuteResult, eventResult } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = (): Promise>> => - Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); - -interface InvocationOptions { - flagOn?: boolean; -} - -const invocation = (payload: AnthropicMessagesPayload, { flagOn = true }: InvocationOptions = {}): AnthropicMessagesInvocation => ({ - payload, - candidate: stubModelCandidate({ - model: { endpoints: { anthropicMessages: {} } }, - enabledFlags: flagOn ? new Set(['strip-billing-attribution']) : new Set(), - }), - targetApi: 'anthropicMessages', - headers: new Headers(), -}); - -test('strips billing-header lines and cch hashes from a string system prompt while preserving the rest', async () => { - const input = invocation({ - model: 'm', - max_tokens: 1, - messages: [], - system: 'You are a helpful assistant.\nx-anthropic-billing-header: per-turn-token\ncch=deadbeef1234;\nKeep going.', - }); - - await stripBillingAttribution(input, stubCtx, okEvents); - - assertEquals(input.payload.system, 'You are a helpful assistant.\n\n\nKeep going.'); -}); - -test('strips per-block from an array-form system prompt and filters blocks that become empty', async () => { - const input = invocation({ - model: 'm', - max_tokens: 1, - messages: [], - system: [ - { type: 'text', text: 'You are a helpful assistant.' }, - { type: 'text', text: 'x-anthropic-billing-header: token\ncch=abcdef12345' }, - { type: 'text', text: 'Keep going. cch=99fffaa1;' }, - ], - }); - - await stripBillingAttribution(input, stubCtx, okEvents); - - assertEquals(input.payload.system, [ - { type: 'text', text: 'You are a helpful assistant.' }, - { type: 'text', text: 'Keep going.' }, - ]); -}); - -test('deletes the system field entirely when every array block becomes empty', async () => { - const input = invocation({ - model: 'm', - max_tokens: 1, - messages: [], - system: [ - { type: 'text', text: 'x-anthropic-billing-header: token' }, - { type: 'text', text: 'cch=deadbeef1234;' }, - ], - }); - - await stripBillingAttribution(input, stubCtx, okEvents); - - assertEquals('system' in input.payload, false); -}); - -test('deletes a string system field that becomes empty after stripping', async () => { - const input = invocation({ - model: 'm', - max_tokens: 1, - messages: [], - system: 'x-anthropic-billing-header: token\ncch=deadbeef1234;', - }); - - await stripBillingAttribution(input, stubCtx, okEvents); - - assertEquals('system' in input.payload, false); -}); - -test('is a no-op when system is absent', async () => { - const input = invocation({ - model: 'm', - max_tokens: 1, - messages: [], - }); - - await stripBillingAttribution(input, stubCtx, okEvents); - - assertEquals('system' in input.payload, false); -}); - -test('leaves a system prompt without billing markers untouched', async () => { - const original = 'You are a helpful assistant. Respond in markdown and use code fences for snippets.'; - const input = invocation({ - model: 'm', - max_tokens: 1, - messages: [], - system: original, - }); - - await stripBillingAttribution(input, stubCtx, okEvents); - - assertEquals(input.payload.system, original); -}); - -test('leaves the billing block intact when the strip flag is off (claude-code default)', async () => { - const system = 'You are a helpful assistant.\nx-anthropic-billing-header: per-turn-token\ncch=deadbeef1234;\nKeep going.'; - const input = invocation( - { - model: 'm', - max_tokens: 1, - messages: [], - system, - }, - { flagOn: false }, - ); - - await stripBillingAttribution(input, stubCtx, okEvents); - - assertEquals(input.payload.system, system); -}); - -test('leaves an array-form billing block intact when the strip flag is off', async () => { - const system = [ - { type: 'text' as const, text: 'You are a helpful assistant.' }, - { type: 'text' as const, text: 'x-anthropic-billing-header: token\ncch=abcdef12345' }, - { type: 'text' as const, text: 'Keep going. cch=99fffaa1;' }, - ]; - const input = invocation( - { - model: 'm', - max_tokens: 1, - messages: [], - system, - }, - { flagOn: false }, - ); - - await stripBillingAttribution(input, stubCtx, okEvents); - - assertEquals(input.payload.system, system); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/respond_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/respond_test.ts deleted file mode 100644 index 1dc47d3876..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/respond_test.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { Hono } from 'hono'; -import { test } from 'vitest'; - -import { respondAnthropicMessages } from '../../../../src/data-plane/chat/anthropic-messages/respond.ts'; -import type { ChatGatewayCtx } from '../../../../src/data-plane/chat/shared/gateway-ctx.ts'; -import { initRepo } from '../../../../src/repo/index.ts'; -import { InMemoryRepo } from '../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import { eventResult, type ExecuteResult } from '@floway-dev/provider'; -import { assert, assertEquals, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -// --- header forwarding --- - -const forwardedHeadersFixture = (): Headers => new Headers({ - // forwardable: vendor traces, plan billing, vendor `x-*`, arbitrary custom - 'anthropic-ratelimit-unified-status': 'allowed_warning', - 'anthropic-ratelimit-unified-fallback-percentage': '50', - 'request-id': 'req_anthropic_abc', - 'cf-ray': 'cf_ray_xyz', - 'openai-version': '2024-10-21', - 'x-custom-thing': 'ok', - // blocked: hop-by-hop, body framing, cookies. Distinctive values so we can - // tell the upstream's header from anything Hono's writers add. - 'connection': 'close', - 'transfer-encoding': 'gzip', - 'content-length': '999', - 'content-encoding': 'br', - 'content-type': 'application/x-upstream-quirk', - 'set-cookie': 'session=secret', -}); - -const makeRespondCtx = (): ChatGatewayCtx => mockChatGatewayCtx({ apiKeyId: 'key_respond_test' }); - -const anthropicMessagesEventsForRespond = (): readonly AnthropicMessagesStreamEvent[] => [ - { - type: 'message_start', - message: { - id: 'msg_1', type: 'message', role: 'assistant', content: [], model: 'claude-test', - stop_reason: null, stop_sequence: null, - usage: { input_tokens: 3, output_tokens: 0 }, - }, - }, - { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, - { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'hi' } }, - { type: 'content_block_stop', index: 0 }, - { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 1 } }, - { type: 'message_stop' }, -]; - -const anthropicMessagesProtocolFrames = async function* (): AsyncGenerator> { - for (const event of anthropicMessagesEventsForRespond()) yield eventFrame(event); - yield doneFrame(); -}; - -const callRespond = async (wantsStream: boolean): Promise => { - initRepo(new InMemoryRepo()); - const app = new Hono(); - let captured: Response | undefined; - app.get('/', async c => { - const result: ExecuteResult> = eventResult( - anthropicMessagesProtocolFrames(), - testTelemetryModelIdentity, - { headers: forwardedHeadersFixture() }, - ); - const response = await respondAnthropicMessages(c, result, wantsStream, makeRespondCtx()); - captured = response; - return response; - }); - await app.request('/'); - if (!captured) throw new Error('respondAnthropicMessages did not produce a Response'); - return captured; -}; - -test('respondAnthropicMessages forwards upstream headers and strips hop-by-hop / framing / cookie headers on the non-streaming JSON response', async () => { - const response = await callRespond(false); - // forwarded verbatim - assertEquals(response.headers.get('anthropic-ratelimit-unified-status'), 'allowed_warning'); - assertEquals(response.headers.get('anthropic-ratelimit-unified-fallback-percentage'), '50'); - assertEquals(response.headers.get('request-id'), 'req_anthropic_abc'); - assertEquals(response.headers.get('cf-ray'), 'cf_ray_xyz'); - assertEquals(response.headers.get('openai-version'), '2024-10-21'); - assertEquals(response.headers.get('x-custom-thing'), 'ok'); - // hop-by-hop and cookies dropped - assertEquals(response.headers.get('connection'), null); - assertEquals(response.headers.get('transfer-encoding'), null); - assertEquals(response.headers.get('set-cookie'), null); - // framing headers dropped — upstream values would mis-frame the response; - // Response.json sets its own content-type, which must not echo upstream's - assertEquals(response.headers.get('content-length'), null); - assertEquals(response.headers.get('content-encoding'), null); - assertEquals(response.headers.get('content-type'), 'application/json'); -}); - -test('respondAnthropicMessages forwards upstream headers and strips hop-by-hop / framing / cookie headers on the streaming SSE response', async () => { - const response = await callRespond(true); - // forwarded verbatim - assertEquals(response.headers.get('anthropic-ratelimit-unified-status'), 'allowed_warning'); - assertEquals(response.headers.get('anthropic-ratelimit-unified-fallback-percentage'), '50'); - assertEquals(response.headers.get('request-id'), 'req_anthropic_abc'); - assertEquals(response.headers.get('cf-ray'), 'cf_ray_xyz'); - assertEquals(response.headers.get('openai-version'), '2024-10-21'); - assertEquals(response.headers.get('x-custom-thing'), 'ok'); - // hop-by-hop and cookies dropped. `connection` and `transfer-encoding` - // are special-cased: Hono's streamSSE writer sets its own `keep-alive` / - // `chunked`, so we assert upstream's distinctive values did not survive - // rather than asserting absence. - assert(response.headers.get('connection') !== 'close'); - assert(response.headers.get('transfer-encoding') !== 'gzip'); - assertEquals(response.headers.get('set-cookie'), null); - // framing headers dropped; streamSSE writes its own text/event-stream and - // never emits content-length or content-encoding for a streamed body - assertEquals(response.headers.get('content-length'), null); - assertEquals(response.headers.get('content-encoding'), null); - assertEquals(response.headers.get('content-type')?.split(';')[0], 'text/event-stream'); - // Drain the body so the lazy generator releases its resources and the - // background `finally` block in `streamSSE` doesn't keep the test runner - // alive. - await response.text(); -}); - -// --- partial usage checkpointing on client disconnect --- - -// A generator whose next() resolves only when emit() supplies the next event. -// Lets a test interleave "upstream emitted frame X" with "downstream cancels", -// so the streaming finally block fires while message_stop is still in flight. diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/serve_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/serve_test.ts deleted file mode 100644 index d57fc9c9c2..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/serve_test.ts +++ /dev/null @@ -1,743 +0,0 @@ -import { afterEach, test, vi } from 'vitest'; - -import { initRepo } from '../../../../src/repo/index.ts'; -import { InMemoryRepo } from '../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import { type AliasRules, doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { type AnthropicMessagesUpstreamCallOptions, type ModelCandidate, directFetcher, type ProviderCallResult, type ProviderOpenAIResponsesResult, type ProviderStreamResult, type OpenAIResponsesAction, type UpstreamCallOptions, type FlagId } from '@floway-dev/provider'; -import { assert, assertEquals, stubProvider, stubInternalModel, stubProviderModel } from '@floway-dev/test-utils'; - -// Mock the resolver seam so each test hands the serve exactly the provider -// candidates it wants, optionally with an alias-rules overlay attached. -interface QueuedResolution { - readonly candidates: readonly ModelCandidate[]; - readonly sawModel: boolean; - readonly failedUpstreams: readonly string[]; -} -const resolutionsQueue: QueuedResolution[] = []; -const lastResolveCall: { model?: string } = {}; -vi.mock('../../../../src/data-plane/providers/resolution.ts', async importOriginal => { - const original = await importOriginal(); - return { - ...original, - enumerateModelCandidates: vi.fn(async ({ model }: { model: string }) => { - lastResolveCall.model = model; - const next = resolutionsQueue.shift(); - if (next === undefined) throw new Error('serve_test: no resolution enqueued'); - return next; - }), - }; -}); - -const { anthropicMessagesServe } = await import('../../../../src/data-plane/chat/anthropic-messages/serve.ts'); - -const API_KEY_ID = 'key_messages_serve_test'; - -const queueResolution = ( - candidates: readonly ModelCandidate[], - extra: { sawModel?: boolean; aliasRules?: AliasRules } = {}, -): void => { - const rules = extra.aliasRules; - resolutionsQueue.push({ - candidates: rules !== undefined ? candidates.map(c => ({ ...c, rules })) : candidates, - sawModel: extra.sawModel ?? candidates.length > 0, - failedUpstreams: [], - }); -}; - -afterEach(() => { resolutionsQueue.length = 0; }); - -const installRepo = (): InMemoryRepo => { - const repo = new InMemoryRepo(); - initRepo(repo); - return repo; -}; - -const makeGatewayCtx = () => mockChatGatewayCtx({ apiKeyId: API_KEY_ID, wantsStream: true }); - -const makePayload = (overrides: Partial = {}): AnthropicMessagesPayload => ({ - model: 'test-model', - max_tokens: 32, - messages: [{ role: 'user', content: 'hello' }], - ...overrides, -}); - -const makeAnthropicMessagesResultEvents = (id = 'msg_test'): readonly AnthropicMessagesStreamEvent[] => [ - { - type: 'message_start', - message: { - id, - type: 'message', - role: 'assistant', - content: [], - model: 'test-model', - stop_reason: null, - stop_sequence: null, - usage: { input_tokens: 10, output_tokens: 0 }, - }, - }, - { - type: 'content_block_start', - index: 0, - content_block: { type: 'text', text: '' }, - }, - { - type: 'content_block_delta', - index: 0, - delta: { type: 'text_delta', text: 'hi' }, - }, - { type: 'content_block_stop', index: 0 }, - { - type: 'message_delta', - delta: { stop_reason: 'end_turn', stop_sequence: null }, - usage: { output_tokens: 1 }, - }, - { type: 'message_stop' }, -]; - -const makeOpenAIResponsesResultEvent = (id = 'resp_test'): OpenAIResponsesStreamEvent => { - const response: OpenAIResponsesResult = { - id, - object: 'response', - model: 'test-model', - status: 'completed', - output: [{ - type: 'message', - id: 'msg_resp', - role: 'assistant', - status: 'completed', - content: [{ type: 'output_text', text: 'hi from responses', annotations: [] }], - }], - output_text: 'hi from responses', - error: null, - incomplete_details: null, - }; - return { type: 'response.completed', sequence_number: 0, response }; -}; - -const makeProtocolFrames = async function* (events: readonly TEvent[]): AsyncGenerator> { - for (const event of events) yield eventFrame(event); - yield doneFrame(); -}; - -const makeCandidate = (overrides: { - upstream?: string; - modelId?: string; - endpoints?: ModelEndpoints; - kind?: ModelCandidate['provider']['kind']; - enabledFlags?: ReadonlySet; - callAnthropicMessages?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: AnthropicMessagesUpstreamCallOptions) => Promise>; - callOpenAIResponses?: (model: unknown, body: unknown, action: OpenAIResponsesAction, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise; - callAnthropicMessagesCountTokens?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: AnthropicMessagesUpstreamCallOptions) => Promise; -} = {}): ModelCandidate => { - const upstream = overrides.upstream ?? 'up_test'; - const modelId = overrides.modelId ?? 'test-model'; - const kind = overrides.kind ?? 'custom'; - const provider = stubProvider({ - callAnthropicMessages: overrides.callAnthropicMessages, - callOpenAIResponses: overrides.callOpenAIResponses, - callAnthropicMessagesCountTokens: overrides.callAnthropicMessagesCountTokens, - }); - return { - provider: { - upstreamId: upstream, kind, name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: provider, - }, - model: stubInternalModel({ - id: modelId, - ...(overrides.endpoints ? { endpoints: overrides.endpoints } : {}), - providerModels: { - [upstream]: stubProviderModel({ - id: modelId, - ...(overrides.endpoints ? { endpoints: overrides.endpoints } : {}), - ...(overrides.enabledFlags ? { enabledFlags: overrides.enabledFlags } : {}), - }), - }, - }, upstream), - fetcher: directFetcher, - }; -}; - -const collectEvents = async (events: AsyncIterable>): Promise => { - const out: TEvent[] = []; - for await (const frame of events) { - if (frame.type === 'event') out.push(frame.event); - } - return out; -}; - -// `assertEquals(result.type, X)` does not narrow the union in the TS type -// checker; the manual `if (result.type !== X) throw` follow-up was pure -// type-narrowing scaffold. This helper asserts the variant and returns the -// narrowed value so call sites stay on a single line. -const assertResultType = ( - result: U, - type: T, -): Extract => { - assertEquals(result.type, type); - return result as Extract; -}; - -function assertIsArray(value: unknown): asserts value is readonly T[] { - assert(Array.isArray(value)); -} - -test('generate routes a native Anthropic Messages candidate end to end', async () => { - installRepo(); - let callOptions: AnthropicMessagesUpstreamCallOptions | undefined; - const callAnthropicMessages = vi.fn(async (_model, _body, _signal, opts): Promise> => { - callOptions = opts; - return { - ok: true, - events: makeProtocolFrames(makeAnthropicMessagesResultEvents()), - modelKey: 'test-model-key', - headers: new Headers(), - }; - }); - queueResolution([makeCandidate({ upstream: 'up_a', callAnthropicMessages })]); - - const result = await anthropicMessagesServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - headers: new Headers({ 'anthropic-beta': 'context-1m-2025-08-07, advanced-tool-use-2025-11-20' }), - }); - - const events = await collectEvents(assertResultType(result, 'events').events); - assert(events.length >= 1); - assertEquals(callAnthropicMessages.mock.calls.length, 1); - assertEquals(callOptions?.anthropicBeta, ['context-1m-2025-08-07', 'advanced-tool-use-2025-11-20']); - assertEquals(callOptions?.headers.has('anthropic-beta'), false); -}); - -test('generate translates through the OpenAI Responses target when only that endpoint is exposed', async () => { - installRepo(); - const callOpenAIResponses = vi.fn(async (): Promise => ({ - action: 'generate', ok: true, - events: makeProtocolFrames([makeOpenAIResponsesResultEvent()]), - modelKey: 'responses-model-key', - headers: new Headers(), - })); - queueResolution([makeCandidate({ upstream: 'up_r', endpoints: { openaiResponses: {} }, callOpenAIResponses })]); - - const result = await anthropicMessagesServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - await collectEvents(assertResultType(result, 'events').events); - assertEquals(callOpenAIResponses.mock.calls.length, 1); -}); - -test('generate falls through to the next candidate when the first yields an upstream error', async () => { - installRepo(); - const firstError = new Response(JSON.stringify({ error: { message: 'nope' } }), { - status: 502, headers: new Headers({ 'content-type': 'application/json' }), - }); - const firstCall = vi.fn(async (): Promise> => ({ - ok: false, response: firstError, modelKey: 'first-key', - })); - const secondCall = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeAnthropicMessagesResultEvents('msg_second')), modelKey: 'second-key', headers: new Headers(), - })); - queueResolution([ - makeCandidate({ upstream: 'up_a', callAnthropicMessages: firstCall }), - makeCandidate({ upstream: 'up_b', callAnthropicMessages: secondCall }), - ]); - - const result = await anthropicMessagesServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - // The narrowed candidate list exists exactly so a transient upstream - // failure (5xx/429/network) on one entry rolls over to the next. The - // second candidate's success is the request's final answer. - assertEquals(result.type, 'events'); - assertEquals(firstCall.mock.calls.length, 1); - assertEquals(secondCall.mock.calls.length, 1); -}); - -test('generate surfaces the last upstream error verbatim when every candidate fails', async () => { - installRepo(); - const firstError = new Response('first', { status: 503 }); - const lastError = new Response('last', { status: 502 }); - queueResolution([ - makeCandidate({ upstream: 'up_a', callAnthropicMessages: async () => ({ ok: false, response: firstError, modelKey: 'first-key' }) }), - makeCandidate({ upstream: 'up_b', callAnthropicMessages: async () => ({ ok: false, response: lastError, modelKey: 'last-key' }) }), - ]); - - const result = await anthropicMessagesServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - const failure = assertResultType(result, 'api-error'); - assertEquals(failure.status, 502); -}); - -test('generate stops at the first candidate when the payload has no reasoning carriers to route on', async () => { - installRepo(); - const callAnthropicMessages = vi.fn(async (): Promise> => ({ - ok: true, - events: makeProtocolFrames(makeAnthropicMessagesResultEvents()), - modelKey: 'test-model-key', - headers: new Headers(), - })); - queueResolution([ - makeCandidate({ upstream: 'up_a', callAnthropicMessages }), - makeCandidate({ upstream: 'up_b', callAnthropicMessages }), - ]); - - const result = await anthropicMessagesServe.generate({ - payload: makePayload({ messages: [{ role: 'user', content: 'hi' }] }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - await collectEvents(assertResultType(result, 'events').events); - assertEquals(callAnthropicMessages.mock.calls.length, 1); -}); - -test('generate renders model-missing when no candidates are available', async () => { - installRepo(); - queueResolution([]); - - const result = await anthropicMessagesServe.generate({ - payload: makePayload({ model: 'unknown-model' }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - const failure = assertResultType(result, 'api-error'); - assertEquals(failure.status, 404); - const body = JSON.parse(new TextDecoder().decode(failure.body)); - assertEquals(body.error.type, 'not_found_error'); - assertEquals(body.error.message, 'Model unknown-model is not available on any configured upstream.'); -}); - -test('generate filters out candidates whose endpoints do not satisfy the messages-generate preference and renders model-unsupported as a 400', async () => { - installRepo(); - const callAnthropicMessages = vi.fn(); - // anthropicMessagesGenerateTarget prefers messages > responses > openai-chat-completions; an - // endpoints-only `openaiCompletions` candidate matches none and is filtered out. - queueResolution([makeCandidate({ upstream: 'up_x', endpoints: { openaiCompletions: {} }, callAnthropicMessages })]); - - const result = await anthropicMessagesServe.generate({ - payload: makePayload({ model: 'wrong-endpoint-model' }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - const failure = assertResultType(result, 'api-error'); - assertEquals(failure.status, 400); - const body = JSON.parse(new TextDecoder().decode(failure.body)); - assertEquals(body.error.type, 'invalid_request_error'); - assert(typeof body.error.message === 'string' && body.error.message.includes('does not support')); - assertEquals(callAnthropicMessages.mock.calls.length, 0); -}); - -test('countTokens proxies the upstream measurement response as a plain result', async () => { - installRepo(); - const observedModelIds: string[] = []; - let callOptions: AnthropicMessagesUpstreamCallOptions | undefined; - const callAnthropicMessagesCountTokens = vi.fn(async (model: unknown, _body, _signal, opts): Promise => { - observedModelIds.push((model as { id: string }).id); - callOptions = opts; - return { - response: new Response(JSON.stringify({ input_tokens: 42 }), { - status: 200, - headers: new Headers({ 'content-type': 'application/json' }), - }), - modelKey: 'test-model-key', - }; - }); - const candidate = makeCandidate({ upstream: 'up_a', modelId: 'claude-target', callAnthropicMessagesCountTokens }); - queueResolution([candidate]); - const payload = makePayload({ model: 'claude-alias' }); - - const result = await anthropicMessagesServe.countTokens({ - payload, - ctx: makeGatewayCtx(), - headers: new Headers({ 'anthropic-beta': 'context-1m-2025-08-07, advanced-tool-use-2025-11-20' }), - }); - - const plain = assertResultType(result, 'plain'); - assertEquals(plain.status, 200); - const body = JSON.parse(new TextDecoder().decode(plain.body)); - assertEquals(body.input_tokens, 42); - assertEquals(callAnthropicMessagesCountTokens.mock.calls.length, 1); - assertEquals(callOptions?.anthropicBeta, ['context-1m-2025-08-07', 'advanced-tool-use-2025-11-20']); - assertEquals(callOptions?.headers.has('anthropic-beta'), false); - assertEquals(observedModelIds, ['claude-target']); - assertEquals(payload.model, 'claude-alias'); -}); - -test('countTokens renders model-missing as a 404 when no candidates are available', async () => { - installRepo(); - queueResolution([]); - - const result = await anthropicMessagesServe.countTokens({ - payload: makePayload({ model: 'unknown-model' }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - const failure = assertResultType(result, 'api-error'); - assertEquals(failure.status, 404); - const body = JSON.parse(new TextDecoder().decode(failure.body)); - assertEquals(body.error.type, 'not_found_error'); - assertEquals(body.error.message, 'Model unknown-model is not available on any configured upstream.'); -}); - -test('countTokens filters out candidates whose endpoints do not satisfy the messages-countTokens preference and renders model-unsupported as a 400', async () => { - installRepo(); - const callAnthropicMessagesCountTokens = vi.fn(); - // anthropicMessagesCountTokensTarget = chatTargetPicker(['anthropicMessages']); a candidate - // exposing only openaiChatCompletions matches none and is filtered out. - queueResolution([makeCandidate({ upstream: 'up_x', endpoints: { openaiChatCompletions: {} }, callAnthropicMessagesCountTokens })]); - - const result = await anthropicMessagesServe.countTokens({ - payload: makePayload({ model: 'wrong-endpoint-model' }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - const failure = assertResultType(result, 'api-error'); - assertEquals(failure.status, 400); - const body = JSON.parse(new TextDecoder().decode(failure.body)); - assertEquals(body.error.type, 'invalid_request_error'); - assert(typeof body.error.message === 'string' && body.error.message.includes('does not support')); - assertEquals(callAnthropicMessagesCountTokens.mock.calls.length, 0); -}); - -// strip-billing-attribution defaults OFF for claude-code, so a request whose -// system prompt carries the `x-anthropic-billing-header:` block must reach -// the claude-code provider's callAnthropicMessages with the block intact — otherwise -// Anthropic loses the plan-tier attribution it bills against. -test('claude-code candidate preserves x-anthropic-billing-header system block through the interceptor chain', async () => { - installRepo(); - - // Match the `strip-billing-attribution: false` decision the claude-code - // provider ships (see provider-claude-code/src/defaults.ts): the plan-tier - // block must reach Anthropic verbatim. - const claudeCodeDefaults: ReadonlySet = new Set(); - - const billingBlock = 'x-anthropic-billing-header: per-turn-token\ncch=deadbeef1234;\ncc_entrypoint=cli'; - - const capturedBodies: Omit[] = []; - const callAnthropicMessages = vi.fn(async ( - _model: unknown, - body: unknown, - ): Promise> => { - capturedBodies.push(body as Omit); - return { - ok: true, - events: makeProtocolFrames(makeAnthropicMessagesResultEvents()), - modelKey: 'claude-sonnet-4-5-20250929', - }; - }); - - queueResolution([ - makeCandidate({ - upstream: 'up_cc', - kind: 'claude-code', - enabledFlags: claudeCodeDefaults, - callAnthropicMessages, - }), - ]); - - const result = await anthropicMessagesServe.generate({ - payload: makePayload({ - system: [ - { type: 'text', text: billingBlock }, - { type: 'text', text: "You are Claude Code, Anthropic's official CLI for Claude." }, - ], - }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - await collectEvents(assertResultType(result, 'events').events); - - assertEquals(callAnthropicMessages.mock.calls.length, 1); - const observed = capturedBodies[0]!; - assertIsArray<{ text: string }>(observed.system); - assertEquals(observed.system.length, 2); - assertEquals(observed.system[0].text, billingBlock); - assertEquals(observed.system[1].text, "You are Claude Code, Anthropic's official CLI for Claude."); -}); - -// The same request routed to a copilot candidate (which carries the -// strip-billing-attribution default-on flag) must have the billing block -// stripped before the upstream call — the mirror image of the claude-code -// assertion above. -test('copilot candidate strips x-anthropic-billing-header system block via the default-on flag', async () => { - installRepo(); - - // Match `strip-billing-attribution: true` on the copilot provider default - // (see provider-copilot/src/defaults.ts). - const copilotDefaults: ReadonlySet = new Set(['strip-billing-attribution']); - - const billingBlock = 'x-anthropic-billing-header: per-turn-token\ncch=deadbeef1234;'; - - const capturedBodies: Omit[] = []; - const callAnthropicMessages = vi.fn(async ( - _model: unknown, - body: unknown, - ): Promise> => { - capturedBodies.push(body as Omit); - return { - ok: true, - events: makeProtocolFrames(makeAnthropicMessagesResultEvents()), - modelKey: 'claude-sonnet-4-5', - }; - }); - - queueResolution([ - makeCandidate({ - upstream: 'up_co', - kind: 'copilot', - enabledFlags: copilotDefaults, - callAnthropicMessages, - }), - ]); - - const result = await anthropicMessagesServe.generate({ - payload: makePayload({ - system: [ - { type: 'text', text: billingBlock }, - { type: 'text', text: 'You are a helpful assistant.' }, - ], - }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - await collectEvents(assertResultType(result, 'events').events); - - assertEquals(callAnthropicMessages.mock.calls.length, 1); - const observed = capturedBodies[0]!; - assertIsArray<{ text: string }>(observed.system); - assertEquals(observed.system.length, 1); - assertEquals(observed.system[0].text, 'You are a helpful assistant.'); -}); - -test('generate failover preserves billing blocks for a strip-off candidate', async () => { - installRepo(); - const billingBlock = 'x-anthropic-billing-header: per-turn-token\ncch=deadbeef1234;'; - const system = [ - { type: 'text' as const, text: billingBlock }, - { type: 'text' as const, text: "You are Claude Code, Anthropic's official CLI for Claude." }, - ]; - const expectedSystem = structuredClone(system); - const messages = [{ role: 'user' as const, content: [{ type: 'text' as const, text: 'original user text' }] }]; - const expectedMessages = structuredClone(messages); - const firstBodies: Array> = []; - const firstCall = vi.fn(async (_model: unknown, body: unknown): Promise> => { - const firstBody = body as Omit; - firstBodies.push(firstBody); - const message = firstBody.messages[0]; - if (!Array.isArray(message.content) || message.content[0]?.type !== 'text') throw new Error('expected text content'); - message.content[0].text = 'mutated by first provider'; - return { - ok: false, - response: new Response('unavailable', { status: 503 }), - modelKey: 'first-key', - }; - }); - const observedBodies: Array> = []; - const secondCall = vi.fn(async (_model: unknown, body: unknown): Promise> => { - observedBodies.push(body as Omit); - return { - ok: true, - events: makeProtocolFrames(makeAnthropicMessagesResultEvents('msg_claude_code')), - modelKey: 'claude-code-key', - }; - }); - queueResolution([ - makeCandidate({ - upstream: 'up_copilot', - kind: 'copilot', - enabledFlags: new Set(['strip-billing-attribution']), - callAnthropicMessages: firstCall, - }), - makeCandidate({ - upstream: 'up_claude_code', - kind: 'claude-code', - enabledFlags: new Set(), - callAnthropicMessages: secondCall, - }), - ]); - - const payload = makePayload({ system, messages }); - const result = await anthropicMessagesServe.generate({ payload, ctx: makeGatewayCtx(), headers: new Headers() }); - await collectEvents(assertResultType(result, 'events').events); - - assertEquals(firstCall.mock.calls.length, 1); - assertEquals(secondCall.mock.calls.length, 1); - assertEquals(firstBodies[0]?.system, [{ type: 'text', text: "You are Claude Code, Anthropic's official CLI for Claude." }]); - assertEquals(observedBodies[0]?.system, expectedSystem); - assertEquals(observedBodies[0]?.messages, expectedMessages); - assertEquals(payload.system, expectedSystem); - assertEquals(payload.messages, expectedMessages); -}); - -test('countTokens failover preserves billing blocks for a strip-off candidate', async () => { - installRepo(); - const system = [ - { type: 'text' as const, text: 'x-anthropic-billing-header: per-turn-token\ncch=deadbeef1234;' }, - { type: 'text' as const, text: 'Count this prompt.' }, - ]; - const expectedSystem = structuredClone(system); - const messages = [{ role: 'user' as const, content: [{ type: 'text' as const, text: 'original user text' }] }]; - const expectedMessages = structuredClone(messages); - const firstBodies: Array> = []; - const firstCall = vi.fn(async (_model: unknown, body: unknown): Promise => { - const firstBody = body as Omit; - firstBodies.push(firstBody); - const message = firstBody.messages[0]; - if (!Array.isArray(message.content) || message.content[0]?.type !== 'text') throw new Error('expected text content'); - message.content[0].text = 'mutated by first provider'; - return { - response: new Response('unavailable', { status: 503 }), - modelKey: 'first-key', - }; - }); - const observedBodies: Array> = []; - const secondCall = vi.fn(async (_model: unknown, body: unknown): Promise => { - observedBodies.push(body as Omit); - return { - response: Response.json({ input_tokens: 12 }), - modelKey: 'second-key', - }; - }); - queueResolution([ - makeCandidate({ - upstream: 'up_strip_on', - enabledFlags: new Set(['strip-billing-attribution']), - callAnthropicMessagesCountTokens: firstCall, - }), - makeCandidate({ - upstream: 'up_strip_off', - enabledFlags: new Set(), - callAnthropicMessagesCountTokens: secondCall, - }), - ]); - - const payload = makePayload({ system, messages }); - const result = await anthropicMessagesServe.countTokens({ payload, ctx: makeGatewayCtx(), headers: new Headers() }); - - assertEquals(assertResultType(result, 'plain').status, 200); - assertEquals(firstCall.mock.calls.length, 1); - assertEquals(secondCall.mock.calls.length, 1); - assertEquals(firstBodies[0]?.system, [{ type: 'text', text: 'Count this prompt.' }]); - assertEquals(observedBodies[0]?.system, expectedSystem); - assertEquals(observedBodies[0]?.messages, expectedMessages); - assertEquals(payload.system, expectedSystem); - assertEquals(payload.messages, expectedMessages); -}); - -test('alias resolution swaps the inbound model id for the target and overlays rules onto the Anthropic Messages payload', async () => { - installRepo(); - const capturedBodies: AnthropicMessagesPayload[] = []; - const observedModelIds: string[] = []; - const callAnthropicMessages = vi.fn(async (model: unknown, body: unknown): Promise> => { - observedModelIds.push((model as { id: string }).id); - capturedBodies.push({ ...(body as Omit), model: 'claude-opus-4-7' }); - return { ok: true, events: makeProtocolFrames(makeAnthropicMessagesResultEvents()), modelKey: 'claude-opus-4-7' }; - }); - // Alias flow shape: the resolver returns candidates carrying the target's - // upstream catalog id AND the alias's rule overlay on `candidate.rules`. - // The attempt stamps its private clone with `candidate.model.id` and reads - // the overlay directly off `candidate.rules` at wire-call time. - const candidate = makeCandidate({ upstream: 'up_cf', modelId: 'claude-opus-4-7', callAnthropicMessages }); - queueResolution([candidate], { aliasRules: { reasoning: { effort: 'high', budget_tokens: 2048 }, serviceTier: 'fast' } }); - - const payload = makePayload({ model: 'claude-fast' }); - const result = await anthropicMessagesServe.generate({ - payload, - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - await collectEvents(assertResultType(result, 'events').events); - - // The resolver and caller payload retain the inbound alias while dispatch - // uses the resolved target id. - assertEquals(lastResolveCall.model, 'claude-fast'); - assertEquals(observedModelIds, ['claude-opus-4-7']); - assertEquals(payload.model, 'claude-fast'); - const observed = capturedBodies[0]!; - assertEquals(observed.output_config?.effort, 'high'); - assertEquals(observed.thinking?.budget_tokens, 2048); - // The serviceTier=fast → speed=fast bridge lands the alias rule on - // Anthropic's native Fast Mode field. - assertEquals(observed.speed, 'fast'); -}); - -test('alias whose targets have no kind-matching binding surfaces as the regular model-missing 404', async () => { - installRepo(); - queueResolution([], { sawModel: false }); - - const result = await anthropicMessagesServe.generate({ - payload: makePayload({ model: 'claude-fast' }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 404); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assertEquals(body.error.type, 'not_found_error'); - assertEquals(body.error.message, 'Model claude-fast is not available on any configured upstream.'); -}); - -// A mid-attempt throw (interceptor bug / translation error / provider-layer JS -// exception not represented as a ChatServeFailure) must attribute the perf error -// row to the throwing candidate, not the previous one. The serve stamps -// `ctx.attempt.telemetry` synchronously in the iterateCandidates -// callback so the http.ts catch can build an internal-error result carrying -// the correct upstream, and `recordFailedRequest` lands a row rather than -// short-circuiting on missing telemetry. Passthrough's equivalent regression -// lived in passthrough-serve_test.ts, which went with that surface (R3 fix 303c4e89). -test('mid-attempt throw stamps telemetry with the throwing candidate, not the previous one', async () => { - installRepo(); - const firstError = new Response(JSON.stringify({ error: { message: 'nope' } }), { - status: 502, headers: new Headers({ 'content-type': 'application/json' }), - }); - const firstCall = vi.fn(async (): Promise> => ({ - ok: false, response: firstError, modelKey: 'first-key', - })); - const secondCall = vi.fn(async (): Promise> => { - throw new Error('simulated provider-layer JS exception'); - }); - queueResolution([ - makeCandidate({ upstream: 'up_a', callAnthropicMessages: firstCall }), - makeCandidate({ upstream: 'up_b', callAnthropicMessages: secondCall }), - ]); - - const ctx = makeGatewayCtx(); - await anthropicMessagesServe.generate({ - payload: makePayload(), - ctx, - headers: new Headers(), - }).then( - () => { throw new Error('expected anthropicMessagesServe.generate to throw'); }, - (error: unknown) => { - assertEquals((error as Error).message, 'simulated provider-layer JS exception'); - }, - ); - - assertEquals(firstCall.mock.calls.length, 1); - assertEquals(secondCall.mock.calls.length, 1); - // The perf attribution slot reflects the throwing upstream, so the http.ts - // catch synthesizes the internal-error result with performance context and - // the error row lands against up_b. - assertEquals(ctx.attempt.telemetry?.upstream, 'up_b'); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-drive.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-drive.ts new file mode 100644 index 0000000000..965a3d606a --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-drive.ts @@ -0,0 +1,80 @@ +// Driving Anthropic's web-search stage from a test that scripts the dial. +// +// What a test scripts is the *ending* — the one answer the stage descends for. Everything above +// it is the real chain: the preparation rewrites the body, the stage descends, and the frames +// come back rewritten into Anthropic's native shape. +// +// `invocation.payload` is written back on the descent, so a test that asserts on what the +// upstream was asked for reads the same field it always did. + +import { runAnthropicMessagesWebSearchTool } from '../../../../src/data-plane/chat/anthropic-messages/web-search-tool.ts'; +import type { ChatGatewayCtx } from '../../../../src/data-plane/chat/shared/gateway-ctx.ts'; +import { compose, defineStage, move, run } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import type { AnthropicMessagesInvocation, ModelCandidate } from '@floway-dev/provider'; +import { type ExecuteResult } from '@floway-dev/test-utils'; + +const ANSWER = 'response.chat.anthropicMessages'; + +type Result = ExecuteResult>; + +/** A scripted refusal as the failure value a chain carries. */ +const failureOf = (result: Exclude) => ({ + status: result.type === 'internal-error' ? result.status : result.status, + message: result.type === 'internal-error' ? result.error.message : new TextDecoder().decode(result.body), +}); + +/** What the run answered with, in the shape the assertions read. */ +const resultOf = (facts: Record): Result => { + const answer = facts[ANSWER] as { status?: number; message?: string; body?: unknown; kind?: string; frames?: unknown }; + if (answer.kind === 'stream') { + return { type: 'events', events: answer.frames as AsyncIterable>, modelIdentity: undefined } as unknown as Result; + } + return { + type: 'api-error', + source: 'gateway', + status: answer.status!, + headers: new Headers({ 'content-type': 'application/json' }), + body: new TextEncoder().encode(answer.message!), + } as Result; +}; + +export const driveWebSearchStage = async ( + invocation: AnthropicMessagesInvocation, + gatewayCtx: ChatGatewayCtx, + dial: () => Promise, +): Promise => { + const ending = defineStage, Record>({ + name: 'scriptedDial', + return: { provides: [ANSWER, 'response.usage.billable', 'response.http.headers', 'response.http.status'] }, + execute: async facts => { + invocation.payload = facts['request.chat.anthropicMessages'] as AnthropicMessagesPayload; + const result = await dial(); + return move({ + ...facts, + [ANSWER]: result.type === 'events' + ? { kind: 'stream' as const, frames: result.events } + : failureOf(result), + 'response.usage.billable': [], + 'response.http.headers': [], 'response.http.status': result.type === 'events' ? 200 : result.status, + }); + }, + }); + + const chain = compose, Record>('webSearchToolUnderTest', [ + runAnthropicMessagesWebSearchTool({ targetOf: () => invocation.targetApi }) as never, + ending, + ]); + + const { facts } = await run(chain, move({ + 'request.chat.anthropicMessages': invocation.payload, + 'route.attempt': { upstreamId: 'up_test', modelId: invocation.payload.model, flags: [] }, + }), { + gateway: gatewayCtx, + background: () => {}, + resolveAttempt: (): ModelCandidate => invocation.candidate, + } as never); + + return resultOf(facts); +}; diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/web-search-shim_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-shim_test.ts similarity index 87% rename from packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/web-search-shim_test.ts rename to packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-shim_test.ts index 6c3636f6a7..02850c9929 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/interceptors/web-search-shim_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/web-search-shim_test.ts @@ -1,6 +1,7 @@ -import { test } from 'vitest'; +import { expect, test, vi } from 'vitest'; -import type { AnthropicMessagesInvocation } from '../../../../../src/data-plane/chat/anthropic-messages/interceptors/types.ts'; +import { driveWebSearchStage } from './web-search-drive.ts'; +import * as webSearchShim from '../../../../src/data-plane/chat/anthropic-messages/web-search-shim.ts'; import { decodeWebSearchCitationPayload, decodeWebSearchResultPayload, @@ -9,14 +10,14 @@ import { type AnthropicMessagesWebSearchShimState, prepareAnthropicMessagesWebSearchShimRequest, rewriteAnthropicMessagesWebSearchEventsToNative, - withAnthropicMessagesWebSearchRequestPrepared, - withAnthropicMessagesWebSearchShim, -} from '../../../../../src/data-plane/chat/anthropic-messages/interceptors/web-search-shim.ts'; -import { DEFAULT_WEB_SEARCH_CONFIG } from '../../../../../src/data-plane/tools/web-search/config.ts'; -import type { WebSearchProvider, WebSearchProviderResult } from '../../../../../src/data-plane/tools/web-search/types.ts'; -import { initRepo } from '../../../../../src/repo/index.ts'; -import { InMemoryRepo } from '../../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; +} from '../../../../src/data-plane/chat/anthropic-messages/web-search-shim.ts'; +import { runAnthropicMessagesWebSearchTool } from '../../../../src/data-plane/chat/anthropic-messages/web-search-tool.ts'; +import { DEFAULT_WEB_SEARCH_CONFIG } from '../../../../src/data-plane/tools/web-search/config.ts'; +import type { WebSearchProvider, WebSearchProviderResult } from '../../../../src/data-plane/tools/web-search/types.ts'; +import { initRepo } from '../../../../src/repo/index.ts'; +import { InMemoryRepo } from '../../../repo/memory.ts'; +import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; +import { compose, defineStage, move, run, type Event as RunEvent } from '@floway-dev/pipeline'; import { anthropicMessagesProtocolFrameToSSEFrame } from '@floway-dev/protocols/anthropic-messages'; import type { AnthropicMessagesAssistantContentBlock, @@ -30,6 +31,7 @@ import type { AnthropicMessagesUserContentBlock, } from '@floway-dev/protocols/anthropic-messages'; import { type ProtocolFrame, eventFrame } from '@floway-dev/protocols/common'; +import type { AnthropicMessagesInvocation } from '@floway-dev/provider'; import { assertEquals, assertExists, assertRejects, stubModelCandidate } from '@floway-dev/test-utils'; const testTelemetryModelIdentity = { @@ -587,44 +589,34 @@ test('generation and count_tokens prepare identical web-search request payloads' const generationInvocation = invocation(structuredClone(source)); const countInvocation = invocation(structuredClone(source)); - await withAnthropicMessagesWebSearchShim(generationInvocation, mockChatGatewayCtx(), () => Promise.resolve({ + await driveWebSearchStage(generationInvocation, mockChatGatewayCtx(), () => Promise.resolve({ type: 'events', events: toAsyncIterable>([]), modelIdentity: testTelemetryModelIdentity, })); - const countResponse = await withAnthropicMessagesWebSearchRequestPrepared( - countInvocation, - mockChatGatewayCtx(), - () => Promise.resolve(new Response(null, { status: 204 })), - ); - - assertEquals(countResponse.status, 204); - assertEquals(countInvocation.payload, generationInvocation.payload); + // `:countTokens` has no stream to rewrite, so its stage prepares the same request and dials + // it — measuring what generation would have sent rather than what the client wrote. + const counted = prepareAnthropicMessagesWebSearchShimRequest(countInvocation.payload); + assertEquals(counted.type, 'ok'); + assertEquals(counted.type === 'ok' ? counted.payload : undefined, generationInvocation.payload); const rewrittenTool = generationInvocation.payload.tools?.[0] as AnthropicMessagesClientTool; assertEquals(rewrittenTool.name, 'web_search'); assertEquals('type' in rewrittenTool, false); }); test('count_tokens returns a native error response for invalid web-search tools', async () => { - const response = await withAnthropicMessagesWebSearchRequestPrepared( - invocation({ - model: 'claude-test', - max_tokens: 64, - messages: [{ role: 'user', content: 'search' }], - tools: [{ type: 'web_search_20260209' }, { type: 'web_search_20260209' }], - }), - mockChatGatewayCtx(), - () => Promise.reject(new Error('run should not be called')), - ); - - assertEquals(response.status, 400); - assertEquals(await response.json(), { - type: 'error', - error: { - type: 'invalid_request_error', - message: 'Only one native web search tool definition is supported per request.', - }, + const prepared = prepareAnthropicMessagesWebSearchShimRequest({ + model: 'claude-test', + max_tokens: 64, + messages: [{ role: 'user', content: 'search' }], + tools: [{ type: 'web_search_20260209' }, { type: 'web_search_20260209' }], }); + + assertEquals(prepared.type, 'invalid-request'); + assertEquals( + prepared.type === 'invalid-request' ? prepared.message : undefined, + 'Only one native web search tool definition is supported per request.', + ); }); const runReplayOnlyShim = async (messageId: string): Promise[]> => { @@ -632,7 +624,7 @@ const runReplayOnlyShim = async (messageId: string): Promise + const result = await driveWebSearchStage(invocation(payload), mockChatGatewayCtx(), () => Promise.resolve({ type: 'events', events: toAsyncIterable( @@ -672,23 +664,30 @@ const runReplayOnlyShim = async (messageId: string): Promise { +// A turn that needs a search backend the operator never configured is unanswerable, and that is +// this gateway's own fault rather than a refusal to fail over — so it raises, and the seam turns +// it into the internal-error envelope with the sentence and the stack on it. +test('the web-search stage raises when the request needs a search provider the operator disabled', async () => { await initDisabledSearchRepo(); - const result = await withAnthropicMessagesWebSearchShim( - invocation({ - model: 'claude-test', - max_tokens: 64, - messages: [{ role: 'user', content: 'latest React docs' }], - tools: [{ type: 'web_search_20260209' }], - }), mockChatGatewayCtx(), - () => Promise.reject(new Error('run should not be called')), - ); - - assertEquals(result.type, 'internal-error'); + let raised: unknown; + try { + await driveWebSearchStage( + invocation({ + model: 'claude-test', + max_tokens: 64, + messages: [{ role: 'user', content: 'latest React docs' }], + tools: [{ type: 'web_search_20260209' }], + }), mockChatGatewayCtx(), + () => Promise.reject(new Error('run should not be called')), + ); + } catch (error) { raised = error; } + + assertEquals(raised instanceof Error, true); + assertEquals((raised as Error).message, 'Native Anthropic Messages web search requires an enabled search provider.'); }); -test('withAnthropicMessagesWebSearchShim allows replay-only history when the search provider is disabled', async () => { +test('the web-search stage allows replay-only history when the search provider is disabled', async () => { const collected = await runReplayOnlyShim('msg_replay_only'); const events = collected.flatMap(frame => (frame.type === 'event' ? [frame.event] : [])); @@ -696,7 +695,7 @@ test('withAnthropicMessagesWebSearchShim allows replay-only history when the sea assertEquals(citationsDelta?.delta.type === 'citations_delta' ? citationsDelta.delta.citation.type : undefined, 'web_search_result_location'); }); -test('withAnthropicMessagesWebSearchShim emits native-like citation deltas for replay-only history', async () => { +test('the web-search stage emits native-like citation deltas for replay-only history', async () => { const collected = await runReplayOnlyShim('msg_replay_only_stream'); const frames = collected.map(anthropicMessagesProtocolFrameToSSEFrame).filter(frame => frame !== null); @@ -830,12 +829,14 @@ const runStreamingShim = ( events: AnthropicMessagesStreamEvent[], state: AnthropicMessagesWebSearchShimState, provider?: ReturnType, + reportFailure: (error: unknown) => Promise = async () => {}, ) => collectStreamEvents( rewriteAnthropicMessagesWebSearchEventsToNative( toAsyncIterable(events.map(event => ({ type: 'event' as const, event }))), state, provider, + reportFailure, ), ); @@ -867,8 +868,8 @@ test('rewriteAnthropicMessagesWebSearchEventsToNative single web search emits se const indexed = events.filter(event => event.type === 'content_block_start' || event.type === 'content_block_stop' || event.type === 'content_block_delta'); assertEquals(indexed.map(event => (event as { index: number }).index), [0, 0, 0, 1, 1, 2, 2, 3, 3, 3]); - const serverToolUse = events.find(event => event.type === 'content_block_start' && event.content_block.type === 'server_tool_use'); - assertEquals(serverToolUse?.type === 'content_block_start' ? serverToolUse.content_block : undefined, { + const hostedToolUse = events.find(event => event.type === 'content_block_start' && event.content_block.type === 'server_tool_use'); + assertEquals(hostedToolUse?.type === 'content_block_start' ? hostedToolUse.content_block : undefined, { type: 'server_tool_use', id: 'srvtoolu_1', name: 'web_search', @@ -1332,3 +1333,43 @@ test('rewriteAnthropicMessagesWebSearchEventsToNative requires a provider when m 'Active messages web-search rewrite requires a provider.', ); }); + +test('the native web-search tool records its original backend error before serving unavailable', async () => { + initRepo(new InMemoryRepo()); + const fault = new Error('search backend unavailable', { cause: new TypeError('backend connection state') }); + const provider = activeProvider(searchOnlyProvider(async () => { throw fault; })); + const selected = stubModelCandidate({ enabledFlags: new Set(['anthropic-messages-web-search-shim']) }); + const backend = vi.spyOn(webSearchShim, 'resolveActiveAnthropicMessagesWebSearchProvider').mockResolvedValue(provider); + try { + const terminal = defineStage, Record>({ + name: 'nativeSearchSource', + return: { provides: ['response.chat.anthropicMessages', 'response.chat.anthropicMessages.streamedUsage', 'response.usage.billable', 'response.http.headers', 'response.http.status', 'response.http.body'] }, + execute: async facts => move({ + ...facts, 'response.chat.anthropicMessages': { + kind: 'stream', frames: toAsyncIterable([ + upstreamMessageStart(), ...upstreamWebSearchBlock(0, 'tool_1', 'question'), ...upstreamMessageEnd('tool_use'), + ].map(event => eventFrame(event))), + }, + 'response.chat.anthropicMessages.streamedUsage': null, 'response.usage.billable': [], + 'response.http.headers': [], 'response.http.status': 200, 'response.http.body': null, + }), + }); + const logs: RunEvent[] = []; + const outcome = await run(compose, Record>('nativeSearch', [ + runAnthropicMessagesWebSearchTool({ targetOf: () => 'anthropicMessages' }), terminal, + ]), move({ + 'request.chat.anthropicMessages': { model: 'model', max_tokens: 32, messages: [{ role: 'user', content: 'question' }], tools: [{ type: 'web_search_20260209' }] }, + 'route.attempt': { candidateId: 0, upstreamId: selected.provider.upstreamId, modelId: selected.model.id, flags: ['anthropic-messages-web-search-shim'] }, + }), { gateway: mockChatGatewayCtx(), resolveAttempt: () => selected, dump: async (event: RunEvent) => { logs.push(event); } }); + const answer = outcome.facts['response.chat.anthropicMessages'] as { frames: AsyncIterable> }; + const events = await collectStreamEvents(answer.frames); + expect(events).toContainEqual(expect.objectContaining({ type: 'content_block_start', content_block: expect.objectContaining({ type: 'web_search_tool_result', content: { type: 'web_search_tool_result_error', error_code: 'unavailable' } }) })); + const entry = logs.find(event => event.type === 'stage.log' && event.level === 'error'); + expect(entry).toMatchObject({ context: 'runAnthropicMessagesWebSearchTool', message: 'web search failed' }); + expect(entry?.type === 'stage.log' ? entry.fields?.error : undefined).toBe(fault); + expect(fault.cause).toBeInstanceOf(TypeError); + await outcome.drain(); + } finally { + backend.mockRestore(); + } +}); diff --git a/packages/gateway/__tests__/data-plane/chat/client-errors_test.ts b/packages/gateway/__tests__/data-plane/chat/client-errors_test.ts new file mode 100644 index 0000000000..f58a2e8b12 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/client-errors_test.ts @@ -0,0 +1,101 @@ +import { Hono } from 'hono'; +import { expect, test, vi } from 'vitest'; + +import { emitAnthropicMessages } from '../../../src/data-plane/chat/anthropic-messages/emit.ts'; +import { meterUsage as meterAnthropic } from '../../../src/data-plane/chat/anthropic-messages/meter.ts'; +import { emitGeminiGenerateContent } from '../../../src/data-plane/chat/gemini-generate-content/emit.ts'; +import { emitOpenAIChatCompletions } from '../../../src/data-plane/chat/openai-chat-completions/emit.ts'; +import { meterUsage as meterChat } from '../../../src/data-plane/chat/openai-chat-completions/meter.ts'; +import { prologueFor, serveThrough } from '../../../src/data-plane/pipeline/serve.ts'; +import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts'; +import { openRunDump } from '../../../src/dump/run-sink.ts'; +import { initRepo } from '../../../src/repo/index.ts'; +import type { ApiKey } from '../../../src/repo/types.ts'; +import { eventsOf, installDumpStubs } from '../../dump/test-fixtures.ts'; +import { InMemoryRepo } from '../../repo/memory.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { compose, createRunReader, defineStage, move, defer, type DumpEvent } from '@floway-dev/pipeline'; +import { eventFrame, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { testTelemetryModelIdentity } from '@floway-dev/test-utils'; + +const key: ApiKey = { id: 'client-key', userId: 1, name: 'Client key', key: 'client-key', serverSecret: '00'.repeat(32), createdAt: '2026-01-01T00:00:00Z', upstreamIds: null, deletedAt: null, dumpRetentionSeconds: 3600, openaiResponsesRetentionSeconds: 0 }; +const fixtures = [ + { protocol: 'openaiChatCompletions', emit: emitOpenAIChatCompletions, meter: meterChat, initial: { id: 'chat', object: 'chat.completion.chunk', created: 1, model: 'model', choices: [{ index: 0, delta: { content: 'partial' }, finish_reason: null }] } }, + { protocol: 'anthropicMessages', emit: emitAnthropicMessages, meter: meterAnthropic, initial: { type: 'content_block_start', index: 0, content_block: { type: 'text', text: 'partial' } } }, + { protocol: 'geminiGenerateContent', emit: emitGeminiGenerateContent, meter: null, initial: { candidates: [{ index: 0, content: { role: 'model', parts: [{ text: 'partial' }] } }] } }, +]; +type Exit = Record & { 'response.http.status': number; 'response.http.headers': readonly (readonly [string, string])[] }; + +for (const fixture of fixtures) for (const failureAt of ['read', 'render'] as const) { + test(`actual ${fixture.protocol} SSE records and exposes ${failureAt} failure with its original error chain`, async () => { + initRepo(new InMemoryRepo()); + const stubs = installDumpStubs(initDumpStore, initDumpBroker); + const pending: Promise[] = []; + const schedule = (work: Promise) => { pending.push(work); }; + const dump = openRunDump(key, { method: 'POST', path: '/v1/chat', body: { bytes: new Uint8Array(), streamError: null } }, schedule, true, { upstreamCallStartedAt: null, firstOutputTokenAt: null }); + if (dump === null) throw new Error('retained request has no recording'); + const gateway = mockChatGatewayCtx({ apiKeyId: key.id, dump, wantsStream: true, backgroundScheduler: schedule }); + const fault = new Error('stream socket broke', { cause: new Error('socket reset') }); + const logged = vi.spyOn(console, 'error').mockImplementation(() => {}); + let completed = false; + let settle!: (outcome: { billable: readonly never[]; failed: boolean }) => void; + const reading = defer(new Promise<{ billable: readonly never[]; failed: boolean }>(resolve => { settle = resolve; })); + const responseKey = `response.chat.${fixture.protocol}`; + const usageKey = `${responseKey}.streamedUsage`; + const dial = defineStage, Record>({ + name: 'scriptedDial', return: { provides: [responseKey, usageKey, 'response.http.headers', 'response.http.status', 'response.usage.billable'] }, + execute: async facts => move({ + ...facts, 'response.http.headers': [], 'response.http.status': 200, + 'response.usage.billable': [{ identity: testTelemetryModelIdentity, quantities: {} }], + [usageKey]: fixture.meter === null ? reading : null, + [responseKey]: { + kind: 'stream', frames: { + async *[Symbol.asyncIterator]() { + try { + yield eventFrame({ ...fixture.initial, ...(failureAt === 'render' ? { impossibleJson: 1n } : {}) }); + if (failureAt === 'read') throw fault; + } finally { + completed = true; + settle({ billable: [], failed: true }); + } + }, + }, + }, + }), + }); + const pipeline = compose, Exit>('clientError', [fixture.emit, ...(fixture.meter === null ? [] : [fixture.meter(usageKey)]), dial]); + const app = new Hono(); + app.post('/v1/chat', c => serveThrough(c, prologueFor(gateway, { body: { bytes: new Uint8Array(), streamError: null }, headers: [] }, dump), pipeline, + move({ [`ingress.chat.${fixture.protocol}.wantsStream`]: true, 'ingress.chat.openaiChatCompletions.wantsUsageChunk': false }), + facts => ({ frames: facts[`${responseKey}.rendered`] as AsyncIterable }))); + const response = await app.request('/v1/chat', { method: 'POST' }); + const body = await response.text(); + await Promise.all(pending); + expect(completed).toBe(true); + expect(logged).not.toHaveBeenCalled(); + logged.mockRestore(); + const actual = body.split('\n').filter(line => line.startsWith('data: ')).map(line => JSON.parse(line.slice(6)) as { error?: { message?: string; cause?: { message: string } } }); + const error = actual.find(event => event.error !== undefined)?.error; + expect(error?.message).toBe(failureAt === 'read' ? fault.message : 'Do not know how to serialize a BigInt'); + if (failureAt === 'read') expect(error?.cause?.message).toBe('socket reset'); + expect(body).not.toContain('[DONE]'); + expect(body).not.toContain('impossibleJson'); + const record = stubs.stored[0]!.record; + const read = createRunReader(); + const streams = new Map(); + let client: number | null = null; + for (const item of eventsOf(record)) { + const event = item as unknown as DumpEvent; + const decoded = read(event); + if (decoded?.facts && 'response.chat.clientFrames' in decoded.facts) client = (decoded.facts['response.chat.clientFrames'] as { stream: number }).stream; + if (event.type === 'stream.frame') { + const frames = streams.get(event.streamId) ?? []; + frames.push(...(decoded?.frames as ProtocolFrame[]).map(frame => frame.type === 'event' ? frame.event : '[DONE]')); + streams.set(event.streamId, frames); + } + } + expect(client).not.toBeNull(); + expect(streams.get(client!)).toEqual(actual); + expect(record.meta.error).toMatchObject({ kind: 'failed' }); + }); +} diff --git a/packages/gateway/__tests__/data-plane/chat/client-json_test.ts b/packages/gateway/__tests__/data-plane/chat/client-json_test.ts new file mode 100644 index 0000000000..ec7a4a86ff --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/client-json_test.ts @@ -0,0 +1,50 @@ +import { Hono } from 'hono'; +import { expect, test } from 'vitest'; + +import { emitOpenAIChatCompletions } from '../../../src/data-plane/chat/openai-chat-completions/emit.ts'; +import { serializeClientJson } from '../../../src/data-plane/pipeline/serialize-client-json.ts'; +import { prologueFor, serveThrough } from '../../../src/data-plane/pipeline/serve.ts'; +import { writeSettlement } from '../../../src/data-plane/pipeline/settlement.ts'; +import { initRepo } from '../../../src/repo/index.ts'; +import { InMemoryRepo } from '../../repo/memory.ts'; +import { flushBackground, trackBackground } from '../../test-utils/background-tracker.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { compose, defineStage, move } from '@floway-dev/pipeline'; +import { testTelemetryModelIdentity } from '@floway-dev/test-utils'; + +test('a client JSON formatting fault is inside the run settlement boundary', async () => { + const repo = new InMemoryRepo(); + initRepo(repo); + const now = performance.now(); + const gateway = mockChatGatewayCtx({ + backgroundScheduler: trackBackground, attempt: { + timing: { upstreamCallStartedAt: now - 10, firstOutputTokenAt: now - 5 }, + telemetry: { keyId: 'json-key', model: 'model', upstream: 'upstream', operation: 'chat', runtimeLocation: 'TEST' }, + }, + }); + const ending = defineStage, Record>({ + name: 'scriptedValue', + return: { provides: ['response.chat.openaiChatCompletions', 'response.chat.openaiChatCompletions.streamedUsage', 'response.http.headers', 'response.http.status', 'response.usage.billable'] }, + execute: async facts => move({ + ...facts, + 'response.chat.openaiChatCompletions': { kind: 'value', body: { id: 'completion', extension: 1n } }, + 'response.chat.openaiChatCompletions.streamedUsage': null, + 'response.http.headers': [], 'response.http.status': 200, + 'response.usage.billable': [{ identity: testTelemetryModelIdentity, quantities: { input_tokens: '3', output_tokens: '2' } }], + }), + }); + const pipeline = compose, Record & { 'response.http.status': number; 'response.http.headers': readonly (readonly [string, string])[] }>('clientJson', [ + writeSettlement(facts => Number(facts['response.http.status']) >= 400, 'response.chat.openaiChatCompletions.streamedUsage'), + serializeClientJson('response.chat.openaiChatCompletions.rendered'), emitOpenAIChatCompletions, ending, + ]); + const app = new Hono(); + app.post('/v1/chat/completions', c => serveThrough(c, prologueFor(gateway, { body: { bytes: new Uint8Array(), streamError: null }, headers: [] }), pipeline, + move({ 'ingress.chat.openaiChatCompletions.wantsStream': false, 'ingress.chat.openaiChatCompletions.wantsUsageChunk': false }), + facts => ({ body: facts['response.http.jsonBody'] as Uint8Array, contentType: 'application/json' }))); + const response = await app.request('/v1/chat/completions', { method: 'POST' }); + expect(response.status).toBe(500); + expect(await response.json()).toMatchObject({ error: { name: 'TypeError', message: 'Do not know how to serialize a BigInt' } }); + await flushBackground(); + expect(await repo.usage.listAll()).toMatchObject([{ requests: 1 }]); + expect(await repo.performance.listAll()).toMatchObject([{ requests: 1, errorsWithOutput: 1, errorsNoOutput: 0 }]); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/client-streams_test.ts b/packages/gateway/__tests__/data-plane/chat/client-streams_test.ts new file mode 100644 index 0000000000..876f188f52 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/client-streams_test.ts @@ -0,0 +1,102 @@ +import { expect, test } from 'vitest'; + +import { emitAnthropicMessages } from '../../../src/data-plane/chat/anthropic-messages/emit.ts'; +import { emitGeminiGenerateContent } from '../../../src/data-plane/chat/gemini-generate-content/emit.ts'; +import { emitOpenAIChatCompletions } from '../../../src/data-plane/chat/openai-chat-completions/emit.ts'; +import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts'; +import { openRunDump } from '../../../src/dump/run-sink.ts'; +import type { ApiKey } from '../../../src/repo/types.ts'; +import { eventsOf, installDumpStubs } from '../../dump/test-fixtures.ts'; +import { flushBackground, trackBackground } from '../../test-utils/background-tracker.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { compose, createRunReader, defineStage, move, run, type DumpEvent } from '@floway-dev/pipeline'; +import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import { doneFrame, eventFrame, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import type { GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; +import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; + +const key: ApiKey = { id: 'client-key', userId: 1, name: 'Client key', key: 'client-key', serverSecret: '00'.repeat(32), createdAt: '2026-01-01T00:00:00Z', upstreamIds: null, deletedAt: null, dumpRetentionSeconds: 3600, openaiResponsesRetentionSeconds: 0 }; +const usage: OpenAIChatCompletionsStreamEvent = { id: 'chat', object: 'chat.completion.chunk', created: 1, model: 'model', choices: [], usage: { prompt_tokens: 2, completion_tokens: 1, total_tokens: 3 } }; +const chat: OpenAIChatCompletionsStreamEvent = { id: 'chat', object: 'chat.completion.chunk', created: 1, model: 'model', choices: [{ index: 0, delta: { content: 'answer' }, finish_reason: 'stop' }] }; +const citation: AnthropicMessagesStreamEvent = { + type: 'content_block_start', index: 0, + content_block: { + type: 'text', text: 'answer', + citations: [{ type: 'search_result_location', url: 'https://example.com/source', title: 'Source', search_result_index: 0, start_block_index: 0, end_block_index: 1 }], + }, +}; +const gemini: GeminiGenerateContentStreamEvent = { candidates: [{ index: 0, content: { role: 'model', parts: [{ text: 'answer' }] }, finishReason: 'STOP' }] }; + +const cases = [ + ...[false, true].flatMap(includeUsage => [false, true].map(done => ({ + name: `Chat Completions ${includeUsage ? 'included' : 'hidden'} usage with ${done ? 'DONE' : 'normal EOF'}`, + emit: emitOpenAIChatCompletions, + entry: { 'ingress.chat.openaiChatCompletions.wantsStream': true, 'ingress.chat.openaiChatCompletions.wantsUsageChunk': includeUsage }, + response: 'response.chat.openaiChatCompletions', rendered: 'response.chat.openaiChatCompletions.rendered', + upstream: [eventFrame(chat), eventFrame(usage), ...(done ? [doneFrame()] : [])], + verify: (events: unknown[]) => { + expect(events.some(event => typeof event === 'object' && event !== null && 'usage' in event)).toBe(includeUsage); + expect(events.includes('[DONE]')).toBe(done); + }, + }))), + { + name: 'Anthropic citations and transport terminator', emit: emitAnthropicMessages, + entry: { 'ingress.chat.anthropicMessages.wantsStream': true }, + response: 'response.chat.anthropicMessages', rendered: 'response.chat.anthropicMessages.rendered', + upstream: [eventFrame(citation), eventFrame({ type: 'content_block_stop', index: 0 }), eventFrame({ type: 'message_stop' }), doneFrame()], + verify: (events: unknown[]) => { + expect(events).not.toContain('[DONE]'); + expect(events).toContainEqual({ ...citation, index: 1, content_block: { type: 'text', text: 'answer', citations: [{ type: 'search_result_location', source: 'https://example.com/source', title: 'Source', search_result_index: 0, start_block_index: 0, end_block_index: 1 }] } }); + }, + }, + { + name: 'Gemini transport terminator', emit: emitGeminiGenerateContent, + entry: { 'ingress.chat.geminiGenerateContent.wantsStream': true }, + response: 'response.chat.geminiGenerateContent', rendered: 'response.chat.geminiGenerateContent.rendered', + upstream: [eventFrame(gemini), doneFrame()], + verify: (events: unknown[]) => { expect(events).not.toContain('[DONE]'); expect(events).toMatchObject([gemini]); }, + }, +]; + +for (const fixture of cases) { + test(`selected recorded client protocol frames equal actual SSE for ${fixture.name}`, async () => { + const stubs = installDumpStubs(initDumpStore, initDumpBroker); + const dump = openRunDump(key, { method: 'POST', path: '/v1/chat', body: { bytes: new Uint8Array(), streamError: null } }, trackBackground, true, { upstreamCallStartedAt: null, firstOutputTokenAt: null }); + if (dump === null) throw new Error('retained client request has no run recording'); + const dial = defineStage, Record>({ + name: 'scriptedDial', return: { provides: [fixture.response, 'response.http.headers', 'response.http.status', `${fixture.response}.streamedUsage`] }, + execute: async facts => move({ + ...facts, 'response.http.headers': [], 'response.http.status': 200, [`${fixture.response}.streamedUsage`]: null, + [fixture.response]: { kind: 'stream', frames: { async *[Symbol.asyncIterator]() { yield* fixture.upstream; } } }, + }), + }); + const outcome = await run(compose, Record>('clientStream', [fixture.emit, dial]), move(fixture.entry), { gateway: mockChatGatewayCtx({ dump, wantsStream: true }), dump: dump.sink }); + const wire: unknown[] = []; + for await (const frame of outcome.facts[fixture.rendered] as AsyncIterable) wire.push(frame.data === '[DONE]' ? frame.data : JSON.parse(frame.data)); + await outcome.drain(); + dump.finalize(200, 0); + await flushBackground(); + const read = createRunReader(); + const streams = new Map[]>(); + let client: number | null = null; + const record = stubs.stored[0]?.record; + if (record === undefined) throw new Error('completed client stream has no persisted record'); + for (const item of eventsOf(record)) { + const event = item as unknown as DumpEvent; + const decoded = read(event); + if (decoded?.facts && 'response.chat.clientFrames' in decoded.facts) client = (decoded.facts['response.chat.clientFrames'] as { stream: number }).stream; + if (event.type === 'stream.frame') { + const frames = streams.get(event.streamId) ?? []; + frames.push(...decoded?.frames as ProtocolFrame[]); + streams.set(event.streamId, frames); + } + } + const frames = client === null ? [] : streams.get(client) ?? []; + expect(frames.length).toBeGreaterThan(0); + const recorded = frames.map(frame => frame.type === 'done' ? '[DONE]' : frame.event); + expect(recorded).toEqual(wire); + expect(eventsOf(record).some(event => event.type === 'stream.end' && event.streamId === client)).toBe(true); + expect(record.meta.error).toBeNull(); + fixture.verify(recorded); + }); +} diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts new file mode 100644 index 0000000000..65e08e38aa --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-count-tokens_test.ts @@ -0,0 +1,235 @@ +// The Gemini generateContent count-tokens chain, run. What is written down here is what only running it can +// say: that the question is asked in Anthropic Messages and the answer read back out as Google's, that +// what no translation can carry is gone before the question is asked, that a refusal keeps +// the upstream's own status, and that measuring bills nothing. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { geminiGenerateContentCountTokensPipeline } from '../../../src/data-plane/chat/gemini-generate-content/count-tokens.ts'; +import { createCandidateRegistry } from '../../../src/data-plane/pipeline/candidates.ts'; +import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; +import { initRepo } from '../../../src/repo/index.ts'; +import { chatFixtureHttpServices, stubChatProviderPipelines } from '../../test-utils/chat-provider-pipelines.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { move, run } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import type { ModelEndpoints } from '@floway-dev/protocols/common'; +import type { GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; +import type { AnthropicMessagesUpstreamCallOptions, ModelCandidate } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel, type ProviderCallResult } from '@floway-dev/test-utils'; + +vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ + ...(await importOriginal()), + enumerateModelCandidates: vi.fn(), +})); + +let live: readonly ModelCandidate[] = []; + +type CountTokens = ( + model: unknown, + body: unknown, + signal: AbortSignal | undefined, + opts: AnthropicMessagesUpstreamCallOptions, +) => Promise; + +const candidate = ( + callAnthropicMessagesCountTokens: CountTokens, + overrides: { upstreamId?: string; endpoints?: ModelEndpoints } = {}, +): ModelCandidate => { + const upstreamId = overrides.upstreamId ?? 'up_a'; + const endpoints = overrides.endpoints ?? { anthropicMessages: {} }; + return { + provider: { + upstreamId, kind: 'claude-code', name: upstreamId, + // Wide enough that a header reaching the provider proves the ending let it through, + // rather than proving only that the allowlist did. + inboundHeaderAllowlist: [/^(anthropic-beta|x-trace)$/], + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + pipelines: stubChatProviderPipelines({ callAnthropicMessagesCountTokens: callAnthropicMessagesCountTokens as never }), + instance: stubProvider(), + }, + model: stubInternalModel( + { + id: 'claude-model', + endpoints, + limits: { max_output_tokens: 4096 }, + providerModels: { [upstreamId]: stubProviderModel({ id: 'claude-model', endpoints }) }, + }, + upstreamId, + ), + } as unknown as ModelCandidate; +}; + +const resolves = (candidates: readonly ModelCandidate[]): void => { + live = candidates; + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] } as never); +}; + +const answered = (body: unknown): ProviderCallResult => ({ + response: Response.json(body), + modelKey: 'claude-model-key', +}); + +const payload: GeminiGenerateContentPayload = { contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }; + +const count = async ( + request: GeminiGenerateContentPayload = payload, + headers: readonly (readonly [string, string])[] = [], +) => { + const gateway = mockChatGatewayCtx({ wantsStream: false }); + return await run( + geminiGenerateContentCountTokensPipeline(), + move({ + 'ingress.http.headers': headers, + 'ingress.chat.sourceProtocol': 'geminiGenerateContent', + 'request.chat.geminiGenerateContent': request, + 'serve.model': 'gemini-model', + }) as never, + { + gateway, + background: () => {}, + ...chatFixtureHttpServices(gateway), + ...createCandidateRegistry(), + selectAffinity: () => { throw new Error('a measurement pins nothing; it must not select affinity'); }, + resolveAttempt: (selector: { readonly upstreamId: string }) => { + const found = live.find(c => c.provider.upstreamId === selector.upstreamId); + if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); + return found; + }, + } as never, + ); +}; + +beforeEach(() => { + vi.mocked(enumerateModelCandidates).mockReset(); + initRepo({ + apiKeys: { update: async () => {} }, + usage: { record: async () => {} }, + performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, + } as never); +}); + +describe('the Gemini generateContent count-tokens chain', () => { + // Gemini generateContent has no endpoint that answers this, so the question is asked in Anthropic Messages and the + // counts are read back out under the name this protocol gives them. + it('asks the question in Anthropic Messages and answers in Google-s own envelope', async () => { + let sent: AnthropicMessagesPayload | undefined; + let seenModel: { readonly id: string } | undefined; + resolves([candidate(async (model, body) => { + seenModel = model as { readonly id: string }; + sent = body as AnthropicMessagesPayload; + return answered({ input_tokens: 42 }); + })]); + + const { facts } = await count({ ...payload, systemInstruction: { parts: [{ text: 'system' }] } }); + + expect(facts['response.http.status']).toBe(200); + expect(facts['response.chat.geminiGenerateContent.rendered']).toEqual({ totalTokens: 42 }); + expect(seenModel?.id).toBe('claude-model'); + expect(sent?.system).toBeDefined(); + expect(sent?.messages).toMatchObject([{ role: 'user', content: [{ type: 'text', text: 'hi' }] }]); + // `stream` says how an answer would be delivered, and a measurement delivers none. + expect(sent).not.toHaveProperty('stream'); + }); + + // Anthropic's own endpoint answers `input_tokens`; a Copilot upstream's translated count + // answers `total_tokens`. Either is the number this protocol calls `totalTokens`. + it('reads the other dialect an upstream states its counts in', async () => { + resolves([candidate(async () => answered({ total_tokens: 19 }))]); + + const { facts } = await count(); + + expect(facts['response.chat.geminiGenerateContent.rendered']).toEqual({ totalTokens: 19 }); + }); + + // A body carrying neither figure is an answer this gateway cannot read, which is the + // gateway failing to serve — and it is a failure the fork can move past, so another + // upstream gets asked before the client is told. + it('fails an unreadable count over, then reports it as the gateway-s own', async () => { + const tried: string[] = []; + resolves([ + candidate(async () => { tried.push('odd'); return answered({ unexpected: true }); }, { upstreamId: 'up_odd' }), + candidate(async () => { tried.push('odder'); return answered({ unexpected: true }); }, { upstreamId: 'up_odder' }), + ]); + + const { facts } = await count(); + + expect(tried).toEqual(['odd', 'odder']); + expect(facts['response.http.status']).toBe(502); + expect(facts['response.chat.geminiGenerateContent.rendered']).toEqual({ + error: { code: 502, message: 'Invalid upstream token counting response.', status: 'UNAVAILABLE' }, + }); + }); + + // Gemini generateContent has no wire of its own, so what no translation can carry cannot be sent whichever + // candidate answers — and a count taken before the strip would measure a request the + // translator would refuse. + it('strips what no translation carries before asking', async () => { + let sent: AnthropicMessagesPayload | undefined; + resolves([candidate(async (_model, body) => { sent = body as AnthropicMessagesPayload; return answered({ input_tokens: 5 }); })]); + + const { facts } = await count({ + contents: [{ role: 'user', parts: [{ text: 'hi' }, { fileData: { fileUri: 'gs://x', mimeType: 'text/plain' } }] }], + tools: [{ googleSearch: {} }], + safetySettings: [{ category: 'HARM_CATEGORY_HARASSMENT', threshold: 'BLOCK_NONE' }], + } as unknown as GeminiGenerateContentPayload); + + expect(facts['response.chat.geminiGenerateContent.rendered']).toEqual({ totalTokens: 5 }); + // A tool group left declaring no function is not a tool group, so `tools` goes with it. + expect(sent).not.toHaveProperty('tools'); + expect(JSON.stringify(sent)).not.toContain('fileData'); + expect(sent).not.toHaveProperty('safetySettings'); + }); + + // Anthropic's beta flags are the client's own only when the client spoke that protocol, and + // a Gemini generateContent turn asked for nothing on that wire. + it('does not carry another protocol-s beta flags across the boundary', async () => { + let seen: AnthropicMessagesUpstreamCallOptions | undefined; + resolves([candidate(async (_model, _body, _signal, opts) => { seen = opts; return answered({ input_tokens: 1 }); })]); + + await count(payload, [['anthropic-beta', 'must-not-cross-source-protocols'], ['x-trace', 'abc']]); + + expect(seen?.anthropicBeta).toEqual([]); + expect(seen?.headers.get('anthropic-beta')).toBeNull(); + expect(seen?.headers.get('x-trace')).toBe('abc'); + }); + + it('keeps the status of an upstream that refused, and names a refusal that said nothing', async () => { + resolves([candidate(async () => ({ response: new Response('', { status: 503 }), modelKey: 'k' }))]); + + const { facts } = await count(); + + expect(facts['response.http.status']).toBe(503); + expect(facts['response.chat.geminiGenerateContent.rendered']).toEqual({ + error: { code: 503, message: 'Upstream token counting request failed.', status: 'UNAVAILABLE' }, + }); + }); + + // Only an upstream's own Anthropic Messages endpoint measures, so a candidate that would serve + // generation over a translated wire cannot serve this. + it('refuses a candidate no wire can measure on, naming the action', async () => { + resolves([candidate(async () => answered({ input_tokens: 0 }), { endpoints: { openaiChatCompletions: {} } })]); + + const { facts } = await count(); + + expect(facts['response.http.status']).toBe(400); + expect(facts['response.chat.geminiGenerateContent.rendered']).toEqual({ + error: { code: 400, message: 'Model gemini-model does not support countTokens.', status: 'INVALID_ARGUMENT' }, + }); + }); + + it('bills nothing for a measurement', async () => { + resolves([candidate(async () => answered({ input_tokens: 3 }))]); + + const { facts } = await count(); + + expect((facts as Record)['response.usage.billable']).toEqual([]); + }); +}); + +it('preserves the accepted native counting HTTP status', async () => { + resolves([candidate(async () => ({ response: new Response(JSON.stringify({ input_tokens: 3 }), { status: 201, headers: { 'content-type': 'application/json' } }), modelKey: 'counted-model' }))]); + const { facts, drain } = await count(); + expect(facts['response.http.status']).toBe(201); + await drain(); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts new file mode 100644 index 0000000000..2d4549cc59 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content-pipeline_test.ts @@ -0,0 +1,411 @@ +// Gemini generateContent's pipeline, assembled and run. Gemini generateContent is the source-only family and the one with no +// wire of its own, so what a run of it has to show beyond the entry contract is the round +// trip: a turn translated out to OpenAI Chat Completions, an answer translated back, the turn's own +// state handed back on a Part for the client to carry, and the reading taken on the dialect +// the upstream actually spoke rather than on the one the client did. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { geminiGenerateContentServePipeline } from '../../../src/data-plane/chat/gemini-generate-content/pipeline.ts'; +import { createCandidateRegistry } from '../../../src/data-plane/pipeline/candidates.ts'; +import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; +import { initRepo } from '../../../src/repo/index.ts'; +import { chatFixtureHttpServices, stubChatProviderPipelines } from '../../test-utils/chat-provider-pipelines.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { move, run } from '@floway-dev/pipeline'; +import { doneFrame, eventFrame, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { GEMINI_GENERATE_CONTENT_MISSING_TERMINAL_MESSAGE, type GeminiGenerateContentPayload, type GeminiGenerateContentResult } from '@floway-dev/protocols/gemini-generate-content'; +import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; +import { directFetcher, type ModelCandidate } from '@floway-dev/provider'; +import { type StubChatProviderCall, type ProviderStreamResult } from '@floway-dev/test-utils'; +import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; + +vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ + ...(await importOriginal()), + enumerateModelCandidates: vi.fn(), +})); + +/** The live candidates the resolver hands back. They never enter the record: a candidate + * carries the provider's instance, its fetcher and its models cache, and freezing those is + * what putting one in the record would do. What travels is the selector. */ +vi.mock('../../../src/data-plane/chat/gemini-generate-content/affinity/ingress.ts', async importOriginal => ({ + ...(await importOriginal()), + analyzeGeminiGenerateContentAffinity: async () => ({ + requiredTargets: [], + evaluateCandidate: () => ({ kind: 'accepted', degrades: false, preferred: true, materialize: () => { asked += 1; return affinityPayload; } }), + }), +})); + +let live: readonly ModelCandidate[] = []; + +const resolves = (candidates: readonly ModelCandidate[]): void => { + live = candidates; + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] }); +}; + +const resolveAttempt = (selector: { readonly upstreamId: string }): ModelCandidate => { + const found = live.find(candidate => candidate.provider.upstreamId === selector.upstreamId); + if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); + return found; +}; + +type CallOpenAIChatCompletions = StubChatProviderCall<'openaiChatCompletions'>; + +const candidate = (upstream: string, callOpenAIChatCompletions: CallOpenAIChatCompletions): ModelCandidate => { + const endpoints = { openaiChatCompletions: {} }; + return { + provider: { + upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + pipelines: stubChatProviderPipelines({ callOpenAIChatCompletions: callOpenAIChatCompletions as never }), + instance: stubProvider(), + }, + model: stubInternalModel({ id: 'gemini-2.5-pro', endpoints, providerModels: { [upstream]: stubProviderModel({ id: 'gemini-2.5-pro', endpoints }) } }, upstream), + fetcher: directFetcher, + }; +}; + +const chunk = (delta: Record, finishReason?: string): ProtocolFrame => + eventFrame({ + id: 'chatcmpl_1', object: 'chat.completion.chunk', created: 1, model: 'gemini-2.5-pro', + choices: [{ index: 0, delta, ...(finishReason === undefined ? {} : { finish_reason: finishReason }) }], + } as OpenAIChatCompletionsStreamEvent); + +const usageChunk: ProtocolFrame = eventFrame({ + id: 'chatcmpl_1', object: 'chat.completion.chunk', created: 1, model: 'gemini-2.5-pro', + choices: [], + usage: { prompt_tokens: 5, completion_tokens: 7, total_tokens: 12 }, +} as OpenAIChatCompletionsStreamEvent); + +/** One whole turn on the wire below: two content deltas, the figures the upstream metered, + * and the sentinel that ends it. */ +const turn = (): AsyncIterable> => ({ + [Symbol.asyncIterator]: () => (async function* () { + yield chunk({ content: 'he' }); + yield chunk({ content: 'llo' }, 'stop'); + yield usageChunk; + yield doneFrame(); + })(), +}); + +/** The wire below closed cleanly and never finished the turn: no choice ever carried a + * finish reason, so nothing that comes out of the translation is a Gemini generateContent terminal event. */ +const unfinishedTurn = (): AsyncIterable> => ({ + [Symbol.asyncIterator]: () => (async function* () { + yield chunk({ content: 'he' }); + yield doneFrame(); + })(), +}); + +const streamed = ( + events: AsyncIterable>, + headers?: Headers, +): ProviderStreamResult => ({ + ok: true, events, modelKey: 'gemini-2.5-pro-key', ...(headers === undefined ? {} : { headers }), +}); + +const payload = { contents: [{ role: 'user' as const, parts: [{ text: 'hello' }] }] }; + +/** What affinity materialized for the candidate about to be dialled. It differs from the + * turn the client sent, which is the point: carried state is rewritten per candidate. */ +let affinityPayload: GeminiGenerateContentPayload = payload; + +/** How many times the chain asked the resolver for that turn. One stage owns the reading; + * anything below it reads the record. */ +let asked = 0; + +/** Which candidates the run said a follow-up turn must come back to. */ +const selected: ModelCandidate[] = []; + +// Settlement is part of every serve pipeline, and it writes. A test that drives a whole +// pipeline therefore needs somewhere for the row to go. +const recorded: { usage: unknown[]; performance: unknown[] } = { usage: [], performance: [] }; + +beforeEach(() => { + vi.mocked(enumerateModelCandidates).mockReset(); + recorded.usage = []; + recorded.performance = []; + selected.length = 0; + affinityPayload = payload; + asked = 0; + initRepo({ + apiKeys: { update: async () => {} }, + usage: { record: async (row: unknown) => { recorded.usage.push(row); } }, + performance: { + recordNeutral: async (dims: unknown) => { recorded.performance.push(dims); }, + recordZeroOutputError: async (dims: unknown) => { recorded.performance.push(dims); }, + }, + } as never); +}); + +const serve = async (facts: Record) => + await serveWith(mockChatGatewayCtx({ wantsStream: facts['ingress.chat.geminiGenerateContent.wantsStream'] === true }), facts); + +const serveWith = async (gateway: ReturnType, facts: Record) => await run( + geminiGenerateContentServePipeline(), + move(facts) as never, + { + gateway, + background: () => {}, + ...chatFixtureHttpServices(gateway), + ...createCandidateRegistry(), + // Wired where the app wires it: the carrier the edge writes is addressed to whatever the + // chain named here. + selectAffinity: (answered: ModelCandidate) => { selected.push(answered); gateway.affinity.select(answered); }, + resolveAttempt, + } as never, +); + +const entryFacts = (overrides: Record = {}) => ({ + 'ingress.chat.geminiGenerateContent.wantsStream': true, + 'ingress.chat.sourceProtocol': 'geminiGenerateContent', + 'ingress.http.headers': [], + 'request.chat.geminiGenerateContent': payload, + 'serve.model': 'gemini-2.5-pro', + ...overrides, +}); + +const collect = async (rendered: unknown): Promise => { + const frames: SseFrame[] = []; + for await (const frame of rendered as AsyncIterable) frames.push(frame); + return frames; +}; + +/** The turn's own answer. The carrier the edge writes back rides as a `thoughtSignature` on + * a Part rather than as an event of its own, so taking it off leaves what Gemini generateContent itself + * said. */ +const withoutCarrier = (event: GeminiGenerateContentResult): GeminiGenerateContentResult => ({ + ...event, + ...(event.candidates === undefined ? {} : { + candidates: event.candidates.map(candidate => ({ + ...candidate, + content: { + ...candidate.content, + parts: candidate.content.parts.map(({ thoughtSignature: _carried, ...part }) => part), + }, + })), + }), +}); + +/** Every piece of this turn's own state the client was handed back. */ +const affinityCarriers = (events: readonly GeminiGenerateContentResult[]): readonly string[] => + events.flatMap(event => (event.candidates ?? []).flatMap(candidate => + candidate.content.parts.flatMap(part => part.thoughtSignature ?? []))); + +describe('the Gemini generateContent pipeline', () => { + it('assembles, and asks its caller only for what the descending stages need', () => { + expect([...geminiGenerateContentServePipeline().entryNeeds].sort()).toEqual([ + 'ingress.chat.geminiGenerateContent.wantsStream', + 'ingress.chat.sourceProtocol', + 'ingress.http.headers', + 'request.chat.geminiGenerateContent', + 'serve.model', + ]); + }); + + it('translates the turn out, dials the OpenAI Chat Completions wire, and streams Gemini generateContent frames back', async () => { + let sent: Record | undefined; + resolves([candidate('up_a', async (_model, body) => { + sent = body as Record; + return streamed(turn()); + })]); + + const { facts, drain } = await serve(entryFacts()); + + // The answer comes back before the drain runs, which is what lets a streaming family + // hand its stream on: the frames are still there to read. + expect((await collect(facts['response.chat.geminiGenerateContent.rendered'])) + .map(frame => withoutCarrier(JSON.parse(frame.data) as GeminiGenerateContentResult))).toEqual([ + { candidates: [{ index: 0, content: { role: 'model', parts: [{ text: 'he' }] } }] }, + { + candidates: [{ index: 0, content: { role: 'model', parts: [{ text: 'llo' }] }, finishReason: 'STOP' }], + usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 7, totalTokenCount: 12 }, + }, + ]); + // What went out is the translated turn, addressed to the candidate's own id — which the + // ending then strips, because the provider re-stamps what it resolved upstream. + expect(sent).toMatchObject({ stream: true, messages: [{ role: 'user', content: 'hello' }] }); + expect(sent).not.toHaveProperty('model'); + await drain(); + }); + + // Gemini generateContent carries its state in `thoughtSignature` parts, which only mean anything to the + // upstream that issued them — so the turn that is translated is the one affinity rewrote + // for this candidate, and the candidate that answered is marked for the next turn. That + // turn enters the record below the fork because everything between there and the dial + // rewrites the request as a fact, and one stage owns the reading: a wire that asked the + // resolver again would translate a turn no stage in the chain had touched. + it('translates what the record holds, and marks the candidate that answered', async () => { + let sent: Record | undefined; + affinityPayload = { contents: [{ role: 'user', parts: [{ text: 'rewritten' }] }] }; + const dialled = candidate('up_a', async (_model, body) => { + sent = body as Record; + return streamed(turn()); + }); + resolves([dialled]); + + const { facts, drain } = await serve(entryFacts()); + await collect(facts['response.chat.geminiGenerateContent.rendered']); + + // The handoff consumed `request.chat.geminiGenerateContent` and provided the wire's own key, so what the + // record still carries below the fork is the translated turn and not the Gemini generateContent one — the + // whole of what a translation is, said in the fact space. + expect((facts as Record)['request.chat.geminiGenerateContent']).toBeUndefined(); + expect((facts as Record)['request.chat.openaiChatCompletions']) + .toMatchObject({ messages: [{ role: 'user', content: 'rewritten' }] }); + expect(asked).toBe(1); + expect(sent).toMatchObject({ messages: [{ role: 'user', content: 'rewritten' }] }); + expect(selected).toEqual([dialled]); + await drain(); + }); + + it('folds the frames into one Gemini generateContent result for a client that did not ask to stream', async () => { + resolves([candidate('up_a', async () => streamed(turn()))]); + + const { facts, drain } = await serve(entryFacts({ 'ingress.chat.geminiGenerateContent.wantsStream': false })); + + // The two chunks stay two Parts: the carrier signs the Part it rides on, and merging a + // signed Part into its neighbour would spread that signature over text it does not cover. + expect(withoutCarrier(facts['response.chat.geminiGenerateContent.rendered'] as GeminiGenerateContentResult)).toEqual({ + candidates: [{ index: 0, content: { role: 'model', parts: [{ text: 'he' }, { text: 'llo' }] }, finishReason: 'STOP' }], + usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 7, totalTokenCount: 12 }, + }); + expect(facts['response.http.status']).toBe(200); + await drain(); + }); + + // The other half of affinity: the resolver reads client-carried state on the way down, and + // this is what the client is given to carry back. A turn that handed back nothing would + // leave the follow-up quoting it with no way to name the upstream that issued it. + it('hands the turn-s own state back on the Part that answered', async () => { + resolves([candidate('up_a', async () => streamed(turn()))]); + const gateway = mockChatGatewayCtx({ wantsStream: true }); + + const { facts, drain } = await serveWith(gateway, entryFacts()); + const frames = await collect(facts['response.chat.geminiGenerateContent.rendered']); + const carried = affinityCarriers(frames.map(frame => JSON.parse(frame.data) as GeminiGenerateContentResult)); + + expect(carried).toHaveLength(1); + // It names the upstream that answered, sealed under this run's own secret — which is what + // lets the next turn be pinned to it. + expect(await gateway.affinity.codec.unwrap(carried[0]!, 'gemini-generate-content.part.thoughtSignature')).toMatchObject({ + kind: 'owned', + affinity: { upstreamId: 'up_a', modelId: 'gemini-2.5-pro' }, + }); + await drain(); + }); + + it('fails a refusal over to the next candidate, and answers in the last upstream-s own words', async () => { + const tried: string[] = []; + resolves([ + candidate('up_a', async () => { + tried.push('up_a'); + return { ok: false, response: Response.json({ error: { message: 'slow down' } }, { status: 429 }), modelKey: 'k' }; + }), + candidate('up_b', async () => { + tried.push('up_b'); + return { ok: false, response: Response.json({ error: { message: 'no' } }, { status: 400 }), modelKey: 'k' }; + }), + ]); + + const { facts, drain } = await serve(entryFacts()); + + expect(tried).toEqual(['up_a', 'up_b']); + // Every candidate failed, so the last failure is the base: the client sees the status the + // last upstream actually returned, and its words. The shape around them is this protocol's, + // because every Gemini generateContent wire is a translation and the object that came back was written by + // whatever protocol the candidate was dialled on. + expect(facts['response.http.status']).toBe(400); + const rendered = facts['response.chat.geminiGenerateContent.rendered'] as { error: { code: number; message: string; status: string } }; + expect(rendered.error.code).toBe(400); + expect(rendered.error.status).toBe('INVALID_ARGUMENT'); + expect(rendered.error.message).toContain('no'); + // An upstream that was called and reported nothing, which is a different statement from + // reporting zero. + expect(facts['response.usage.billable']).toEqual([ + { identity: { model: 'gemini-2.5-pro', upstream: 'up_a', modelKey: 'k', pricing: null }, quantities: {} }, + { identity: { model: 'gemini-2.5-pro', upstream: 'up_b', modelKey: 'k', pricing: null }, quantities: {} }, + ]); + await drain(); + }); + + it('turns a dial that threw into a failure value rather than a thrown run', async () => { + resolves([candidate('up_a', async () => ({ type: 'transportFailure', error: new Error('socket hang up') }))]); + + const { facts, drain } = await serve(entryFacts()); + + expect(facts['response.http.status']).toBe(502); + expect(facts['response.chat.geminiGenerateContent.rendered']).toEqual({ error: { code: 502, message: 'socket hang up', status: 'UNAVAILABLE' } }); + // A dial that never completed reached no upstream, so nothing was billed and there are no + // headers to carry. + expect(facts['response.usage.billable']).toEqual([]); + expect(facts['response.http.headers']).toEqual([]); + await drain(); + }); + + it('forwards the upstream headers a client may see and drops the ones it may not', async () => { + resolves([candidate('up_a', async () => streamed(turn(), new Headers({ + 'content-type': 'text/event-stream', + 'transfer-encoding': 'chunked', + 'x-request-id': 'req_1', + })))]); + + const { facts, drain } = await serve(entryFacts()); + + expect(facts['response.http.headers']).toEqual([['x-request-id', 'req_1']]); + await collect(facts['response.chat.geminiGenerateContent.rendered']); + await drain(); + }); + + // The wire below always streams, whatever the client asked for, so this family's numbers + // always arrive with the last chunk — which is after the run has answered. Settling in the + // settlement stage registers that promise once and writes the measured result at completion. + it('defers settlement to the promise it hands up, metered on the dialect the upstream spoke', async () => { + resolves([candidate('up_a', async () => streamed(turn()))]); + + const { facts, drain } = await serve(entryFacts()); + await collect(facts['response.chat.geminiGenerateContent.rendered']); + await drain(); + + expect(recorded.usage).toHaveLength(1); + const usage = facts['response.chat.geminiGenerateContent.streamedUsage']; + expect(usage).not.toBeNull(); + const outcome = await usage!; + const billable = outcome.billable; + expect(billable[0]).toMatchObject({ + identity: { model: 'gemini-2.5-pro', upstream: 'up_a', modelKey: 'gemini-2.5-pro-key', pricing: null }, + quantities: { input_tokens: '5', output_tokens: '7' }, + }); + // A rate can depend on the tier and on how much input there was; both are selector + // coordinates, so they travel as pricing facts rather than as quantities. + expect(billable[0]!.pricingFacts).toMatchObject({ inputTokens: 5 }); + }); + + // Time to first token is measured where the token is, and on the dialect the upstream + // actually spoke rather than the one the client did. A run that never stamps it is recorded + // as having produced nothing to time. + it('stamps the first frame that carried a generated token', async () => { + resolves([candidate('up_a', async () => streamed(turn()))]); + const gateway = mockChatGatewayCtx({ wantsStream: true }); + + const { facts, drain } = await serveWith(gateway, entryFacts()); + expect(gateway.attempt.timing.firstOutputTokenAt).toBeNull(); + await collect(facts['response.chat.geminiGenerateContent.rendered']); + + expect(gateway.attempt.timing.firstOutputTokenAt).toBeTypeOf('number'); + await drain(); + }); + + // A wire that closes cleanly has not thereby finished the turn: with no finish reason on + // any choice, nothing that comes out of the translation says the answer is over. Serving + // those frames would report a truncated answer as a whole one. + it('fails a stream that ended without its terminal frame', async () => { + resolves([candidate('up_a', async () => streamed(unfinishedTurn()))]); + + const { facts, drain } = await serve(entryFacts()); + + const frames = await collect(facts['response.chat.geminiGenerateContent.rendered']); + expect(JSON.parse(frames.at(-1)!.data)).toMatchObject({ error: { code: 500, status: 'INTERNAL', message: GEMINI_GENERATE_CONTENT_MISSING_TERMINAL_MESSAGE } }); + await drain(); + }); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/affinity/egress_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/affinity/egress_test.ts index 408f4fecd1..029231aa79 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/affinity/egress_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/affinity/egress_test.ts @@ -531,4 +531,50 @@ describe('Gemini generateContent affinity egress', () => { }); await expect(iterator.next()).rejects.toThrow('upstream failed'); }); + + // The property the whole shape of this transducer rests on: a frame it is handed belongs to + // the layer that produced it. The events used to be deep-cloned on the way in precisely + // because the rules then wrote into them; nothing writes now, so a frozen source has to + // survive the pass unchanged — and the relocations are exercised here rather than asserted + // about, because a rewrite that reached for `delete` or an index assignment throws in strict + // mode instead of quietly corrupting the caller's event. + test('rewrites a deeply frozen source without writing into it', async () => { + const freeze = (value: T): T => { + if (value === null || typeof value !== 'object') return value; + for (const entry of Object.values(value as Record)) freeze(entry); + return Object.freeze(value); + }; + const frozen = (event: GeminiGenerateContentStreamEvent): ProtocolFrame => eventFrame(freeze(event)); + const source: ProtocolFrame[] = [ + // A signature-only leader, relocated forward onto the next event's content. + frozen({ + candidates: [{ index: 0, content: { role: 'model', parts: [{ thoughtSignature: 'lead' }] } }], + }), + // Content that continues into the third event, whose signature is pulled back onto it. + frozen({ + candidates: [{ index: 0, content: { role: 'model', parts: [{ text: 'hello' }] } }], + usageMetadata: { totalTokenCount: 1 }, + }), + frozen({ + candidates: [{ + index: 0, + content: { role: 'model', parts: [{ text: ' world', thoughtSignature: 'tail' }] }, + finishReason: 'STOP', + }], + }), + ]; + const before = JSON.stringify(source); + + const output: ProtocolFrame[] = []; + for await (const frame of wrapGeminiGenerateContentAffinityEgress(frames(source), { codec: immediateCodec, affinity })) { + output.push(frame); + } + + // The source is what it was, and the pass still did its work. + expect(JSON.stringify(source)).toBe(before); + const signatures = output.flatMap(frame => + (frame.type === 'event' && 'candidates' in frame.event ? frame.event.candidates ?? [] : []) + .flatMap(candidate => candidate.content.parts.flatMap(part => part.thoughtSignature ?? []))); + expect(signatures.some(signature => signature.startsWith('wrapped:'))).toBe(true); + }); }); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/attempt_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/attempt_test.ts deleted file mode 100644 index 5547742f6b..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/attempt_test.ts +++ /dev/null @@ -1,274 +0,0 @@ -import { test, vi } from 'vitest'; - -import { geminiGenerateContentAttempt } from '../../../../src/data-plane/chat/gemini-generate-content/attempt.ts'; -import { initRepo } from '../../../../src/repo/index.ts'; -import { InMemoryRepo } from '../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import { doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; -import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import type { OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { type AnthropicMessagesUpstreamCallOptions, type ModelCandidate, directFetcher, type ProviderCallResult, type ProviderOpenAIResponsesResult, type ProviderStreamResult, type OpenAIResponsesAction, type UpstreamCallOptions } from '@floway-dev/provider'; -import { assertEquals, stubProvider, stubInternalModel } from '@floway-dev/test-utils'; - -const API_KEY_ID = 'key_gemini_attempt_test'; - -const makeGatewayCtx = () => mockChatGatewayCtx({ apiKeyId: API_KEY_ID, wantsStream: true }); - -const makePayload = (overrides: Partial = {}): GeminiGenerateContentPayload => ({ - contents: [{ role: 'user', parts: [{ text: 'hello' }] }], - ...overrides, -}); - -const installRepo = (): InMemoryRepo => { - const repo = new InMemoryRepo(); - initRepo(repo); - return repo; -}; - -const makeProtocolFrames = async function* (events: readonly TEvent[]): AsyncGenerator> { - for (const event of events) yield eventFrame(event); - yield doneFrame(); -}; - -const makeAnthropicMessagesEvents = (): readonly AnthropicMessagesStreamEvent[] => [ - { - type: 'message_start', - message: { - id: 'msg_1', type: 'message', role: 'assistant', content: [], - model: 'test-model', stop_reason: null, stop_sequence: null, - usage: { input_tokens: 4, output_tokens: 0 }, - }, - }, - { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, - { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'hi' } }, - { type: 'content_block_stop', index: 0 }, - { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 1 } }, - { type: 'message_stop' }, -]; - -const makeOpenAIResponsesResultEvent = (id = 'resp_test'): OpenAIResponsesStreamEvent => { - const response: OpenAIResponsesResult = { - id, object: 'response', model: 'test-model', status: 'completed', - output: [{ - type: 'message', id: 'msg_resp', role: 'assistant', status: 'completed', - content: [{ type: 'output_text', text: 'hi from responses', annotations: [] }], - }], - output_text: 'hi from responses', error: null, incomplete_details: null, - }; - return { type: 'response.completed', sequence_number: 0, response }; -}; - -const makeOpenAIChatCompletionsEvents = (): readonly OpenAIChatCompletionsStreamEvent[] => [ - { - id: 'chatcmpl_1', object: 'chat.completion.chunk', created: 1, model: 'test-model', - choices: [{ index: 0, delta: { content: 'hi' }, finish_reason: null }], - }, - { - id: 'chatcmpl_1', object: 'chat.completion.chunk', created: 1, model: 'test-model', - choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], - }, -]; - -const makeCandidate = (overrides: { - upstream?: string; - endpoints?: ModelEndpoints; - callAnthropicMessages?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: AnthropicMessagesUpstreamCallOptions) => Promise>; - callOpenAIResponses?: (model: unknown, body: unknown, action: OpenAIResponsesAction, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise; - callOpenAIChatCompletions?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise>; - callAnthropicMessagesCountTokens?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: AnthropicMessagesUpstreamCallOptions) => Promise; -} = {}): ModelCandidate => { - const upstream = overrides.upstream ?? 'up_test'; - const provider = stubProvider({ - callAnthropicMessages: overrides.callAnthropicMessages, - callOpenAIResponses: overrides.callOpenAIResponses, - callOpenAIChatCompletions: overrides.callOpenAIChatCompletions, - callAnthropicMessagesCountTokens: overrides.callAnthropicMessagesCountTokens, - }); - return { - provider: { - upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: provider, - }, - model: stubInternalModel(overrides.endpoints ? { endpoints: overrides.endpoints } : {}, upstream), - fetcher: directFetcher, - }; -}; - -const collectEvents = async (events: AsyncIterable>): Promise => { - const out: TEvent[] = []; - for await (const frame of events) { - if (frame.type === 'event') out.push(frame.event); - } - return out; -}; - -test('generate translates through OpenAI Chat Completions when targetApi is openai-chat-completions', async () => { - installRepo(); - const callOpenAIChatCompletions = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeOpenAIChatCompletionsEvents()), modelKey: 'k', headers: new Headers(), - })); - const result = await geminiGenerateContentAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callOpenAIChatCompletions }), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callOpenAIChatCompletions.mock.calls.length, 1); -}); - -test('generate translates through Anthropic Messages when targetApi is messages', async () => { - installRepo(); - let callOptions: AnthropicMessagesUpstreamCallOptions | undefined; - const callAnthropicMessages = vi.fn(async (_model, _body, _signal, opts): Promise> => { - callOptions = opts; - return { ok: true, events: makeProtocolFrames(makeAnthropicMessagesEvents()), modelKey: 'k', headers: new Headers() }; - }); - const result = await geminiGenerateContentAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callAnthropicMessages, endpoints: { anthropicMessages: {} } }), - headers: new Headers({ 'anthropic-beta': 'must-not-cross-source-protocols' }), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callAnthropicMessages.mock.calls.length, 1); - assertEquals(callOptions?.anthropicBeta, []); - assertEquals(callOptions?.headers.has('anthropic-beta'), false); -}); - -test('generate translates through OpenAI Responses when targetApi is responses', async () => { - installRepo(); - const callOpenAIResponses = vi.fn(async (): Promise => ({ - action: 'generate', ok: true, events: makeProtocolFrames([makeOpenAIResponsesResultEvent()]), modelKey: 'k', headers: new Headers(), - })); - const result = await geminiGenerateContentAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callOpenAIResponses, endpoints: { openaiResponses: {} } }), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callOpenAIResponses.mock.calls.length, 1); -}); - -test('countTokens translates Gemini generateContent to Anthropic Messages count_tokens and reshapes to totalTokens envelope', async () => { - installRepo(); - let upstreamBody: Record | undefined; - let callOptions: AnthropicMessagesUpstreamCallOptions | undefined; - const callAnthropicMessagesCountTokens = vi.fn(async (_model, body, _signal, opts): Promise => { - upstreamBody = body as Record; - callOptions = opts; - return { - response: new Response(JSON.stringify({ input_tokens: 42 }), { - status: 200, - headers: new Headers({ 'content-type': 'application/json' }), - }), - modelKey: 'k', - }; - }); - const result = await geminiGenerateContentAttempt.countTokens({ - payload: makePayload({ systemInstruction: { parts: [{ text: 'system' }] } }), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callAnthropicMessagesCountTokens }), - headers: new Headers({ 'anthropic-beta': 'must-not-cross-source-protocols' }), - }); - - assertEquals(result.type, 'plain'); - if (result.type !== 'plain') throw new Error('unreachable'); - assertEquals(result.status, 200); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assertEquals(body, { totalTokens: 42 }); - assertEquals(callAnthropicMessagesCountTokens.mock.calls.length, 1); - assertEquals(callOptions?.anthropicBeta, []); - assertEquals(callOptions?.headers.has('anthropic-beta'), false); - // The Anthropic Messages count_tokens body should never carry the translation-time - // `stream: true` flag — that field belongs to the streaming path only. - if (upstreamBody === undefined) throw new Error('upstreamBody not captured'); - assertEquals('stream' in upstreamBody, false); -}); - -test('countTokens accepts the upstream total_tokens dialect and refuses unknown shapes with a 502', async () => { - installRepo(); - const totalTokensResp = await geminiGenerateContentAttempt.countTokens({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ - callAnthropicMessagesCountTokens: async () => ({ - response: new Response(JSON.stringify({ total_tokens: 19 }), { status: 200, headers: new Headers({ 'content-type': 'application/json' }) }), - modelKey: 'k', - }), - }), - headers: new Headers(), - }); - assertEquals(totalTokensResp.type, 'plain'); - if (totalTokensResp.type !== 'plain') throw new Error('unreachable'); - assertEquals(JSON.parse(new TextDecoder().decode(totalTokensResp.body)), { totalTokens: 19 }); - - const unexpectedResp = await geminiGenerateContentAttempt.countTokens({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ - callAnthropicMessagesCountTokens: async () => ({ - response: new Response(JSON.stringify({ unexpected: true }), { status: 200, headers: new Headers({ 'content-type': 'application/json' }) }), - modelKey: 'k', - }), - }), - headers: new Headers(), - }); - assertEquals(unexpectedResp.type, 'plain'); - if (unexpectedResp.type !== 'plain') throw new Error('unreachable'); - assertEquals(unexpectedResp.status, 502); - const body = JSON.parse(new TextDecoder().decode(unexpectedResp.body)); - assertEquals(body.error.code, 502); - assertEquals(body.error.status, 'UNAVAILABLE'); -}); - -test('countTokens refuses a non-anthropic-messages candidate', async () => { - installRepo(); - let thrown: unknown = null; - try { - await geminiGenerateContentAttempt.countTokens({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ endpoints: { openaiResponses: {} } }), - headers: new Headers(), - }); - } catch (error) { - thrown = error; - } - if (!(thrown instanceof Error)) throw new Error('expected an Error to be thrown'); - assertEquals(thrown.message.includes('chatTargetPicker.pick'), true); -}); - -test('generate propagates upstream response headers through the openai-chat-completions translation', async () => { - installRepo(); - const upstreamHeaders = new Headers({ - 'anthropic-ratelimit-unified-status': 'allowed', - 'x-request-id': 'req_gemini_xyz', - }); - const callOpenAIChatCompletions = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeOpenAIChatCompletionsEvents()), modelKey: 'k', headers: upstreamHeaders, - })); - const result = await geminiGenerateContentAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callOpenAIChatCompletions }), - headers: new Headers(), - }); - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - assertEquals(result.headers?.get('anthropic-ratelimit-unified-status'), 'allowed'); - assertEquals(result.headers?.get('x-request-id'), 'req_gemini_xyz'); - await collectEvents(result.events); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts index 887168f737..62194793b7 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts @@ -5,11 +5,12 @@ import type { AuthVars } from '../../../../src/middleware/auth.ts'; import { initRepo } from '../../../../src/repo/index.ts'; import type { ApiKey, User } from '../../../../src/repo/types.ts'; import { InMemoryRepo } from '../../../repo/memory.ts'; +import { stubChatProviderPipelines } from '../../../test-utils/chat-provider-pipelines.ts'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import { type ModelCandidate, directFetcher, type ProviderCallResult, type ProviderStreamResult, type UpstreamCallOptions } from '@floway-dev/provider'; -import { assert, assertEquals, stubProvider, stubInternalModel } from '@floway-dev/test-utils'; +import { type ModelCandidate, directFetcher, type UpstreamCallOptions } from '@floway-dev/provider'; +import { assert, assertEquals, stubProvider, stubInternalModel, type ProviderCallResult, type ProviderStreamResult } from '@floway-dev/test-utils'; const candidatesQueue: { readonly candidates: readonly ModelCandidate[]; readonly sawModel: boolean; readonly failedUpstreams: readonly string[] }[] = []; vi.mock('../../../../src/data-plane/providers/resolution.ts', async importOriginal => { @@ -102,7 +103,7 @@ const makeCandidate = (overrides: { const targetApi = overrides.targetApi ?? 'openaiChatCompletions'; // When the test fixes `endpoints` directly, use it verbatim — that lets a // test pin a wrong-endpoint shape the picker rejects. Otherwise synthesize - // a single-endpoint map from `targetApi` so the gemini serve layer's + // a single-endpoint map from `targetApi` so the Gemini generateContent serve layer's // picker (openai-chat-completions first, then messages, then responses) lands on // the requested wire. const endpoints = overrides.endpoints ?? (targetApi === 'openaiChatCompletions' @@ -110,15 +111,16 @@ const makeCandidate = (overrides: { : targetApi === 'anthropicMessages' ? { anthropicMessages: {} } : { openaiResponses: {} }); - const provider = stubProvider({ + const calls = { callOpenAIChatCompletions: overrides.callOpenAIChatCompletions, callAnthropicMessages: overrides.callAnthropicMessages, callAnthropicMessagesCountTokens: overrides.callAnthropicMessagesCountTokens, - }); + }; + const provider = stubProvider(); return { provider: { upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: provider, + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: stubChatProviderPipelines(calls), instance: provider, }, model: stubInternalModel({ endpoints }, upstream), fetcher: directFetcher, diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/interceptors/strip-safety-settings_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/interceptors/strip-safety-settings_test.ts deleted file mode 100644 index 5b79bc5473..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/interceptors/strip-safety-settings_test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { test } from 'vitest'; - -import { stripSafetySettings } from '../../../../../src/data-plane/chat/gemini-generate-content/interceptors/strip-safety-settings.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiGenerateContentPayload, GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; -import { type ExecuteResult, eventResult, type GeminiGenerateContentInvocation } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = (): Promise>> => - Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); - -const invocation = (payload: GeminiGenerateContentPayload): GeminiGenerateContentInvocation => ({ - payload, - candidate: stubModelCandidate(), - targetApi: 'anthropicMessages', - headers: new Headers(), -}); - -test('removes safetySettings without inventing missing defaults and preserves siblings', async () => { - const input = invocation({ - cachedContent: 'cachedContents/example', - safetySettings: [ - { category: 'HARM_CATEGORY_DANGEROUS_CONTENT', threshold: 'BLOCK_ONLY_HIGH' }, - ], - }); - - await stripSafetySettings(input, stubCtx, okEvents); - - assertEquals(input.payload, { cachedContent: 'cachedContents/example' }); -}); - -test('is a no-op when safetySettings is absent', async () => { - const input = invocation({ - contents: [{ role: 'user', parts: [{ text: 'hi' }] }], - }); - - await stripSafetySettings(input, stubCtx, okEvents); - - assertEquals(input.payload, { contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/interceptors/strip-unsupported-part-fields_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/interceptors/strip-unsupported-part-fields_test.ts deleted file mode 100644 index 721bfd9367..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/interceptors/strip-unsupported-part-fields_test.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { test } from 'vitest'; - -import { stripUnsupportedPartFields } from '../../../../../src/data-plane/chat/gemini-generate-content/interceptors/strip-unsupported-part-fields.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiGenerateContentPayload, GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; -import { type ExecuteResult, eventResult, type GeminiGenerateContentInvocation } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = (): Promise>> => - Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); - -const invocation = (payload: GeminiGenerateContentPayload): GeminiGenerateContentInvocation => ({ - payload, - candidate: stubModelCandidate(), - targetApi: 'anthropicMessages', - headers: new Headers(), -}); - -test('strips fileData, executableCode, and codeExecutionResult while preserving supported part fields', async () => { - const input = invocation({ - contents: [ - { - role: 'user', - parts: [ - { - text: 'hello', - thought: true, - thoughtSignature: 'thought-signature', - inlineData: { mimeType: 'image/png', data: 'aW1hZ2U=' }, - functionCall: { id: 'call-1', name: 'lookup', args: { query: 'docs' } }, - functionResponse: { id: 'call-1', name: 'lookup', response: { ok: true } }, - fileData: { mimeType: 'text/plain', fileUri: 'gs://bucket/file.txt' }, - executableCode: { language: 'python', code: 'print(1)' }, - codeExecutionResult: { outcome: 'OUTCOME_OK', output: '1' }, - }, - ], - }, - ], - systemInstruction: { - parts: [ - { - text: 'system', - fileData: { mimeType: 'text/plain', fileUri: 'gs://bucket/system.txt' }, - }, - ], - }, - }); - - await stripUnsupportedPartFields(input, stubCtx, okEvents); - - assertEquals(input.payload, { - contents: [ - { - role: 'user', - parts: [ - { - text: 'hello', - thought: true, - thoughtSignature: 'thought-signature', - inlineData: { mimeType: 'image/png', data: 'aW1hZ2U=' }, - functionCall: { id: 'call-1', name: 'lookup', args: { query: 'docs' } }, - functionResponse: { id: 'call-1', name: 'lookup', response: { ok: true } }, - }, - ], - }, - ], - systemInstruction: { - parts: [{ text: 'system' }], - }, - }); -}); - -test('removes parts that only contain unsupported file or code fields', async () => { - const input = invocation({ - contents: [ - { - role: 'user', - parts: [ - { fileData: { mimeType: 'text/plain', fileUri: 'gs://bucket/file.txt' } }, - { text: 'keep me' }, - { - executableCode: { language: 'python', code: 'print(1)' }, - codeExecutionResult: { outcome: 'OUTCOME_OK', output: '1' }, - }, - ], - }, - ], - systemInstruction: { - parts: [ - { fileData: { mimeType: 'text/plain', fileUri: 'gs://bucket/system.txt' } }, - { text: 'system' }, - ], - }, - }); - - await stripUnsupportedPartFields(input, stubCtx, okEvents); - - assertEquals(input.payload, { - contents: [ - { - role: 'user', - parts: [{ text: 'keep me' }], - }, - ], - systemInstruction: { - parts: [{ text: 'system' }], - }, - }); -}); - -test('is a no-op when no unsupported part fields are present', async () => { - const input = invocation({ - contents: [ - { role: 'user', parts: [{ text: 'hello' }] }, - ], - }); - - await stripUnsupportedPartFields(input, stubCtx, okEvents); - - assertEquals(input.payload, { - contents: [ - { role: 'user', parts: [{ text: 'hello' }] }, - ], - }); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/interceptors/strip-unsupported-tools_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/interceptors/strip-unsupported-tools_test.ts deleted file mode 100644 index ca60a5d0b1..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/interceptors/strip-unsupported-tools_test.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { test } from 'vitest'; - -import { stripUnsupportedTools } from '../../../../../src/data-plane/chat/gemini-generate-content/interceptors/strip-unsupported-tools.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiGenerateContentPayload, GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; -import { type ExecuteResult, eventResult, type GeminiGenerateContentInvocation } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = (): Promise>> => - Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); - -const invocation = (payload: GeminiGenerateContentPayload): GeminiGenerateContentInvocation => ({ - payload, - candidate: stubModelCandidate(), - targetApi: 'anthropicMessages', - headers: new Headers(), -}); - -test('strips non-functionDeclarations capabilities and drops groups that become empty', async () => { - const input = invocation({ - tools: [ - { - functionDeclarations: [ - { - name: 'lookup', - description: 'Look up a value', - parameters: { type: 'object', properties: { query: { type: 'string' } } }, - }, - ], - googleSearch: {}, - googleSearchRetrieval: {}, - codeExecution: {}, - computerUse: {}, - urlContext: {}, - fileSearch: {}, - mcpServers: [{ name: 'server' }], - googleMaps: {}, - }, - { googleSearch: {} }, - { codeExecution: {} }, - ], - }); - - await stripUnsupportedTools(input, stubCtx, okEvents); - - assertEquals(input.payload, { - tools: [ - { - functionDeclarations: [ - { - name: 'lookup', - description: 'Look up a value', - parameters: { type: 'object', properties: { query: { type: 'string' } } }, - }, - ], - }, - ], - }); -}); - -test('removes the tools field entirely when every group becomes empty', async () => { - const input = invocation({ - tools: [ - { googleSearch: {} }, - { codeExecution: {} }, - ], - }); - - await stripUnsupportedTools(input, stubCtx, okEvents); - - assertEquals(input.payload, {}); -}); - -test('is a no-op when tools is absent', async () => { - const input = invocation({ - contents: [{ role: 'user', parts: [{ text: 'hi' }] }], - }); - - await stripUnsupportedTools(input, stubCtx, okEvents); - - assertEquals(input.payload, { - contents: [{ role: 'user', parts: [{ text: 'hi' }] }], - }); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/interceptors/suppress-thought-parts_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/interceptors/suppress-thought-parts_test.ts deleted file mode 100644 index 4944cbc21d..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/interceptors/suppress-thought-parts_test.ts +++ /dev/null @@ -1,151 +0,0 @@ -import { test } from 'vitest'; - -import { suppressThoughtParts } from '../../../../../src/data-plane/chat/gemini-generate-content/interceptors/suppress-thought-parts.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiGenerateContentPayload, GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; -import { type ExecuteResult, eventResult, type GeminiGenerateContentInvocation } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const invocation = (payload: GeminiGenerateContentPayload): GeminiGenerateContentInvocation => ({ - payload, - candidate: stubModelCandidate(), - targetApi: 'anthropicMessages', - headers: new Headers(), -}); - -const streamingResult = (events: GeminiGenerateContentStreamEvent[]) => (): Promise>> => - Promise.resolve(eventResult( - (async function* (): AsyncGenerator> { - for (const event of events) yield eventFrame(event); - })(), - testTelemetryModelIdentity, - )); - -const collect = async (events: AsyncIterable>): Promise => { - const out: GeminiGenerateContentStreamEvent[] = []; - for await (const frame of events) if (frame.type === 'event') out.push(frame.event); - return out; -}; - -test('drops thought parts and suppresses candidates that become empty without a finishReason', async () => { - const upstream: GeminiGenerateContentStreamEvent[] = [ - { - candidates: [ - { - index: 0, - content: { - role: 'model', - parts: [ - { thought: true, text: 'reasoning...' }, - { text: 'final answer' }, - ], - }, - }, - ], - }, - { - candidates: [ - { - index: 0, - content: { role: 'model', parts: [{ thought: true, text: 'more reasoning' }] }, - }, - ], - }, - ]; - - const res = await suppressThoughtParts(invocation({}), stubCtx, streamingResult(upstream)); - if (res.type !== 'events') throw new Error('expected events'); - - assertEquals(await collect(res.events), [ - { - candidates: [ - { - index: 0, - content: { role: 'model', parts: [{ text: 'final answer' }] }, - }, - ], - }, - ]); -}); - -test('preserves an empty candidate when it carries a finishReason', async () => { - const upstream: GeminiGenerateContentStreamEvent[] = [ - { - candidates: [ - { - index: 0, - content: { role: 'model', parts: [{ thought: true, text: 'reasoning...' }] }, - finishReason: 'STOP', - }, - ], - }, - ]; - - const res = await suppressThoughtParts(invocation({}), stubCtx, streamingResult(upstream)); - if (res.type !== 'events') throw new Error('expected events'); - - assertEquals(await collect(res.events), [ - { - candidates: [ - { - index: 0, - content: { role: 'model', parts: [] }, - finishReason: 'STOP', - }, - ], - }, - ]); -}); - -test('passes thought parts through when includeThoughts is opted in', async () => { - const upstream: GeminiGenerateContentStreamEvent[] = [ - { - candidates: [ - { - index: 0, - content: { - role: 'model', - parts: [ - { thought: true, text: 'reasoning...' }, - { text: 'final answer' }, - ], - }, - }, - ], - }, - ]; - - const res = await suppressThoughtParts( - invocation({ generationConfig: { thinkingConfig: { includeThoughts: true } } }), - stubCtx, - streamingResult(upstream), - ); - if (res.type !== 'events') throw new Error('expected events'); - - assertEquals(await collect(res.events), upstream); -}); - -test('passes error frames through unchanged', async () => { - const upstream: GeminiGenerateContentStreamEvent[] = [ - { error: { code: 500, message: 'oops', status: 'INTERNAL' } }, - ]; - - const res = await suppressThoughtParts(invocation({}), stubCtx, streamingResult(upstream)); - if (res.type !== 'events') throw new Error('expected events'); - - assertEquals(await collect(res.events), upstream); -}); - -test('preserves usageMetadata-only frames that have no candidates', async () => { - const upstream: GeminiGenerateContentStreamEvent[] = [ - { usageMetadata: { promptTokenCount: 3, candidatesTokenCount: 5, totalTokenCount: 8 } }, - ]; - - const res = await suppressThoughtParts(invocation({}), stubCtx, streamingResult(upstream)); - if (res.type !== 'events') throw new Error('expected events'); - - assertEquals(await collect(res.events), upstream); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/respond_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/respond_test.ts deleted file mode 100644 index f368a9c2e7..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/respond_test.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { Hono } from 'hono'; -import { test } from 'vitest'; - -import { respondGeminiGenerateContent } from '../../../../src/data-plane/chat/gemini-generate-content/respond.ts'; -import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { eventFrame } from '@floway-dev/protocols/common'; -import type { GeminiGenerateContentErrorResponse } from '@floway-dev/protocols/gemini-generate-content'; -import type { ExecuteResult, InternalDebugError } from '@floway-dev/provider'; -import { assertEquals, assertExists } from '@floway-dev/test-utils'; - -const encoder = new TextEncoder(); - -const testTelemetryModelIdentity = { - model: 'test-model', - upstream: 'test-upstream', - modelKey: 'test-model-key', - pricing: null, -}; - -const ctx = () => mockChatGatewayCtx(); - -const requestGeminiGenerateContentResponse = async (result: ExecuteResult>): Promise => { - const app = new Hono(); - app.get('/', async c => await respondGeminiGenerateContent(c, result, false, ctx())); - return await app.request('/'); -}; - -test('respondGeminiGenerateContent preserves non-stream Gemini generateContent error event HTTP code', async () => { - const errorEvent: GeminiGenerateContentErrorResponse = { - error: { - code: 504, - status: 'DEADLINE_EXCEEDED', - message: 'timeout', - }, - }; - - const response = await requestGeminiGenerateContentResponse({ - type: 'events', - events: (async function* () { - yield eventFrame(errorEvent); - })(), - modelIdentity: testTelemetryModelIdentity, - }); - - assertEquals(response.status, 504); - assertEquals(await response.json(), errorEvent); -}); - -test('respondGeminiGenerateContent preserves upstream Google RPC Status body', async () => { - const upstreamBody: GeminiGenerateContentErrorResponse = { - error: { - code: 412, - status: 'FAILED_PRECONDITION', - message: 'account is not ready', - }, - }; - - const response = await requestGeminiGenerateContentResponse({ - type: 'api-error', - source: 'upstream', - status: 400, - headers: new Headers({ 'content-type': 'application/json' }), - body: encoder.encode(JSON.stringify(upstreamBody)), - }); - - assertEquals(response.status, 412); - assertEquals(await response.json(), upstreamBody); -}); - -test('respondGeminiGenerateContent internal errors include debug fields in Google RPC Status', async () => { - const error: InternalDebugError = { - type: 'internal_error', - name: 'TypeError', - message: 'boom', - stack: 'TypeError: boom\n at test', - cause: { upstream: 'bad shape' }, - target_api: 'responses', - }; - - const response = await requestGeminiGenerateContentResponse({ - type: 'internal-error', - status: 502, - error, - }); - const body = await response.json(); - - assertEquals(response.status, 502); - assertEquals(body.error.code, 502); - assertEquals(body.error.status, 'UNAVAILABLE'); - assertEquals(body.error.message, 'boom'); - assertEquals(body.error.stack, error.stack); - assertEquals(body.error.target_api, 'responses'); - assertExists(body.error.cause); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/serve_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/serve_test.ts deleted file mode 100644 index b428789a8b..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/serve_test.ts +++ /dev/null @@ -1,479 +0,0 @@ -import { afterEach, test, vi } from 'vitest'; - -import { initRepo } from '../../../../src/repo/index.ts'; -import { InMemoryRepo } from '../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import { type AliasRules, doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; -import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import type { OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { type ModelCandidate, directFetcher, type ProviderCallResult, type ProviderOpenAIResponsesResult, type ProviderStreamResult, type OpenAIResponsesAction, type UpstreamCallOptions } from '@floway-dev/provider'; -import { assert, assertEquals, stubProvider, stubInternalModel } from '@floway-dev/test-utils'; - -// Mock the resolver seam so each test hands the serve exactly the model -// candidates it wants, optionally with an alias-rules overlay attached. -interface QueuedResolution { - readonly candidates: readonly ModelCandidate[]; - readonly sawModel: boolean; - readonly failedUpstreams: readonly string[]; -} -const resolutionsQueue: QueuedResolution[] = []; -const lastResolveCall: { model?: string } = {}; -vi.mock('../../../../src/data-plane/providers/resolution.ts', async importOriginal => { - const original = await importOriginal(); - return { - ...original, - enumerateModelCandidates: vi.fn(async ({ model }: { model: string }) => { - lastResolveCall.model = model; - const next = resolutionsQueue.shift(); - if (next === undefined) throw new Error('serve_test: no resolution enqueued'); - return next; - }), - }; -}); - -const { geminiGenerateContentServe } = await import('../../../../src/data-plane/chat/gemini-generate-content/serve.ts'); - -const API_KEY_ID = 'key_gemini_serve_test'; - -const queueResolution = ( - candidates: readonly ModelCandidate[], - extra: { sawModel?: boolean; aliasRules?: AliasRules } = {}, -): void => { - const rules = extra.aliasRules; - resolutionsQueue.push({ - candidates: rules !== undefined ? candidates.map(c => ({ ...c, rules })) : candidates, - sawModel: extra.sawModel ?? candidates.length > 0, - failedUpstreams: [], - }); -}; - -afterEach(() => { resolutionsQueue.length = 0; }); - -const installRepo = (): InMemoryRepo => { - const repo = new InMemoryRepo(); - initRepo(repo); - return repo; -}; - -const makeGatewayCtx = () => mockChatGatewayCtx({ apiKeyId: API_KEY_ID, wantsStream: true }); - -const makePayload = (overrides: Partial = {}): GeminiGenerateContentPayload => ({ - contents: [{ role: 'user', parts: [{ text: 'hello' }] }], - ...overrides, -}); - -const makeProtocolFrames = async function* (events: readonly TEvent[]): AsyncGenerator> { - for (const event of events) yield eventFrame(event); - yield doneFrame(); -}; - -const makeOpenAIChatCompletionsEvents = (text = 'hi'): readonly OpenAIChatCompletionsStreamEvent[] => [ - { - id: 'chatcmpl_1', object: 'chat.completion.chunk', created: 1, model: 'test-model', - choices: [{ index: 0, delta: { content: text }, finish_reason: null }], - }, - { - id: 'chatcmpl_1', object: 'chat.completion.chunk', created: 1, model: 'test-model', - choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], - }, -]; - -const makeAnthropicMessagesEvents = (id = 'msg_1'): readonly AnthropicMessagesStreamEvent[] => [ - { - type: 'message_start', - message: { - id, type: 'message', role: 'assistant', content: [], - model: 'test-model', stop_reason: null, stop_sequence: null, - usage: { input_tokens: 4, output_tokens: 0 }, - }, - }, - { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, - { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'hi' } }, - { type: 'content_block_stop', index: 0 }, - { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 1 } }, - { type: 'message_stop' }, -]; - -const makeOpenAIResponsesResultEvent = (id = 'resp_test'): OpenAIResponsesStreamEvent => { - const response: OpenAIResponsesResult = { - id, object: 'response', model: 'test-model', status: 'completed', - output: [{ - type: 'message', id: 'msg_resp', role: 'assistant', status: 'completed', - content: [{ type: 'output_text', text: 'hi from responses', annotations: [] }], - }], - output_text: 'hi from responses', error: null, incomplete_details: null, - }; - return { type: 'response.completed', sequence_number: 0, response }; -}; - -const makeCandidate = (overrides: { - upstream?: string; - targetApi?: 'openaiChatCompletions' | 'anthropicMessages' | 'openaiResponses'; - endpoints?: ModelEndpoints; - callOpenAIChatCompletions?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise>; - callAnthropicMessages?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise>; - callOpenAIResponses?: (model: unknown, body: unknown, action: OpenAIResponsesAction, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise; - callAnthropicMessagesCountTokens?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise; -} = {}): ModelCandidate => { - const upstream = overrides.upstream ?? 'up_test'; - const targetApi = overrides.targetApi ?? 'openaiChatCompletions'; - // The gemini serve layer picks the target from model.endpoints (openai-chat-completions - // first, then messages, then responses). Narrow endpoints to the target this - // test wants so the gemini serve layer picks the expected target via - // geminiGenerateContentGenerateTarget.canServe. An explicit `endpoints` override beats the - // default `targetApi`-derived map, for tests that need a candidate whose - // endpoints satisfy none of the target preferences. - const endpoints = overrides.endpoints ?? (targetApi === 'openaiChatCompletions' - ? { openaiChatCompletions: {} } - : targetApi === 'anthropicMessages' - ? { anthropicMessages: {} } - : { openaiResponses: {} }); - const provider = stubProvider({ - callOpenAIChatCompletions: overrides.callOpenAIChatCompletions, - callAnthropicMessages: overrides.callAnthropicMessages, - callOpenAIResponses: overrides.callOpenAIResponses, - callAnthropicMessagesCountTokens: overrides.callAnthropicMessagesCountTokens, - }); - return { - provider: { - upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: provider, - }, - model: stubInternalModel({ endpoints }, upstream), - fetcher: directFetcher, - }; -}; - -const collectEvents = async (events: AsyncIterable>): Promise => { - const out: TEvent[] = []; - for await (const frame of events) { - if (frame.type === 'event') out.push(frame.event); - } - return out; -}; - -const expectType = (r: T, k: K): Extract => { - assertEquals(r.type, k); - return r as Extract; -}; - -test('generate translates through native OpenAI Chat Completions target end to end', async () => { - installRepo(); - const callOpenAIChatCompletions = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeOpenAIChatCompletionsEvents()), modelKey: 'k', headers: new Headers(), - })); - queueResolution([makeCandidate({ targetApi: 'openaiChatCompletions', callOpenAIChatCompletions })]); - - const result = await geminiGenerateContentServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - model: 'test-model', - headers: new Headers(), - }); - - const events = expectType(result, 'events').events; - await collectEvents(events); - assertEquals(callOpenAIChatCompletions.mock.calls.length, 1); -}); - -test('generate translates through Anthropic Messages when only that endpoint is exposed', async () => { - installRepo(); - const callAnthropicMessages = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeAnthropicMessagesEvents()), modelKey: 'k', headers: new Headers(), - })); - queueResolution([makeCandidate({ targetApi: 'anthropicMessages', callAnthropicMessages })]); - - const result = await geminiGenerateContentServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - model: 'test-model', - headers: new Headers(), - }); - - const events = expectType(result, 'events').events; - await collectEvents(events); - assertEquals(callAnthropicMessages.mock.calls.length, 1); -}); - -test('generate translates through OpenAI Responses when only that endpoint is exposed', async () => { - installRepo(); - const callOpenAIResponses = vi.fn(async (): Promise => ({ - action: 'generate', ok: true, events: makeProtocolFrames([makeOpenAIResponsesResultEvent()]), modelKey: 'k', headers: new Headers(), - })); - queueResolution([makeCandidate({ targetApi: 'openaiResponses', callOpenAIResponses })]); - - const result = await geminiGenerateContentServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - model: 'test-model', - headers: new Headers(), - }); - - const events = expectType(result, 'events').events; - await collectEvents(events); - assertEquals(callOpenAIResponses.mock.calls.length, 1); -}); - -test('generate falls through to the next candidate when the first yields an upstream error', async () => { - installRepo(); - const payload = makePayload({ - contents: [{ - role: 'user', - parts: [{ text: 'hello', thought: true, thoughtSignature: 'opaque-signature' }], - }], - }); - const firstError = new Response(JSON.stringify({ error: { message: 'nope' } }), { - status: 502, headers: new Headers({ 'content-type': 'application/json' }), - }); - const firstCall = vi.fn(async (): Promise> => ({ - ok: false, response: firstError, modelKey: 'first-key', - })); - const secondCall = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeOpenAIChatCompletionsEvents()), modelKey: 'second-key', headers: new Headers(), - })); - queueResolution([ - makeCandidate({ upstream: 'up_a', targetApi: 'openaiChatCompletions', callOpenAIChatCompletions: firstCall }), - makeCandidate({ upstream: 'up_b', targetApi: 'openaiChatCompletions', callOpenAIChatCompletions: secondCall }), - ]); - - const result = await geminiGenerateContentServe.generate({ - payload, - ctx: makeGatewayCtx(), - model: 'test-model', - headers: new Headers(), - }); - - // The narrowed candidate list exists exactly so a transient upstream - // failure (5xx/429/network) on one entry rolls over to the next. The - // second candidate's success is the request's final answer. - expectType(result, 'events'); - assertEquals(firstCall.mock.calls.length, 1); - assertEquals(secondCall.mock.calls.length, 1); - assertEquals(payload.contents?.[0]?.parts, [{ - text: 'hello', - thought: true, - thoughtSignature: 'opaque-signature', - }]); -}); - -// A mid-attempt throw (interceptor bug / translation error / provider-layer -// JS exception not represented as a ChatServeFailure) must attribute the perf -// error row to the throwing candidate, not the previous one that already -// failed cleanly with a 5xx. -test('mid-attempt throw stamps telemetry with the throwing candidate, not the previous one', async () => { - installRepo(); - const firstError = new Response(JSON.stringify({ error: { message: 'nope' } }), { - status: 502, headers: new Headers({ 'content-type': 'application/json' }), - }); - const firstCall = vi.fn(async (): Promise> => ({ - ok: false, response: firstError, modelKey: 'first-key', - })); - const secondCall = vi.fn(async (): Promise> => { - throw new Error('simulated provider-layer JS exception'); - }); - queueResolution([ - makeCandidate({ upstream: 'up_a', targetApi: 'openaiChatCompletions', callOpenAIChatCompletions: firstCall }), - makeCandidate({ upstream: 'up_b', targetApi: 'openaiChatCompletions', callOpenAIChatCompletions: secondCall }), - ]); - - const ctx = makeGatewayCtx(); - await geminiGenerateContentServe.generate({ - payload: makePayload(), - ctx, - model: 'test-model', - headers: new Headers(), - }).then( - () => { throw new Error('expected geminiGenerateContentServe.generate to throw'); }, - (error: unknown) => { - assertEquals((error as Error).message, 'simulated provider-layer JS exception'); - }, - ); - - assertEquals(firstCall.mock.calls.length, 1); - assertEquals(secondCall.mock.calls.length, 1); - assertEquals(ctx.attempt.telemetry?.upstream, 'up_b'); -}); - -test('generate is a routing no-op for a bare user-text request (degenerate path)', async () => { - installRepo(); - const callOpenAIChatCompletions = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeOpenAIChatCompletionsEvents()), modelKey: 'k', headers: new Headers(), - })); - queueResolution([ - makeCandidate({ upstream: 'up_a', targetApi: 'openaiChatCompletions', callOpenAIChatCompletions }), - makeCandidate({ upstream: 'up_b', targetApi: 'openaiChatCompletions', callOpenAIChatCompletions }), - ]); - - const result = await geminiGenerateContentServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - model: 'test-model', - headers: new Headers(), - }); - - const events = expectType(result, 'events').events; - await collectEvents(events); - assertEquals(callOpenAIChatCompletions.mock.calls.length, 1); -}); - -test('generate renders model-missing as a Google RPC 404 when no candidates are available', async () => { - installRepo(); - queueResolution([]); - - const result = await geminiGenerateContentServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - model: 'unknown-model', - headers: new Headers(), - }); - - const upstreamError = expectType(result, 'api-error'); - assertEquals(upstreamError.status, 404); - const body = JSON.parse(new TextDecoder().decode(upstreamError.body)); - assertEquals(body.error.code, 404); - assertEquals(body.error.status, 'NOT_FOUND'); - assert(typeof body.error.message === 'string' && body.error.message.includes('unknown-model')); -}); - -test('generate filters out candidates whose endpoints do not satisfy the gemini-generate-content-generate preference and renders model-unsupported as a 400', async () => { - installRepo(); - const callOpenAIChatCompletions = vi.fn(); - // geminiGenerateContentGenerateTarget prefers openai-chat-completions > messages > responses; - // an endpoints-only `openaiCompletions` candidate matches none and is filtered. - queueResolution([makeCandidate({ upstream: 'up_x', endpoints: { openaiCompletions: {} }, callOpenAIChatCompletions })]); - - const result = await geminiGenerateContentServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - model: 'wrong-endpoint-model', - headers: new Headers(), - }); - - const upstreamError = expectType(result, 'api-error'); - assertEquals(upstreamError.status, 400); - const body = JSON.parse(new TextDecoder().decode(upstreamError.body)); - assertEquals(body.error.code, 400); - assertEquals(body.error.status, 'INVALID_ARGUMENT'); - assert(typeof body.error.message === 'string' && body.error.message.includes('does not support')); - // Defense-in-depth: even though `pick` would also throw on this candidate's - // endpoints, the spy pins that the filter happens BEFORE pick is reached, - // not after a fallback. Sibling tests in messages/responses/openai-chat-completions - // serve_test.ts mirror this assertion for the same reason. - assertEquals(callOpenAIChatCompletions.mock.calls.length, 0); -}); - -test('countTokens translates Gemini generateContent to Anthropic Messages count_tokens and returns the Gemini generateContent envelope', async () => { - installRepo(); - const callAnthropicMessagesCountTokens = vi.fn(async (): Promise => ({ - response: new Response(JSON.stringify({ input_tokens: 17 }), { - status: 200, headers: new Headers({ 'content-type': 'application/json' }), - }), - modelKey: 'k', - })); - queueResolution([makeCandidate({ targetApi: 'anthropicMessages', callAnthropicMessagesCountTokens })]); - - const result = await geminiGenerateContentServe.countTokens({ - payload: makePayload(), - ctx: makeGatewayCtx(), - model: 'test-model', - headers: new Headers(), - }); - - const plain = expectType(result, 'plain'); - assertEquals(plain.status, 200); - const body = JSON.parse(new TextDecoder().decode(plain.body)); - assertEquals(body, { totalTokens: 17 }); - assertEquals(callAnthropicMessagesCountTokens.mock.calls.length, 1); -}); - -test('countTokens renders a Google RPC NOT_FOUND when no Anthropic-Messages-capable candidate exists', async () => { - installRepo(); - queueResolution([]); - - const result = await geminiGenerateContentServe.countTokens({ - payload: makePayload(), - ctx: makeGatewayCtx(), - model: 'no-messages-model', - headers: new Headers(), - }); - - const upstreamError = expectType(result, 'api-error'); - assertEquals(upstreamError.status, 404); - const body = JSON.parse(new TextDecoder().decode(upstreamError.body)); - assertEquals(body.error.code, 404); - assertEquals(body.error.status, 'NOT_FOUND'); - assert(typeof body.error.message === 'string' && body.error.message.includes('no-messages-model is not')); -}); - -test('countTokens filters out candidates whose endpoints do not satisfy the gemini-countTokens preference and renders model-unsupported as a 400', async () => { - installRepo(); - const callAnthropicMessagesCountTokens = vi.fn(); - // geminiGenerateContentCountTokensTarget = chatTargetPicker(['anthropicMessages']); a candidate - // exposing only openaiChatCompletions matches none and is filtered out. - queueResolution([makeCandidate({ upstream: 'up_x', endpoints: { openaiChatCompletions: {} }, callAnthropicMessagesCountTokens })]); - - const result = await geminiGenerateContentServe.countTokens({ - payload: makePayload(), - ctx: makeGatewayCtx(), - model: 'wrong-endpoint-model', - headers: new Headers(), - }); - - const upstreamError = expectType(result, 'api-error'); - assertEquals(upstreamError.status, 400); - const body = JSON.parse(new TextDecoder().decode(upstreamError.body)); - assertEquals(body.error.code, 400); - assertEquals(body.error.status, 'INVALID_ARGUMENT'); - assert(typeof body.error.message === 'string' && body.error.message.includes('does not support')); - assertEquals(callAnthropicMessagesCountTokens.mock.calls.length, 0); -}); - -test('alias resolution overlays rules onto the target IR at the wire call', async () => { - installRepo(); - const capturedBodies: Record[] = []; - const callOpenAIChatCompletions = vi.fn(async (_model: unknown, body: unknown): Promise> => { - capturedBodies.push(body as Record); - return { ok: true, events: makeProtocolFrames(makeOpenAIChatCompletionsEvents()), modelKey: 'gpt-5.4', headers: new Headers() }; - }); - queueResolution([makeCandidate({ targetApi: 'openaiChatCompletions', callOpenAIChatCompletions })], { - aliasRules: { reasoning: { effort: 'high', budget_tokens: 1024 }, verbosity: 'low' }, - }); - - const result = await geminiGenerateContentServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - model: 'gemini-fast', - headers: new Headers(), - }); - await collectEvents(expectType(result, 'events').events); - - // The resolver sees the inbound alias id verbatim. Target-id walking - // happens inside the resolver. - assertEquals(lastResolveCall.model, 'gemini-fast'); - // Alias rules land on the terminal wire target's native slots — for a - // Gemini generateContent→OpenAI Chat Completions traversal that is `reasoning_effort` and - // `verbosity`. Rules that lack a native CC slot (budget_tokens) drop. - const observed = capturedBodies[0]!; - assertEquals(observed.reasoning_effort, 'high'); - assertEquals(observed.verbosity, 'low'); -}); - -test('alias whose targets have no kind-matching binding surfaces as the regular model-missing 404', async () => { - installRepo(); - queueResolution([], { sawModel: false }); - - const result = await geminiGenerateContentServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - model: 'gemini-fast', - headers: new Headers(), - }); - - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 404); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assertEquals(body.error.message, 'Model gemini-fast is not available on any configured upstream.'); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts new file mode 100644 index 0000000000..b09be27ffb --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions-pipeline_test.ts @@ -0,0 +1,474 @@ +// The OpenAI Chat Completions chain, run. Assembly is checked where every family's is; what is +// written down here is what only running it can say — that the edge folds a stream into one +// object when the client did not ask to stream, that it hands the turn's own state back for +// the client to carry, that a refusal keeps the upstream's own status and words, and that a +// dial nobody answered is a value the fork can move past. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { openaiChatCompletionsServePipeline } from '../../../src/data-plane/chat/openai-chat-completions/pipeline.ts'; +import { createCandidateRegistry } from '../../../src/data-plane/pipeline/candidates.ts'; +import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; +import { initRepo } from '../../../src/repo/index.ts'; +import { chatFixtureHttpServices, stubChatProviderPipelines } from '../../test-utils/chat-provider-pipelines.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { move, run } from '@floway-dev/pipeline'; +import type { SseFrame } from '@floway-dev/protocols/common'; +import { type OpenAIChatCompletionsPayload, type OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; +import { directFetcher, type FlagId, type ModelCandidate } from '@floway-dev/provider'; +import { type StubChatProviderCall, type ProviderStreamResult } from '@floway-dev/test-utils'; +import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; + +vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ + ...(await importOriginal()), + enumerateModelCandidates: vi.fn(), +})); + +vi.mock('../../../src/data-plane/chat/openai-chat-completions/affinity/ingress.ts', async importOriginal => ({ + ...(await importOriginal()), + analyzeOpenAIChatCompletionsAffinity: async () => ({ + requiredTargets: [], + evaluateCandidate: () => ({ kind: 'accepted', degrades: false, preferred: true, materialize: () => affinityPayload }), + }), +})); + +let live: readonly ModelCandidate[] = []; + +const candidate = ( + callOpenAIChatCompletions: StubChatProviderCall<'openaiChatCompletions'>, + upstreamId = 'up_a', + flags: readonly FlagId[] = [], +): ModelCandidate => { + const endpoints = { openaiChatCompletions: {} }; + return { + provider: { + upstreamId, kind: 'custom', name: upstreamId, inboundHeaderAllowlist: [], + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + pipelines: stubChatProviderPipelines({ callOpenAIChatCompletions }), + instance: stubProvider(), + }, + model: stubInternalModel( + { + id: 'chat-model', + endpoints, + providerModels: { [upstreamId]: stubProviderModel({ id: 'chat-model', endpoints, enabledFlags: new Set(flags) }) }, + }, + upstreamId, + ), + fetcher: directFetcher, + } as unknown as ModelCandidate; +}; + +const resolves = (candidates: readonly ModelCandidate[]): void => { + live = candidates; + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] } as never); +}; + +const chunk = (text: string): OpenAIChatCompletionsStreamEvent => ({ + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'chat-model', + choices: [{ index: 0, delta: { content: text }, finish_reason: null }], +}); + +/** What this frame carries of the turn's own state, if it is the carrier the edge writes: a + * chunk whose delta holds nothing but `reasoning_opaque`. */ +const affinityCarrier = (frame: SseFrame): string | undefined => { + if (frame.data === '[DONE]') return undefined; + return (JSON.parse(frame.data) as OpenAIChatCompletionsStreamEvent).choices[0]?.delta.reasoning_opaque ?? undefined; +}; + +/** A stream that stops without ever saying it ended — what a dropped upstream looks like. */ +const truncated = (...events: readonly OpenAIChatCompletionsStreamEvent[]): ProviderStreamResult => ({ + ok: true, + modelKey: 'chat-model-key', + headers: new Headers(), + events: (async function* () { + for (const event of events) yield { type: 'event' as const, event }; + })(), +}); + +const stream = (...events: readonly OpenAIChatCompletionsStreamEvent[]): ProviderStreamResult => ({ + ok: true, + modelKey: 'chat-model-key', + headers: new Headers({ 'x-request-id': 'req-1', 'content-length': '99' }), + events: (async function* () { + for (const event of events) yield { type: 'event' as const, event }; + yield { type: 'done' as const }; + })(), +}); + +const payload = { model: 'chat-model', messages: [{ role: 'user', content: 'hi' }] } as unknown as OpenAIChatCompletionsPayload; + +/** What affinity materialized for the candidate about to be dialled. It differs from the + * payload the client sent, which is the point: carried state is rewritten per candidate. */ +let affinityPayload: OpenAIChatCompletionsPayload = payload; + +const serve = async (wantsStream: boolean) => await serveWith(mockChatGatewayCtx({ wantsStream }), wantsStream); + +const serveWith = async ( + gateway: ReturnType, + wantsStream: boolean, + wantsUsageChunk = false, +) => await run( + openaiChatCompletionsServePipeline(), + move({ + 'ingress.http.headers': [] as readonly (readonly [string, string])[], + 'ingress.chat.sourceProtocol': 'openaiChatCompletions', + 'ingress.chat.openaiChatCompletions.wantsStream': wantsStream, + 'ingress.chat.openaiChatCompletions.wantsUsageChunk': wantsUsageChunk, + 'request.chat.openaiChatCompletions': payload, + 'serve.model': 'chat-model', + }) as never, + { + gateway, + background: () => {}, + ...chatFixtureHttpServices(gateway), + ...createCandidateRegistry(), + // Wired where the app wires it: the carrier the edge writes is addressed to whatever the + // chain named here. + selectAffinity: (selected: ModelCandidate) => { gateway.affinity.select(selected); }, + resolveAttempt: (selector: { readonly upstreamId: string }) => { + const found = live.find(c => c.provider.upstreamId === selector.upstreamId); + if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); + return found; + }, + } as never, +); + +beforeEach(() => { + vi.mocked(enumerateModelCandidates).mockReset(); + initRepo({ + apiKeys: { update: async () => {} }, + usage: { record: async () => {} }, + performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, + } as never); +}); + +describe('the chat completions chain', () => { + // The wire's rules rewrite the request as a fact, so what they rewrite has to be what the + // ending sends. An ending that asked the resolver for the payload instead would read one no + // rule had touched, and every rewrite in the chain would go nowhere. + it('sends what the rules rewrote, not what the resolver materialized', async () => { + let sent: { messages: { role: string }[] } | undefined; + affinityPayload = { + ...payload, + messages: [{ role: 'system', content: 'be brief' }, { role: 'user', content: 'hi' }], + } as unknown as OpenAIChatCompletionsPayload; + resolves([candidate(async (_model, body) => { + sent = body as { messages: { role: string }[] }; + return stream(chunk('hi')); + }, 'up_a', ['rewrite-system-to-developer'])]); + + await serve(false); + + expect(sent!.messages.map(message => message.role)).toEqual(['developer', 'user']); + affinityPayload = payload; + }); + + // Affinity rewrites client-carried state for the upstream that will see it, so the body + // that goes out is the one it materialized rather than the one the client sent. + it('sends the payload affinity materialized for the candidate it dialled', async () => { + let sent: Record | undefined; + affinityPayload = { ...payload, messages: [{ role: 'user', content: 'rewritten' }] } as OpenAIChatCompletionsPayload; + resolves([candidate(async (_model, body) => { + sent = body as Record; + return stream(chunk('hi')); + })]); + + await serve(false); + + expect(sent).toMatchObject({ messages: [{ role: 'user', content: 'rewritten' }] }); + expect(sent).not.toHaveProperty('model'); + affinityPayload = payload; + }); + + it('writes the frames out when the client asked to stream', async () => { + resolves([candidate(async () => stream(chunk('he'), chunk('llo')))]); + + const { facts } = await serve(true); + const frames: SseFrame[] = []; + for await (const frame of facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable) frames.push(frame); + + expect(facts['response.http.status']).toBe(200); + // The carrier the edge writes has a test of its own below; what the protocol itself + // answered with is these frames, in this order. + expect(frames.filter(frame => affinityCarrier(frame) === undefined).map(frame => frame.data)).toEqual([ + JSON.stringify(chunk('he')), + JSON.stringify(chunk('llo')), + '[DONE]', + ]); + }); + + // The other half of affinity: the resolver reads client-carried state on the way down, and + // this is what the client is given to carry back. A turn that handed back nothing would + // leave the follow-up quoting it with no way to name the upstream that issued it. + it('hands the turn-s own state back on a chunk of its own', async () => { + resolves([candidate(async () => stream(chunk('hi')))]); + const gateway = mockChatGatewayCtx({ wantsStream: true }); + + const { facts } = await serveWith(gateway, true); + const frames: SseFrame[] = []; + for await (const frame of facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable) frames.push(frame); + const carried = frames.flatMap(frame => affinityCarrier(frame) ?? []); + + expect(carried).toHaveLength(1); + // It names the upstream that answered, sealed under this run's own secret — which is what + // lets the next turn be pinned to it. + expect(await gateway.affinity.codec.unwrap(carried[0]!, 'openai-chat-completions.reasoning_opaque')).toMatchObject({ + kind: 'owned', + affinity: { upstreamId: 'up_a', modelId: 'chat-model' }, + }); + }); + + // The upstream speaks SSE whatever the client asked for, so a client that did not ask to + // stream is answered from the same frames — folded here rather than read a second time. + it('folds the frames into one object when the client did not', async () => { + resolves([candidate(async () => stream(chunk('he'), chunk('llo')))]); + + const { facts } = await serve(false); + const rendered = facts['response.chat.openaiChatCompletions.rendered'] as { choices: { message: { content: string } }[] }; + + expect(facts['response.http.status']).toBe(200); + expect(rendered.choices[0]!.message.content).toBe('hello'); + expect((await facts['response.chat.openaiChatCompletions.streamedUsage']!).failed).toBe(false); + }); + + // Content-length would misdescribe a body this gateway serialized itself; a vendor trace + // is what a client and an operator both need to correlate a turn. + it('forwards the upstream headers a client may see and drops the ones it may not', async () => { + resolves([candidate(async () => stream(chunk('hi')))]); + + const { facts } = await serve(true); + + expect(Object.fromEntries(facts['response.http.headers'])).toMatchObject({ 'x-request-id': 'req-1' }); + expect(Object.fromEntries(facts['response.http.headers'])).not.toHaveProperty('content-length'); + }); + + it('answers an upstream refusal with its own status and words', async () => { + resolves([candidate(async () => ({ + ok: false, + modelKey: 'chat-model-key', + response: new Response(JSON.stringify({ error: { message: 'slow down' } }), { + status: 429, headers: { 'content-type': 'application/json' }, + }), + }))]); + + const { facts } = await serve(false); + + expect(facts['response.http.status']).toBe(429); + expect(facts['response.chat.openaiChatCompletions.rendered']).toEqual({ error: { message: 'slow down' } }); + }); + + // What the upstream reported is per token category and what is billed is per metric name; + // the two are not interchangeable, and a cast between them writes a row with no metrics. + it('bills the metrics the upstream-s own usage adds up to', async () => { + resolves([candidate(async () => stream(chunk('hi'), { + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'chat-model', + choices: [], usage: { prompt_tokens: 11, completion_tokens: 4, total_tokens: 15 }, + } as unknown as OpenAIChatCompletionsStreamEvent))]); + + const { facts } = await serve(true); + for await (const _frame of facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable) { /* drain */ } + const outcome = await facts['response.chat.openaiChatCompletions.streamedUsage']!; + + expect(outcome.billable[0]!.quantities).toMatchObject({ input_tokens: '11', output_tokens: '4' }); + // A rate can depend on the tier and on how much input there was; both are selector + // coordinates, so they travel as pricing facts rather than as quantities. + expect(outcome.billable[0]!.pricingFacts).toMatchObject({ inputTokens: 11 }); + // The stream reached its terminator, so the turn produced what it said it would. + expect(outcome.failed).toBe(false); + }); + + // The test above hands over a usage chunk whatever was asked for, which is a stream no + // OpenAI-compatible upstream would have sent: the final usage chunk arrives only when + // `stream_options.include_usage` is on. This upstream behaves like a real one, so what it + // states is the whole billing path — a chain that stopped asking would meter a stream that + // reported nothing and bill the turn zero, with nothing else in the suite noticing. + it('asks the upstream for the usage chunk, and so bills a streaming turn at all', async () => { + let asked: { include_usage?: boolean } | null | undefined; + resolves([candidate(async (_model, body) => { + asked = (body as { stream_options?: { include_usage?: boolean } }).stream_options; + return asked?.include_usage === true + ? stream(chunk('hi'), { + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'chat-model', + choices: [], usage: { prompt_tokens: 11, completion_tokens: 4, total_tokens: 15 }, + } as unknown as OpenAIChatCompletionsStreamEvent) + : stream(chunk('hi')); + })]); + + // The client did not ask to see the chunk, which is the ordinary case and the one where + // the two questions come apart: the upstream is asked for it regardless. + const { facts } = await serve(true); + const frames: SseFrame[] = []; + for await (const frame of facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable) frames.push(frame); + const outcome = await facts['response.chat.openaiChatCompletions.streamedUsage']!; + + expect(asked).toEqual({ include_usage: true }); + expect(outcome.billable[0]!.quantities).toMatchObject({ input_tokens: '11', output_tokens: '4' }); + // And the client, who asked for none of this, is shown none of it. + expect(frames.some(frame => frame.data !== '[DONE]' && 'usage' in (JSON.parse(frame.data) as object))).toBe(false); + }); + + // The wire's response-direction rules, on the wire that owns them, in the order it composes + // them — which is a property of the array in `openaiChatCompletionsWire` and of nothing else. A + // DeepSeek-shaped upstream reporting the exclusive cache convention exercises all three at + // once, and each one's input is what the one below it left: the vendor rename, then the fold + // that reads the renamed field, then the split that puts the settled usage on a carrier. + it('serves a vendor dialect through the wire-s rules, in the order the wire composes them', async () => { + resolves([candidate(async () => stream( + { + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'chat-model', + choices: [{ index: 0, delta: { reasoning_content: 'pondering' }, finish_reason: null }], + } as unknown as OpenAIChatCompletionsStreamEvent, + { + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'chat-model', + // Hung off the final delta rather than sent on a carrier, in DeepSeek's own cache + // field names, counting the cached prefix outside the input total: 479 + 13312 + 373 + // is the 14164 it states. + choices: [{ index: 0, delta: { content: 'hi' }, finish_reason: 'stop' }], + usage: { + prompt_tokens: 479, completion_tokens: 373, total_tokens: 14164, + prompt_cache_hit_tokens: 13312, prompt_cache_miss_tokens: 479, + }, + } as unknown as OpenAIChatCompletionsStreamEvent, + ), 'up_a', ['vendor-deepseek'])]); + + const { facts } = await serveWith(mockChatGatewayCtx({ wantsStream: true }), true, true); + const written: OpenAIChatCompletionsStreamEvent[] = []; + for await (const frame of facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable) { + if (frame.data !== '[DONE]' && affinityCarrier(frame) === undefined) { + written.push(JSON.parse(frame.data) as OpenAIChatCompletionsStreamEvent); + } + } + + // The vendor's reasoning field left under the canonical name. + expect(written[0]!.choices[0]!.delta).toEqual({ reasoning_text: 'pondering' }); + // The generated text is still the client's, whatever the rules did around it. + expect(written.some(event => event.choices[0]?.delta.content === 'hi')).toBe(true); + // Exactly one frame carries usage, and it is a carrier rather than a chunk that also + // carried a delta — which is the split, and it happened once. + const carriers = written.filter(event => 'usage' in event) as unknown as { choices: unknown[]; usage: Record }[]; + expect(carriers).toHaveLength(1); + expect(carriers[0]!.choices).toEqual([]); + // In OpenAI's names, with the cached prefix folded back into the input total the way + // OpenAI's contract states it: neither would be true if either rule ran out of turn. + expect(carriers[0]!.usage).toEqual({ + prompt_tokens: 13791, + completion_tokens: 373, + total_tokens: 14164, + prompt_tokens_details: { cached_tokens: 13312 }, + }); + // And the row is written from that same frame rather than from the vendor's own. The + // reading is taken above every rule, so what is billed and what is shown cannot disagree — + // metering at the dial would bill the 479 DeepSeek called the input and lose the cached + // prefix entirely, against the 13791 the client was just shown. + const outcome = await facts['response.chat.openaiChatCompletions.streamedUsage']!; + expect(outcome.billable[0]!.quantities).toEqual({ + input_tokens: '479', + input_cache_read_tokens: '13312', + output_tokens: '373', + }); + }); + + // The other two dialects the wire composes, each stating it in a different direction: Qwen + // rewrites the request, Kimi the usage that comes back. The flags are independent, so one + // candidate carrying both is what runs the pair in one turn — an upstream in the field + // carries one of them. + it('serves the request-side and response-side vendor dialects the wire also composes', async () => { + let sent: Record | undefined; + affinityPayload = { ...payload, reasoning_effort: 'none' } as OpenAIChatCompletionsPayload; + resolves([candidate(async (_model, body) => { + sent = body as Record; + return stream(chunk('hi'), { + id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'chat-model', + choices: [], usage: { prompt_tokens: 100, completion_tokens: 20, total_tokens: 120, cached_tokens: 30 }, + } as unknown as OpenAIChatCompletionsStreamEvent); + }, 'up_a', ['vendor-qwen', 'vendor-kimi'])]); + + const { facts } = await serveWith(mockChatGatewayCtx({ wantsStream: true }), true, true); + const written: OpenAIChatCompletionsStreamEvent[] = []; + for await (const frame of facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable) { + if (frame.data !== '[DONE]' && affinityCarrier(frame) === undefined) { + written.push(JSON.parse(frame.data) as OpenAIChatCompletionsStreamEvent); + } + } + affinityPayload = payload; + + // Qwen's own spelling of "no reasoning" went out on the body this wire sent. + expect('reasoning_effort' in sent!).toBe(false); + expect(sent!.enable_thinking).toBe(false); + // And Kimi's flat cached count came back under the name every rule above it reads. The + // totals say the cache was counted inside the input here, so nothing is folded. + const carrier = written.find(event => 'usage' in event) as unknown as { usage: Record }; + expect(carrier.usage).toEqual({ + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + prompt_tokens_details: { cached_tokens: 30 }, + }); + }); + + // Time to first token is measured where the token is — an envelope frame is not one, and a + // run that never stamps it is recorded as having produced nothing to time. + it('stamps the first frame that carried a generated token', async () => { + resolves([candidate(async () => stream(chunk('hi')))]); + const gateway = mockChatGatewayCtx({ wantsStream: true }); + + const { facts } = await serveWith(gateway, true); + expect(gateway.attempt.timing.firstOutputTokenAt).toBeNull(); + for await (const _frame of facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable) { /* drain */ } + + expect(gateway.attempt.timing.firstOutputTokenAt).toBeTypeOf('number'); + }); + + // A refusal the gateway made itself has no upstream body to forward, and that is the one + // case where the client's own protocol decides the shape. An OpenAI client reads + // error.type, and `api_error` would say the gateway broke rather than that the request did. + it('names its own refusal in the type an OpenAI client reads', async () => { + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates: [], sawModel: true, failedUpstreams: [] } as never); + + const { facts } = await serve(false); + + expect(facts['response.http.status']).toBe(400); + expect(facts['response.chat.openaiChatCompletions.rendered']).toEqual({ + error: { message: 'Model chat-model does not support the /chat/completions endpoint.', type: 'invalid_request_error' }, + }); + }); + + it('accepts normal EOF without a terminal marker', async () => { + resolves([candidate(async () => truncated(chunk('he'), chunk('llo')))]); + + const { facts } = await serve(true); + const drain = async (): Promise => { + for await (const _frame of facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable) { /* to the end */ } + }; + + await expect(drain()).resolves.toBeUndefined(); + expect((await facts['response.chat.openaiChatCompletions.streamedUsage']!).failed).toBe(false); + }); + + // Anything an upstream keeps sending after its terminator is not part of the answer. + it('stops reading at the terminator, and writes it', async () => { + resolves([candidate(async () => stream(chunk('hi')))]); + + const { facts } = await serve(true); + const frames: SseFrame[] = []; + for await (const frame of facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable) frames.push(frame); + + expect(frames.at(-1)!.data).toBe('[DONE]'); + }); + + // A refused connection is an outcome the fork has to be able to see, not a fault that ends + // the run — so the second candidate is tried and its answer is the one served. + it('fails a dial that never connected over to the next candidate', async () => { + const tried: string[] = []; + resolves([ + candidate(async () => { tried.push('dead'); return { type: 'transportFailure' as const, error: new Error('ECONNREFUSED') }; }, 'up_dead'), + candidate(async () => { tried.push('alive'); return stream(chunk('hi')); }, 'up_alive'), + ]); + + const { facts } = await serve(false); + + expect(tried).toEqual(['dead', 'alive']); + expect(facts['response.http.status']).toBe(200); + }); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/attempt_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/attempt_test.ts deleted file mode 100644 index 77a22b00d2..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/attempt_test.ts +++ /dev/null @@ -1,318 +0,0 @@ -import { test, vi } from 'vitest'; - -import { openaiChatCompletionsAttempt } from '../../../../src/data-plane/chat/openai-chat-completions/attempt.ts'; -import { initRepo } from '../../../../src/repo/index.ts'; -import { InMemoryRepo } from '../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import { initExternalResourceFetcher } from '@floway-dev/platform'; -import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import { doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import type { OpenAIResponsesPayload, OpenAIResponsesResult } from '@floway-dev/protocols/openai-responses'; -import { type AnthropicMessagesUpstreamCallOptions, type ModelCandidate, directFetcher, type ProviderOpenAIResponsesResult, type ProviderStreamResult, type OpenAIResponsesAction, type UpstreamCallOptions } from '@floway-dev/provider'; -import type { FlagId } from '@floway-dev/provider/flags'; -import { assert, assertEquals, stubProvider, stubInternalModel, stubProviderModel } from '@floway-dev/test-utils'; - -const API_KEY_ID = 'key_chat_completions_attempt_test'; - -const makeGatewayCtx = () => mockChatGatewayCtx({ apiKeyId: API_KEY_ID, wantsStream: true }); - -const makePayload = (overrides: Partial = {}): OpenAIChatCompletionsPayload => ({ - model: 'test-model', - messages: [{ role: 'user', content: 'hello' }], - ...overrides, -}); - -const makeOpenAIChatCompletionsEvents = (): readonly OpenAIChatCompletionsStreamEvent[] => [ - { - id: 'chatcmpl_test', object: 'chat.completion.chunk', created: 0, model: 'test-model', - choices: [{ index: 0, delta: { role: 'assistant' }, finish_reason: null }], - }, - { - id: 'chatcmpl_test', object: 'chat.completion.chunk', created: 0, model: 'test-model', - choices: [{ index: 0, delta: { content: 'hi' }, finish_reason: null }], - }, - { - id: 'chatcmpl_test', object: 'chat.completion.chunk', created: 0, model: 'test-model', - choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], - }, -]; - -const makeAnthropicMessagesEvents = (): readonly AnthropicMessagesStreamEvent[] => [ - { - type: 'message_start', - message: { - id: 'msg_cc_via_m', type: 'message', role: 'assistant', content: [], - model: 'test-model', stop_reason: null, stop_sequence: null, - usage: { input_tokens: 4, output_tokens: 0 }, - }, - }, - { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, - { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'hi' } }, - { type: 'content_block_stop', index: 0 }, - { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 1 } }, - { type: 'message_stop' }, -]; - -const makeProtocolFrames = async function* (events: readonly TEvent[]): AsyncGenerator> { - for (const event of events) yield eventFrame(event); - yield doneFrame(); -}; - -const makeCandidate = (overrides: { - upstream?: string; - endpoints?: ModelEndpoints; - callOpenAIChatCompletions?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise>; - callAnthropicMessages?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: AnthropicMessagesUpstreamCallOptions) => Promise>; - callOpenAIResponses?: (model: unknown, body: unknown, action: OpenAIResponsesAction, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise; - enabledFlags?: ReadonlySet; -} = {}): ModelCandidate => { - const upstream = overrides.upstream ?? 'up_test'; - const endpoints = overrides.endpoints ?? { openaiChatCompletions: {}, openaiResponses: {}, anthropicMessages: {} }; - const provider = stubProvider({ - callOpenAIChatCompletions: overrides.callOpenAIChatCompletions, - callAnthropicMessages: overrides.callAnthropicMessages, - callOpenAIResponses: overrides.callOpenAIResponses, - }); - return { - provider: { - upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: provider, - }, - model: stubInternalModel({ - endpoints, - providerModels: { - [upstream]: stubProviderModel({ endpoints, enabledFlags: new Set(overrides.enabledFlags ?? []) }), - }, - }, upstream), - fetcher: directFetcher, - }; -}; - -const collectEvents = async (events: AsyncIterable>): Promise => { - const out: TEvent[] = []; - for await (const frame of events) { - if (frame.type === 'event') out.push(frame.event); - } - return out; -}; - -const installRepo = (): InMemoryRepo => { - const repo = new InMemoryRepo(); - initRepo(repo); - return repo; -}; - -test('generate native openai-chat-completions target calls provider.callOpenAIChatCompletions', async () => { - installRepo(); - const callOpenAIChatCompletions = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeOpenAIChatCompletionsEvents()), modelKey: 'k', headers: new Headers(), - })); - const result = await openaiChatCompletionsAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callOpenAIChatCompletions }), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callOpenAIChatCompletions.mock.calls.length, 1); -}); - -test('generate native target applies role compatibility flags in target-chain order', async () => { - installRepo(); - let observedBody: Omit | undefined; - const callOpenAIChatCompletions = vi.fn(async (_model, body): Promise> => { - observedBody = body as Omit; - return { - ok: true, - events: makeProtocolFrames(makeOpenAIChatCompletionsEvents()), - modelKey: 'k', - headers: new Headers(), - }; - }); - const result = await openaiChatCompletionsAttempt.generate({ - payload: makePayload({ - messages: [ - { role: 'system', content: 'base instructions' }, - { role: 'user', content: 'hello' }, - { role: 'system', content: 'inline instructions' }, - ], - }), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ - callOpenAIChatCompletions, - endpoints: { openaiChatCompletions: {} }, - enabledFlags: new Set([ - 'rewrite-developer-to-system', - 'rewrite-mid-conv-system-to-user', - 'rewrite-system-to-developer', - ]), - }), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(observedBody?.messages, [ - { role: 'system', content: 'base instructions' }, - { role: 'user', content: 'hello' }, - { role: 'user', content: 'inline instructions' }, - ]); -}); - -test('generate translates through the Anthropic Messages target when only that endpoint is exposed', async () => { - installRepo(); - let anthropicBeta: readonly string[] | undefined; - const callAnthropicMessages = vi.fn(async (_model, _body, _signal, opts): Promise> => { - anthropicBeta = opts?.anthropicBeta; - return { ok: true, events: makeProtocolFrames(makeAnthropicMessagesEvents()), modelKey: 'k', headers: new Headers() }; - }); - const result = await openaiChatCompletionsAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callAnthropicMessages, endpoints: { anthropicMessages: {} } }), - headers: new Headers({ 'anthropic-beta': 'must-not-cross-source-protocols' }), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callAnthropicMessages.mock.calls.length, 1); - assertEquals(anthropicBeta, []); -}); - -test('generate injects the platform external-image loader into OpenAI-Chat-Completions-to-Anthropic-Messages translation', async () => { - installRepo(); - initExternalResourceFetcher(url => { - assertEquals(url.href, 'https://example.com/image.png'); - return Promise.resolve(new Response(Uint8Array.of(1, 2, 3), { headers: { 'content-type': 'image/png' } })); - }); - let observedBody: Omit | undefined; - const callAnthropicMessages = vi.fn(async (_model, body): Promise> => { - observedBody = body as Omit; - return { ok: true, events: makeProtocolFrames(makeAnthropicMessagesEvents()), modelKey: 'k', headers: new Headers() }; - }); - const result = await openaiChatCompletionsAttempt.generate({ - payload: makePayload({ - messages: [{ - role: 'user', - content: [{ type: 'image_url', image_url: { url: 'https://example.com/image.png' } }], - }], - }), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callAnthropicMessages, endpoints: { anthropicMessages: {} } }), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - const message = observedBody?.messages[0]; - assert(message?.role === 'user' && Array.isArray(message.content)); - const image = message.content.find(block => block.type === 'image'); - assert(image?.type === 'image'); - assertEquals(image.source, { type: 'base64', media_type: 'image/png', data: 'AQID' }); -}); - -test('generate translates through the OpenAI Responses target when only that endpoint is exposed', async () => { - installRepo(); - const respResp: OpenAIResponsesResult = { - id: 'resp_x', object: 'response', model: 'test-model', status: 'completed', - output: [{ - type: 'message', id: 'msg_resp', role: 'assistant', status: 'completed', - content: [{ type: 'output_text', text: 'hi', annotations: [] }], - }], - output_text: 'hi', error: null, incomplete_details: null, - }; - const callOpenAIResponses = vi.fn(async (): Promise => ({ - action: 'generate', ok: true, - events: makeProtocolFrames([{ type: 'response.completed', sequence_number: 0, response: respResp }]), - modelKey: 'k', - headers: new Headers(), - })); - const result = await openaiChatCompletionsAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callOpenAIResponses, endpoints: { openaiResponses: {} } }), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callOpenAIResponses.mock.calls.length, 1); -}); - -test('generate preserves translated instructions before rewriting inline system messages', async () => { - installRepo(); - const observedBodies: Omit[] = []; - const callOpenAIResponses = vi.fn(async (_model, body): Promise => { - observedBodies.push(body as Omit); - return { - action: 'generate', - ok: true, - events: makeProtocolFrames([{ - type: 'response.completed', sequence_number: 0, response: { - id: 'resp_x', object: 'response', model: 'test-model', status: 'completed', - output: [], output_text: '', error: null, incomplete_details: null, - }, - }]), - modelKey: 'k', - headers: new Headers(), - }; - }); - - const result = await openaiChatCompletionsAttempt.generate({ - payload: makePayload({ - messages: [ - { role: 'system', content: 'base instructions' }, - { role: 'user', content: 'hello' }, - { role: 'system', content: 'inline instructions' }, - ], - }), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ - callOpenAIResponses, - endpoints: { openaiResponses: {} }, - enabledFlags: new Set(['rewrite-system-to-developer']), - }), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - const observedBody = observedBodies[0]; - if (!observedBody) throw new Error('expected observed OpenAI Responses body'); - assertEquals(observedBody.instructions, 'base instructions'); - const input = observedBody.input; - if (!Array.isArray(input)) throw new Error('expected OpenAI Responses input array'); - assertEquals(input[0], { type: 'message', role: 'user', content: 'hello' }); - assertEquals(input[1], { type: 'message', role: 'developer', content: 'inline instructions' }); -}); - -test('generate propagates upstream response headers onto the EventResult so respond can forward them', async () => { - installRepo(); - const upstreamHeaders = new Headers({ - 'anthropic-ratelimit-unified-status': 'allowed', - 'cf-ray': 'cf_ray_cc', - }); - const callOpenAIChatCompletions = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeOpenAIChatCompletionsEvents()), modelKey: 'k', headers: upstreamHeaders, - })); - const result = await openaiChatCompletionsAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - candidate: makeCandidate({ callOpenAIChatCompletions }), - headers: new Headers(), - }); - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - assertEquals(result.headers?.get('anthropic-ratelimit-unified-status'), 'allowed'); - assertEquals(result.headers?.get('cf-ray'), 'cf_ray_cc'); - await collectEvents(result.events); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/errors_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/errors_test.ts deleted file mode 100644 index a2c24ebc05..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/errors_test.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { test } from 'vitest'; - -import { translatorInputErrorResult } from '../../../../src/data-plane/chat/openai-chat-completions/errors.ts'; -import type { ApiErrorResult } from '@floway-dev/provider'; -import { assertEquals } from '@floway-dev/test-utils'; -import { TranslatorInputError } from '@floway-dev/translate'; - -const apiErrorOf = (result: ReturnType): ApiErrorResult => result as ApiErrorResult; -const bodyOf = (result: ReturnType): unknown => - JSON.parse(new TextDecoder().decode(apiErrorOf(result).body)); - -test('translatorInputErrorResult renders an OpenAI 400 invalid_request_error envelope with default `messages` param', () => { - const result = translatorInputErrorResult( - new TranslatorInputError("Missing required field 'tool_call_id' on a 'tool' role message."), - ); - const apiError = apiErrorOf(result); - - assertEquals(apiError.type, 'api-error'); - assertEquals(apiError.source, 'gateway'); - assertEquals(apiError.status, 400); - assertEquals(bodyOf(result), { - error: { - message: "Missing required field 'tool_call_id' on a 'tool' role message.", - type: 'invalid_request_error', - param: 'messages', - code: null, - }, - }); -}); - -test('translatorInputErrorResult honors an explicit param from the translator', () => { - const result = translatorInputErrorResult( - new TranslatorInputError('content part not supported', { param: 'messages[2].content[1]' }), - ); - - assertEquals(bodyOf(result), { - error: { - message: 'content part not supported', - type: 'invalid_request_error', - param: 'messages[2].content[1]', - code: null, - }, - }); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts index 4fb01b74b7..87440ece2c 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts @@ -1,5 +1,5 @@ import { type Context, Hono } from 'hono'; -import { test, vi } from 'vitest'; +import { expect, test, vi } from 'vitest'; import { initDumpBroker, initDumpStore } from '../../../../src/dump/registry.ts'; import type { AuthVars } from '../../../../src/middleware/auth.ts'; @@ -8,10 +8,11 @@ import type { ApiKey, User } from '../../../../src/repo/types.ts'; import { installDumpStubs } from '../../../dump/test-fixtures.ts'; import { InMemoryRepo } from '../../../repo/memory.ts'; import { flushBackground } from '../../../test-utils/background-tracker.ts'; +import { stubChatProviderPipelines } from '../../../test-utils/chat-provider-pipelines.ts'; import { doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import { type ModelCandidate, directFetcher, type ProviderStreamResult, type UpstreamCallOptions } from '@floway-dev/provider'; -import { assert, assertEquals, stubProvider, stubInternalModel } from '@floway-dev/test-utils'; +import { type ModelCandidate, directFetcher, type UpstreamCallOptions } from '@floway-dev/provider'; +import { assert, assertEquals, stubProvider, stubInternalModel, type ProviderStreamResult } from '@floway-dev/test-utils'; const candidatesQueue: { readonly candidates: readonly ModelCandidate[]; readonly sawModel: boolean; readonly failedUpstreams: readonly string[] }[] = []; vi.mock('../../../../src/data-plane/providers/resolution.ts', async importOriginal => { @@ -108,11 +109,12 @@ const makeCandidate = (overrides: { callOpenAIChatCompletions?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise>; } = {}): ModelCandidate => { const upstream = overrides.upstream ?? 'up_test'; - const provider = stubProvider({ callOpenAIChatCompletions: overrides.callOpenAIChatCompletions }); + const calls = { callOpenAIChatCompletions: overrides.callOpenAIChatCompletions }; + const provider = stubProvider(); return { provider: { upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: provider, + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: stubChatProviderPipelines(calls), instance: provider, }, model: stubInternalModel(overrides.endpoints ? { endpoints: overrides.endpoints } : {}, upstream), fetcher: directFetcher, @@ -358,7 +360,7 @@ test('POST /v1/chat/completions renders the OpenAI-shaped model-unsupported 400 installRepo(); // Queue a chat-kind candidate whose endpoints expose only `openaiCompletions` — // the openaiChatCompletionsTarget picker rejects it (its preference list is - // `openai-chat-completions` > `messages` > `responses`), leaving zero viable + // `openaiChatCompletions` > `anthropicMessages` > `openaiResponses`), leaving zero viable // candidates, and with sawModel=true the serve renders model-unsupported // as a 400. queueCandidates([makeCandidate({ endpoints: { openaiCompletions: {} } })]); @@ -465,3 +467,70 @@ test('POST /v1/chat/completions leaves TTFT absent on a failure before output', const meta = dumpStubs.stored[0]!.record.meta; assertEquals(meta.ttftMs, null); }); + +test('actual HTTP error-body read failure preserves its original error and called observation', async () => { + const repo = installRepo(); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); + const fault = new Error('upstream error body read failed', { cause: new TypeError('socket reading failed') }); + const logged = vi.spyOn(console, 'error').mockImplementation(() => {}); + const call = vi.fn(async (): Promise> => ({ + ok: false, modelKey: 'called-model', + response: new Response(new ReadableStream({ pull: controller => controller.error(fault) }), { status: 503, headers: { 'content-type': 'application/json' } }), + })); + queueCandidates([makeCandidate({ callOpenAIChatCompletions: call })]); + const response = await makeApp(undefined, { dumpRetentionSeconds: 3600 }).request('/v1/chat/completions', { + method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ model: 'test-model', messages: [{ role: 'user', content: 'hello' }] }), + }); + expect(response.status).toBe(500); + expect(await response.json()).toMatchObject({ error: { name: fault.name, message: fault.message, stack: fault.stack, cause: { name: 'TypeError', message: 'socket reading failed' } } }); + await flushBackground(); + expect(call).toHaveBeenCalledOnce(); + expect(logged).toHaveBeenCalledOnce(); + expect(logged).toHaveBeenCalledWith(fault); + logged.mockRestore(); + expect(dumps.stored).toHaveLength(1); + expect(dumps.stored[0]!.record.meta).toMatchObject({ status: 500, error: { kind: 'failed', reason: fault.message } }); + const usage = await repo.usage.listAll(); + expect(usage).toHaveLength(1); + expect(usage[0]).toMatchObject({ modelKey: 'called-model', requests: 1, metrics: [] }); + expect(await repo.performance.listAll()).toMatchObject([{ requests: 1, errorsNoOutput: 1, errorsWithOutput: 0 }]); +}); + +test('actual HTTP cancellation before the first client frame releases the called provider and settles failure once', async () => { + const repo = installRepo(); + let unblock!: () => void; + const gate = new Promise(resolve => { unblock = resolve; }); + let closed!: () => void; + const stopped = new Promise(resolve => { closed = resolve; }); + let started = false; + let signal: AbortSignal | undefined; + const call = vi.fn(async (_model: unknown, _body: unknown, calledSignal?: AbortSignal): Promise> => { + signal = calledSignal; + return ({ + ok: true, modelKey: 'called-model', events: (async function* () { + started = true; + try { + await gate; + yield eventFrame(makeOpenAIChatCompletionsEvents()[1]!); + } finally { closed(); } + })(), + }); + }); + queueCandidates([makeCandidate({ callOpenAIChatCompletions: call })]); + const response = await makeApp().request('/v1/chat/completions', { + method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ model: 'test-model', stream: true, messages: [{ role: 'user', content: 'hello' }] }), + }); + expect(response.status).toBe(200); + expect(started).toBe(true); + if (response.body === null) throw new Error('Streaming response has no client body'); + await response.body.cancel(); + expect(signal?.aborted).toBe(true); + unblock(); + await stopped; + await flushBackground(); + expect(call).toHaveBeenCalledTimes(1); + expect(await repo.usage.listAll()).toMatchObject([{ modelKey: 'called-model', requests: 1 }]); + expect(await repo.performance.listAll()).toMatchObject([{ requests: 1, errorsNoOutput: 1, errorsWithOutput: 0 }]); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/apply-role-compatibility_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/apply-role-compatibility_test.ts deleted file mode 100644 index e886e11902..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/apply-role-compatibility_test.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { test } from 'vitest'; - -import { withRoleCompatibilityApplied } from '../../../../../src/data-plane/chat/openai-chat-completions/interceptors/apply-role-compatibility.ts'; -import type { OpenAIChatCompletionsInvocation } from '../../../../../src/data-plane/chat/openai-chat-completions/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { OpenAIChatCompletionsMessage, OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; -import { eventResult, type FlagId } from '@floway-dev/provider'; -import { assert, assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const gatewayCtx = mockChatGatewayCtx(); -const okEvents = () => Promise.resolve(eventResult((async function* () {})(), testTelemetryModelIdentity)); - -const applyRoles = async ( - messages: OpenAIChatCompletionsMessage[], - enabledFlags: ReadonlySet, - targetApi: OpenAIChatCompletionsInvocation['targetApi'] = 'openaiChatCompletions', -): Promise => { - const payload: OpenAIChatCompletionsPayload = { model: 'test-model', messages }; - const invocation: OpenAIChatCompletionsInvocation = { - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi, - headers: new Headers(), - }; - await withRoleCompatibilityApplied(invocation, gatewayCtx, okEvents); - return invocation.payload.messages; -}; - -test('leaves roles unchanged without flags or at a translated target', async () => { - const messages: OpenAIChatCompletionsMessage[] = [ - { role: 'system', content: 'rules' }, - { role: 'developer', content: 'developer rules' }, - ]; - - assertEquals(await applyRoles(messages, new Set()), messages); - assertEquals( - await applyRoles(messages, new Set(['rewrite-system-to-developer']), 'openaiResponses'), - messages, - ); -}); - -test('applies the system and developer rewrites independently', async () => { - assertEquals( - await applyRoles( - [{ role: 'system', content: 'rules' }, { role: 'user', content: 'hello' }], - new Set(['rewrite-system-to-developer']), - ), - [{ role: 'developer', content: 'rules' }, { role: 'user', content: 'hello' }], - ); - assertEquals( - await applyRoles( - [{ role: 'developer', content: 'rules' }, { role: 'user', content: 'hello' }], - new Set(['rewrite-developer-to-system']), - ), - [{ role: 'system', content: 'rules' }, { role: 'user', content: 'hello' }], - ); -}); - -test('preserves the leading system run and rewrites later system messages', async () => { - assertEquals( - await applyRoles( - [ - { role: 'system', content: 'base A' }, - { role: 'system', content: 'base B' }, - { role: 'user', content: 'hello' }, - { role: 'system', content: 'inline rules' }, - ], - new Set(['rewrite-mid-conv-system-to-user']), - ), - [ - { role: 'system', content: 'base A' }, - { role: 'system', content: 'base B' }, - { role: 'user', content: 'hello' }, - { role: 'user', content: 'inline rules' }, - ], - ); -}); - -test('keeps a leading-only system run and an empty input unchanged', async () => { - const leading: OpenAIChatCompletionsMessage[] = [ - { role: 'system', content: 'base A' }, - { role: 'system', content: 'base B' }, - ]; - assertEquals(await applyRoles(leading, new Set(['rewrite-mid-conv-system-to-user'])), leading); - assertEquals(await applyRoles([], new Set(['rewrite-mid-conv-system-to-user'])), []); -}); - -test('preserves multipart content identity when rewriting a mid-conversation system message', async () => { - const content = [ - { type: 'text' as const, text: 'one' }, - { type: 'text' as const, text: 'two' }, - ]; - const result = await applyRoles( - [{ role: 'user', content: 'hello' }, { role: 'system', content }], - new Set(['rewrite-mid-conv-system-to-user']), - ); - assertEquals(result, [{ role: 'user', content: 'hello' }, { role: 'user', content }]); - assert(result[1]?.content === content); -}); - -test('applies overlapping flags in system-to-developer then developer-to-system order', async () => { - assertEquals( - await applyRoles( - [ - { role: 'system', content: 'base rules' }, - { role: 'user', content: 'hello' }, - { role: 'system', content: 'inline rules' }, - ], - new Set([ - 'rewrite-system-to-developer', - 'rewrite-developer-to-system', - 'rewrite-mid-conv-system-to-user', - ]), - ), - [ - { role: 'system', content: 'base rules' }, - { role: 'user', content: 'hello' }, - { role: 'user', content: 'inline rules' }, - ], - ); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/disable-reasoning-on-forced-tool-choice_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/disable-reasoning-on-forced-tool-choice_test.ts deleted file mode 100644 index ddf1d8d745..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/disable-reasoning-on-forced-tool-choice_test.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { test } from 'vitest'; - -import { withReasoningDisabledOnForcedToolChoice } from '../../../../../src/data-plane/chat/openai-chat-completions/interceptors/disable-reasoning-on-forced-tool-choice.ts'; -import type { OpenAIChatCompletionsInvocation } from '../../../../../src/data-plane/chat/openai-chat-completions/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; -import { eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = () => Promise.resolve(eventResult((async function* () {})(), testTelemetryModelIdentity)); - -const invocation = ( - payload: OpenAIChatCompletionsPayload, - enabledFlags: ReadonlySet = new Set(['disable-reasoning-on-forced-tool-choice']), -): OpenAIChatCompletionsInvocation => ({ - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi: 'openaiChatCompletions', - headers: new Headers(), -}); - -test('required tool_choice sets reasoning_effort to the canonical none sentinel', async () => { - const input = invocation({ - model: 'm', - messages: [], - reasoning_effort: 'high', - tool_choice: 'required', - }); - - await withReasoningDisabledOnForcedToolChoice(input, stubCtx, okEvents); - - assertEquals(input.payload.reasoning_effort, 'none'); - const out = input.payload as unknown as Record; - assertEquals(out.thinking, undefined); - assertEquals(out.enable_thinking, undefined); -}); - -test('object tool_choice is treated as forced', async () => { - const input = invocation({ - model: 'm', - messages: [], - reasoning_effort: 'high', - tool_choice: { type: 'function', function: { name: 'x' } }, - }); - - await withReasoningDisabledOnForcedToolChoice(input, stubCtx, okEvents); - - assertEquals(input.payload.reasoning_effort, 'none'); -}); - -test('non-forced tool_choice leaves reasoning_effort untouched', async () => { - for (const tool_choice of ['auto', 'none', null] as const) { - const input = invocation({ - model: 'm', - messages: [], - reasoning_effort: 'high', - tool_choice, - }); - - await withReasoningDisabledOnForcedToolChoice(input, stubCtx, okEvents); - - assertEquals(input.payload.reasoning_effort, 'high'); - } -}); - -test('is a no-op when the flag is not set on the candidate', async () => { - const input = invocation( - { model: 'm', messages: [], reasoning_effort: 'high', tool_choice: 'required' }, - new Set(), - ); - - await withReasoningDisabledOnForcedToolChoice(input, stubCtx, okEvents); - - assertEquals(input.payload.reasoning_effort, 'high'); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/include-usage-stream-options_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/include-usage-stream-options_test.ts deleted file mode 100644 index 86fb0f201e..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/include-usage-stream-options_test.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { test } from 'vitest'; - -import { withUsageStreamOptionsIncluded } from '../../../../../src/data-plane/chat/openai-chat-completions/interceptors/include-usage-stream-options.ts'; -import type { OpenAIChatCompletionsInvocation } from '../../../../../src/data-plane/chat/openai-chat-completions/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; -import { eventResult } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = () => Promise.resolve(eventResult((async function* () {})(), testTelemetryModelIdentity)); - -const invocation = (payload: OpenAIChatCompletionsPayload): OpenAIChatCompletionsInvocation => ({ - payload, - candidate: stubModelCandidate(), - targetApi: 'openaiChatCompletions', - headers: new Headers(), -}); - -test('adds stream_options.include_usage when stream_options is absent', async () => { - const input = invocation({ model: 'm', messages: [] }); - - await withUsageStreamOptionsIncluded(input, stubCtx, okEvents); - - assertEquals(input.payload.stream_options, { include_usage: true }); -}); - -test('overrides include_usage:false on an existing stream_options object', async () => { - const input = invocation({ - model: 'm', - messages: [], - stream_options: { include_usage: false }, - }); - - await withUsageStreamOptionsIncluded(input, stubCtx, okEvents); - - assertEquals(input.payload.stream_options, { include_usage: true }); -}); - -test('preserves sibling stream_options keys while forcing include_usage on', async () => { - // Upstreams that pass through unknown fields can carry extra `stream_options` - // keys past our typed surface; the interceptor must not drop them when it - // flips include_usage. - const input = invocation({ - model: 'm', - messages: [], - stream_options: { extra: 'keep-me', include_usage: false } as unknown as OpenAIChatCompletionsPayload['stream_options'], - }); - - await withUsageStreamOptionsIncluded(input, stubCtx, okEvents); - - assertEquals(input.payload.stream_options as unknown as Record, { - extra: 'keep-me', - include_usage: true, - }); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/normalize-empty-tools-tool-choice_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/normalize-empty-tools-tool-choice_test.ts deleted file mode 100644 index 013f5dfb5e..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/normalize-empty-tools-tool-choice_test.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { test } from 'vitest'; - -import { withEmptyToolsToolChoiceNormalized } from '../../../../../src/data-plane/chat/openai-chat-completions/interceptors/normalize-empty-tools-tool-choice.ts'; -import type { OpenAIChatCompletionsInvocation } from '../../../../../src/data-plane/chat/openai-chat-completions/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; -import { eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); -const okEvents = () => Promise.resolve(eventResult((async function* () {})(), testTelemetryModelIdentity)); - -const invocation = ( - payload: OpenAIChatCompletionsPayload, - enabledFlags: ReadonlySet = new Set(['empty-tools-tool-choice-none']), -): OpenAIChatCompletionsInvocation => ({ - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi: 'openaiChatCompletions', - headers: new Headers(), -}); - -test('empty tools rewrites tool_choice to none', async () => { - const input = invocation({ model: 'm', messages: [], tools: [], tool_choice: 'auto' }); - - await withEmptyToolsToolChoiceNormalized(input, stubCtx, okEvents); - - assertEquals(input.payload.tools, []); - assertEquals(input.payload.tool_choice, 'none'); -}); - -test('empty tools stays unchanged when the flag is disabled', async () => { - const input = invocation({ model: 'm', messages: [], tools: [], tool_choice: 'auto' }, new Set()); - - await withEmptyToolsToolChoiceNormalized(input, stubCtx, okEvents); - - assertEquals(input.payload.tool_choice, 'auto'); -}); - -test('missing or non-empty tools does not rewrite tool_choice', async () => { - const missing = invocation({ model: 'm', messages: [], tool_choice: 'auto' }); - const present = invocation({ - model: 'm', - messages: [], - tools: [{ type: 'function', function: { name: 'x', parameters: {} } }], - tool_choice: 'auto', - }); - - await withEmptyToolsToolChoiceNormalized(missing, stubCtx, okEvents); - await withEmptyToolsToolChoiceNormalized(present, stubCtx, okEvents); - - assertEquals(missing.payload.tool_choice, 'auto'); - assertEquals(present.payload.tool_choice, 'auto'); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/normalize-exclusive-cached-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/normalize-exclusive-cached-tokens_test.ts deleted file mode 100644 index f463e451d8..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/normalize-exclusive-cached-tokens_test.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { expect, test } from 'vitest'; - -import { withExclusiveCachedTokensNormalized } from '../../../../../src/data-plane/chat/openai-chat-completions/interceptors/normalize-exclusive-cached-tokens.ts'; -import type { OpenAIChatCompletionsInvocation } from '../../../../../src/data-plane/chat/openai-chat-completions/interceptors/types.ts'; -import { billableUsageFromOpenAIChatCompletionsUsage } from '../../../../../src/data-plane/chat/openai-chat-completions/usage.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import { type ExecuteResult, eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const invocation = ( - enabledFlags: ReadonlySet = new Set(['usage-exclusive-cached-tokens']), - targetApi: OpenAIChatCompletionsInvocation['targetApi'] = 'openaiChatCompletions', -): OpenAIChatCompletionsInvocation => ({ - payload: { model: 'kimi-k3', messages: [{ role: 'user', content: 'hi' }] } satisfies OpenAIChatCompletionsPayload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi, - headers: new Headers(), -}); - -const collectFrames = async (result: ExecuteResult>): Promise[]> => { - if (result.type !== 'events') throw new Error('expected events result'); - const out: ProtocolFrame[] = []; - for await (const frame of result.events) out.push(frame); - return out; -}; - -const usageChunk = (usage: Record): OpenAIChatCompletionsStreamEvent => ({ - id: 'x', - object: 'chat.completion.chunk', - created: 0, - model: 'kimi-k3', - choices: [], - usage: usage as unknown as OpenAIChatCompletionsStreamEvent['usage'], -}); - -const run = async (ctx: OpenAIChatCompletionsInvocation, usage: Record): Promise> => { - const result = await withExclusiveCachedTokensNormalized(ctx, stubCtx, () => - Promise.resolve(eventResult( - (async function* () { - yield eventFrame(usageChunk(usage)); - })(), - testTelemetryModelIdentity, - ))); - const frames = await collectFrames(result); - assertEquals(frames.length, 1); - const frame = frames[0]; - if (frame.type !== 'event') throw new Error('expected event frame'); - return frame.event.usage as unknown as Record; -}; - -// Verbatim from a Charm Hyper kimi-k3 turn: 479 + 13312 + 373 = 14164, so -// `prompt_tokens` counts only what the cache did not serve, and total_tokens -// witnesses it. -const HYPER_USAGE = { - prompt_tokens: 479, - completion_tokens: 373, - total_tokens: 14164, - prompt_tokens_details: { cached_tokens: 13312 }, -}; - -// The same accounting with the total withheld, which is the only state the -// flag exists to settle. -const HYPER_USAGE_WITHOUT_TOTAL = { ...HYPER_USAGE, total_tokens: undefined }; - -// An ordinary OpenAI-shaped turn: 1000 + 50 = 1050, cached inside the input. -const INCLUSIVE_USAGE = { - prompt_tokens: 1000, - completion_tokens: 50, - total_tokens: 1050, - prompt_tokens_details: { cached_tokens: 400 }, -}; - -test('folds on the totals alone, with no flag set', async () => { - const usage = await run(invocation(new Set()), HYPER_USAGE); - assertEquals(usage.prompt_tokens, 13791); - // total_tokens already counted the real input, and now agrees with it. - assertEquals(usage.total_tokens, 14164); - assertEquals(usage.prompt_tokens_details, { cached_tokens: 13312 }); -}); - -test('folds on the flag when the totals witness nothing', async () => { - const usage = await run(invocation(), HYPER_USAGE_WITHOUT_TOTAL); - assertEquals(usage.prompt_tokens, 13791); -}); - -test('leaves a normalized chunk billable instead of underflowing', async () => { - const raw = HYPER_USAGE as unknown as NonNullable; - expect(() => billableUsageFromOpenAIChatCompletionsUsage(raw, null)).toThrowError(RangeError); - - const usage = await run(invocation(new Set()), HYPER_USAGE); - assertEquals( - billableUsageFromOpenAIChatCompletionsUsage(usage as unknown as NonNullable, null), - { input: 479, cacheRead: 13312, cacheWrite: 0, cacheWrite1h: 0, output: 373 }, - ); -}); - -test('folds cache writes back as well', async () => { - const usage = await run(invocation(), { - prompt_tokens: 100, - completion_tokens: 10, - prompt_tokens_details: { cached_tokens: 120, cache_creation_input_tokens: 80 }, - }); - assertEquals(usage.prompt_tokens, 300); -}); - -test('leaves an inclusive chunk alone when no flag claims otherwise', async () => { - const usage = await run(invocation(new Set()), INCLUSIVE_USAGE); - assertEquals(usage.prompt_tokens, 1000); -}); - -test('leaves a chunk with no cache counts untouched', async () => { - const usage = await run(invocation(), { prompt_tokens: 40, completion_tokens: 1, total_tokens: 41 }); - assertEquals(usage.prompt_tokens, 40); -}); - -test('raises when the flag claims exclusive and the totals say inclusive', async () => { - await expect(run(invocation(), INCLUSIVE_USAGE)).rejects.toThrowError(/usage-exclusive-cached-tokens is enabled/); -}); - -test('raises naming the flag when the cache counts underflow with no verdict', async () => { - await expect(run(invocation(new Set()), HYPER_USAGE_WITHOUT_TOTAL)) - .rejects.toThrowError(/enable usage-exclusive-cached-tokens/); -}); - -test('stands down entirely when the wire it speaks about is elsewhere', async () => { - // A OpenAI Chat Completions request whose upstream speaks OpenAI Responses: the OpenAI Responses - // entry owns that wire, and by the time these events reach this chain they - // are a translation. Neither the fold nor its errors apply here, so even a - // payload that would raise on the wire passes through untouched. - const usage = await run(invocation(undefined, 'openaiResponses'), HYPER_USAGE_WITHOUT_TOTAL); - assertEquals(usage.prompt_tokens, 479); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/normalize-usage_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/normalize-usage_test.ts deleted file mode 100644 index 4dfdd2c4d8..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/normalize-usage_test.ts +++ /dev/null @@ -1,149 +0,0 @@ -import { test } from 'vitest'; - -import { withUsageNormalized } from '../../../../../src/data-plane/chat/openai-chat-completions/interceptors/normalize-usage.ts'; -import type { OpenAIChatCompletionsInvocation } from '../../../../../src/data-plane/chat/openai-chat-completions/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import { type ExecuteResult, eventResult } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const invocation = (payload: OpenAIChatCompletionsPayload = { model: 'test-model', messages: [] }): OpenAIChatCompletionsInvocation => ({ - payload, - candidate: stubModelCandidate(), - targetApi: 'openaiChatCompletions', - headers: new Headers(), -}); - -const collectFrames = async (result: ExecuteResult>): Promise[]> => { - if (result.type !== 'events') throw new Error('expected events result'); - const out: ProtocolFrame[] = []; - for await (const frame of result.events) out.push(frame); - return out; -}; - -const runWithFrames = async (...frames: ProtocolFrame[]): Promise[]> => { - const result = await withUsageNormalized(invocation(), stubCtx, () => - Promise.resolve( - eventResult( - (async function* () { - for (const frame of frames) yield frame; - })(), - testTelemetryModelIdentity, - ), - )); - return await collectFrames(result); -}; - -const usageRecord = (usage: NonNullable): Record => usage as unknown as Record; - -test('leaves a spec-compliant carrier usage chunk untouched', async () => { - const frames = await runWithFrames( - eventFrame({ - id: 'x', - object: 'chat.completion.chunk', - created: 0, - model: 'gpt-test', - choices: [], - usage: { - prompt_tokens: 100, - completion_tokens: 20, - total_tokens: 120, - prompt_tokens_details: { cached_tokens: 60, audio_tokens: 0 }, - } as unknown as OpenAIChatCompletionsStreamEvent['usage'], - }), - ); - - assertEquals(frames.length, 1); - const frame = frames[0]; - if (frame.type !== 'event') throw new Error('expected event frame'); - const usage = usageRecord(frame.event.usage!); - assertEquals(usage.prompt_tokens_details, { cached_tokens: 60, audio_tokens: 0 }); -}); - -test('relocates usage from a non-empty choices chunk onto a synthesized carrier', async () => { - const frames = await runWithFrames( - eventFrame({ - id: 'chatcmpl_1', - object: 'chat.completion.chunk', - created: 1, - model: 'gpt-test', - choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], - usage: { - prompt_tokens: 100, - completion_tokens: 20, - total_tokens: 120, - prompt_tokens_details: { cached_tokens: 70 }, - } as unknown as OpenAIChatCompletionsStreamEvent['usage'], - }), - ); - - assertEquals(frames.length, 2); - - const first = frames[0]; - if (first.type !== 'event') throw new Error('expected event frame'); - assertEquals(first.event.choices, [{ index: 0, delta: {}, finish_reason: 'stop' }]); - assertEquals(first.event.usage, undefined); - - const carrier = frames[1]; - if (carrier.type !== 'event') throw new Error('expected event frame'); - assertEquals(carrier.event.id, 'chatcmpl_1'); - assertEquals(carrier.event.model, 'gpt-test'); - assertEquals(carrier.event.choices, []); - const usage = usageRecord(carrier.event.usage!); - assertEquals(usage.prompt_tokens, 100); - assertEquals(usage.prompt_tokens_details, { cached_tokens: 70 }); -}); - -test('passes raw vendor cache fields through unchanged so vendor interceptors stay responsible for the rewrite', async () => { - const frames = await runWithFrames( - eventFrame({ - id: 'x', - object: 'chat.completion.chunk', - created: 0, - model: 'unknown-vendor', - choices: [], - usage: { - prompt_tokens: 100, - completion_tokens: 20, - total_tokens: 120, - prompt_cache_hit_tokens: 70, - cached_tokens: 25, - } as unknown as OpenAIChatCompletionsStreamEvent['usage'], - }), - ); - - assertEquals(frames.length, 1); - const frame = frames[0]; - if (frame.type !== 'event') throw new Error('expected event frame'); - const usage = usageRecord(frame.event.usage!); - assertEquals(usage.prompt_cache_hit_tokens, 70); - assertEquals(usage.cached_tokens, 25); - assertEquals('prompt_tokens_details' in usage, false); -}); - -test('leaves chunks without usage untouched', async () => { - const original = eventFrame({ - id: 'chatcmpl_3', - object: 'chat.completion.chunk', - created: 1, - model: 'gpt-test', - choices: [ - { index: 0, delta: { content: 'hi' }, finish_reason: null }, - ], - } satisfies OpenAIChatCompletionsStreamEvent); - - const frames = await runWithFrames(original); - - assertEquals(frames, [original]); -}); - -test('passes protocol done frames through verbatim', async () => { - const done = doneFrame(); - - const frames = await runWithFrames(done); - - assertEquals(frames, [done]); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/strip-prompt-cache-key_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/strip-prompt-cache-key_test.ts deleted file mode 100644 index b9cfe8dac3..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/strip-prompt-cache-key_test.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { test } from 'vitest'; - -import { withPromptCacheKeyStripped } from '../../../../../src/data-plane/chat/openai-chat-completions/interceptors/strip-prompt-cache-key.ts'; -import type { OpenAIChatCompletionsInvocation } from '../../../../../src/data-plane/chat/openai-chat-completions/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; -import { eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = () => Promise.resolve(eventResult((async function* () {})(), testTelemetryModelIdentity)); - -const invocation = ( - payload: OpenAIChatCompletionsPayload, - enabledFlags: ReadonlySet = new Set(['strip-prompt-cache-key']), -): OpenAIChatCompletionsInvocation => ({ - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi: 'openaiChatCompletions', - headers: new Headers(), -}); - -test('drops top-level prompt_cache_key when the flag is on', async () => { - const input = invocation({ - model: 'm', - messages: [], - prompt_cache_key: 'thread-42', - }); - - await withPromptCacheKeyStripped(input, stubCtx, okEvents); - - assertEquals(Object.hasOwn(input.payload, 'prompt_cache_key'), false); -}); - -test('leaves the payload untouched when prompt_cache_key is absent', async () => { - const input = invocation({ model: 'm', messages: [] }); - const before = input.payload; - - await withPromptCacheKeyStripped(input, stubCtx, okEvents); - - assertEquals(input.payload, before); -}); - -test('is a no-op when the flag is not set on the candidate', async () => { - const input = invocation( - { model: 'm', messages: [], prompt_cache_key: 'thread-42' }, - new Set(), - ); - - await withPromptCacheKeyStripped(input, stubCtx, okEvents); - - assertEquals(input.payload.prompt_cache_key, 'thread-42'); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/vendor-deepseek-normalize_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/vendor-deepseek-normalize_test.ts deleted file mode 100644 index a891acdf28..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/vendor-deepseek-normalize_test.ts +++ /dev/null @@ -1,393 +0,0 @@ -import { expect, test } from 'vitest'; - -import type { OpenAIChatCompletionsInvocation } from '../../../../../src/data-plane/chat/openai-chat-completions/interceptors/types.ts'; -import { withVendorDeepSeekOpenAIChatCompletionsNormalize } from '../../../../../src/data-plane/chat/openai-chat-completions/interceptors/vendor-deepseek-normalize.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import { type ExecuteResult, eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -type DeepSeekReasoningDelta = OpenAIChatCompletionsStreamEvent['choices'][number]['delta'] & { - reasoning_content?: string; -}; - -const stubCtx = mockChatGatewayCtx(); - -const invocation = (payload: OpenAIChatCompletionsPayload, enabledFlags: ReadonlySet = new Set(['vendor-deepseek'])): OpenAIChatCompletionsInvocation => ({ - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi: 'openaiChatCompletions', - headers: new Headers(), -}); - -const baseRequest = (): OpenAIChatCompletionsPayload => ({ - model: 'deepseek-reasoner', - messages: [ - { role: 'user', content: 'first turn' }, - { - role: 'assistant', - content: null, - reasoning_text: 'let me check the docs', - reasoning_opaque: 'opaque-blob', - reasoning_items: [{ type: 'reasoning', summary: [] }], - tool_calls: [ - { id: 'call_1', type: 'function', function: { name: 'lookup', arguments: '{}' } }, - ], - }, - { role: 'tool', tool_call_id: 'call_1', content: 'result' }, - { role: 'user', content: 'next turn' }, - ], -}); - -const collectFrames = async (result: ExecuteResult>): Promise[]> => { - if (result.type !== 'events') throw new Error('expected events result'); - const out: ProtocolFrame[] = []; - for await (const frame of result.events) out.push(frame); - return out; -}; - -const usageRecord = (usage: NonNullable): Record => usage as unknown as Record; - -const okEvents = () => Promise.resolve(eventResult((async function* () {})(), testTelemetryModelIdentity)); - -// ── Outbound: assistant reasoning field rewrite ── - -test('renames outbound reasoning_text to reasoning_content on assistant messages', async () => { - const ctx = invocation(baseRequest()); - - let observed: OpenAIChatCompletionsPayload | null = null; - await withVendorDeepSeekOpenAIChatCompletionsNormalize(ctx, stubCtx, () => { - observed = ctx.payload; - return okEvents(); - }); - - const assistant = observed!.messages[1] as unknown as Record; - assertEquals(assistant.reasoning_content, 'let me check the docs'); - assertEquals(assistant.reasoning_text, undefined); - assertEquals(assistant.reasoning_opaque, undefined); - assertEquals(assistant.reasoning_items, undefined); - assertEquals((assistant.tool_calls as unknown[]).length, 1); -}); - -test('synthesizes reasoning_content from reasoning_items when reasoning_text is absent', async () => { - const ctx = invocation({ - model: 'deepseek-reasoner', - messages: [ - { role: 'user', content: 'first turn' }, - { - role: 'assistant', - content: null, - reasoning_items: [ - { - type: 'reasoning', - id: 'rs_1', - summary: [ - { type: 'summary_text', text: 'step one. ' }, - { type: 'summary_text', text: 'step two.' }, - ], - }, - ], - tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'lookup', arguments: '{}' } }], - }, - { role: 'tool', tool_call_id: 'call_1', content: 'result' }, - ], - }); - - let observed: OpenAIChatCompletionsPayload | null = null; - await withVendorDeepSeekOpenAIChatCompletionsNormalize(ctx, stubCtx, () => { - observed = ctx.payload; - return okEvents(); - }); - - const assistant = observed!.messages[1] as unknown as Record; - assertEquals(assistant.reasoning_content, 'step one. step two.'); - assertEquals(assistant.reasoning_text, undefined); - assertEquals(assistant.reasoning_opaque, undefined); - assertEquals(assistant.reasoning_items, undefined); -}); - -test('strips reasoning_items even when no summaries are available', async () => { - const ctx = invocation({ - model: 'deepseek-reasoner', - messages: [ - { role: 'user', content: 'first turn' }, - { - role: 'assistant', - content: 'answer', - reasoning_items: [{ type: 'reasoning' }], - reasoning_opaque: 'opaque-chain', - }, - ], - }); - - let observed: OpenAIChatCompletionsPayload | null = null; - await withVendorDeepSeekOpenAIChatCompletionsNormalize(ctx, stubCtx, () => { - observed = ctx.payload; - return okEvents(); - }); - - const assistant = observed!.messages[1] as unknown as Record; - assertEquals(assistant.reasoning_content, undefined); - assertEquals(assistant.reasoning_items, undefined); - assertEquals(assistant.reasoning_opaque, undefined); - assertEquals(assistant.content, 'answer'); -}); - -// ── Outbound: reasoning_effort === 'none' canonical sentinel ── - -test("translates canonical reasoning_effort: 'none' into top-level thinking:{type:'disabled'}", async () => { - const ctx = invocation({ - model: 'deepseek-reasoner', - messages: [{ role: 'user', content: 'hi' }], - reasoning_effort: 'none', - }); - - let observed: OpenAIChatCompletionsPayload | null = null; - await withVendorDeepSeekOpenAIChatCompletionsNormalize(ctx, stubCtx, () => { - observed = ctx.payload; - return okEvents(); - }); - - const out = observed! as unknown as Record; - assertEquals(out.reasoning_effort, undefined); - assertEquals(out.thinking, { type: 'disabled' }); -}); - -test('leaves a real reasoning_effort value untouched (only the none sentinel triggers the rewrite)', async () => { - const ctx = invocation({ - model: 'deepseek-reasoner', - messages: [{ role: 'user', content: 'hi' }], - reasoning_effort: 'high', - }); - - let observed: OpenAIChatCompletionsPayload | null = null; - await withVendorDeepSeekOpenAIChatCompletionsNormalize(ctx, stubCtx, () => { - observed = ctx.payload; - return okEvents(); - }); - - assertEquals(observed!.reasoning_effort, 'high'); - const out = observed! as unknown as Record; - assertEquals(out.thinking, undefined); -}); - -// ── Outbound: structured-output json_schema downgrade ── - -test('downgrades response_format json_schema to json_object (schema body dropped)', async () => { - const ctx = invocation({ - model: 'deepseek-chat', - messages: [{ role: 'user', content: 'hi' }], - response_format: { - type: 'json_schema', - json_schema: { - name: 'messages_response', - strict: true, - schema: { type: 'object', properties: { test: { type: 'string' } }, required: ['test'], additionalProperties: false }, - }, - }, - }); - - let observed: OpenAIChatCompletionsPayload | null = null; - await withVendorDeepSeekOpenAIChatCompletionsNormalize(ctx, stubCtx, () => { - observed = ctx.payload; - return okEvents(); - }); - - assertEquals(observed!.response_format, { type: 'json_object' }); -}); - -test('leaves an already-json_object response_format untouched', async () => { - const ctx = invocation({ - model: 'deepseek-chat', - messages: [{ role: 'user', content: 'hi' }], - response_format: { type: 'json_object' }, - }); - - let observed: OpenAIChatCompletionsPayload | null = null; - await withVendorDeepSeekOpenAIChatCompletionsNormalize(ctx, stubCtx, () => { - observed = ctx.payload; - return okEvents(); - }); - - assertEquals(observed!.response_format, { type: 'json_object' }); -}); - -// ── Inbound: delta reasoning_content → reasoning_text ── - -test('renames inbound protocol reasoning_content deltas to reasoning_text', async () => { - const ctx = invocation(baseRequest()); - const upstreamChunk: OpenAIChatCompletionsStreamEvent = { - id: 'chunk_1', - object: 'chat.completion.chunk', - created: 1, - model: 'deepseek-reasoner', - choices: [ - { - index: 0, - delta: { reasoning_content: 'thinking...' } as DeepSeekReasoningDelta, - finish_reason: null, - }, - ], - }; - - const result = await withVendorDeepSeekOpenAIChatCompletionsNormalize(ctx, stubCtx, () => - Promise.resolve(eventResult( - (async function* () { yield eventFrame(upstreamChunk); })(), - testTelemetryModelIdentity, - ))); - - const frames = await collectFrames(result); - assertEquals(frames.length, 1); - const frame = frames[0]; - if (frame.type !== 'event') throw new Error('expected event frame'); - const delta = frame.event.choices[0].delta as Record; - assertEquals(delta.reasoning_text, 'thinking...'); - assertEquals(delta.reasoning_content, undefined); -}); - -test('preserves reasoning_content from non-stream JSON responses', async () => { - const ctx = invocation(baseRequest()); - const id = 'chatcmpl_deepseek_json'; - const model = 'deepseek-reasoner'; - const chunk = (delta: OpenAIChatCompletionsStreamEvent['choices'][number]['delta'], finish_reason: 'stop' | null = null): OpenAIChatCompletionsStreamEvent => ({ - id, - object: 'chat.completion.chunk', - created: 1, - model, - choices: [{ index: 0, delta, finish_reason }], - }); - - const result = await withVendorDeepSeekOpenAIChatCompletionsNormalize(ctx, stubCtx, () => - Promise.resolve(eventResult( - (async function* () { - yield eventFrame(chunk({ role: 'assistant' })); - yield eventFrame(chunk({ reasoning_content: 'json thinking' } as OpenAIChatCompletionsStreamEvent['choices'][number]['delta'])); - yield eventFrame(chunk({ content: 'answer' })); - yield eventFrame(chunk({}, 'stop')); - yield doneFrame(); - })(), - testTelemetryModelIdentity, - ))); - - const frames = await collectFrames(result); - const reasoningFrame = frames.find(frame => frame.type === 'event' && frame.event.choices[0]?.delta.reasoning_text !== undefined); - assertEquals(reasoningFrame?.type === 'event' ? reasoningFrame.event.choices[0]?.delta.reasoning_text : undefined, 'json thinking'); -}); - -// ── Inbound: usage cache-token field rewrite ── - -test('rewrites prompt_cache_hit_tokens/prompt_cache_miss_tokens into prompt_tokens_details.cached_tokens', async () => { - const ctx = invocation(baseRequest()); - const result = await withVendorDeepSeekOpenAIChatCompletionsNormalize(ctx, stubCtx, () => - Promise.resolve(eventResult( - (async function* () { - yield eventFrame({ - id: 'x', - object: 'chat.completion.chunk', - created: 0, - model: 'deepseek-test', - choices: [], - usage: { - prompt_tokens: 100, - completion_tokens: 20, - total_tokens: 120, - prompt_cache_hit_tokens: 70, - prompt_cache_miss_tokens: 30, - prompt_tokens_details: { upstream_metric: 'preserved' }, - } as unknown as OpenAIChatCompletionsStreamEvent['usage'], - }); - })(), - testTelemetryModelIdentity, - ))); - - const frames = await collectFrames(result); - assertEquals(frames.length, 1); - const frame = frames[0]; - if (frame.type !== 'event') throw new Error('expected event frame'); - const usage = usageRecord(frame.event.usage!); - assertEquals(usage.prompt_tokens, 100); - assertEquals(usage.prompt_tokens_details, { upstream_metric: 'preserved', cached_tokens: 70 }); - assertEquals('prompt_cache_hit_tokens' in usage, false); - assertEquals('prompt_cache_miss_tokens' in usage, false); -}); - -test('replaces array-shaped prompt_tokens_details without copying array indices', async () => { - const ctx = invocation(baseRequest()); - const result = await withVendorDeepSeekOpenAIChatCompletionsNormalize(ctx, stubCtx, () => - Promise.resolve(eventResult( - (async function* () { - yield eventFrame({ - id: 'x', - object: 'chat.completion.chunk', - created: 0, - model: 'deepseek-test', - choices: [], - usage: { - prompt_tokens: 100, - completion_tokens: 20, - total_tokens: 120, - prompt_cache_hit_tokens: 70, - prompt_cache_miss_tokens: 30, - prompt_tokens_details: [{ cached_tokens: 1 }], - } as unknown as OpenAIChatCompletionsStreamEvent['usage'], - }); - })(), - testTelemetryModelIdentity, - ))); - - const frames = await collectFrames(result); - const frame = frames[0]; - if (frame.type !== 'event') throw new Error('expected event frame'); - const usage = usageRecord(frame.event.usage!); - assertEquals(usage.prompt_tokens_details, { cached_tokens: 70 }); -}); - -test('propagates upstream stream errors unchanged', async () => { - const failure = new Error('upstream stream failed'); - const result = await withVendorDeepSeekOpenAIChatCompletionsNormalize(invocation(baseRequest()), stubCtx, () => - Promise.resolve(eventResult( - (async function* () { - throw failure; - })(), - testTelemetryModelIdentity, - ))); - - await expect(collectFrames(result)).rejects.toBe(failure); -}); - -// ── Pass-through ── - -test('leaves protocol done frames untouched', async () => { - const ctx = invocation(baseRequest()); - const done = doneFrame(); - - const result = await withVendorDeepSeekOpenAIChatCompletionsNormalize(ctx, stubCtx, () => - Promise.resolve(eventResult( - (async function* () { yield done; })(), - testTelemetryModelIdentity, - ))); - - assertEquals(await collectFrames(result), [done]); -}); - -test('early-returns when its flag is not set on the candidate', async () => { - const ctx = invocation( - { - model: 'deepseek-reasoner', - messages: [{ role: 'user', content: 'hi' }], - reasoning_effort: 'none', - }, - new Set(), - ); - - let observed: OpenAIChatCompletionsPayload | null = null; - await withVendorDeepSeekOpenAIChatCompletionsNormalize(ctx, stubCtx, () => { - observed = ctx.payload; - return okEvents(); - }); - - assertEquals(observed!.reasoning_effort, 'none'); - const out = observed! as unknown as Record; - assertEquals(out.thinking, undefined); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/vendor-kimi-normalize_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/vendor-kimi-normalize_test.ts deleted file mode 100644 index da35e75a2c..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/vendor-kimi-normalize_test.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { expect, test } from 'vitest'; - -import type { OpenAIChatCompletionsInvocation } from '../../../../../src/data-plane/chat/openai-chat-completions/interceptors/types.ts'; -import { withVendorKimiOpenAIChatCompletionsNormalize } from '../../../../../src/data-plane/chat/openai-chat-completions/interceptors/vendor-kimi-normalize.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import { type ExecuteResult, eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const invocation = (payload: OpenAIChatCompletionsPayload, enabledFlags: ReadonlySet = new Set(['vendor-kimi'])): OpenAIChatCompletionsInvocation => ({ - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi: 'openaiChatCompletions', - headers: new Headers(), -}); - -const collectFrames = async (result: ExecuteResult>): Promise[]> => { - if (result.type !== 'events') throw new Error('expected events result'); - const out: ProtocolFrame[] = []; - for await (const frame of result.events) out.push(frame); - return out; -}; - -const usageRecord = (usage: NonNullable): Record => usage as unknown as Record; - -const baseRequest = (): OpenAIChatCompletionsPayload => ({ model: 'kimi-k2', messages: [{ role: 'user', content: 'hi' }] }); - -test('rewrites flat cached_tokens into prompt_tokens_details.cached_tokens', async () => { - const ctx = invocation(baseRequest()); - const result = await withVendorKimiOpenAIChatCompletionsNormalize(ctx, stubCtx, () => - Promise.resolve(eventResult( - (async function* () { - yield eventFrame({ - id: 'x', - object: 'chat.completion.chunk', - created: 0, - model: 'kimi-test', - choices: [], - usage: { - prompt_tokens: 100, - completion_tokens: 20, - total_tokens: 120, - cached_tokens: 50, - prompt_tokens_details: { upstream_metric: 'preserved' }, - } as unknown as OpenAIChatCompletionsStreamEvent['usage'], - }); - })(), - testTelemetryModelIdentity, - ))); - - const frames = await collectFrames(result); - assertEquals(frames.length, 1); - const frame = frames[0]; - if (frame.type !== 'event') throw new Error('expected event frame'); - const usage = usageRecord(frame.event.usage!); - assertEquals(usage.prompt_tokens_details, { upstream_metric: 'preserved', cached_tokens: 50 }); - assertEquals('cached_tokens' in usage, false); -}); - -test('replaces array-shaped prompt_tokens_details without copying array indices', async () => { - const ctx = invocation(baseRequest()); - const result = await withVendorKimiOpenAIChatCompletionsNormalize(ctx, stubCtx, () => - Promise.resolve(eventResult( - (async function* () { - yield eventFrame({ - id: 'x', - object: 'chat.completion.chunk', - created: 0, - model: 'kimi-test', - choices: [], - usage: { - prompt_tokens: 100, - completion_tokens: 20, - total_tokens: 120, - cached_tokens: 50, - prompt_tokens_details: [{ cached_tokens: 1 }], - } as unknown as OpenAIChatCompletionsStreamEvent['usage'], - }); - })(), - testTelemetryModelIdentity, - ))); - - const frames = await collectFrames(result); - const frame = frames[0]; - if (frame.type !== 'event') throw new Error('expected event frame'); - const usage = usageRecord(frame.event.usage!); - assertEquals(usage.prompt_tokens_details, { cached_tokens: 50 }); -}); - -test('propagates upstream stream errors unchanged', async () => { - const failure = new Error('upstream stream failed'); - const result = await withVendorKimiOpenAIChatCompletionsNormalize(invocation(baseRequest()), stubCtx, () => - Promise.resolve(eventResult( - (async function* () { - throw failure; - })(), - testTelemetryModelIdentity, - ))); - - await expect(collectFrames(result)).rejects.toBe(failure); -}); - -test('early-returns when its flag is not set on the candidate', async () => { - const ctx = invocation(baseRequest(), new Set()); - const result = await withVendorKimiOpenAIChatCompletionsNormalize(ctx, stubCtx, () => - Promise.resolve(eventResult( - (async function* () { - yield eventFrame({ - id: 'x', - object: 'chat.completion.chunk', - created: 0, - model: 'kimi-test', - choices: [], - usage: { - prompt_tokens: 100, - completion_tokens: 20, - total_tokens: 120, - cached_tokens: 50, - } as unknown as OpenAIChatCompletionsStreamEvent['usage'], - }); - })(), - testTelemetryModelIdentity, - ))); - - const frames = await collectFrames(result); - const frame = frames[0]; - if (frame.type !== 'event') throw new Error('expected event frame'); - const usage = usageRecord(frame.event.usage!); - assertEquals(usage.cached_tokens, 50); - assertEquals('prompt_tokens_details' in usage, false); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/vendor-qwen-normalize_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/vendor-qwen-normalize_test.ts deleted file mode 100644 index d3367723d2..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/interceptors/vendor-qwen-normalize_test.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { test } from 'vitest'; - -import type { OpenAIChatCompletionsInvocation } from '../../../../../src/data-plane/chat/openai-chat-completions/interceptors/types.ts'; -import { withVendorQwenOpenAIChatCompletionsNormalize } from '../../../../../src/data-plane/chat/openai-chat-completions/interceptors/vendor-qwen-normalize.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; -import { eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const invocation = (payload: OpenAIChatCompletionsPayload, enabledFlags: ReadonlySet = new Set(['vendor-qwen'])): OpenAIChatCompletionsInvocation => ({ - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi: 'openaiChatCompletions', - headers: new Headers(), -}); - -const okEvents = () => Promise.resolve(eventResult((async function* () {})(), testTelemetryModelIdentity)); - -test("translates canonical reasoning_effort: 'none' into top-level enable_thinking:false", async () => { - const ctx = invocation({ - model: 'qwen-max', - messages: [{ role: 'user', content: 'hi' }], - reasoning_effort: 'none', - }); - - let observed: OpenAIChatCompletionsPayload | null = null; - await withVendorQwenOpenAIChatCompletionsNormalize(ctx, stubCtx, () => { - observed = ctx.payload; - return okEvents(); - }); - - const out = observed! as unknown as Record; - assertEquals(out.reasoning_effort, undefined); - assertEquals(out.enable_thinking, false); -}); - -test('leaves a real reasoning_effort value untouched (only the none sentinel triggers the rewrite)', async () => { - const ctx = invocation({ - model: 'qwen-max', - messages: [{ role: 'user', content: 'hi' }], - reasoning_effort: 'high', - }); - - let observed: OpenAIChatCompletionsPayload | null = null; - await withVendorQwenOpenAIChatCompletionsNormalize(ctx, stubCtx, () => { - observed = ctx.payload; - return okEvents(); - }); - - assertEquals(observed!.reasoning_effort, 'high'); - const out = observed! as unknown as Record; - assertEquals(out.enable_thinking, undefined); -}); - -test('early-returns when its flag is not set on the candidate', async () => { - const ctx = invocation( - { - model: 'qwen-max', - messages: [{ role: 'user', content: 'hi' }], - reasoning_effort: 'none', - }, - new Set(), - ); - - let observed: OpenAIChatCompletionsPayload | null = null; - await withVendorQwenOpenAIChatCompletionsNormalize(ctx, stubCtx, () => { - observed = ctx.payload; - return okEvents(); - }); - - assertEquals(observed!.reasoning_effort, 'none'); - const out = observed! as unknown as Record; - assertEquals(out.enable_thinking, undefined); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/respond_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/respond_test.ts deleted file mode 100644 index 8302558221..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/respond_test.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { Hono } from 'hono'; -import { expect, test } from 'vitest'; - -import { respondOpenAIChatCompletions } from '../../../../src/data-plane/chat/openai-chat-completions/respond.ts'; -import type { DumpAccumulator } from '../../../../src/dump/accumulator.ts'; -import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import { eventResult } from '@floway-dev/provider'; -import { assert, assertEquals, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const recordingDump = () => { - const frames: ProtocolFrame[] = []; - return { - frames, - dump: { - failed: () => {}, - success: () => {}, - frame: (frame: ProtocolFrame) => { frames.push(frame); }, - } as unknown as DumpAccumulator, - }; -}; - -const chunk = (text: string): OpenAIChatCompletionsStreamEvent => ({ - id: 'x', object: 'chat.completion.chunk', created: 0, model: 'm', - choices: [{ index: 0, delta: { content: text }, finish_reason: null }], -}); - -const serve = async (dump: DumpAccumulator, frames: AsyncGenerator>): Promise => { - const ctx = mockChatGatewayCtx({ wantsStream: true, dump }); - const app = new Hono().get('/', c => - respondOpenAIChatCompletions(c, eventResult(frames, testTelemetryModelIdentity), true, true, ctx)); - return await (await app.request('/')).text(); -}; - -test('the error frame the client is sent is recorded like every other frame', async () => { - const { dump, frames } = recordingDump(); - const body = await serve(dump, (async function* () { - yield eventFrame(chunk('hi')); - throw new RangeError('cache token counts exceed inclusive input tokens: 479 - 13312 - 0'); - })()); - - // What the client receives is unchanged by this. - expect(body).toContain('event: error'); - expect(body).toContain('cache token counts exceed inclusive input tokens'); - expect(body).not.toContain('data: [DONE]'); - - // And the recorded turn now ends on the same event rather than on the last - // good chunk. - assertEquals(frames.length, 2); - const last = frames[1]; - assert(last.type === 'event', 'expected an event frame'); - const payload = last.event as unknown as { error?: { message?: string } }; - assertEquals(payload.error?.message, 'cache token counts exceed inclusive input tokens: 479 - 13312 - 0'); -}); - -test('a stream that completes records only its own frames', async () => { - const { dump, frames } = recordingDump(); - const body = await serve(dump, (async function* () { - yield eventFrame(chunk('hi')); - yield doneFrame(); - })()); - - expect(body).toContain('data: [DONE]'); - assertEquals(frames.length, 2); -}); - -test('natural EOF forwards useful content without a missing-DONE error', async () => { - const { dump, frames } = recordingDump(); - const body = await serve(dump, (async function* () { yield eventFrame(chunk('usable')); })()); - expect(body).toContain('usable'); - expect(body).not.toContain('event: error'); - expect(frames).toHaveLength(1); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/serve_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/serve_test.ts deleted file mode 100644 index 3c3a9609e5..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/serve_test.ts +++ /dev/null @@ -1,398 +0,0 @@ -import { afterEach, test, vi } from 'vitest'; - -import { initRepo } from '../../../../src/repo/index.ts'; -import { InMemoryRepo } from '../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import { type AliasRules, doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import { type ModelCandidate, directFetcher, type ProviderStreamResult, type UpstreamCallOptions } from '@floway-dev/provider'; -import { assert, assertEquals, stubProvider, stubInternalModel } from '@floway-dev/test-utils'; - -// Mock the resolver seam so each test hands the serve exactly the model -// candidates it wants, optionally with an alias-rules overlay attached. -interface QueuedResolution { - readonly candidates: readonly ModelCandidate[]; - readonly sawModel: boolean; - readonly failedUpstreams: readonly string[]; -} -const resolutionsQueue: QueuedResolution[] = []; -const lastResolveCall: { model?: string } = {}; -vi.mock('../../../../src/data-plane/providers/resolution.ts', async importOriginal => { - const original = await importOriginal(); - return { - ...original, - enumerateModelCandidates: vi.fn(async ({ model }: { model: string }) => { - lastResolveCall.model = model; - const next = resolutionsQueue.shift(); - if (next === undefined) throw new Error('serve_test: no resolution enqueued'); - return next; - }), - }; -}); - -const { openaiChatCompletionsServe } = await import('../../../../src/data-plane/chat/openai-chat-completions/serve.ts'); - -const API_KEY_ID = 'key_chat_completions_serve_test'; - -const queueResolution = ( - candidates: readonly ModelCandidate[], - extra: { sawModel?: boolean; aliasRules?: AliasRules } = {}, -): void => { - const rules = extra.aliasRules; - resolutionsQueue.push({ - candidates: rules !== undefined ? candidates.map(c => ({ ...c, rules })) : candidates, - sawModel: extra.sawModel ?? candidates.length > 0, - failedUpstreams: [], - }); -}; - -afterEach(() => { resolutionsQueue.length = 0; }); - -const installRepo = (): InMemoryRepo => { - const repo = new InMemoryRepo(); - initRepo(repo); - return repo; -}; - -const makeGatewayCtx = () => mockChatGatewayCtx({ apiKeyId: API_KEY_ID, wantsStream: true }); - -const makePayload = (overrides: Partial = {}): OpenAIChatCompletionsPayload => ({ - model: 'test-model', - messages: [{ role: 'user', content: 'hello' }], - ...overrides, -}); - -const makeOpenAIChatCompletionsEvents = (): readonly OpenAIChatCompletionsStreamEvent[] => [ - { - id: 'chatcmpl_test', object: 'chat.completion.chunk', created: 0, model: 'test-model', - choices: [{ index: 0, delta: { role: 'assistant' }, finish_reason: null }], - }, - { - id: 'chatcmpl_test', object: 'chat.completion.chunk', created: 0, model: 'test-model', - choices: [{ index: 0, delta: { content: 'hi' }, finish_reason: null }], - }, - { - id: 'chatcmpl_test', object: 'chat.completion.chunk', created: 0, model: 'test-model', - choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], - }, -]; - -const makeProtocolFrames = async function* (events: readonly TEvent[]): AsyncGenerator> { - for (const event of events) yield eventFrame(event); - yield doneFrame(); -}; - -const makeCandidate = (overrides: { - upstream?: string; - modelId?: string; - endpoints?: ModelEndpoints; - callOpenAIChatCompletions?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise>; -} = {}): ModelCandidate => { - const upstream = overrides.upstream ?? 'up_test'; - const provider = stubProvider({ - callOpenAIChatCompletions: overrides.callOpenAIChatCompletions, - }); - return { - provider: { - upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: provider, - }, - model: stubInternalModel({ - id: overrides.modelId ?? 'test-model', - ...(overrides.endpoints ? { endpoints: overrides.endpoints } : {}), - }, upstream), - fetcher: directFetcher, - }; -}; - -const collectEvents = async (events: AsyncIterable>): Promise => { - const out: TEvent[] = []; - for await (const frame of events) { - if (frame.type === 'event') out.push(frame.event); - } - return out; -}; - -test('generate routes a native OpenAI Chat Completions candidate end to end', async () => { - installRepo(); - const callOpenAIChatCompletions = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeOpenAIChatCompletionsEvents()), modelKey: 'test-model-key', headers: new Headers(), - })); - queueResolution([makeCandidate({ upstream: 'up_a', callOpenAIChatCompletions })]); - - const result = await openaiChatCompletionsServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - const events = await collectEvents(result.events); - assert(events.length >= 1); - assertEquals(callOpenAIChatCompletions.mock.calls.length, 1); -}); - -test('generate filters out candidates that do not expose any chat-completions-target endpoint', async () => { - installRepo(); - const callOpenAIChatCompletions = vi.fn(); - // `openaiCompletions:{}` is not in the openaiChatCompletionsTarget preference list - // (`openai-chat-completions` > `messages` > `responses`), so the picker rejects - // this candidate. - queueResolution([makeCandidate({ upstream: 'up_m', endpoints: { openaiCompletions: {} }, callOpenAIChatCompletions })]); - - const result = await openaiChatCompletionsServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - // canServe drops messages-only candidates; with no viable candidate the - // serve renders model-unsupported as a 400 api-error (distinct from the - // model-missing 404) without ever reaching the upstream call. - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 400); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assert(typeof body.error.message === 'string' && body.error.message.includes('does not support')); - assertEquals(callOpenAIChatCompletions.mock.calls.length, 0); -}); - -test('generate falls through to the next candidate when the first yields an upstream error', async () => { - installRepo(); - const originalImageUrl = 'data:image/png;base64,AQID'; - const payload = makePayload({ - messages: [{ - role: 'user', - content: [{ type: 'image_url', image_url: { url: originalImageUrl, detail: 'auto' } }], - }], - }); - const firstError = new Response(JSON.stringify({ error: { message: 'nope' } }), { - status: 502, headers: new Headers({ 'content-type': 'application/json' }), - }); - const firstCall = vi.fn(async (_model: unknown, body: unknown): Promise> => { - const message = (body as Omit).messages[0]; - if (!Array.isArray(message.content) || message.content[0]?.type !== 'image_url') throw new Error('expected image content'); - message.content[0].image_url.url = 'data:image/webp;base64,COMPRESSED'; - return { ok: false, response: firstError, modelKey: 'first-key' }; - }); - let fallbackImageUrl: string | undefined; - const secondCall = vi.fn(async (_model: unknown, body: unknown): Promise> => { - const message = (body as Omit).messages[0]; - if (!Array.isArray(message.content) || message.content[0]?.type !== 'image_url') throw new Error('expected image content'); - fallbackImageUrl = message.content[0].image_url.url; - return { ok: true, events: makeProtocolFrames(makeOpenAIChatCompletionsEvents()), modelKey: 'second-key', headers: new Headers() }; - }); - queueResolution([ - makeCandidate({ upstream: 'up_a', callOpenAIChatCompletions: firstCall }), - makeCandidate({ upstream: 'up_b', callOpenAIChatCompletions: secondCall }), - ]); - - const result = await openaiChatCompletionsServe.generate({ - payload, - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - // The narrowed candidate list exists exactly so a transient upstream - // failure (5xx/429/network) on one entry rolls over to the next. The - // second candidate's success is the request's final answer. - assertEquals(result.type, 'events'); - assertEquals(firstCall.mock.calls.length, 1); - assertEquals(secondCall.mock.calls.length, 1); - assertEquals(fallbackImageUrl, originalImageUrl); - const sourceMessage = payload.messages[0]; - if (!Array.isArray(sourceMessage.content) || sourceMessage.content[0]?.type !== 'image_url') throw new Error('expected source image content'); - assertEquals(sourceMessage.content[0].image_url.url, originalImageUrl); -}); - -// A mid-attempt throw (interceptor bug / translation error / provider-layer -// JS exception not represented as a ChatServeFailure) must attribute the perf -// error row to the throwing candidate, not the previous one that already -// failed cleanly with a 5xx. -test('mid-attempt throw stamps telemetry with the throwing candidate, not the previous one', async () => { - installRepo(); - const firstError = new Response(JSON.stringify({ error: { message: 'nope' } }), { - status: 502, headers: new Headers({ 'content-type': 'application/json' }), - }); - const firstCall = vi.fn(async (): Promise> => ({ - ok: false, response: firstError, modelKey: 'first-key', - })); - const secondCall = vi.fn(async (): Promise> => { - throw new Error('simulated provider-layer JS exception'); - }); - queueResolution([ - makeCandidate({ upstream: 'up_a', callOpenAIChatCompletions: firstCall }), - makeCandidate({ upstream: 'up_b', callOpenAIChatCompletions: secondCall }), - ]); - - const ctx = makeGatewayCtx(); - await openaiChatCompletionsServe.generate({ - payload: makePayload(), - ctx, - headers: new Headers(), - }).then( - () => { throw new Error('expected openaiChatCompletionsServe.generate to throw'); }, - (error: unknown) => { - assertEquals((error as Error).message, 'simulated provider-layer JS exception'); - }, - ); - - assertEquals(firstCall.mock.calls.length, 1); - assertEquals(secondCall.mock.calls.length, 1); - assertEquals(ctx.attempt.telemetry?.upstream, 'up_b'); -}); - -test('generate surfaces the last upstream error verbatim when every candidate fails', async () => { - installRepo(); - const firstError = new Response('first', { status: 503 }); - const lastError = new Response(JSON.stringify({ error: { message: 'last' } }), { - status: 502, headers: new Headers({ 'content-type': 'application/json' }), - }); - queueResolution([ - makeCandidate({ upstream: 'up_a', callOpenAIChatCompletions: async () => ({ ok: false, response: firstError, modelKey: 'first-key' }) }), - makeCandidate({ upstream: 'up_b', callOpenAIChatCompletions: async () => ({ ok: false, response: lastError, modelKey: 'last-key' }) }), - ]); - - const result = await openaiChatCompletionsServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 502); -}); - -test('generate is a routing no-op when the payload carries no reasoning carriers (degenerate path)', async () => { - installRepo(); - const callOpenAIChatCompletions = vi.fn(async (): Promise> => ({ - ok: true, events: makeProtocolFrames(makeOpenAIChatCompletionsEvents()), modelKey: 'test-model-key', headers: new Headers(), - })); - queueResolution([ - makeCandidate({ upstream: 'up_a', callOpenAIChatCompletions }), - makeCandidate({ upstream: 'up_b', callOpenAIChatCompletions }), - ]); - - const result = await openaiChatCompletionsServe.generate({ - // A bare user message: no reasoning blocks → affinity walk finds no - // refs → both candidates surface in the original order. - payload: makePayload({ messages: [{ role: 'user', content: 'hi' }] }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callOpenAIChatCompletions.mock.calls.length, 1); -}); - -test('generate renders model-missing when no candidates are available', async () => { - installRepo(); - queueResolution([]); - - const result = await openaiChatCompletionsServe.generate({ - payload: makePayload({ model: 'unknown-model' }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 404); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assertEquals(body.error.type, 'invalid_request_error'); - assertEquals(body.error.message, 'Model unknown-model is not available on any configured upstream.'); -}); - -test('alias resolution swaps the inbound model id for the target and overlays rules onto the IR', async () => { - installRepo(); - const capturedBodies: OpenAIChatCompletionsPayload[] = []; - const observedModelIds: string[] = []; - const callOpenAIChatCompletions = vi.fn(async (model: unknown, body: unknown): Promise> => { - observedModelIds.push((model as { id: string }).id); - capturedBodies.push(body as OpenAIChatCompletionsPayload); - return { ok: true, events: makeProtocolFrames(makeOpenAIChatCompletionsEvents()), modelKey: 'gpt-5.4', headers: new Headers() }; - }); - // Alias flow shape: the resolver returns candidates carrying the target's - // upstream catalog id AND the alias's rule overlay on `candidate.rules`. - // The attempt stamps its private clone with `candidate.model.id` and reads - // the overlay directly off `candidate.rules` at wire-call time. - const candidate = makeCandidate({ upstream: 'up_a', modelId: 'gpt-5.4', callOpenAIChatCompletions }); - queueResolution([candidate], { aliasRules: { reasoning: { effort: 'low' }, verbosity: 'low' } }); - - const payload = makePayload({ model: 'gpt-fast' }); - const result = await openaiChatCompletionsServe.generate({ - payload, - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - - // The resolver saw the inbound alias id verbatim — target-id walking - // happens inside the resolver, not in serve. - assertEquals(lastResolveCall.model, 'gpt-fast'); - assertEquals(observedModelIds, ['gpt-5.4']); - assertEquals(payload.model, 'gpt-fast'); - // Alias rules land on the IR through candidate.rules → the attempt's - // applyRulesToUpstreamOpenAIChatCompletions call. - const observed = capturedBodies[0]!; - assertEquals(observed.reasoning_effort, 'low'); - assertEquals(observed.verbosity, 'low'); -}); - -test('direct dispatch uses the resolved public id without mutating the addressed model', async () => { - // A prefix-addressable id ('cop/gpt-5.4') resolves to the catalog's - // 'gpt-5.4' — the resolver strips the prefix internally. The attempt pins - // only its private clone to that canonical id. - installRepo(); - const observedModelIds: string[] = []; - const callOpenAIChatCompletions = vi.fn(async (model: unknown): Promise> => { - observedModelIds.push((model as { id: string }).id); - return { ok: true, events: makeProtocolFrames(makeOpenAIChatCompletionsEvents()), modelKey: 'gpt-5.4', headers: new Headers() }; - }); - const candidate = makeCandidate({ upstream: 'up_a', modelId: 'gpt-5.4', callOpenAIChatCompletions }); - queueResolution([candidate]); - - const payload = makePayload({ model: 'cop/gpt-5.4' }); - const result = await openaiChatCompletionsServe.generate({ - payload, - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - - // Resolver and caller payload retain the prefixed address while dispatch - // uses the catalog id. - assertEquals(lastResolveCall.model, 'cop/gpt-5.4'); - assertEquals(observedModelIds, ['gpt-5.4']); - assertEquals(payload.model, 'cop/gpt-5.4'); -}); - -test('alias whose targets have no kind-matching binding surfaces as the regular model-missing 404', async () => { - installRepo(); - // The resolver walks the alias's targets, finds no candidates for any, - // and returns empty candidates + sawModel=false. Serve renders that as - // a regular model-missing 404 with the alias name in the wording. - queueResolution([], { sawModel: false }); - - const result = await openaiChatCompletionsServe.generate({ - payload: makePayload({ model: 'gpt-fast' }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 404); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assertEquals(body.error.type, 'invalid_request_error'); - assertEquals(body.error.message, 'Model gpt-fast is not available on any configured upstream.'); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts new file mode 100644 index 0000000000..c4afaca684 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses-compact_test.ts @@ -0,0 +1,515 @@ +// The OpenAI Responses compaction chain, run. Assembly is checked where every family's is; what is +// written down here is what only running it can say — that an upstream whose own endpoint +// compacts is asked to and answered under an id this gateway minted, that one with no +// compaction wire is sent the compactor's turn over the wires generation would have used and +// answered with an envelope of this gateway's own, that a prior simulated compaction the +// client echoed back is expanded before the history is summarized, that a summarization +// producing nothing is a candidate the fork moves past, and that a compaction whose turn +// failed settles as a failure. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { SUMMARY_PREFIX } from '../../../src/data-plane/chat/openai-responses/compact-shim.ts'; +import { openaiResponsesCompactPipeline } from '../../../src/data-plane/chat/openai-responses/compact.ts'; +import { createCandidateRegistry } from '../../../src/data-plane/pipeline/candidates.ts'; +import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; +import { initRepo } from '../../../src/repo/index.ts'; +import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../src/shared/base64url-json.ts'; +import { chatFixtureHttpServices, stubChatProviderPipelines } from '../../test-utils/chat-provider-pipelines.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { move, run } from '@floway-dev/pipeline'; +import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import type { ModelEndpoints } from '@floway-dev/protocols/common'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesCompactionResult, OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import { directFetcher, type FlagId, type ModelCandidate } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel, type ProviderOpenAIResponsesResult, type ProviderStreamResult } from '@floway-dev/test-utils'; + +vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ + ...(await importOriginal()), + enumerateModelCandidates: vi.fn(), +})); + +let live: readonly ModelCandidate[] = []; + +const candidate = ( + calls: { + callOpenAIResponses?: (model: unknown, body: unknown, action: unknown) => Promise; + callAnthropicMessages?: (model: unknown, body: unknown) => Promise>; + }, + overrides: { upstreamId?: string; endpoints?: ModelEndpoints; enabledFlags?: ReadonlySet } = {}, +): ModelCandidate => { + const upstreamId = overrides.upstreamId ?? 'up_a'; + const endpoints = overrides.endpoints ?? { openaiResponses: {} }; + const enabledFlags = overrides.enabledFlags ?? new Set(); + return { + provider: { + upstreamId, kind: 'custom', name: upstreamId, inboundHeaderAllowlist: [], + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + pipelines: stubChatProviderPipelines(calls as never), + instance: stubProvider(), + }, + model: stubInternalModel( + { + id: 'responses-model', + endpoints, + limits: { max_output_tokens: 4096 }, + providerModels: { [upstreamId]: stubProviderModel({ id: 'responses-model', endpoints, enabledFlags }) }, + }, + upstreamId, + ), + fetcher: directFetcher, + } as unknown as ModelCandidate; +}; + +const resolves = (candidates: readonly ModelCandidate[]): void => { + live = candidates; + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] } as never); +}; + +const USAGE = { input_tokens: 12, output_tokens: 3, total_tokens: 15 }; + +/** What a native compaction upstream answers with: `CompactResource` states no `status`, no + * `model` and no `error`. */ +const compaction = (overrides: Partial = {}): OpenAIResponsesCompactionResult => ({ + id: 'resp_upstream_compaction', + object: 'response.compaction', + output: [{ type: 'compaction', id: 'cmp_1', encrypted_content: 'OPAQUE_NATIVE_BLOB' } as unknown as never], + usage: USAGE, + ...overrides, +}); + +const compacts = ( + seen: { body?: Record; action?: unknown }, + overrides: Partial = {}, +) => async (_model: unknown, body: unknown, action: unknown): Promise => { + seen.body = body as Record; + seen.action = action; + return { action: 'compact', ok: true, result: compaction(overrides), modelKey: 'responses-model-key' }; +}; + +/** A generate turn over the OpenAI Responses wire that produces one assistant message. */ +const generates = (text: string, seen: { body?: Record; action?: unknown } = {}) => + async (_model: unknown, body: unknown, action: unknown): Promise => { + seen.body = body as Record; + seen.action = action; + const response: OpenAIResponsesResult = { + id: 'resp_summary', + object: 'response', + model: 'responses-model', + status: 'completed', + output: [{ + type: 'message', id: 'msg_1', role: 'assistant', status: 'completed', + content: [{ type: 'output_text', text, annotations: [] }], + }], + error: null, + incomplete_details: null, + usage: USAGE, + }; + return { + action: 'generate', ok: true, modelKey: 'responses-model-key', headers: new Headers(), + events: (async function* () { + yield { type: 'event' as const, event: { type: 'response.completed', sequence_number: 0, response } as OpenAIResponsesStreamEvent }; + yield { type: 'done' as const }; + })(), + }; + }; + +/** The same turn over the Anthropic Messages wire, which is the only way an Anthropic Messages-only candidate can + * be reached: the compaction crosses into that protocol and its frames come back translated. */ +const anthropicMessagesTurn = (text: string, seen: { body?: Record } = {}) => + async (_model: unknown, body: unknown): Promise> => { + seen.body = body as Record; + const events: unknown[] = [ + { + type: 'message_start', + message: { + id: 'msg_1', type: 'message', role: 'assistant', content: [], model: 'responses-model', + stop_reason: null, stop_sequence: null, usage: { input_tokens: 12, output_tokens: 0 }, + }, + }, + { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, + { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text } }, + { type: 'content_block_stop', index: 0 }, + { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 3 } }, + { type: 'message_stop' }, + ]; + return { + ok: true, modelKey: 'responses-model-key', headers: new Headers(), + events: (async function* () { + for (const event of events) yield { type: 'event' as const, event: event as AnthropicMessagesStreamEvent }; + yield { type: 'done' as const }; + })(), + }; + }; + +const payload = { + model: 'responses-model', + input: [{ type: 'message', role: 'user', content: 'a long conversation' }], + store: false, +} as unknown as CanonicalOpenAIResponsesPayload; + +const settlements: { billable: unknown; failed: boolean }[] = []; + +let gateway = mockChatGatewayCtx({ wantsStream: false }); + +const compact = async (request: CanonicalOpenAIResponsesPayload = payload, signal?: AbortSignal) => { + gateway = mockChatGatewayCtx({ wantsStream: false, abortSignal: signal }); + const outcome = await run( + openaiResponsesCompactPipeline(), + move({ + 'ingress.http.headers': [] as readonly (readonly [string, string])[], + 'ingress.chat.sourceProtocol': 'openaiResponses', + 'request.chat.openaiResponses': request, + 'serve.model': request.model, + }) as never, + { + gateway, + background: () => {}, + ...chatFixtureHttpServices(gateway), + ...createCandidateRegistry(), + selectAffinity: (selected: ModelCandidate) => { gateway.affinity.select(selected); }, + resolveAttempt: (selector: { readonly upstreamId: string }) => { + const found = live.find(c => c.provider.upstreamId === selector.upstreamId); + if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); + return found; + }, + } as never, + ); + // What the epilogue would settle from, once the run has answered. + const pending = outcome.facts['response.chat.openaiResponses.streamedUsage']; + if (pending !== null) settlements.push(await pending); + await outcome.drain(); + return outcome; +}; + +const rendered = (facts: Record): Record => + facts['response.chat.openaiResponses.rendered'] as Record; + +/** The summary a simulated compaction packed into the item the client is handed. The turn's + * own state is sealed around it on the way out — that is what pins the next turn to the + * upstream that issued this one — so the carrier is opened before the blob is read. */ +const summaryIn = (encryptedContent: string): string => { + const items = decodeBase64UrlJson(encryptedContent) as { content: { text: string }[] }[] | null; + if (items === null) throw new Error('expected a portable compaction envelope'); + return items[0]!.content[0]!.text; +}; + +beforeEach(() => { + vi.mocked(enumerateModelCandidates).mockReset(); + settlements.length = 0; + initRepo({ + apiKeys: { update: async () => {} }, + usage: { record: async () => {} }, + performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, + openaiResponsesSnapshots: { lookup: async () => null, put: async () => {} }, + openaiResponsesItems: { lookupMany: async () => [], putMany: async () => {} }, + } as never); +}); + +describe('the responses compaction chain', () => { + // An upstream whose own endpoint compacts is asked to compact. What comes back is expanded + // into the events the stateful half reads, which is what replaces the upstream's response id + // with one this gateway minted and completes the resource this endpoint answers with. + it('dials a compaction, and answers the compaction resource under its own id', async () => { + const seen: { body?: Record; action?: unknown } = {}; + resolves([candidate({ callOpenAIResponses: compacts(seen) })]); + + const { facts } = await compact(); + + expect(seen.action).toBe('compact'); + expect(facts['response.http.status']).toBe(200); + expect(rendered(facts).object).toBe('response.compaction'); + expect(rendered(facts).id).not.toBe('resp_upstream_compaction'); + expect(rendered(facts).created_at).toBeTypeOf('number'); + expect(rendered(facts).usage).toMatchObject({ input_tokens: 12, output_tokens: 3, total_tokens: 15 }); + // A compaction is not a response resource, so none of the twenty-odd keys that one + // requires is decorated onto it. + expect(rendered(facts)).not.toHaveProperty('tools'); + expect(rendered(facts)).not.toHaveProperty('truncation'); + }); + + // Neither field belongs on the compaction endpoint: `store` is a gateway-only snapshot hint + // it rejects, and `stream` describes a delivery this operation does not make. + it('sends neither store nor stream on the compaction wire', async () => { + const seen: { body?: Record } = {}; + resolves([candidate({ callOpenAIResponses: compacts(seen) })]); + + await compact({ ...payload, stream: false } as unknown as CanonicalOpenAIResponsesPayload); + + expect(seen.body).not.toHaveProperty('store'); + expect(seen.body).not.toHaveProperty('stream'); + expect(seen.body).not.toHaveProperty('model'); + }); + + // No translation carries a compaction, so an Anthropic Messages candidate has none to dial. The turn + // it is sent instead is an ordinary generate turn against the compactor's prompt, over the + // wire generation would have used — which is the whole of what the action pivot was. + it('simulates a compaction over a candidate with no compaction wire', async () => { + const seen: { body?: Record } = {}; + resolves([candidate({ callAnthropicMessages: anthropicMessagesTurn('CONDENSED SUMMARY', seen) }, { endpoints: { anthropicMessages: {} } })]); + + const { facts } = await compact({ + ...payload, + input: [ + { type: 'message', role: 'user', content: 'real history' }, + { type: 'compaction_trigger' }, + ], + } as unknown as CanonicalOpenAIResponsesPayload); + + // The compactor's prompt reached the upstream, and the control item that asked for the + // compaction did not. + const sent = JSON.stringify(seen.body); + expect(sent).toContain('CONTEXT CHECKPOINT COMPACTION'); + expect(sent).not.toContain('compaction_trigger'); + + expect(facts['response.http.status']).toBe(200); + expect(rendered(facts).object).toBe('response.compaction'); + const output = rendered(facts).output as { type: string; encrypted_content: string }[]; + expect(output.some(item => item.type === 'compaction')).toBe(true); + expect(summaryIn(output.find(item => item.type === 'compaction')!.encrypted_content)).toBe(`${SUMMARY_PREFIX}\nCONDENSED SUMMARY`); + }); + + // The flag is the operator's opt-in for an upstream that would answer a compaction itself. + it('simulates rather than dials when the operator asked it to', async () => { + const seen: { body?: Record; action?: unknown } = {}; + resolves([candidate( + { callOpenAIResponses: generates('SIMULATED SUMMARY', seen) }, + { enabledFlags: new Set(['openai-responses-compact-shim']) }, + )]); + + const { facts } = await compact(); + + expect(seen.action).toBe('generate'); + // The ephemeral summarization turn is not persisted in the upstream's own history. + expect(seen.body).toMatchObject({ store: false }); + expect(rendered(facts).object).toBe('response.compaction'); + const output = rendered(facts).output as { type: string; encrypted_content: string }[]; + expect(summaryIn(output.find(item => item.type === 'compaction')!.encrypted_content)).toBe(`${SUMMARY_PREFIX}\nSIMULATED SUMMARY`); + }); + + // A compaction this gateway synthesized carries the history it stood for, so a turn that + // echoes it back is summarized from that history rather than from an opaque blob. + it('expands a compaction it wrote before summarizing again', async () => { + const seen: { body?: Record } = {}; + const encoded = encodeBase64UrlJson([ + { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'THE EARLIER HISTORY' }] }, + ]); + resolves([candidate( + { callOpenAIResponses: generates('S', seen) }, + { enabledFlags: new Set(['openai-responses-compact-shim']) }, + )]); + + await compact({ + ...payload, + input: [{ type: 'compaction', id: 'cmp_prior', encrypted_content: encoded }], + } as unknown as CanonicalOpenAIResponsesPayload); + + expect(JSON.stringify(seen.body)).toContain('THE EARLIER HISTORY'); + }); + + // The blob is the whole of what the next turn inherits, so a summarization that closed no + // text produced nothing to inherit — a candidate that did not do the job rather than a + // fault that ends the request. + it('fails a summarization that produced no text over to the next candidate', async () => { + const tried: string[] = []; + resolves([ + candidate( + { callOpenAIResponses: async (...args) => { tried.push('mute'); return await generates('')(...args as [unknown, unknown, unknown]); } }, + { upstreamId: 'up_mute', enabledFlags: new Set(['openai-responses-compact-shim']) }, + ), + candidate( + { callOpenAIResponses: async (...args) => { tried.push('talkative'); return await generates('A SUMMARY')(...args as [unknown, unknown, unknown]); } }, + { upstreamId: 'up_talkative', enabledFlags: new Set(['openai-responses-compact-shim']) }, + ), + ]); + + const { facts } = await compact(); + + expect(tried).toEqual(['mute', 'talkative']); + const outcome = settlements.at(-1) as { billable: Array<{ identity: { upstream: string }; quantities: Record }> }; + expect(outcome.billable.map(entity => [entity.identity.upstream, entity.quantities.input_tokens])).toEqual([['up_mute', '12'], ['up_talkative', '12']]); + expect(facts['response.http.status']).toBe(200); + expect(rendered(facts).object).toBe('response.compaction'); + }); + + it('fails a refused compaction over to the next candidate, and keeps the last refusal-s own status', async () => { + resolves([ + candidate({ + callOpenAIResponses: async (): Promise => ({ + action: 'compact', ok: false, modelKey: 'k', + response: new Response(JSON.stringify({ error: { message: 'slow down', type: 'rate_limit' } }), { + status: 429, headers: { 'content-type': 'application/json' }, + }), + }), + }, { upstreamId: 'up_busy' }), + candidate({ + callOpenAIResponses: async (): Promise => ({ + action: 'compact', ok: false, modelKey: 'k', + response: new Response(JSON.stringify({ error: { message: 'still no', type: 'rate_limit' } }), { + status: 503, headers: { 'content-type': 'application/json' }, + }), + }), + }, { upstreamId: 'up_busier' }), + ]); + + const { facts } = await compact(); + + expect(facts['response.http.status']).toBe(503); + expect(rendered(facts)).toEqual({ error: { message: 'still no', type: 'rate_limit' } }); + }); + + // `Usage` has no `null` alternative on this resource and zeros are not its spelling, so an + // upstream that stated no counts has left the gateway unable to answer. + it('reports a compaction the upstream stated no counts for as the gateway-s own failure', async () => { + resolves([candidate({ callOpenAIResponses: compacts({}, { usage: undefined }) })]); + + const { facts } = await compact(); + + expect(facts['response.http.status']).toBe(502); + const body = rendered(facts).error as { type: string; message: string }; + expect(body.type).toBe('internal_error'); + expect(body.message).toContain('reported no token usage'); + }); + + // `status` is not a key the compaction resource declares — it rides through from the turn + // the upstream actually ran — and a compaction that surfaced as failed belongs in the error + // column rather than masquerading as a success. + it('settles a compaction whose turn failed as a failure', async () => { + resolves([candidate({ callOpenAIResponses: compacts({}, { status: 'failed' } as Partial) })]); + + await compact(); + + expect(settlements).toEqual([{ billable: expect.anything(), failed: true }]); + }); + + it('settles a compaction that ran as a success, billing what its envelope stated', async () => { + resolves([candidate({ callOpenAIResponses: compacts({}) })]); + + await compact(); + + expect(settlements).toHaveLength(1); + expect(settlements[0]!.failed).toBe(false); + expect(settlements[0]!.billable).toMatchObject([ + { quantities: { input_tokens: '12', output_tokens: '3' } }, + ]); + }); + + // A provider owns the body it is dialled with and shapes it in place, down to nested nodes — + // Copilot marks individual items for caching — while the record that body is built from is + // deep-frozen. A dial that handed over a shallow copy throws at the first nested write, and + // this operation would answer 502 where the upstream had said nothing at all. + // The record is frozen all the way down, and nothing copies it on the way out — a provider + // shapes the body it sends by rebuilding, which every boundary rule here does. A rule that + // reached into a child instead would throw, and this is what says so rather than leaving it + // to be discovered as a 502 from the one place that cannot explain it. + it('hands the provider a body no one can write into', async () => { + let refused: unknown; + resolves([candidate({ + callOpenAIResponses: async (model, body, action) => { + const item = (body as { input: Record[] }).input[0]!; + try { item.copilot_cache_control = { type: 'ephemeral' }; } catch (error) { refused = error; } + return await compacts({})(model, body, action); + }, + })]); + + const { facts } = await compact(); + + expect(facts['response.http.status']).toBe(200); + expect(refused).toBeInstanceOf(TypeError); + }); +}); + +describe('native compaction decryption', () => { + const decryptFlags = new Set(['openai-responses-compact-decrypt']); + it('replays both compaction aliases on the same candidate and preserves each call cost', async () => { + const calls: Array<{ model: unknown; body: Record; action: unknown }> = []; + const original = compaction({ + output: [ + { type: 'message', id: 'kept', role: 'assistant', status: 'completed', content: [{ type: 'output_text', text: 'keep me', annotations: [] }] }, + { type: 'compaction', id: 'cmp_first', encrypted_content: 'OPAQUE_A' }, + { type: 'compaction_summary', id: 'cmp_second', encrypted_content: 'OPAQUE_B' }, + ] as never, + }); + resolves([candidate({ + callOpenAIResponses: async (model, body, action) => { + calls.push({ model, body: body as Record, action }); + if (action === 'compact') return { action: 'compact', ok: true, result: original, modelKey: 'responses-model-key' }; + const item = (body as CanonicalOpenAIResponsesPayload).input[1]; + return await generates('id' in item && item.id === 'cmp_first' ? 'first plaintext' : 'second plaintext')(model, body, action); + }, + }, { enabledFlags: decryptFlags })]); + const { facts } = await compact({ ...payload, instructions: 'original instructions', tools: [{ type: 'function', name: 'f' }] }); + expect(calls.map(call => call.action)).toEqual(['compact', 'generate', 'generate']); + expect(calls[1].model).toStrictEqual(calls[0].model); + expect(calls[2].model).toStrictEqual(calls[0].model); + for (const call of calls.slice(1)) { + expect(call.body).not.toHaveProperty('instructions'); + expect(call.body).not.toHaveProperty('tools'); + expect(call.body).toMatchObject({ store: false, input: [{ role: 'system' }, { type: expect.stringMatching(/^compaction/) }, { role: 'system' }] }); + } + const output = rendered(facts).output as Array<{ id: string; type: string; encrypted_content?: string }>; + expect(output.some(item => item.id === 'kept')).toBe(true); + expect(summaryIn(output.find(item => 'id' in item && item.id === 'cmp_first')!.encrypted_content!)).toBe('first plaintext'); + expect(summaryIn(output.find(item => item.id === 'cmp_second')!.encrypted_content!)).toBe('second plaintext'); + expect(rendered(facts).object).toBe('response.compaction'); + expect(rendered(facts).usage).toMatchObject({ input_tokens: 36, output_tokens: 9, total_tokens: 45 }); + const outcome = settlements.at(-1) as { billable: Array<{ quantities: unknown }> }; + expect(outcome.billable.map(entity => entity.quantities)).toEqual(Array.from({ length: 3 }, () => ({ input_tokens: '12', output_tokens: '3' }))); + }); + + it('retains native cost when a replay refuses and a later candidate succeeds', async () => { + const initial = candidate({ + callOpenAIResponses: async (_model, _body, action) => action === 'compact' + ? { action: 'compact', ok: true, result: compaction(), modelKey: 'responses-model-key' } + : { action: 'generate', ok: false, response: Response.json({ error: { message: 'busy' } }, { status: 429 }), modelKey: 'responses-model-key' }, + }, { upstreamId: 'up_first', enabledFlags: decryptFlags }); + const fallback = candidate({ callOpenAIResponses: compacts({}) }, { upstreamId: 'up_fallback' }); + resolves([initial, fallback]); + const { facts } = await compact(); + expect(facts['response.http.status']).toBe(200); + const outcome = settlements.at(-1) as { billable: Array<{ identity: { upstream: string }; quantities: Record }> }; + expect(outcome.billable.filter(entity => entity.quantities.input_tokens !== undefined).map(entity => [entity.identity.upstream, entity.quantities.input_tokens])).toEqual([['up_first', '12'], ['up_fallback', '12']]); + }); + + it('preserves observed replay usage and the original failure when a replay stream breaks', async () => { + const broken = new Error('replay stream broke'); + let released = false; + resolves([candidate({ + callOpenAIResponses: async (_model, _body, action) => action === 'compact' + ? { action: 'compact', ok: true, result: compaction(), modelKey: 'responses-model-key' } + : { + action: 'generate', ok: true, modelKey: 'responses-model-key', headers: new Headers(), events: (async function* () { + try { + yield { type: 'event' as const, event: { type: 'response.in_progress', sequence_number: 0, response: { id: 'replay', object: 'response', model: 'responses-model', status: 'in_progress', output: [], error: null, incomplete_details: null, usage: { input_tokens: 7, output_tokens: 2, total_tokens: 9 } } } as OpenAIResponsesStreamEvent }; + throw broken; + } finally { released = true; } + })(), + }, + }, { enabledFlags: decryptFlags })]); + const { facts } = await compact(); + expect(facts['response.http.status']).toBe(500); + expect(JSON.stringify(rendered(facts))).toContain(broken.message); + expect(released).toBe(true); + const calls = facts['response.usage.billable']; + expect(calls.map(entity => entity.quantities)).toEqual([{ input_tokens: '12', output_tokens: '3' }, { input_tokens: '7', output_tokens: '2' }]); + }); + + it('stops before replay after cancellation and keeps the completed native reading', async () => { + const controller = new AbortController(); + const cancelled = new Error('compaction cancelled'); + let replayCalls = 0; + resolves([candidate({ + callOpenAIResponses: async (_model, _body, action) => { + if (action !== 'compact') { replayCalls++; throw new Error('Unexpected replay'); } + controller.abort(cancelled); + return { action: 'compact', ok: true, result: compaction(), modelKey: 'responses-model-key' }; + }, + }, { enabledFlags: decryptFlags })]); + const { facts } = await compact(payload, controller.signal); + expect(replayCalls).toBe(0); + expect(facts['response.http.status']).toBe(500); + expect(JSON.stringify(rendered(facts))).toContain(cancelled.message); + const calls = facts['response.usage.billable']; + expect(calls.map(entity => entity.quantities)).toEqual([{ input_tokens: '12', output_tokens: '3' }]); + }); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts new file mode 100644 index 0000000000..61c87b89b4 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses-pipeline_test.ts @@ -0,0 +1,783 @@ +// The OpenAI Responses chain, run. Assembly is checked where every family's is; what is written +// down here is what only running it can say — that the edge terminates the client's stream +// on `[DONE]` however the upstream's ended, that it folds a stream into one response object +// when the client did not ask to stream, that it hands the turn's own state back for the +// client to carry, that the stored-items membrane's other half runs there too so the client +// is answered under an envelope this gateway minted, that a turn the upstream answered with +// one envelope instead of a stream is served as that envelope, that a turn asking for a +// compaction is answered with one wherever the shim is this candidate's to run and travels on +// untouched where it is not, that a refusal keeps the upstream's own status and words, and +// that a dial nobody answered is a value the fork can move past. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { SUMMARY_PREFIX } from '../../../src/data-plane/chat/openai-responses/compact-shim.ts'; +import { openaiResponsesServePipeline } from '../../../src/data-plane/chat/openai-responses/pipeline.ts'; +import { createCandidateRegistry } from '../../../src/data-plane/pipeline/candidates.ts'; +import type { StreamOutcome } from '../../../src/data-plane/pipeline/serve.ts'; +import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; +import { initRepo } from '../../../src/repo/index.ts'; +import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../src/shared/base64url-json.ts'; +import { chatFixtureHttpServices, stubChatProviderPipelines } from '../../test-utils/chat-provider-pipelines.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { move, run, type Deferred } from '@floway-dev/pipeline'; +import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import type { ModelEndpoints, SseFrame } from '@floway-dev/protocols/common'; +import { OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesCompactionResult, type OpenAIResponsesOutputItem, type OpenAIResponsesResult, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import { directFetcher, type FlagId, type ModelCandidate } from '@floway-dev/provider'; +import { type StubChatProviderCall, type ProviderOpenAIResponsesResult, type ProviderStreamResult } from '@floway-dev/test-utils'; +import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; + +vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ + ...(await importOriginal()), + enumerateModelCandidates: vi.fn(), +})); + +vi.mock('../../../src/data-plane/chat/openai-responses/affinity/ingress.ts', async importOriginal => ({ + ...(await importOriginal()), + analyzeOpenAIResponsesAffinity: async () => ({ + requiredTargets: [], + evaluateCandidate: () => ({ kind: 'accepted', degrades: false, preferred: true, materialize: () => { asked += 1; return affinityPayload; } }), + }), +})); + +let live: readonly ModelCandidate[] = []; + +const candidate = ( + calls: { + callOpenAIResponses?: StubChatProviderCall<'openaiResponses'>; + callAnthropicMessages?: (model: unknown, body: unknown) => Promise>; + }, + overrides: { upstreamId?: string; endpoints?: ModelEndpoints; enabledFlags?: ReadonlySet } = {}, +): ModelCandidate => { + const upstreamId = overrides.upstreamId ?? 'up_a'; + const endpoints = overrides.endpoints ?? { openaiResponses: {} }; + const enabledFlags = overrides.enabledFlags ?? new Set(); + return { + provider: { + upstreamId, kind: 'custom', name: upstreamId, inboundHeaderAllowlist: [], + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + pipelines: stubChatProviderPipelines(calls as never), + instance: stubProvider(), + }, + model: stubInternalModel( + { + id: 'responses-model', + endpoints, + limits: { max_output_tokens: 4096 }, + providerModels: { [upstreamId]: stubProviderModel({ id: 'responses-model', endpoints, enabledFlags }) }, + }, + upstreamId, + ), + fetcher: directFetcher, + } as unknown as ModelCandidate; +}; + +const resolves = (candidates: readonly ModelCandidate[]): void => { + live = candidates; + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] } as never); +}; + +const delta = (text: string): OpenAIResponsesStreamEvent => ({ + type: 'response.output_text.delta', + sequence_number: 1, + item_id: 'msg_1', + output_index: 0, + content_index: 0, + delta: text, +}); + +const result = (text: string, usage?: OpenAIResponsesResult['usage']): OpenAIResponsesResult => ({ + id: 'resp_1', + object: 'response', + model: 'responses-model', + status: 'completed', + output: [{ + type: 'message', id: 'msg_1', role: 'assistant', status: 'completed', + content: [{ type: 'output_text', text, annotations: [] }], + }], + error: null, + incomplete_details: null, + ...(usage === undefined ? {} : { usage }), +}); + +const completed = (text: string, usage?: OpenAIResponsesResult['usage']): OpenAIResponsesStreamEvent => ({ + type: 'response.completed', + sequence_number: 2, + response: result(text, usage), +}); + +// The upstream's own stream, as the provider hands it up: typed frames, and the `done` frame +// its SSE transport ended on. +const stream = (...events: readonly OpenAIResponsesStreamEvent[]): ProviderOpenAIResponsesResult => ({ + action: 'generate', + ok: true, + modelKey: 'responses-model-key', + headers: new Headers({ 'x-request-id': 'req-1', 'content-length': '99' }), + events: (async function* () { + for (const event of events) yield { type: 'event' as const, event }; + yield { type: 'done' as const }; + })(), +}); + +/** One assistant message over the Anthropic Messages wire, which is the only way an Anthropic Messages-only + * candidate can be reached: the turn crosses into that protocol and comes back translated. */ +const anthropicMessagesTurn = (text: string, seen: { body?: Record } = {}) => + async (_model: unknown, body: unknown): Promise> => { + seen.body = body as Record; + const events: unknown[] = [ + { + type: 'message_start', + message: { + id: 'msg_1', type: 'message', role: 'assistant', content: [], model: 'responses-model', + stop_reason: null, stop_sequence: null, usage: { input_tokens: 12, output_tokens: 0 }, + }, + }, + { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, + { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text } }, + { type: 'content_block_stop', index: 0 }, + { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 3 } }, + { type: 'message_stop' }, + ]; + return { + ok: true, modelKey: 'responses-model-key', headers: new Headers(), + events: (async function* () { + for (const event of events) yield { type: 'event' as const, event: event as AnthropicMessagesStreamEvent }; + yield { type: 'done' as const }; + })(), + }; + }; + +const payload = { + model: 'responses-model', + input: [{ type: 'message', role: 'user', content: 'hi' }], +} as unknown as CanonicalOpenAIResponsesPayload; + +/** What affinity materialized for the candidate about to be dialled. It differs from the + * payload the client sent, which is the point: carried state is rewritten per candidate. */ +let affinityPayload: CanonicalOpenAIResponsesPayload = payload; + +/** How many times the chain asked the resolver for that payload. One stage owns the reading; + * anything below it reads the record. */ +let asked = 0; + +const serve = async (wantsStream: boolean, request: CanonicalOpenAIResponsesPayload = payload) => + await serveWith(mockChatGatewayCtx({ wantsStream }), wantsStream, request); + +const serveWith = async ( + gateway: ReturnType, + wantsStream: boolean, + request: CanonicalOpenAIResponsesPayload = payload, +) => await run( + openaiResponsesServePipeline(), + move({ + 'ingress.http.headers': [] as readonly (readonly [string, string])[], + 'ingress.chat.sourceProtocol': 'openaiResponses', + 'ingress.chat.openaiResponses.wantsStream': wantsStream, + 'request.chat.openaiResponses': request, + 'serve.model': 'responses-model', + }) as never, + { + gateway, + background: () => {}, + ...chatFixtureHttpServices(gateway), + ...createCandidateRegistry(), + // Wired where the app wires it: the carrier the edge writes is addressed to whatever the + // chain named here. + selectAffinity: (selected: ModelCandidate) => { gateway.affinity.select(selected); }, + resolveAttempt: (selector: { readonly upstreamId: string }) => { + const found = live.find(c => c.provider.upstreamId === selector.upstreamId); + if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); + return found; + }, + } as never, +); + +const drain = async (rendered: unknown): Promise => { + const frames: SseFrame[] = []; + for await (const frame of rendered as AsyncIterable) frames.push(frame); + return frames; +}; + +// The carrier the edge writes back is a reasoning item holding this turn's own state. The +// message this turn answers with cannot hold it, so the carrier gets an output slot of its +// own at the head of the answer: two frames, and a slot in every snapshot that follows them. +const isCarrierItem = (item: OpenAIResponsesOutputItem): boolean => item.type === 'reasoning'; + +/** The turn's own answer, with the carrier's item taken back out. */ +const withoutCarrierItem = (response: OpenAIResponsesResult): OpenAIResponsesResult => + ({ ...response, output: response.output.filter(item => !isCarrierItem(item)) }); + +/** The turn's own frames. The two sequence numbers the carrier's frames spent stay spent — + * they are numbers the client saw. */ +const withoutCarrier = (frames: readonly SseFrame[]): readonly SseFrame[] => + frames.flatMap(frame => { + if (frame.data === '[DONE]') return [frame]; + const event = JSON.parse(frame.data) as OpenAIResponsesStreamEvent; + if ( + (event.type === 'response.output_item.added' || event.type === 'response.output_item.done') + && isCarrierItem(event.item) + ) return []; + if (!('response' in event)) return [frame]; + return [{ ...frame, data: JSON.stringify({ ...event, response: withoutCarrierItem(event.response) }) }]; + }); + +/** A turn asking for a compaction, which on this protocol is an ordinary turn whose input + * ends in the control item that requests one. */ +const asksForCompaction = { + ...payload, + input: [ + { type: 'message', role: 'user', content: 'a long conversation' }, + { type: 'compaction_trigger' }, + ], +} as unknown as CanonicalOpenAIResponsesPayload; + +/** The compaction item out of what the client was answered with. */ +const compactionItem = (answered: Record): { readonly encrypted_content: string } => { + const item = (answered.output as { type: string }[]).find(candidateItem => candidateItem.type === 'compaction'); + if (item === undefined) throw new Error('expected the answer to carry a compaction item'); + return item as unknown as { readonly encrypted_content: string }; +}; + +/** The summary a simulated compaction packed into the item the client is handed. The turn's + * own state is sealed around it on the way out — that is what pins the next turn to the + * upstream that issued this one — so the carrier is opened before the blob is read. */ +const summaryIn = (encryptedContent: string): string => { + const items = decodeBase64UrlJson(encryptedContent) as { content: { text: string }[] }[] | null; + if (items === null) throw new Error('expected a portable compaction envelope'); + return items[0]!.content[0]!.text; +}; + +beforeEach(() => { + affinityPayload = payload; + asked = 0; + vi.mocked(enumerateModelCandidates).mockReset(); + initRepo({ + apiKeys: { update: async () => {} }, + usage: { record: async () => {} }, + performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, + } as never); +}); + +describe('the responses chain', () => { + // Affinity rewrites client-carried state for the upstream that will see it, so the body + // that goes out is the one it materialized rather than the one the client sent — and this + // chain dials the generate action, whatever else the protocol can ask an upstream for. The + // payload enters the record below the fork because everything between there and the dial + // rewrites the request as a fact, and one stage owns that reading: an ending that asked the + // resolver again would send a payload no stage in the chain had touched. + it('puts the payload this attempt is owed into the record, and sends that', async () => { + let sent: Record | undefined; + let action: unknown; + affinityPayload = { + ...payload, + input: [{ type: 'message', role: 'user', content: 'rewritten' }], + } as CanonicalOpenAIResponsesPayload; + resolves([candidate({ + callOpenAIResponses: async (_model, body, asks) => { + sent = body as Record; + action = asks; + return stream(completed('hi')); + }, + })]); + + const { facts } = await serve(false); + + // The exit contract names what the run answers with, and the request the chain sent is + // not that — but the record carries every key in flight, so what was sent is still there + // to read. + expect((facts as Record)['request.chat.openaiResponses']).toBe(affinityPayload); + expect(asked).toBe(1); + expect(sent).toMatchObject({ input: [{ type: 'message', role: 'user', content: 'rewritten' }] }); + expect(sent).not.toHaveProperty('model'); + expect(action).toBe('generate'); + }); + + // Copilot's compaction translation and Azure-native compaction both emit assistant messages + // whose content blocks say `input_text`, and both then refuse those same items echoed back + // as input. The wire puts the canonical type back on whatever reaches it, however the + // history got there — a client echo, or the snapshot the membrane replayed. + it('sends an assistant item back in the content type this wire accepts', async () => { + let sent: Record | undefined; + affinityPayload = { + ...payload, + input: [ + { type: 'message', role: 'assistant', content: [{ type: 'input_text', text: 'summary so far' }] }, + { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'and then?' }] }, + ], + } as unknown as CanonicalOpenAIResponsesPayload; + resolves([candidate({ + callOpenAIResponses: async (_model, body) => { + sent = body as Record; + return stream(completed('hi')); + }, + })]); + + await serve(false); + + // Only the assistant's: `input_text` IS the correct type on a user message. + expect(sent).toMatchObject({ + input: [ + { role: 'assistant', content: [{ type: 'output_text', text: 'summary so far' }] }, + { role: 'user', content: [{ type: 'input_text', text: 'and then?' }] }, + ], + }); + }); + + // Every event goes out under its own SSE name, and the client's stream ends on the literal + // `[DONE]` the transport reads as the turn being over. + it('writes the frames out when the client asked to stream', async () => { + resolves([candidate({ callOpenAIResponses: async () => stream(delta('he'), delta('llo'), completed('hello')) })]); + + const { facts } = await serve(true); + const frames = withoutCarrier(await drain(facts['response.chat.openaiResponses.rendered'])); + + expect(facts['response.http.status']).toBe(200); + expect(frames.map(frame => frame.event)).toEqual([ + 'response.output_text.delta', + 'response.output_text.delta', + 'response.completed', + undefined, + ]); + expect(frames.slice(0, 2).map(frame => frame.data)).toEqual([ + JSON.stringify(delta('he')), + JSON.stringify(delta('llo')), + ]); + expect(frames[3]!.data).toBe('[DONE]'); + + // The terminal event is the upstream's, restated: the membrane at the edge stamps the + // envelope id this gateway minted for the turn and completes the resource to what the + // schema requires of it, so the deltas ride through untouched and this one does not. + const terminal = JSON.parse(frames[2]!.data) as { sequence_number: number; response: OpenAIResponsesResult }; + // The carrier's own two frames went out ahead of it under two sequence numbers of their + // own, so the terminal event is numbered where they left it. + expect(terminal.sequence_number).toBe(4); + expect(terminal.response).toMatchObject({ + status: 'completed', + output: [{ content: [{ type: 'output_text', text: 'hello' }] }], + }); + expect(terminal.response.id).not.toBe('resp_1'); + }); + + // The other half of affinity: the resolver reads client-carried state on the way down, and + // this is what the client is given to carry back. A turn that handed back nothing would + // leave the follow-up quoting it with no way to name the upstream that issued it. + it('hands the turn-s own state back on an item of its own', async () => { + resolves([candidate({ callOpenAIResponses: async () => stream(delta('hi'), completed('hi')) })]); + const gateway = mockChatGatewayCtx({ wantsStream: true }); + + const { facts } = await serveWith(gateway, true); + const frames = await drain(facts['response.chat.openaiResponses.rendered']); + const answered = frames.find(frame => frame.event === 'response.completed'); + const carrier = (JSON.parse(answered!.data) as { readonly response: OpenAIResponsesResult }).response.output + .filter(item => item.type === 'reasoning'); + + expect(carrier).toHaveLength(1); + // It names the upstream that answered, sealed under this run's own secret — which is what + // lets the next turn be pinned to it. The item is ours rather than the upstream's, and it + // says so, so a turn quoting it back is not read as reasoning the upstream produced. + expect(await gateway.affinity.codec.unwrap(carrier[0]!.encrypted_content!, 'openai-responses.reasoning.encrypted_content')).toMatchObject({ + kind: 'owned', + syntheticItem: true, + affinity: { upstreamId: 'up_a', modelId: 'responses-model' }, + }); + }); + + // A stream's numbers arrive with its last chunk, so what the run hands up is a promise and + // the reading is taken off the frames the client itself drove. + it('bills what the terminal event reported, once the frames have run out', async () => { + resolves([candidate({ + callOpenAIResponses: async () => stream( + delta('hi'), + completed('hi', { input_tokens: 11, output_tokens: 7, total_tokens: 18 }), + ), + })]); + + const { facts } = await serve(true); + await drain(facts['response.chat.openaiResponses.rendered']); + const outcome = await facts['response.chat.openaiResponses.streamedUsage']!; + const billable = outcome.billable; + + expect(billable).toEqual([expect.objectContaining({ + quantities: { input_tokens: '11', output_tokens: '7' }, + })]); + // A rate can depend on the tier and on how much input there was; both are selector + // coordinates, so they travel as pricing facts rather than as quantities. + expect(billable[0]!.pricingFacts).toMatchObject({ inputTokens: 11 }); + }); + + // The reading is taken above the wire's rules, so what the row is written from is the same + // frame the client was shown. An upstream counting its cached prefix outside the input total + // is where the two would otherwise come apart: 479 + 13312 + 373 is the 14164 it states, so + // the fold restores OpenAI's inclusive contract before anything reads the numbers. + it('bills the usage its rules settled rather than the one the upstream sent', async () => { + resolves([candidate({ + callOpenAIResponses: async () => stream( + delta('hi'), + completed('hi', { + input_tokens: 479, output_tokens: 373, total_tokens: 14164, + input_tokens_details: { cached_tokens: 13312 }, + } as OpenAIResponsesResult['usage']), + ), + })]); + + const { facts } = await serve(true); + const frames = withoutCarrier(await drain(facts['response.chat.openaiResponses.rendered'])); + const terminal = frames + .filter(frame => frame.event === 'response.completed') + .map(frame => JSON.parse(frame.data) as { response: OpenAIResponsesResult }); + + // What the client was shown: the cached prefix folded back into the input total. + expect(terminal[0]!.response.usage).toMatchObject({ input_tokens: 13791, output_tokens: 373 }); + // And what the row is written from, which is that same figure split onto the two metrics a + // cache read is priced on. Metering at the dial would bill 479 and lose the prefix. + const outcome = await facts['response.chat.openaiResponses.streamedUsage']!; + expect(outcome.billable[0]!.quantities).toEqual({ + input_tokens: '479', + input_cache_read_tokens: '13312', + output_tokens: '373', + }); + }); + + // Time to first token is measured where the token is — a lifecycle envelope is not one, and + // a run that never stamps it is recorded as having produced nothing to time. + it('stamps the first frame that carried a generated token', async () => { + resolves([candidate({ callOpenAIResponses: async () => stream(delta('hi'), completed('hi')) })]); + const gateway = mockChatGatewayCtx({ wantsStream: true }); + + const { facts } = await serveWith(gateway, true); + expect(gateway.attempt.timing.firstOutputTokenAt).toBeNull(); + await drain(facts['response.chat.openaiResponses.rendered']); + + expect(gateway.attempt.timing.firstOutputTokenAt).toBeTypeOf('number'); + }); + + // The turn is over at its terminal event, so what an upstream writes after it is not part + // of the answer and the client is not shown it. Reading stops there, which also closes the + // upstream rather than leaving the client's stream open behind a connection nobody reads. + it('stops reading at the terminal event', async () => { + resolves([candidate({ callOpenAIResponses: async () => stream(delta('hi'), completed('hi'), delta(' and more')) })]); + + const { facts } = await serve(true); + const frames = withoutCarrier(await drain(facts['response.chat.openaiResponses.rendered'])); + + expect(frames.map(frame => frame.event)).toEqual([ + 'response.output_text.delta', + 'response.completed', + undefined, + ]); + }); + + // A stream that ran out before its terminal event is a turn nobody can answer from: the + // response was never stated complete, incomplete or failed, so serving what did arrive + // would report a truncated answer as a whole one. By then the client is already being + // streamed to and the status went out with the headers, so it is told in the protocol's own + // words — and the stream ends on that rather than on the terminator, because a stream that + // ended on `[DONE]` is a stream that finished. + it('tells a client already being streamed to that the stream never ended', async () => { + resolves([candidate({ callOpenAIResponses: async () => stream(delta('hi')) })]); + + const { facts } = await serve(true); + const frames = await drain(facts['response.chat.openaiResponses.rendered']); + + expect(frames.map(frame => frame.event)).toEqual(['response.output_text.delta', 'error']); + expect(JSON.parse(frames[1]!.data)).toMatchObject({ + type: 'error', + error: { message: OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE }, + }); + // The client was answered, and the run still failed: what the frames said on the way out + // is not what the row says the turn was. + expect((await facts['response.chat.openaiResponses.streamedUsage']!).failed).toBe(true); + }); + + // The upstream speaks SSE whatever the client asked for, so a client that did not ask to + // stream is answered from the same frames — folded here rather than read a second time. + it('folds the frames into one response when the client did not', async () => { + resolves([candidate({ callOpenAIResponses: async () => stream(delta('he'), delta('llo'), completed('hello')) })]); + + const { facts } = await serve(false); + + expect(facts['response.http.status']).toBe(200); + const answered = withoutCarrierItem(facts['response.chat.openaiResponses.rendered'] as unknown as OpenAIResponsesResult); + expect(answered).toMatchObject({ + status: 'completed', + output: [{ content: [{ type: 'output_text', text: 'hello' }] }], + }); + // One client response can span several upstream calls, so the envelope it is answered + // under is the one the membrane minted rather than the one the upstream sent. + expect(answered.id).not.toBe('resp_1'); + }); + + // Native compaction counts pass through the same normalization and metering as generation. + it('serves an upstream that answered with one envelope as that envelope', async () => { + const compaction: OpenAIResponsesCompactionResult = { + id: 'resp_compact_1', + object: 'response.compaction', + output: [{ + type: 'message', id: 'msg_c', role: 'assistant', status: 'completed', + content: [{ type: 'output_text', text: 'summary so far', annotations: [] }], + }], + usage: { input_tokens: 900, output_tokens: 40, total_tokens: 940 }, + }; + resolves([candidate({ + callOpenAIResponses: async () => ({ + action: 'compact', ok: true, modelKey: 'responses-model-key', result: compaction, + }), + })]); + + const { facts } = await serve(false); + + expect(facts['response.http.status']).toBe(200); + expect(facts['response.chat.openaiResponses.rendered']).toMatchObject({ object: 'response.compaction', usage: compaction.usage }); + expect(facts['response.chat.openaiResponses.streamedUsage']).not.toBeNull(); + expect((await (facts['response.chat.openaiResponses.streamedUsage'] as Deferred)).billable).toEqual([expect.objectContaining({ quantities: { input_tokens: '900', output_tokens: '40' } })]); + }); + + // Content-length would misdescribe a body this gateway serialized itself; a vendor trace + // is what a client and an operator both need to correlate a turn. + it('forwards the upstream headers a client may see and drops the ones it may not', async () => { + resolves([candidate({ callOpenAIResponses: async () => stream(completed('hi')) })]); + + const { facts } = await serve(true); + + expect(Object.fromEntries(facts['response.http.headers'])).toMatchObject({ 'x-request-id': 'req-1' }); + expect(Object.fromEntries(facts['response.http.headers'])).not.toHaveProperty('content-length'); + }); + + it('answers an upstream refusal with its own status and words', async () => { + resolves([candidate({ + callOpenAIResponses: async () => ({ + action: 'generate', + ok: false, + modelKey: 'responses-model-key', + response: new Response(JSON.stringify({ error: { message: 'slow down' } }), { + status: 429, headers: { 'content-type': 'application/json' }, + }), + }), + })]); + + const { facts } = await serve(false); + + expect(facts['response.http.status']).toBe(429); + expect(facts['response.chat.openaiResponses.rendered']).toEqual({ error: { message: 'slow down' } }); + }); + + // A refused connection is an outcome the fork has to be able to see, not a fault that ends + // the run — so the second candidate is tried and its answer is the one served. + it('fails a dial that never connected over to the next candidate', async () => { + const tried: string[] = []; + resolves([ + candidate({ callOpenAIResponses: async () => { tried.push('dead'); return { type: 'transportFailure' as const, error: new Error('ECONNREFUSED') }; } }, { upstreamId: 'up_dead' }), + candidate({ callOpenAIResponses: async () => { tried.push('alive'); return stream(completed('hi')); } }, { upstreamId: 'up_alive' }), + ]); + + const { facts } = await serve(false); + + expect(tried).toEqual(['dead', 'alive']); + expect(facts['response.http.status']).toBe(200); + }); + + // A provider owns the body it is dialled with and shapes it in place, down to nested nodes — + // Copilot marks individual items for caching, and its Anthropic Messages boundary writes into a nested + // field — while the record that body is built from is deep-frozen. A dial that handed over a + // shallow copy satisfies every type in the system and throws at the first nested write, + // answering the client with a 502 raised where nothing can explain it. + // The record is frozen all the way down, and nothing copies it on the way out — a provider + // shapes the body it sends by rebuilding, which every boundary rule here does. A rule that + // reached into a child instead would throw, and this is what says so rather than leaving it + // to be discovered as a 502 from the one place that cannot explain it. + it('hands the provider a body no one can write into', async () => { + let refused: unknown; + resolves([candidate({ + callOpenAIResponses: async (_model, body) => { + const item = (body as { input: Record[] }).input[0]!; + try { item.copilot_cache_control = { type: 'ephemeral' }; } catch (error) { refused = error; } + return stream(completed('hi')); + }, + })]); + + const { facts } = await serve(false); + + expect(facts['response.http.status']).toBe(200); + expect(refused).toBeInstanceOf(TypeError); + }); + + // Codex asks for a compaction inside an ordinary turn, by ending its input with the control + // item that requests one. Where the shim is this candidate's to run, that item never reaches + // the upstream: what is sent instead is the compactor's own turn, and what comes back is an + // envelope this gateway packed the summary into. + it('answers a turn that asked for a compaction with one, where the shim is engaged', async () => { + let sent: Record | undefined; + affinityPayload = asksForCompaction; + resolves([candidate( + { + callOpenAIResponses: async (_model, body) => { + sent = body as Record; + return stream(completed('CONDENSED SUMMARY')); + }, + }, + { enabledFlags: new Set(['openai-responses-compact-shim']) }, + )]); + const gateway = mockChatGatewayCtx({ wantsStream: false }); + + const { facts } = await serveWith(gateway, false, asksForCompaction); + + // The compactor's prompt reached the upstream, the item that asked for the compaction did + // not, and the ephemeral summarization turn is not persisted in the upstream's history. + expect(JSON.stringify(sent)).toContain('CONTEXT CHECKPOINT COMPACTION'); + expect(JSON.stringify(sent)).not.toContain('compaction_trigger'); + expect(sent).toMatchObject({ store: false }); + + expect(facts['response.http.status']).toBe(200); + const answered = facts['response.chat.openaiResponses.rendered'] as Record; + expect(answered.object).toBe('response.compaction'); + expect(summaryIn(compactionItem(answered).encrypted_content)).toBe(`${SUMMARY_PREFIX}\nCONDENSED SUMMARY`); + }); + + // The flag says where a compaction would be simulated, not that this turn asked for one — + // so an ordinary turn on an opted-in upstream is dialled as itself and answered as a + // response, with nothing of the compactor's in the body that went out. + it('dials an ordinary turn as itself, where the shim is engaged but nothing asked for one', async () => { + let sent: Record | undefined; + resolves([candidate( + { + callOpenAIResponses: async (_model, body) => { + sent = body as Record; + return stream(completed('hello')); + }, + }, + { enabledFlags: new Set(['openai-responses-compact-shim']) }, + )]); + + const { facts } = await serve(false); + + expect(sent).toMatchObject({ input: [{ type: 'message', role: 'user', content: 'hi' }] }); + expect(JSON.stringify(sent)).not.toContain('CONTEXT CHECKPOINT COMPACTION'); + const answered = withoutCarrierItem(facts['response.chat.openaiResponses.rendered'] as unknown as OpenAIResponsesResult); + expect(answered.object).toBe('response'); + expect(answered).toMatchObject({ output: [{ content: [{ type: 'output_text', text: 'hello' }] }] }); + }); + + // The flag is the operator's opt-in, so an upstream that did not get it answers the item + // itself: it reaches the wire as the client wrote it, and the compaction that comes back is + // the upstream's own — its id, and a blob only it can read. + it('sends a turn that asked for a compaction on unchanged, where the shim is not engaged', async () => { + let sent: Record | undefined; + const upstreamCompaction: OpenAIResponsesCompactionResult = { + id: 'resp_upstream_compaction', + object: 'response.compaction', + output: [{ type: 'compaction', id: 'cmp_1', encrypted_content: 'OPAQUE_NATIVE_BLOB' } as unknown as never], + usage: { input_tokens: 900, output_tokens: 40, total_tokens: 940 }, + }; + affinityPayload = asksForCompaction; + resolves([candidate({ + callOpenAIResponses: async (_model, body) => { + sent = body as Record; + return { action: 'compact', ok: true, modelKey: 'responses-model-key', result: upstreamCompaction }; + }, + })]); + + const { facts } = await serve(false, asksForCompaction); + + expect(sent).toMatchObject({ input: [{ role: 'user' }, { type: 'compaction_trigger' }] }); + expect(JSON.stringify(sent)).not.toContain('CONTEXT CHECKPOINT COMPACTION'); + expect(facts['response.chat.openaiResponses.rendered']).toMatchObject({ object: 'response.compaction', usage: upstreamCompaction.usage }); + }); + + // No translation carries a compaction, and neither translator models the item that asks for + // one — so on a candidate this protocol can only reach through one, the shim is structurally + // required rather than opted into, and the flag has nothing to say. + it('answers a turn that asked for a compaction over a candidate with no OpenAI Responses wire', async () => { + const seen: { body?: Record } = {}; + affinityPayload = asksForCompaction; + resolves([candidate( + { callAnthropicMessages: anthropicMessagesTurn('CONDENSED SUMMARY', seen) }, + { endpoints: { anthropicMessages: {} } }, + )]); + const gateway = mockChatGatewayCtx({ wantsStream: false }); + + const { facts } = await serveWith(gateway, false, asksForCompaction); + + expect(JSON.stringify(seen.body)).toContain('CONTEXT CHECKPOINT COMPACTION'); + expect(JSON.stringify(seen.body)).not.toContain('compaction_trigger'); + const answered = facts['response.chat.openaiResponses.rendered'] as Record; + expect(answered.object).toBe('response.compaction'); + expect(summaryIn(compactionItem(answered).encrypted_content)).toBe(`${SUMMARY_PREFIX}\nCONDENSED SUMMARY`); + }); + + // A compaction this gateway simulated carries the history it stood for, so an ordinary turn + // that echoes one back is sent that history — the upstream has no key for the blob, and + // would otherwise continue from nothing. + it('expands a compaction it wrote before the turn quoting it goes out', async () => { + let sent: Record | undefined; + const continues = { + ...payload, + input: [ + { + type: 'compaction', + id: 'cmp_prior', + encrypted_content: encodeBase64UrlJson([ + { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'THE EARLIER HISTORY' }] }, + ]), + }, + { type: 'message', role: 'user', content: 'and then?' }, + ], + } as unknown as CanonicalOpenAIResponsesPayload; + affinityPayload = continues; + resolves([candidate( + { + callOpenAIResponses: async (_model, body) => { + sent = body as Record; + return stream(completed('hi')); + }, + }, + { enabledFlags: new Set(['openai-responses-compact-shim']) }, + )]); + + await serve(false, continues); + + expect(JSON.stringify(sent)).toContain('THE EARLIER HISTORY'); + expect(JSON.stringify(sent)).not.toContain('cmp_prior'); + }); + + // Only an upstream whose compactions this gateway simulates can be holding one of ours, and + // an upstream that compacts natively is owed its own blob byte for byte — so where the shim + // is not engaged the item travels as the client wrote it. + it('expands portable compactions even when both compaction flags are disabled', async () => { + let sent: Record | undefined; + const encoded = encodeBase64UrlJson([ + { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'THE EARLIER HISTORY' }] }, + ]); + const continues = { + ...payload, + input: [{ type: 'compaction', id: 'cmp_prior', encrypted_content: encoded }], + } as unknown as CanonicalOpenAIResponsesPayload; + affinityPayload = continues; + resolves([candidate({ + callOpenAIResponses: async (_model, body) => { + sent = body as Record; + return stream(completed('hi')); + }, + })]); + + await serve(false, continues); + + expect(sent).toMatchObject({ input: [{ type: 'message', role: 'user', content: [{ type: 'input_text', text: 'THE EARLIER HISTORY' }] }] }); + }); +}); + +it('HTTP generate:false reaches the upstream without the WebSocket-local warmup', async () => { + let seen: Record | undefined; + resolves([candidate({ + callOpenAIResponses: async (_model, body) => { + seen = body as Record; + return stream(completed('HTTP answer')); + }, + })]); + const request = { ...payload, generate: false }; + affinityPayload = request; + const outcome = await serve(false, request); + expect(seen?.generate).toBe(false); + expect(outcome.facts['response.http.status']).toBe(200); + await outcome.drain(); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/affinity/copilot-roundtrip_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/affinity/copilot-roundtrip_test.ts index 5470fb6c1c..1f96e75b16 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/affinity/copilot-roundtrip_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/affinity/copilot-roundtrip_test.ts @@ -8,7 +8,7 @@ import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesOutputItem, OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; import { initProviderRepo, providerModelOf, type UpstreamRecord } from '@floway-dev/provider'; import { clearInProcessCopilotTokenCache, copilotProviderModule } from '@floway-dev/provider-copilot'; -import { noopUpstreamCallOptions, sseResponse, stubModelCandidate, stubProvider, withMockedFetch } from '@floway-dev/test-utils'; +import { collectChatProviderPipeline, noopUpstreamCallOptions, sseResponse, stubModelCandidate, stubProvider, withMockedFetch } from '@floway-dev/test-utils'; const upstream: UpstreamRecord = { id: 'up-copilot', @@ -117,17 +117,16 @@ test('Copilot item-id and generic affinity trailers compose and unwrap in bounda return sseResponse(); }, async () => { - const first = await provider.instance.callOpenAIResponses( - providerModelOf(candidate), + const first = await collectChatProviderPipeline( + provider, 'openaiResponses', providerModelOf(candidate), { input: [], stream: true, store: false }, - 'generate', undefined, noopUpstreamCallOptions(), ); - if (!first.ok || first.action !== 'generate') throw new Error('expected first Copilot stream'); + if (first.output === null || !('kind' in first.output) || first.output.kind !== 'stream') throw new Error('expected first Copilot stream'); const codec = new AffinityCodec('00'.repeat(32)); - const publicEvents = await collectEvents(wrapOpenAIResponsesAffinityEgress(first.events, { + const publicEvents = await collectEvents(wrapOpenAIResponsesAffinityEgress({ async *[Symbol.asyncIterator]() { yield* first.frames; } }, { codec, affinity: { upstreamId: provider.upstreamId, @@ -155,14 +154,13 @@ test('Copilot item-id and generic affinity trailers compose and unwrap in bounda expect(foreign.input[0]).toEqual({ type: 'reasoning', id: publicItem.id, summary: [] }); const { model: _model, ...exactBody } = exact; - const second = await provider.instance.callOpenAIResponses( - providerModelOf(candidate), + const second = await collectChatProviderPipeline( + provider, 'openaiResponses', providerModelOf(candidate), exactBody, - 'generate', undefined, noopUpstreamCallOptions(), ); - if (!second.ok || second.action !== 'generate') throw new Error('expected replay Copilot stream'); + if (second.output === null || !('kind' in second.output) || second.output.kind !== 'stream') throw new Error('expected replay Copilot stream'); }, ); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/affinity/ingress_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/affinity/ingress_test.ts index b0062fe969..1ccc6fd28c 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/affinity/ingress_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/affinity/ingress_test.ts @@ -1,7 +1,7 @@ import { expect, test } from 'vitest'; import { analyzeOpenAIResponsesAffinity } from '../../../../../src/data-plane/chat/openai-responses/affinity/ingress.ts'; -import { isOpenAIResponsesCompactShimItem } from '../../../../../src/data-plane/chat/openai-responses/interceptors/compact-shim.ts'; +import { isOpenAIResponsesCompactShimItem } from '../../../../../src/data-plane/chat/openai-responses/compact-shim.ts'; import { AffinityCodec, type AffinityIdentity, type AffinityRequestAnalysis, compatibilityIdentityForCandidate, selectAffinityCandidates } from '../../../../../src/data-plane/chat/shared/affinity/index.ts'; import { encodeBase64UrlJson } from '../../../../../src/shared/base64url-json.ts'; import { acceptedAffinityEvaluation } from '../../shared/affinity/helpers.ts'; diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/affinity/roundtrip_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/affinity/roundtrip_test.ts index 20f6446440..f6aabe64cd 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/affinity/roundtrip_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/affinity/roundtrip_test.ts @@ -2,7 +2,7 @@ import { expect, test } from 'vitest'; import { wrapOpenAIResponsesAffinityEgress } from '../../../../../src/data-plane/chat/openai-responses/affinity/egress.ts'; import { analyzeOpenAIResponsesAffinity } from '../../../../../src/data-plane/chat/openai-responses/affinity/ingress.ts'; -import { expandShimCompactionItems } from '../../../../../src/data-plane/chat/openai-responses/interceptors/compact-shim.ts'; +import { expandShimCompactionItems } from '../../../../../src/data-plane/chat/openai-responses/compact-shim.ts'; import { hydrateOpenAIResponsesPayload } from '../../../../../src/data-plane/chat/openai-responses/items/hydrate.ts'; import { wrapOpenAIResponsesClientOutput } from '../../../../../src/data-plane/chat/openai-responses/items/output.ts'; import { createOpenAIResponsesHttpStore } from '../../../../../src/data-plane/chat/openai-responses/items/store.ts'; diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/answer-websocket-warmup_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/answer-websocket-warmup_test.ts new file mode 100644 index 0000000000..3ac37781aa --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/answer-websocket-warmup_test.ts @@ -0,0 +1,54 @@ +import { expect, test, vi } from 'vitest'; + +import { answerOpenAIResponsesWebSocketWarmup } from '../../../../src/data-plane/chat/openai-responses/answer-websocket-warmup.ts'; +import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; +import { compose, defineStage, move, run } from '@floway-dev/pipeline'; +import { doneFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import { stubModelCandidate } from '@floway-dev/test-utils'; + +const exercise = async (generate: boolean | null | undefined) => { + const candidate = stubModelCandidate(); + const gateway = mockChatGatewayCtx(); + const selectAffinity = vi.fn(); + const dial = vi.fn(); + const ending = defineStage, Record>({ + name: 'scriptedWarmupDial', + return: { provides: ['response.chat.openaiResponses', 'response.chat.openaiResponses.streamedUsage', 'response.usage.billable', 'response.http.headers', 'response.http.status', 'response.http.body'] }, + execute: async facts => { + dial(); + return move({ ...facts, 'response.chat.openaiResponses': { kind: 'stream', frames: (async function* () { yield doneFrame(); })() }, 'response.chat.openaiResponses.streamedUsage': null, 'response.usage.billable': [], 'response.http.headers': [], 'response.http.status': 200, 'response.http.body': null }); + }, + }); + const payload: CanonicalOpenAIResponsesPayload = { model: 'client-model', input: [{ type: 'message', role: 'developer', content: 'Base instructions' }], ...(generate === undefined ? {} : { generate }) }; + const outcome = await run(compose, Record>('warmupUnderTest', [answerOpenAIResponsesWebSocketWarmup, ending]), move({ + 'request.chat.openaiResponses': payload, + 'route.attempt': { candidateId: 0, upstreamId: candidate.provider.upstreamId, modelId: candidate.model.id, flags: [] }, + }), { gateway, resolveAttempt: () => candidate, selectAffinity } as never); + const answer = outcome.facts['response.chat.openaiResponses'] as { frames: AsyncIterable> }; + const frames: ProtocolFrame[] = []; + for await (const frame of answer.frames) frames.push(frame); + await outcome.drain(); + return { frames, dial, selectAffinity, candidate, gateway, facts: outcome.facts }; +}; + +test('a prewarm completes locally without dispatch timing and preserves its request identity', async () => { + const { frames, dial, selectAffinity, candidate, gateway, facts } = await exercise(false); + expect(dial).not.toHaveBeenCalled(); + expect(selectAffinity).toHaveBeenCalledWith(candidate); + expect(frames.map(frame => frame.type === 'event' ? frame.event.type : frame.type)).toEqual(['response.created', 'response.in_progress', 'response.completed', 'done']); + const terminal = frames[2]; + expect(terminal).toMatchObject({ type: 'event', event: { response: { model: candidate.model.id, output: [], status: 'completed', usage: { input_tokens: 0, output_tokens: 0, total_tokens: 0 } } } }); + expect(gateway.attempt.telemetry).toBeUndefined(); + expect(gateway.attempt.timing.upstreamCallStartedAt).toBeNull(); + expect(facts['response.usage.billable']).toEqual([]); +}); + +for (const generate of [undefined, null, true]) { + test(`generate:${generate} continues through the normal chain`, async () => { + const { dial, selectAffinity, frames } = await exercise(generate); + expect(dial).toHaveBeenCalledOnce(); + expect(selectAffinity).not.toHaveBeenCalled(); + expect(frames).toEqual([doneFrame()]); + }); +} diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/attempt_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/attempt_test.ts deleted file mode 100644 index 8fd1b0bb01..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/attempt_test.ts +++ /dev/null @@ -1,739 +0,0 @@ -import { test, vi } from 'vitest'; - -import { TEST_OPENAI_RESPONSES_RETENTION_SECONDS, testOpenAIResponsesStatePolicy } from './test-policy.ts'; -import { analyzeOpenAIResponsesAffinity } from '../../../../src/data-plane/chat/openai-responses/affinity/ingress.ts'; -import { openaiResponsesAttempt } from '../../../../src/data-plane/chat/openai-responses/attempt.ts'; -import { withOpenAIResponsesCompactShim } from '../../../../src/data-plane/chat/openai-responses/interceptors/compact-shim.ts'; -import { openaiResponsesInterceptors } from '../../../../src/data-plane/chat/openai-responses/interceptors/index.ts'; -import type { OpenAIResponsesInterceptor } from '../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; -import { hydrateOpenAIResponsesPayload } from '../../../../src/data-plane/chat/openai-responses/items/hydrate.ts'; -import * as outputModule from '../../../../src/data-plane/chat/openai-responses/items/output.ts'; -import { createOpenAIResponsesHttpStore } from '../../../../src/data-plane/chat/openai-responses/items/store.ts'; -import type { ChatGatewayCtx } from '../../../../src/data-plane/chat/shared/gateway-ctx.ts'; -import { initRepo } from '../../../../src/repo/index.ts'; -import type { StoredOpenAIResponsesItem } from '../../../../src/repo/types.ts'; -import { encodeBase64UrlJson } from '../../../../src/shared/base64url-json.ts'; -import { InMemoryRepo } from '../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import { acceptedAffinityEvaluation } from '../shared/affinity/helpers.ts'; -import { initExternalResourceFetcher } from '@floway-dev/platform'; -import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesPayload, OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { type AnthropicMessagesUpstreamCallOptions, type ModelCandidate, directFetcher, type ProviderModel, type ProviderOpenAIResponsesResult, type ProviderStreamResult, type OpenAIResponsesAction, type UpstreamCallOptions, type FlagId } from '@floway-dev/provider'; -import { assert, assertEquals, stubProvider, stubInternalModel, stubProviderModel } from '@floway-dev/test-utils'; - -const API_KEY_ID = 'key_attempt_test'; - -const makeGatewayCtx = (store?: ChatGatewayCtx['store']) => - mockChatGatewayCtx({ apiKeyId: API_KEY_ID, wantsStream: true, ...(store ? { store } : {}) }); - -const makePayload = (overrides: Partial = {}): CanonicalOpenAIResponsesPayload => ({ - model: 'test-model', - input: [{ type: 'message', role: 'user', content: 'hello' }], - ...overrides, -}); - -const makeOpenAIResponsesResult = (id = 'resp_test'): OpenAIResponsesResult => ({ - id, - object: 'response', - model: 'test-model', - status: 'completed', - output: [{ - type: 'message', - id: 'msg_1', - role: 'assistant', - status: 'completed', - content: [{ type: 'output_text', text: 'hi', annotations: [] }], - }], - output_text: 'hi', - error: null, - incomplete_details: null, -}); - -const makeProviderEvents = async function* (events: readonly OpenAIResponsesStreamEvent[]): AsyncGenerator> { - for (const event of events) yield eventFrame(event); - yield doneFrame(); -}; - -const makeCandidate = ( - callOpenAIResponses: (model: ProviderModel, body: Omit, action: OpenAIResponsesAction, signal: AbortSignal | undefined, opts: UpstreamCallOptions) => Promise, - enabledFlags: ReadonlySet = new Set(), -): ModelCandidate => { - const provider = stubProvider({ callOpenAIResponses }); - const upstream = 'up_test'; - return { - provider: { - upstreamId: upstream, - kind: 'custom', - name: upstream, - inboundHeaderAllowlist: [], - disabledPublicModelIds: [], - modelPrefix: null, - modelsCache: null, pipelines: {}, - instance: provider, - }, - model: stubInternalModel({ - providerModels: { [upstream]: stubProviderModel({ enabledFlags }) }, - }, upstream), - fetcher: directFetcher, - }; -}; - -const collectEvents = async (events: AsyncIterable>): Promise => { - const out: OpenAIResponsesStreamEvent[] = []; - for await (const frame of events) { - if (frame.type === 'event') out.push(frame.event); - } - return out; -}; - -const installRepo = () => { - const repo = new InMemoryRepo(); - initRepo(repo); - void repo.apiKeys.save({ - id: API_KEY_ID, userId: 1, name: 'OpenAI Responses test key', key: 'raw-responses-test', - serverSecret: '99'.repeat(32), createdAt: '2026-01-01T00:00:00.000Z', - upstreamIds: null, deletedAt: null, dumpRetentionSeconds: null, - openaiResponsesRetentionSeconds: TEST_OPENAI_RESPONSES_RETENTION_SECONDS, - }); - return repo; -}; - -const insertStoredItem = async (repo: InMemoryRepo, overrides: Partial & Pick & { type: string }): Promise => { - const { type, ...itemOverrides } = overrides; - const row: StoredOpenAIResponsesItem = { - apiKeyId: API_KEY_ID, - itemHash: `hash-${overrides.id}`, - payload: { item: { type, id: overrides.id } }, - refreshedAt: Date.now(), - ...itemOverrides, - }; - await repo.openaiResponsesItems.insertMany([row], 0); - return row; -}; - -test('generate native success leaves source-edge state ownership to the caller', async () => { - installRepo(); - - const completedEvent: OpenAIResponsesStreamEvent = { - type: 'response.completed', - sequence_number: 0, - response: makeOpenAIResponsesResult(), - }; - const callOpenAIResponses = vi.fn(async (): Promise => ({ - action: 'generate', ok: true, - events: makeProviderEvents([completedEvent]), - modelKey: 'test-model-key', - headers: new Headers(), - })); - - const candidate = makeCandidate(callOpenAIResponses); - const store = createOpenAIResponsesHttpStore(testOpenAIResponsesStatePolicy(API_KEY_ID), Date.now(), true); - const ctx = makeGatewayCtx(store); - - const result = await openaiResponsesAttempt.generate({ - payload: makePayload(), - ctx, - candidate, - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - - const events = await collectEvents(result.events); - assert(events.length >= 1, 'expected at least the response.completed event'); - - assertEquals(callOpenAIResponses.mock.calls.length, 1); -}); - -test('generate isolates provider mutations with JSON-safe container cloning', async () => { - installRepo(); - const metadata = JSON.parse('{"__proto__":{"retained":true},"nested":{"value":"source"}}') as Record; - const payload = makePayload({ metadata }); - const sourceItem = payload.input[0]; - const callOpenAIResponses = vi.fn(async (_model, body): Promise => { - const clonedMetadata = body.metadata as { nested: { value: string } } & Record; - assert(Object.hasOwn(clonedMetadata, '__proto__'), 'clone lost the own __proto__ JSON field'); - clonedMetadata.nested.value = 'provider'; - (body.input[0] as { role: string }).role = 'assistant'; - return { - action: 'generate', - ok: true, - events: makeProviderEvents([]), - modelKey: 'test-model-key', - }; - }); - - await openaiResponsesAttempt.generate({ - payload, - ctx: makeGatewayCtx(), - candidate: makeCandidate(callOpenAIResponses), - headers: new Headers(), - }); - - assertEquals((payload.metadata as { nested: { value: string } }).nested.value, 'source'); - assertEquals((sourceItem as { role: string }).role, 'user'); -}); - -test('generate treats a translated OpenAI Responses payload as opaque to native affinity and state', async () => { - installRepo(); - let observedBody: Omit | undefined; - const callOpenAIResponses = vi.fn(async ( - _model: ProviderModel, - body: Omit, - ): Promise => { - observedBody = body; - return { - action: 'generate', - ok: true, - events: makeProviderEvents([{ - type: 'response.completed', - sequence_number: 0, - response: makeOpenAIResponsesResult(), - }]), - modelKey: 'test-model-key', - headers: new Headers(), - }; - }); - const candidate = makeCandidate(callOpenAIResponses); - const store = createOpenAIResponsesHttpStore(testOpenAIResponsesStatePolicy(API_KEY_ID), Date.now(), true); - const ctx = makeGatewayCtx(store); - const carrier = await ctx.affinity.codec.wrap( - undefined, - { - upstreamId: candidate.provider.upstreamId, - modelId: candidate.model.id, - opaqueBlobCompatibilityIdentity: { upstreamId: candidate.provider.upstreamId, key: candidate.model.id }, - }, - 'openai-responses.reasoning.encrypted_content', - ); - const unwrap = vi.spyOn(ctx.affinity.codec, 'unwrap'); - const getStoredItem = vi.spyOn(store, 'getItemById'); - const itemId = 'rs_source_edge'; - - const result = await openaiResponsesAttempt.generate({ - payload: makePayload({ - input: [{ type: 'reasoning', id: itemId, summary: [], encrypted_content: carrier }], - }), - ctx, - candidate, - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(unwrap.mock.calls.length, 0); - assertEquals(getStoredItem.mock.calls.length, 0); - assertEquals(observedBody?.input, [{ type: 'reasoning', id: itemId, summary: [], encrypted_content: carrier }]); -}); - -test('generate applies role compatibility flags in target-chain order', async () => { - installRepo(); - let observedBody: Omit | undefined; - const callOpenAIResponses = vi.fn(async ( - _model: ProviderModel, - body: Omit, - ): Promise => { - observedBody = body; - return { - action: 'generate', - ok: true, - events: makeProviderEvents([{ - type: 'response.completed', - sequence_number: 0, - response: makeOpenAIResponsesResult(), - }]), - modelKey: 'test-model-key', - headers: new Headers(), - }; - }); - const candidate = makeCandidate(callOpenAIResponses, new Set([ - 'rewrite-developer-to-system', - 'rewrite-mid-conv-system-to-user', - 'rewrite-system-to-developer', - ])); - - const result = await openaiResponsesAttempt.generate({ - payload: makePayload({ - input: [ - { type: 'message', role: 'system', content: 'base instructions' }, - { type: 'message', role: 'user', content: 'hello' }, - { type: 'message', role: 'system', content: 'inline instructions' }, - ], - }), - ctx: makeGatewayCtx(createOpenAIResponsesHttpStore(testOpenAIResponsesStatePolicy(API_KEY_ID), Date.now(), false)), - candidate, - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(observedBody?.input, [ - { type: 'message', role: 'system', content: 'base instructions' }, - { type: 'message', role: 'user', content: 'hello' }, - { type: 'message', role: 'user', content: 'inline instructions' }, - ]); -}); - -test('generate defers the role rewrite until after translation to OpenAI Chat Completions', async () => { - installRepo(); - let observedBody: Omit | undefined; - const callOpenAIChatCompletions = vi.fn(async ( - _model: ProviderModel, - body: Omit, - ): Promise> => { - observedBody = body; - return { - ok: true, - events: (async function* () { - yield eventFrame({ - id: 'chatcmpl_test', - object: 'chat.completion.chunk', - created: 0, - model: 'test-model', - choices: [{ index: 0, delta: { role: 'assistant' }, finish_reason: null }], - }); - yield eventFrame({ - id: 'chatcmpl_test', - object: 'chat.completion.chunk', - created: 0, - model: 'test-model', - choices: [{ index: 0, delta: { content: 'hi' }, finish_reason: null }], - }); - yield eventFrame({ - id: 'chatcmpl_test', - object: 'chat.completion.chunk', - created: 0, - model: 'test-model', - choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], - }); - yield doneFrame(); - })(), - modelKey: 'test-model-key', - headers: new Headers(), - }; - }); - const upstream = 'up_chat'; - const endpoints = { openaiChatCompletions: {} }; - const candidate: ModelCandidate = { - provider: { - upstreamId: upstream, - kind: 'custom', - name: upstream, - inboundHeaderAllowlist: [], - disabledPublicModelIds: [], - modelPrefix: null, - modelsCache: null, pipelines: {}, - instance: stubProvider({ callOpenAIChatCompletions }), - }, - model: stubInternalModel({ - endpoints, - providerModels: { - [upstream]: stubProviderModel({ - endpoints, - enabledFlags: new Set(['rewrite-system-to-developer']), - }), - }, - }, upstream), - fetcher: directFetcher, - }; - - const result = await openaiResponsesAttempt.generate({ - payload: makePayload({ - input: [ - { type: 'message', role: 'system', content: 'base instructions' }, - { type: 'message', role: 'user', content: 'hello' }, - { type: 'message', role: 'system', content: 'inline instructions' }, - ], - }), - ctx: makeGatewayCtx(createOpenAIResponsesHttpStore(testOpenAIResponsesStatePolicy(API_KEY_ID), Date.now(), false)), - candidate, - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(observedBody?.messages, [ - { role: 'developer', content: 'base instructions' }, - { role: 'user', content: 'hello' }, - { role: 'developer', content: 'inline instructions' }, - ]); -}); - -test('generate passes non-events provider result through unchanged', async () => { - installRepo(); - const wrapSpy = vi.spyOn(outputModule, 'wrapOpenAIResponsesClientOutput'); - - const upstreamResponse = new Response(JSON.stringify({ error: { message: 'nope' } }), { status: 502, headers: new Headers({ 'content-type': 'application/json' }) }); - const callOpenAIResponses = vi.fn(async (): Promise => ({ - action: 'generate', ok: false, - response: upstreamResponse, - modelKey: 'test-model-key', - })); - - const candidate = makeCandidate(callOpenAIResponses); - const result = await openaiResponsesAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(createOpenAIResponsesHttpStore(testOpenAIResponsesStatePolicy(API_KEY_ID), Date.now(), true)), - candidate, - headers: new Headers(), - }); - - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 502); - // Wrap must not run when the upstream failed before any events flowed. - assertEquals(wrapSpy.mock.calls.length, 0); - wrapSpy.mockRestore(); -}); - -test('compact returns the clean upstream result for source-edge affinity and storage', async () => { - installRepo(); - - // Native /responses/compact returns a fully-shaped compaction envelope — - // the `action: 'compact'` branch of `provider.callOpenAIResponses` does the - // Copilot compaction_trigger reshape internally — so the attempt receives - // a OpenAIResponsesResult, expands it into synthetic frames, and wraps the - // output for storage. The synthesized envelope carries a `compaction` - // output item; wrap observes it and derives the 'replace' snapshot. - const compactionItem = { - type: 'compaction' as const, - id: 'cmp_1', - encrypted_content: 'ENC', - }; - const compactionResult: OpenAIResponsesResult = { - ...makeOpenAIResponsesResult(), - object: 'response.compaction', - // Cast: `compaction` is an input-shaped item type the protocol's - // OpenAIResponsesResult.output type does not include but the runtime accepts. - output: [compactionItem] as unknown as OpenAIResponsesResult['output'], - }; - - const callOpenAIResponses = vi.fn(async (_model: ProviderModel, _body: Omit, action: OpenAIResponsesAction): Promise => { - if (action !== 'compact') throw new Error(`compact candidate received action='${action}'`); - return { action: 'compact', ok: true, result: compactionResult, modelKey: 'test-model-key' }; - }); - - const candidate = makeCandidate(callOpenAIResponses); - const store = createOpenAIResponsesHttpStore(testOpenAIResponsesStatePolicy(API_KEY_ID), Date.now(), true); - const result = await openaiResponsesAttempt.invoke({ - payload: makePayload({ - input: [ - { type: 'message', role: 'user', content: 'kept message' }, - ], - }), - action: 'compact', - ctx: makeGatewayCtx(store), - candidate, - headers: new Headers(), - }); - - assertEquals(result.type, 'result'); - if (result.type !== 'result') throw new Error('unreachable'); - assertEquals(result.result.object, 'response.compaction'); - assertEquals(result.result.output.length, 1); - assertEquals((result.result.output[0] as { id: string }).id, 'cmp_1'); - assertEquals(result.result.id, compactionResult.id); -}); - -test('generate strips disallowed headers and injects external image loading across translation to Anthropic Messages', async () => { - installRepo(); - initExternalResourceFetcher(url => { - assertEquals(url.href, 'https://example.com/image.png'); - return Promise.resolve(new Response(Uint8Array.of(1, 2, 3), { headers: { 'content-type': 'image/png' } })); - }); - let observedHeaders: Headers | undefined; - let observedAnthropicBeta: readonly string[] | undefined; - let observedBody: Omit | undefined; - const upstreamModel = stubInternalModel({ endpoints: { anthropicMessages: {} } }, 'up_test'); - const anthropicMessagesProvider = stubProvider({ - callAnthropicMessages: async (_model, body, _signal, opts): Promise> => { - observedHeaders = opts.headers; - observedAnthropicBeta = (opts as AnthropicMessagesUpstreamCallOptions).anthropicBeta; - observedBody = body as Omit; - return { - ok: true, - events: (async function* () { - yield eventFrame({ - type: 'message_start', - message: { - id: 'msg_1', type: 'message', role: 'assistant', content: [], - model: 'test-model', stop_reason: null, stop_sequence: null, - usage: { input_tokens: 1, output_tokens: 0 }, - }, - }); - yield eventFrame({ type: 'message_stop' }); - yield doneFrame(); - })(), - modelKey: 'k', - headers: new Headers(), - }; - }, - }); - const candidate: ModelCandidate = { - provider: { - upstreamId: 'up_test', kind: 'custom', name: 'up_test', inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: anthropicMessagesProvider, - }, - model: upstreamModel, - fetcher: directFetcher, - }; - - const result = await openaiResponsesAttempt.generate({ - payload: makePayload({ - input: [{ - type: 'message', - role: 'user', - content: [{ type: 'input_image', image_url: 'https://example.com/image.png', detail: 'auto' }], - }], - }), - ctx: makeGatewayCtx(createOpenAIResponsesHttpStore(testOpenAIResponsesStatePolicy(API_KEY_ID), Date.now(), true)), - candidate, - headers: new Headers({ 'anthropic-beta': 'must-not-cross-source-protocols', 'x-test': 'abc' }), - }); - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(observedHeaders?.get('x-test'), null); - assertEquals(observedHeaders?.get('anthropic-beta'), null); - assertEquals(observedAnthropicBeta, []); - const message = observedBody?.messages[0]; - assert(message?.role === 'user' && Array.isArray(message.content)); - const image = message.content.find(block => block.type === 'image'); - assert(image?.type === 'image'); - assertEquals(image.source, { type: 'base64', media_type: 'image/png', data: 'AQID' }); -}); - -test('generate seeds privatePayload before interceptors so the web-search shim replays the prior wsc results on echo', async () => { - // End-to-end contract: when a stateless client (e.g. Codex CLI) echoes a - // prior gateway-created web_search_call by its emitted id, the web-search shim's - // `transformItems` (which runs as part of the interceptor chain) must - // find the persisted `payload.private` and emit the cached function_call - // + function_call_output pair to upstream — NOT the not-preserved - // placeholder. - // - // The wire shape we model here: - // - row.id = the public item id echoed as `wsc.id`. - // - payload.item.id = that same public id. - // - payload.private = WebSearchCallPrivatePayload (v:1, functionCallItem, ir). - // - // This regression caught a prior ordering bug where hydration + beginAttempt - // ran inside the interceptor closure, after the shim's input transform — - // so privatePayload was always empty when the shim looked it up, and - // every echoed wsc collapsed to the placeholder. - const repo = installRepo(); - const storedId = `ws_${'a'.repeat(32)}`; - const storedItem = { - type: 'web_search_call' as const, - id: storedId, - status: 'completed' as const, - action: { type: 'search' as const, query: 'deepseek v4', queries: ['deepseek v4'] }, - }; - await insertStoredItem(repo, { - id: storedId, - type: 'web_search_call', - payload: { - item: storedItem, - private: { - v: 1, - functionCallItem: { - type: 'function_call', - call_id: 'call_orig_xyz', - name: 'web_search', - arguments: '{"search_query":[{"q":"deepseek v4"}]}', - status: 'completed', - }, - ir: { - action: { type: 'search', query: 'deepseek v4', queries: ['deepseek v4'] }, - results: [{ type: 'text_result', url: 'https://example.com', title: 'Example', snippet: 'CACHED_SNIPPET_BODY' }], - }, - }, - }, - }); - - // Capture the upstream-bound body so we can verify what the shim produced - // after the echoed wsc passed through transformItems. - let capturedBody: { input?: unknown[] } | undefined; - const upstreamResponse = makeOpenAIResponsesResult(); - // The shim's multi-turn loop requires `response.created` (carrying a model - // name) before any synthesized terminal envelope. Emit the canonical - // created → in_progress → completed sequence so the shim can wrap. - const upstreamEvents: OpenAIResponsesStreamEvent[] = [ - { type: 'response.created', sequence_number: 0, response: upstreamResponse }, - { type: 'response.in_progress', sequence_number: 1, response: upstreamResponse }, - { type: 'response.completed', sequence_number: 2, response: upstreamResponse }, - ]; - const callOpenAIResponses = vi.fn(async (_model, body): Promise => { - capturedBody = body as { input?: unknown[] }; - return { action: 'generate', ok: true, events: makeProviderEvents(upstreamEvents), modelKey: 'test-model-key', headers: new Headers() }; - }); - const candidate = makeCandidate(callOpenAIResponses, new Set(['openai-responses-web-search-shim'])); - - const store = createOpenAIResponsesHttpStore(testOpenAIResponsesStatePolicy(API_KEY_ID), Date.now(), true); - await store.loadInputItems([{ type: 'web_search_call', id: storedId }], []); - const ctx = makeGatewayCtx(store); - const carrier = await ctx.affinity.codec.wrap( - undefined, - { - upstreamId: candidate.provider.upstreamId, - modelId: candidate.model.id, - opaqueBlobCompatibilityIdentity: { upstreamId: candidate.provider.upstreamId, key: candidate.model.id }, - }, - 'openai-responses.reasoning.encrypted_content', - { syntheticItem: true }, - ); - - const sourcePayload = makePayload({ - input: [ - { type: 'message', role: 'user', content: 'follow-up' }, - { type: 'reasoning', id: 'rs_affinity', summary: [], encrypted_content: carrier }, - { - type: 'web_search_call', - id: storedId, - status: 'completed', - action: { type: 'search', queries: ['deepseek v4'] }, - } as unknown as never, - ], - tools: [{ type: 'web_search' }], - }); - await store.loadInputItems(sourcePayload.input, sourcePayload.input); - const hydrated = hydrateOpenAIResponsesPayload(sourcePayload, store); - const affinity = await analyzeOpenAIResponsesAffinity(hydrated.payload, ctx.affinity.codec); - const result = await openaiResponsesAttempt.generate({ - payload: acceptedAffinityEvaluation(affinity, candidate).materialize(), - sourceState: { - privatePayloads: hydrated.privatePayloads, - }, - ctx, - candidate, - headers: new Headers(), - }); - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - - assert(capturedBody !== undefined, 'callOpenAIResponses was not invoked'); - const input = (capturedBody!.input ?? []) as Array<{ type: string; call_id?: string; output?: string; name?: string; arguments?: string }>; - // The wsc echo MUST be replaced by the recovered function_call + output pair, - // carrying the persisted call_id and the cached snippet body verbatim. - const fc = input.find(i => i.type === 'function_call' && i.call_id === 'call_orig_xyz'); - assert(fc !== undefined, 'expected replayed function_call with the persisted call_id'); - assertEquals(fc!.name, 'web_search'); - assertEquals(fc!.arguments, '{"search_query":[{"q":"deepseek v4"}]}'); - const fco = input.find(i => i.type === 'function_call_output' && i.call_id === 'call_orig_xyz'); - assert(fco !== undefined, 'expected replayed function_call_output'); - assert(fco!.output?.includes('CACHED_SNIPPET_BODY'), `expected cached body in function_call_output, got: ${fco!.output}`); - // And the not-preserved placeholder MUST NOT appear. - assert( - !input.some(i => i.type === 'function_call_output' && typeof i.output === 'string' && i.output.includes('Prior search results were not preserved')), - 'shim emitted the not-preserved placeholder despite a stored private payload', - ); -}); - -test('generate propagates upstream response headers onto the EventResult so respond can forward them', async () => { - installRepo(); - const completedEvent: OpenAIResponsesStreamEvent = { - type: 'response.completed', - sequence_number: 0, - response: makeOpenAIResponsesResult(), - }; - const upstreamHeaders = new Headers({ - 'anthropic-ratelimit-unified-status': 'allowed', - 'request-id': 'req_resp_xyz', - }); - const callOpenAIResponses = vi.fn(async (): Promise => ({ - action: 'generate', ok: true, - events: makeProviderEvents([completedEvent]), - modelKey: 'test-model-key', - headers: upstreamHeaders, - })); - const candidate = makeCandidate(callOpenAIResponses); - const store = createOpenAIResponsesHttpStore(testOpenAIResponsesStatePolicy(API_KEY_ID), Date.now(), true); - const result = await openaiResponsesAttempt.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(store), - candidate, - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - assertEquals(result.headers?.get('anthropic-ratelimit-unified-status'), 'allowed'); - assertEquals(result.headers?.get('request-id'), 'req_resp_xyz'); - await collectEvents(result.events); -}); - -test('namespace wire mapping follows compact expansion and is isolated from outer shims across repeated dispatches', async () => { - installRepo(); - const namespace = { type: 'namespace' as const, name: 'files', description: '', tools: [{ type: 'function' as const, name: 'read', parameters: { type: 'object' } }] }; - const history = { type: 'function_call' as const, name: 'read', namespace: 'files', call_id: 'past', arguments: '{}', status: 'completed' as const }; - const payload = makePayload({ - tools: [{ type: 'function', name: 'files_read' }, namespace], - input: [{ type: 'compaction', encrypted_content: encodeBase64UrlJson([history, { type: 'function_call_output', call_id: 'past', output: 'done' }]) }], - }); - const original = structuredClone(payload); - const providerBodies: Omit[] = []; - const endpoints = { openaiChatCompletions: {} }; - const candidate: ModelCandidate = { - ...makeCandidate(async () => { throw new Error('native Responses must not be called'); }), - model: stubInternalModel({ endpoints, providerModels: { up_test: stubProviderModel({ endpoints }) } }, 'up_test'), - }; - candidate.provider.instance.callOpenAIChatCompletions = async (_model, body) => { - providerBodies.push(structuredClone(body)); - assert(JSON.stringify(body.messages).includes('files_read_2'), 'compact-expanded history must be mapped at dispatch'); - const tool = body.tools?.[1]; - assert(tool?.type === 'function' && tool.function.parameters !== undefined); - tool.function.parameters.providerOnly = true; - return { - ok: true, modelKey: 'test-model-key', events: (async function* () { - yield eventFrame({ id: 'chat_isolation', object: 'chat.completion.chunk', created: 0, model: 'test-model', choices: [{ index: 0, delta: { role: 'assistant', tool_calls: [{ index: 0, id: 'next', type: 'function', function: { name: 'files_read_2', arguments: '{}' } }] }, finish_reason: null }] }); - yield eventFrame({ id: 'chat_isolation', object: 'chat.completion.chunk', created: 0, model: 'test-model', choices: [{ index: 0, delta: {}, finish_reason: 'tool_calls' }] }); - yield doneFrame(); - })(), - }; - }; - let observed = 0; - const observer: OpenAIResponsesInterceptor = async (invocation, _ctx, run) => { - assertEquals(invocation.payload.input[0], history, 'outer reader must observe expanded canonical history'); - const freeze = (value: unknown): void => { - if (typeof value !== 'object' || value === null) return; - Object.freeze(value); - for (const child of Object.values(value)) freeze(child); - }; - freeze(invocation.payload); - const first = await run(); - assertEquals(invocation.payload.tools?.[1], namespace, 'translation must leave outer declarations canonical'); - assertEquals(invocation.payload.input[0], history); - assert(first.type === 'events'); - const events = await collectEvents(first.events); - const done = events.find(event => event.type === 'response.output_item.done'); - assert(done?.type === 'response.output_item.done' && done.item.type === 'function_call'); - assertEquals([done.item.name, done.item.namespace], ['read', 'files']); - observed++; - return await run(); - }; - // Observe the real seam between compact expansion and the outer server-tool - // loop. Dispatch twice like that loop does; both invocations must start from - // Standard rather than the first call's ephemeral names. - const chain = openaiResponsesInterceptors as OpenAIResponsesInterceptor[]; - const compactIndex = chain.indexOf(withOpenAIResponsesCompactShim); - assert(compactIndex >= 0, 'expected the compaction interceptor in the dispatch chain'); - chain.splice(compactIndex + 1, 0, observer); - try { - const result = await openaiResponsesAttempt.generate({ payload, ctx: makeGatewayCtx(), candidate, headers: new Headers() }); - assert(result.type === 'events'); - await collectEvents(result.events); - assertEquals(observed, 1); - assertEquals(providerBodies.length, 2); - assertEquals(providerBodies[0], providerBodies[1]); - assertEquals(payload, original); - } finally { - chain.splice(chain.indexOf(observer), 1); - } -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/collaboration-drive.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/collaboration-drive.ts new file mode 100644 index 0000000000..bba0b7359d --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/collaboration-drive.ts @@ -0,0 +1,45 @@ +import { projectOpenAIResponsesCollaboration } from '../../../../src/data-plane/chat/openai-responses/collaboration-shim.ts'; +import type { ChatGatewayCtx } from '../../../../src/data-plane/chat/shared/gateway-ctx.ts'; +import { compose, defineStage, move, run } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import { providerModelOf, type OpenAIResponsesInvocation } from '@floway-dev/provider'; +import { type ExecuteResult } from '@floway-dev/test-utils'; + +type Result = ExecuteResult>; + +export const driveCollaborationStage = async ( + invocation: OpenAIResponsesInvocation, + gateway: ChatGatewayCtx, + dial: () => Promise, +): Promise => { + let result: Result | undefined; + const ending = defineStage, Record>({ + name: 'scriptedCollaborationDial', + return: { provides: ['response.chat.openaiResponses'] }, + execute: async facts => { + invocation.payload = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; + result = await dial(); + return move({ + ...facts, + 'response.chat.openaiResponses': result.type === 'events' + ? { kind: 'stream', frames: result.events } + : { status: result.type === 'internal-error' ? 500 : result.status, message: 'scripted refusal' }, + }); + }, + }); + const { facts, drain } = await run(compose, Record>('collaborationUnderTest', [projectOpenAIResponsesCollaboration, ending]), move({ + 'request.chat.openaiResponses': invocation.payload, + 'route.attempt': { candidateId: 0, upstreamId: invocation.candidate.provider.upstreamId, modelId: invocation.payload.model, flags: [...providerModelOf(invocation.candidate).enabledFlags] }, + }), { gateway } as never); + if (result === undefined) throw new Error('Collaboration harness did not dispatch'); + if (result.type !== 'events') { await drain(); return result; } + const answer = facts['response.chat.openaiResponses'] as { frames: AsyncIterable> }; + if (answer.frames === result.events) { await drain(); return result; } + return { + ...result, + events: (async function* () { + try { yield* answer.frames; } finally { await drain(); } + })(), + }; +}; diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/collaboration-shim_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/collaboration-shim_test.ts similarity index 98% rename from packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/collaboration-shim_test.ts rename to packages/gateway/__tests__/data-plane/chat/openai-responses/collaboration-shim_test.ts index 07f4ea3384..d304f95f6a 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/collaboration-shim_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/collaboration-shim_test.ts @@ -1,13 +1,12 @@ import { expect, test } from 'vitest'; -import { withOpenAIResponsesCollaborationShim } from '../../../../../src/data-plane/chat/openai-responses/interceptors/collaboration-shim.ts'; -import { withOpenAIResponsesServerToolShim } from '../../../../../src/data-plane/chat/openai-responses/interceptors/server-tool-shim.ts'; -import type { OpenAIResponsesInvocation } from '../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; +import { driveCollaborationStage as withOpenAIResponsesCollaborationShim } from './collaboration-drive.ts'; +import { driveHostedToolStage as withHostedTools } from './hosted-tools/drive.ts'; +import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIResponsesOutputFunctionCall, OpenAIResponsesOutputItem, OpenAIResponsesResult, OpenAIResponsesStreamEvent, OpenAIResponsesTool } from '@floway-dev/protocols/openai-responses'; -import { eventResult } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; +import type { OpenAIResponsesInvocation } from '@floway-dev/provider'; +import { assertEquals, stubModelCandidate, testTelemetryModelIdentity, eventResult } from '@floway-dev/test-utils'; const MESSAGE_SCHEMA = { type: 'object', @@ -490,7 +489,7 @@ test('keeps the namespace across an actual hosted-tool loop and restores only th ctx.payload.tool_choice = 'auto'; const gatewayCtx = mockChatGatewayCtx(); let upstreamCalls = 0; - const serverShim = withOpenAIResponsesServerToolShim([() => ({ + const serverShim = withHostedTools([() => ({ type: 'active', baseToolName: 'search', hosted: { hostedTypes: ['web_search'], canonicalize: tool => tool.type === 'web_search' ? tool : undefined, diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/compact-shim_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/compact-shim_test.ts new file mode 100644 index 0000000000..46ae14bfb6 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/compact-shim_test.ts @@ -0,0 +1,267 @@ +// What a simulated compaction is made of. The rules that use these run as stages and are +// tested where they run — `openai-responses-pipeline_test.ts` for the generate chain and +// `openai-responses-compact_test.ts` for the compaction chain; what is written down here is the +// substance those stages send and pack. + +import { test } from 'vitest'; + +import { + buildCompactionEnvelope, + expandShimCompactionItems, + summarizationTurnFor, + summaryTextFrom, + SUMMARY_PREFIX, +} from '../../../../src/data-plane/chat/openai-responses/compact-shim.ts'; +import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../../src/shared/base64url-json.ts'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesInputItem, OpenAIResponsesOutputItem, OpenAIResponsesResult } from '@floway-dev/protocols/openai-responses'; +import { assertEquals } from '@floway-dev/test-utils'; + +const turnFor = (payload: Partial = {}): CanonicalOpenAIResponsesPayload => + summarizationTurnFor({ model: 'test-model', input: [], ...payload } as CanonicalOpenAIResponsesPayload); + +/** The items a turn was rewritten into, as the assertions below read them. */ +const itemsOf = (payload: CanonicalOpenAIResponsesPayload): { type: string; role?: string; content?: { type: string; text: string }[] }[] => + payload.input as { type: string; role?: string; content?: { type: string; text: string }[] }[]; + +/** What the summarization turn produced, as an upstream states it. */ +const summarized = (text: string, overrides: Partial = {}): OpenAIResponsesResult => ({ + id: 'resp_fake_upstream', + object: 'response', + model: 'test-upstream-model', + status: 'completed', + output: [{ + type: 'message', + id: 'msg_1', + role: 'assistant', + status: 'completed', + content: [{ type: 'output_text', text, annotations: [] }], + }], + error: null, + incomplete_details: null, + usage: { input_tokens: 10, output_tokens: 20, total_tokens: 30 }, + ...overrides, +}); + +/** The items a compaction blob stands for. The blob is this gateway's own base64url-JSON + * marker, so what a test reads back is what a later turn's expansion would. */ +const decodeBlob = (encryptedContent: string): { type: string; role: string; content: { type: string; text: string }[] }[] => { + const decoded = decodeBase64UrlJson(encryptedContent); + if (decoded === null) throw new Error('expected a shim-encoded compaction blob'); + return decoded as { type: string; role: string; content: { type: string; text: string }[] }[]; +}; + +// ── Inbound expansion (expandShimCompactionItems) ──────────────────────────── + +test('inbound: compaction item with a shim-encoded payload expands inline', () => { + const userItem = { type: 'message' as const, role: 'user' as const, content: 'history one' }; + const encoded = encodeBase64UrlJson([userItem]); + + const expanded = expandShimCompactionItems({ + model: 'm', + input: [ + { type: 'compaction', id: 'cmp_1', encrypted_content: encoded } as unknown as never, + { type: 'message', role: 'user', content: 'new turn' }, + ], + }); + + assertEquals(expanded.input.length, 2); + assertEquals(expanded.input[0], userItem); + assertEquals(expanded.input[1], { type: 'message', role: 'user', content: 'new turn' }); +}); + +test('inbound: foreign compaction blob (non-base64url-JSON) round-trips untouched', () => { + const original = { + model: 'm', + input: [ + { type: 'compaction', id: 'cmp_native', encrypted_content: 'OPAQUE_NATIVE_BLOB' } as unknown as never, + ], + }; + const expanded = expandShimCompactionItems(original); + // No items expanded — the foreign blob fails decode and the item passes + // through as-is. + assertEquals(expanded, original); +}); + +test('inbound: foreign compaction blob (valid base64url but wrong shape) round-trips untouched', () => { + // base64url-encoded JSON of an object (not an array) — decode succeeds, + // but the schema check rejects it. + const wrongShape = encodeBase64UrlJson({ not: 'an array' }); + const original = { + model: 'm', + input: [ + { type: 'compaction', id: 'cmp_foreign', encrypted_content: wrongShape } as unknown as never, + ], + }; + const expanded = expandShimCompactionItems(original); + assertEquals(expanded, original); +}); + +// ── The summarization turn (summarizationTurnFor) ──────────────────────────── + +test('the turn carries SUMMARIZATION_PROMPT as a role=system item at the head of the history, and persists nothing', () => { + const turn = turnFor({ input: [{ type: 'message', role: 'user', content: 'long conversation history' }] }); + + const compactorSystem = itemsOf(turn)[0]!; + assertEquals(compactorSystem.type, 'message'); + assertEquals(compactorSystem.role, 'system'); + assertEquals(compactorSystem.content![0]!.text.includes('CONTEXT CHECKPOINT COMPACTION'), true); + // The ephemeral summarization turn does not pollute the upstream's own conversation + // history. + assertEquals(turn.store, false); +}); + +test("the caller's `instructions` field flows through untouched (bug-for-bug parity with native compact)", () => { + // Native `/responses/compact` puts SUMMARIZATION_PROMPT into the compactor + // context as a role=system message AND forwards the caller's `instructions` + // as a developer-role message alongside; both are in scope. Prompt- + // injection extraction against the live Copilot upstream confirmed a + // caller-supplied `instructions="always mention 'quokka'"` reaches the + // compactor as a developer message and shows up inside the produced + // summary. The shim must reproduce that shape — do NOT overwrite the + // caller's instructions with SUMMARIZATION_PROMPT (it now lives as a + // separate role=system input item). + const turn = turnFor({ + input: [{ type: 'message', role: 'user', content: 'history' }], + instructions: 'You are a helpful assistant. Always mention the word quokka.', + }); + + assertEquals(turn.instructions, 'You are a helpful assistant. Always mention the word quokka.'); + const compactorSystem = itemsOf(turn)[0]!; + assertEquals(compactorSystem.role, 'system'); + assertEquals(compactorSystem.content![0]!.text.includes('CONTEXT CHECKPOINT COMPACTION'), true); +}); + +test('a caller with no `instructions` still gets the compactor prompt, and the slot stays absent', () => { + // Baseline: when the caller sends no `instructions` field, the shim still + // injects SUMMARIZATION_PROMPT via the role=system input item — matching + // native compact, which always has its system-role compactor prompt in + // scope regardless of whether the caller supplies a developer message. + // The `instructions` slot stays unset so a downstream translator does not + // synthesize a phantom developer message from thin air. + const turn = turnFor({ input: [{ type: 'message', role: 'user', content: 'history' }] }); + + assertEquals(turn.instructions, undefined); + const compactorSystem = itemsOf(turn)[0]!; + assertEquals(compactorSystem.type, 'message'); + assertEquals(compactorSystem.role, 'system'); + assertEquals(compactorSystem.content![0]!.text.includes('CONTEXT CHECKPOINT COMPACTION'), true); +}); + +test('compaction_trigger items are stripped from the history the compactor is sent', () => { + const turn = turnFor({ + input: [ + { type: 'message', role: 'user', content: 'real history' }, + { type: 'compaction_trigger' } as unknown as never, + ], + }); + + assertEquals(itemsOf(turn).every(item => item.type !== 'compaction_trigger'), true); +}); + +test('a history ending on an assistant message gets a synthetic terminal user prompt appended', () => { + // Anthropic Messages rejects assistant prefill: a conversation that + // ends on an assistant turn returns 400 `This model does not support + // assistant message prefill`. The shim normalizes by appending a + // synthetic user-role nudge so the summarization call always ends on + // a user message — harmless on OpenAI-style upstreams and load-bearing + // for translated Anthropic ones. + const turn = turnFor({ + input: [ + { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'hi' }] }, + { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'hello back', annotations: [] }] } as unknown as never, + ], + }); + + // [SUMMARIZATION_PROMPT system, user 'hi', assistant 'hello back', synthetic user nudge] + const items = itemsOf(turn); + assertEquals(items.length, 4); + assertEquals(items[0]!.role, 'system'); + const tail = items[items.length - 1]!; + assertEquals(tail.type, 'message'); + assertEquals(tail.role, 'user'); + assertEquals(tail.content![0]!.type, 'input_text'); +}); + +// ── The summary, and the envelope it is packed into ────────────────────────── + +test('the summary reads the provider-restored terminal output', () => { + const message: OpenAIResponsesOutputItem = { + type: 'message', + id: 'msg_1', + role: 'assistant', + status: 'completed', + content: [{ type: 'output_text', text: 'CONDENSED SUMMARY', annotations: [] }], + }; + + assertEquals(summaryTextFrom([message]), 'CONDENSED SUMMARY'); + assertEquals(summaryTextFrom([]), ''); +}); + +test('the synthesized encrypted_content decodes to a user message carrying the summary behind the handoff prefix', () => { + const envelope = buildCompactionEnvelope('cmp_1', 'THE SUMMARY', summarized('THE SUMMARY')); + + assertEquals(envelope.object, 'response.compaction'); + const item = envelope.output[0] as unknown as { type: string; id: string; encrypted_content: string }; + assertEquals(item.type, 'compaction'); + assertEquals(item.id, 'cmp_1'); + // The prefix rides inside the blob so a downstream LLM that echoes the compaction back + // reads the message as "another LLM's handoff summary", not as raw user speech. + const decoded = decodeBlob(item.encrypted_content); + assertEquals(decoded.length, 1); + assertEquals(decoded[0]!.type, 'message'); + assertEquals(decoded[0]!.role, 'user'); + assertEquals(decoded[0]!.content[0]!.type, 'input_text'); + assertEquals(decoded[0]!.content[0]!.text, `${SUMMARY_PREFIX}\nTHE SUMMARY`); + // The envelope is answered under an id of this gateway's own rather than the summarization + // turn's. + assertEquals(envelope.id === 'resp_fake_upstream', false); +}); + +test("the upstream's `output_text` SDK alias is dropped from the synthesized envelope", () => { + // Some upstreams (and some OpenAPI implementations) emit the convenience + // `output_text` alias alongside `output`. The synthesized + // `response.compaction` envelope must not forward it — its value is the + // upstream's summary plaintext, which a downstream SDK reading + // `output_text` on a compaction envelope would surface in place of the + // opaque-blob contract `encrypted_content` is supposed to carry. + const envelope = buildCompactionEnvelope( + 'cmp_1', + 'THE SUMMARY', + summarized('THE SUMMARY', { output_text: 'THE SUMMARY' } as Partial), + ); + + assertEquals(envelope.output_text, undefined); +}); + +test('an incomplete summarization turn says so on the envelope it backs', () => { + // A summarization that hit `max_output_tokens` mid-stream comes back with + // `status: 'incomplete'` and `incomplete_details` populated. The synthesized + // envelope must surface that — not pretend the turn ran to completion. + const envelope = buildCompactionEnvelope('cmp_1', 'partial summary', summarized('partial summary', { + status: 'incomplete', + incomplete_details: { reason: 'max_output_tokens' }, + })); + + assertEquals(envelope.status, 'incomplete'); + assertEquals(envelope.incomplete_details, { reason: 'max_output_tokens' }); +}); + +// ── Round-trip ──────────────────────────────────────────────────────────────── + +test('round-trip: what one turn packed is what the next turn is sent', () => { + const envelope = buildCompactionEnvelope('cmp_rt', 'SUMMARY TEXT', summarized('SUMMARY TEXT')); + const item = envelope.output[0] as unknown as { id: string; encrypted_content: string }; + + // The next turn echoes the compaction item back as an input item; expansion replaces it + // with the summary message, carrying the handoff prefix baked in at encode time. + const expanded = expandShimCompactionItems({ + model: 'test-model', + input: [{ type: 'compaction', id: item.id, encrypted_content: item.encrypted_content } as unknown as OpenAIResponsesInputItem], + }); + + const items = expanded.input as { type: string; role: string; content: { type: string; text: string }[] }[]; + assertEquals(items.length, 1); + assertEquals(items[0]!.type, 'message'); + assertEquals(items[0]!.role, 'user'); + assertEquals(items[0]!.content[0]!.text, `${SUMMARY_PREFIX}\nSUMMARY TEXT`); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/emit_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/emit_test.ts new file mode 100644 index 0000000000..a10d98d12e --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/emit_test.ts @@ -0,0 +1,131 @@ +import { afterEach, beforeEach, expect, test, vi } from 'vitest'; + +import { emitOpenAIResponses } from '../../../../src/data-plane/chat/openai-responses/emit.ts'; +import type { Fields } from '../../../../src/data-plane/chat/openai-responses/facts.ts'; +import { LayeredOpenAIResponsesStatefulStore, MemoryOpenAIResponsesStatefulBacking } from '../../../../src/data-plane/chat/openai-responses/items/store.ts'; +import { meterUsage } from '../../../../src/data-plane/chat/openai-responses/meter.ts'; +import { initDumpBroker, initDumpStore } from '../../../../src/dump/registry.ts'; +import { openRunDump } from '../../../../src/dump/run-sink.ts'; +import { initRepo } from '../../../../src/repo/index.ts'; +import type { ApiKey } from '../../../../src/repo/types.ts'; +import { installDumpStubs } from '../../../dump/test-fixtures.ts'; +import { flushBackground, trackBackground } from '../../../test-utils/background-tracker.ts'; +import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; +import { compose, createRunReader, defineStage, move, run, type DumpEvent } from '@floway-dev/pipeline'; +import type { ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; +import type { ClientOpenAIResponsesStreamEvent, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import { testTelemetryModelIdentity } from '@floway-dev/test-utils'; + +const key: ApiKey = { + id: 'key_emit', userId: 1, name: 'Emitter key', key: 'emitter-key', serverSecret: '11'.repeat(32), + createdAt: '2026-01-01T00:00:00.000Z', upstreamIds: null, deletedAt: null, + dumpRetentionSeconds: 3600, openaiResponsesRetentionSeconds: 0, +}; + +type Entry = Fields<'ingress.chat.openaiResponses.wantsStream' | 'request.chat.openaiResponses'>; +type Answer = Entry & Fields<'response.chat.openaiResponses' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status'>; +type Exit = Fields<'response.chat.openaiResponses.rendered' | 'response.chat.openaiResponses.streamedUsage'>; + +beforeEach(() => { + initRepo({ upstreams: { getById: async () => null } } as never); +}); +afterEach(() => { vi.restoreAllMocks(); }); + +for (const failureAt of ['before-response', 'upstream-stream', 'snapshot', 'render'] as const) { + test(`records the exact client protocol failure frames for ${failureAt} failure`, async () => { + const announced = failureAt !== 'before-response'; + const stubs = installDumpStubs(initDumpStore, initDumpBroker); + const timing = { upstreamCallStartedAt: 1, firstOutputTokenAt: null }; + const dump = openRunDump(key, { + method: 'POST', path: '/v1/responses', body: { bytes: new Uint8Array(), streamError: null }, + }, trackBackground, true, timing); + if (dump === null) throw new Error('retained request did not open its recording'); + const fault = new Error('upstream connection broke', { cause: new Error('socket reset') }); + const backing = new MemoryOpenAIResponsesStatefulBacking(); + const store = failureAt === 'snapshot' ? new LayeredOpenAIResponsesStatefulStore({ apiKeyId: key.id, reads: [backing], writes: [backing] }) : undefined; + if (store !== undefined) { + expect(store.writesState).toBe(true); + vi.spyOn(store, 'commitSnapshot').mockRejectedValue(fault); + } + const gateway = mockChatGatewayCtx({ dump, wantsStream: true, backgroundScheduler: trackBackground, attempt: { timing, telemetry: undefined }, ...(store === undefined ? {} : { store }) }); + const upstream = (async function* (): AsyncIterable> { + if (announced) { + yield { + type: 'event', event: { + type: 'response.created', sequence_number: 0, + response: { id: 'upstream-response', object: 'response', model: 'model', status: 'in_progress', output: [], error: null, incomplete_details: null }, + }, + }; + } + if (failureAt === 'render') { + yield { type: 'event', event: { type: 'response.in_progress', response: { id: 'upstream-response', object: 'response', model: 'model', status: 'in_progress', output: [], error: null, incomplete_details: null, ...{ impossibleJson: 1n } } } }; + return; + } + if (failureAt === 'snapshot') { + yield { + type: 'event', event: { + type: 'response.completed', sequence_number: 1, + response: { id: 'upstream-response', object: 'response', model: 'model', status: 'completed', output: [], error: null, incomplete_details: null, usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 } }, + }, + }; + return; + } + throw fault; + })(); + const dial = defineStage({ + name: 'scriptedDial', + return: { provides: ['response.chat.openaiResponses', 'response.usage.billable', 'response.http.headers', 'response.http.status'] }, + execute: async facts => move({ + ...facts, 'response.chat.openaiResponses': { kind: 'stream', frames: upstream }, + 'response.usage.billable': [{ identity: testTelemetryModelIdentity, quantities: {} }], 'response.http.headers': [], 'response.http.status': 200, + }), + }); + const outcome = await run(compose('clientFailure', [ + emitOpenAIResponses('sse'), + meterUsage('response.chat.openaiResponses.streamedUsage'), dial, + ]), move({ 'ingress.chat.openaiResponses.wantsStream': true, 'request.chat.openaiResponses': { model: 'model', input: [], store: false } }), { gateway, dump: dump.sink }); + const wire: SseFrame[] = []; + for await (const frame of outcome.facts['response.chat.openaiResponses.rendered'] as AsyncIterable) wire.push(frame); + expect((await outcome.facts['response.chat.openaiResponses.streamedUsage'])?.failed).toBe(true); + if (store !== undefined) { + expect(store.commitSnapshot).toHaveBeenCalledTimes(1); + expect((await outcome.facts['response.chat.openaiResponses.streamedUsage'])?.billable).toMatchObject([{ quantities: { input_tokens: '10', output_tokens: '5' } }]); + } + await outcome.drain(); + dump.finalize(200, 0); + await flushBackground(); + + const read = createRunReader(); + const streams = new Map[]>(); + let client: number | null = null; + const record = stubs.stored[0]?.record; + if (!record) throw new Error('completed request did not persist its recording'); + for (const line of new TextDecoder().decode(record.events).split('\n').filter(Boolean)) { + const event = JSON.parse(line) as DumpEvent; + const decoded = read(event); + if (decoded?.facts && 'response.chat.clientFrames' in decoded.facts) { + client = (decoded.facts['response.chat.clientFrames'] as { stream: number }).stream; + } + if (event.type === 'stream.frame') { + const frames = streams.get(event.streamId) ?? []; + frames.push(...decoded?.frames as ProtocolFrame[]); + streams.set(event.streamId, frames); + } + } + const frames = client === null ? [] : streams.get(client) ?? []; + expect(frames.length).toBeGreaterThan(0); + expect(frames.map(frame => frame.type === 'done' ? '[DONE]' : frame.event)).toEqual(wire.map(frame => JSON.parse(frame.data))); + const expectedEvents = failureAt === 'snapshot' + ? ['response.created', 'response.output_item.added', 'response.output_item.done', 'error', 'response.failed'] + : announced ? ['response.created', 'error', 'response.failed'] : ['error']; + expect(frames.filter(frame => frame.type === 'event').map(frame => frame.event.type)).toEqual(expectedEvents); + const error = frames.find(frame => frame.type === 'event' && frame.event.type === 'error'); + expect(error).toMatchObject({ event: { error: { message: failureAt === 'render' ? 'Do not know how to serialize a BigInt' : fault.message } } }); + if (failureAt !== 'render') expect(error).toMatchObject({ event: { error: { cause: { message: 'socket reset' } } } }); + if (announced) { + const created = frames[0] as ProtocolFrame>; + if (created.type !== 'event') throw new Error('announced response has no creation event'); + expect(frames.at(-1)).toMatchObject({ event: { response: { id: created.event.response.id, status: 'failed' } } }); + } + }); +} diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/errors_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/errors_test.ts index 94e23fba8a..89b4591d37 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/errors_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/errors_test.ts @@ -1,63 +1,15 @@ +// The OpenAI Responses-only refusal, through the same throw/catch. Its shape is the reason the +// mechanism is generic over the failure type: the store finds the missing item deep inside a +// walk, and the stage that started the walk is what answers. + import { test } from 'vitest'; -import { openaiResponsesInputErrorResult, type OpenAIResponsesServeFailure } from '../../../../src/data-plane/chat/openai-responses/errors.ts'; +import type { OpenAIResponsesServeFailure } from '../../../../src/data-plane/chat/openai-responses/errors.ts'; import { throwChatServeFailure, tryCatchChatServeFailure } from '../../../../src/data-plane/chat/shared/errors.ts'; -import type { ApiErrorResult } from '@floway-dev/provider'; import { assertEquals, assertThrows } from '@floway-dev/test-utils'; -import { TranslatorInputError } from '@floway-dev/translate'; - -const bodyOf = (result: ApiErrorResult): unknown => - JSON.parse(new TextDecoder().decode(result.body)); -test('round-trips the OpenAI-Responses-only item-not-found failure through throw/catch', () => { +test('round-trips the OpenAI Responses-only item-not-found failure through throw/catch', () => { const failure: OpenAIResponsesServeFailure = { kind: 'item-not-found', itemId: 'msg_abc' }; const error = assertThrows(() => throwChatServeFailure(failure)); assertEquals(tryCatchChatServeFailure(error), failure); }); - -test('openaiResponsesInputErrorResult renders an OpenAI 400 invalid_request_error envelope with default `input` param', () => { - const result = openaiResponsesInputErrorResult( - new TranslatorInputError("Invalid input item type 'image_generation_call'."), - ); - assertEquals(result.type, 'api-error'); - assertEquals(result.source, 'gateway'); - assertEquals(result.status, 400); - assertEquals(bodyOf(result), { - error: { - message: "Invalid input item type 'image_generation_call'.", - type: 'invalid_request_error', - param: 'input', - code: null, - }, - }); -}); - -test('openaiResponsesInputErrorResult carries an explicit error code into the envelope', () => { - const result = openaiResponsesInputErrorResult( - new TranslatorInputError("Missing required parameter: 'model'.", { param: 'model', code: 'missing_required_parameter' }), - ); - - assertEquals(bodyOf(result), { - error: { - message: "Missing required parameter: 'model'.", - type: 'invalid_request_error', - param: 'model', - code: 'missing_required_parameter', - }, - }); -}); - -test('openaiResponsesInputErrorResult honors an explicit input param', () => { - const result = openaiResponsesInputErrorResult( - new TranslatorInputError('content block not supported', { param: 'input[1].content[0]' }), - ); - - assertEquals(bodyOf(result), { - error: { - message: 'content block not supported', - type: 'invalid_request_error', - param: 'input[1].content[0]', - code: null, - }, - }); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/drive.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/drive.ts new file mode 100644 index 0000000000..c6c3efdf3a --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/drive.ts @@ -0,0 +1,131 @@ +// Driving the hosted-tool stage from a test that scripts the dial. +// +// The stage descends once per upstream turn, so what a test scripts is the *ending* — one +// answer per descent, in the order the loop asks for them. Everything above the ending is the +// real chain: the registrations prepare, the loop splices, and what comes back is read off the +// facts rather than out of a mock. +// +// `invocation.payload` is written back on every descent, so a test that asserts on what the +// upstream was last asked for reads the same field it always did. + +import type { HostedToolRegistration } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/types.ts'; +import { hostedTools } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools.ts'; +import type { ChatGatewayCtx } from '../../../../../src/data-plane/chat/shared/gateway-ctx.ts'; +import type { Failure } from '../../../../../src/data-plane/pipeline/facts.ts'; +import { mintedErrorEnvelope, renderFailure } from '../../../../../src/data-plane/pipeline/facts.ts'; +import type { StreamOutcome } from '../../../../../src/data-plane/pipeline/serve.ts'; +import { tokenUsageFromBillableUsage, tokenUsageMeasurement } from '../../../../../src/data-plane/shared/telemetry/usage.ts'; +import { compose, defer, defineStage, move, run, type Deferred } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import type { ModelCandidate, OpenAIResponsesInvocation } from '@floway-dev/provider'; +import { type EventResultMetadata, type ExecuteResult } from '@floway-dev/test-utils'; + +const ANSWER = 'response.chat.openaiResponses'; +const STREAMED_USAGE = 'response.chat.openaiResponses.streamedUsage'; + +type Result = ExecuteResult>; + +/** What a scripted turn will have billed, as the wire's meter would have handed it up: one + * entity naming the model that answered, carrying what it reported. A turn that reports + * nothing still names its identity, which is how the loop's own reading shows which upstream + * each of its calls reached. */ +const readingOf = (result: Extract) => defer( + (result.finalMetadata ?? Promise.resolve({ modelIdentity: result.modelIdentity, billableUsage: undefined })) + .then((metadata: EventResultMetadata): StreamOutcome => { + const tokens = tokenUsageFromBillableUsage(metadata.billableUsage); + const measurement = tokens === null ? undefined : tokenUsageMeasurement(tokens); + return { + billable: [{ identity: metadata.modelIdentity, quantities: measurement === undefined ? {} : measurement.quantities, ...(measurement === undefined ? {} : { pricingFacts: measurement.pricingFacts }) }], + failed: false, + }; + }), +); + +/** A scripted refusal as the failure value a chain carries. An internal error has no body of + * its own, so what travels is the sentence it raised. */ +const failureOf = (result: Exclude) => ({ + status: result.type === 'internal-error' ? 500 : result.status, + message: result.type === 'internal-error' ? result.error.message : new TextDecoder().decode(result.body), +}); + +/** What the run answered with, in the shape the assertions read. A refusal is rendered the way + * the edge renders one — through the same three tiers — so a test reads the bytes a client + * would have been sent rather than a restatement of them. */ +const resultOf = (facts: Record): Result => { + const answer = facts[ANSWER] as { status?: number; message?: string; kind?: string; frames?: unknown }; + if (answer.kind === 'stream') { + return { type: 'events', events: answer.frames as AsyncIterable>, modelIdentity: undefined } as unknown as Result; + } + const rendered = renderFailure(answer as Failure, mintedErrorEnvelope); + return { + type: 'api-error', + source: 'gateway', + status: rendered.status, + headers: new Headers({ 'content-type': 'application/json' }), + body: new TextEncoder().encode(JSON.stringify(rendered.body)), + } as Result; +}; + +export const driveHostedToolStage = (registrations: readonly HostedToolRegistration[]) => + async ( + invocation: OpenAIResponsesInvocation, + gatewayCtx: ChatGatewayCtx, + dial: () => Promise, + ): Promise | null }> => { + const ending = defineStage, Record>({ + name: 'scriptedDial', + return: { provides: [ANSWER, STREAMED_USAGE, 'response.usage.billable', 'response.http.headers', 'response.http.status', 'response.http.body'] }, + execute: async facts => { + invocation.payload = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; + const result = await dial(); + const answered = result.type === 'events' + ? { [ANSWER]: { kind: 'stream' as const, frames: result.events }, [STREAMED_USAGE]: readingOf(result) } + : { [ANSWER]: failureOf(result), [STREAMED_USAGE]: null }; + return move({ + ...facts, + ...answered, + 'response.usage.billable': result.type === 'events' + ? [{ identity: result.modelIdentity, quantities: {} }] + : [], + 'response.http.headers': [], 'response.http.status': result.type === 'events' ? 200 : result.status, + 'response.http.body': null, + }); + }, + }); + + const chain = compose, Record>('hostedToolsUnderTest', [ + hostedTools(registrations, { + streamedUsage: STREAMED_USAGE, + targetOf: () => invocation.targetApi, + }) as never, + ending, + ]); + + const outcome = await run(chain, move({ + 'request.chat.openaiResponses': invocation.payload, + 'serve.usage.prior': [], + 'route.attempt': { candidateId: 0, upstreamId: 'up_test', modelId: invocation.payload.model, flags: [] }, + 'ingress.http.headers': [...invocation.headers], + }), { + gateway: gatewayCtx, + background: () => {}, + resolveAttempt: (): ModelCandidate => invocation.candidate, + } as never); + + const { facts } = outcome; + const result = resultOf(facts); + if (result.type === 'events') { + const source = result.events; + return { + ...result, events: (async function* () { + try { yield* source; } finally { await outcome.drain(); } + })(), reading: facts[STREAMED_USAGE] as Deferred | null, + }; + } + await outcome.drain(); + + // The reading the run hands up, which is where every turn's identity and cost land now + // that a stage observes them. + return { ...resultOf(facts), reading: facts[STREAMED_USAGE] as Deferred | null }; + }; diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tools/image-generation-integration_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation-integration_test.ts similarity index 79% rename from packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tools/image-generation-integration_test.ts rename to packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation-integration_test.ts index f4f8a71e80..c21bd557e2 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tools/image-generation-integration_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation-integration_test.ts @@ -1,22 +1,23 @@ import { beforeEach, test, vi } from 'vitest'; -import type { OpenAIResponsesInvocation } from '../../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; -import { initRepo } from '../../../../../../src/repo/index.ts'; -import { InMemoryRepo } from '../../../../../repo/memory.ts'; -import { saveUpstreamForTest } from '../../../../../repo/upstreams.ts'; -import { mockChatGatewayCtx } from '../../../../../test-utils/gateway-ctx.ts'; +import { driveHostedToolStage } from './drive.ts'; +import { initRepo } from '../../../../../src/repo/index.ts'; +import { InMemoryRepo } from '../../../../repo/memory.ts'; +import { saveUpstreamForTest } from '../../../../repo/upstreams.ts'; +import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; +import { stubProviderPipeline } from '../../../../test-utils/provider-pipeline.ts'; import { createInMemoryImageProcessor, initExternalResourceFetcher, initImageProcessor } from '@floway-dev/platform'; import { eventFrame } from '@floway-dev/protocols/common'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIResponsesResult, OpenAIResponsesStreamEvent, OpenAIResponsesTool, OpenAIResponsesToolChoice } from '@floway-dev/protocols/openai-responses'; -import { type EventResult, type ExecuteResult, type FlagId, type OpenAIImagesEditsRequest } from '@floway-dev/provider'; -import { assert, assertEquals, assertStringIncludes, stubModelCandidate } from '@floway-dev/test-utils'; +import { type FlagId, type OpenAIImagesEditsRequest, type OpenAIResponsesInvocation } from '@floway-dev/provider'; +import { assert, assertEquals, assertStringIncludes, stubModelCandidate, type EventResult, type ExecuteResult } from '@floway-dev/test-utils'; // Dirty integration harness: mock the model registry so the image backend is a -// pair of in-test stubs, then drive the whole shim (function-tool rewrite, +// pair of in-test stubs, then drive the whole dispatcher (function-tool rewrite, // ReAct loop, dispatch, streaming relay, output feedback) over a scripted // upstream. The stubs record every backend call so a test can assert what the -// shim actually forwarded — most importantly that an image generated in turn 1 +// dispatcher actually forwarded — most importantly that an image generated in turn 1 // is re-collected as an edit source in turn 2. interface BackendStub { @@ -26,7 +27,7 @@ interface BackendStub { nextEdits: Response[]; // When set, the next `enumerateModelCandidates` call returns this // shape verbatim instead of the default single in-test candidate; lets - // a test drive `resolveImageCandidate`'s failure branches. + // a test drive shared resolution failure branches. nextResolutionOverride: { candidates: readonly unknown[]; sawModel: boolean; failedUpstreams: readonly string[] } | null; } @@ -35,7 +36,7 @@ interface BackendStub { const stub = vi.hoisted((): BackendStub => ({ generationsCalls: [], editsRequests: [], nextGenerations: [], nextEdits: [], nextResolutionOverride: null })); // Assigned per test in beforeEach and captured so the perf-attribution test -// can read `repo.performance.listAll()` after the shim completes. +// can read `repo.performance.listAll()` after the dispatcher completes. let repo: InMemoryRepo; const defaultCandidates = vi.hoisted(() => () => [{ @@ -43,10 +44,25 @@ const defaultCandidates = vi.hoisted(() => () => [{ upstreamId: 'u', kind: 'custom', name: 'mock-image', + inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, hue: 210, + pipelines: { + openaiImagesGenerations: stubProviderPipeline('openaiImagesGenerations', async (_model, body) => { + stub.generationsCalls.push(body); + const response = stub.nextGenerations.shift(); + if (response === undefined) throw new Error('test did not enqueue a generations response'); + return { response, modelKey: 'gpt-image-2' }; + }), + openaiImagesEdits: stubProviderPipeline('openaiImagesEdits', async (_model, request) => { + stub.editsRequests.push(request); + const response = stub.nextEdits.shift(); + if (response === undefined) throw new Error('test did not enqueue an edits response'); + return { response, modelKey: 'gpt-image-2' }; + }), + }, instance: { callOpenAIImagesGenerations: async (_model: unknown, body: Record) => { stub.generationsCalls.push(body); @@ -76,7 +92,7 @@ const defaultCandidates = vi.hoisted(() => () => [{ fetcher: (request: Request) => fetch(request), }]); -vi.mock('../../../../../../src/data-plane/providers/resolution.ts', () => ({ +vi.mock('../../../../../src/data-plane/providers/resolution.ts', () => ({ enumerateModelCandidates: vi.fn(async () => { const override = stub.nextResolutionOverride; if (override !== null) { @@ -88,10 +104,9 @@ vi.mock('../../../../../../src/data-plane/providers/resolution.ts', () => ({ })); // Imported AFTER vi.mock so the mocked registry is in effect. -const { withOpenAIResponsesServerToolShim } = await import('../../../../../../src/data-plane/chat/openai-responses/interceptors/server-tool-shim.ts'); -const { imageGenerationServerTool } = await import('../../../../../../src/data-plane/chat/openai-responses/interceptors/server-tools/image-generation.ts'); +const { imageGenerationHostedTool } = await import('../../../../../src/data-plane/chat/openai-responses/hosted-tools/image-generation.ts'); -const shim = withOpenAIResponsesServerToolShim([imageGenerationServerTool]); +const dispatcher = driveHostedToolStage([imageGenerationHostedTool]); const MODEL_IDENTITY = { model: 'orchestrator', upstream: 'u', modelKey: 'orchestrator', pricing: null }; @@ -99,8 +114,8 @@ const emptyResult = (status: OpenAIResponsesResult['status']): OpenAIResponsesRe id: 'upstream', object: 'response', model: 'orchestrator', output: [], output_text: '', status, error: null, incomplete_details: null, }); -const jsonResponse = (b64: string): Response => - new Response(JSON.stringify({ data: [{ b64_json: b64 }] }), { status: 200, headers: new Headers({ 'content-type': 'application/json' }) }); +const jsonResponse = (b64: string, usage?: Record): Response => + new Response(JSON.stringify({ data: [{ b64_json: b64 }], ...(usage === undefined ? {} : { usage }) }), { status: 200, headers: new Headers({ 'content-type': 'application/json' }) }); const REMOTE_PNG_B64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBAQDJ/wEAAAAASUVORK5CYII='; const remotePngResponse = (): Response => new Response(Uint8Array.from(atob(REMOTE_PNG_B64), c => c.charCodeAt(0))); @@ -193,10 +208,7 @@ const drain = async (result: ExecuteResult { repo = new InMemoryRepo(); - // resolveImageCandidate still calls createPerRequestFetcher to satisfy the - // production code path. Seed the in-memory repo with the mocked candidate's - // upstream id so the fetcher mapper resolves it instead of throwing - // "unknown upstream id: u". + // Attribution resolves the candidate's upstream metadata while writing the child run. await saveUpstreamForTest(repo.upstreams, { id: 'u', kind: 'custom', @@ -233,7 +245,7 @@ beforeEach(async () => { test('generates an image end-to-end and emits the native lifecycle', async () => { stub.nextGenerations = [jsonResponse('R0VO')]; // "GEN" - const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw a cat' }], 'auto', { + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw a cat' }], 'auto', { size: '1024x1024', quality: 'low', }), gatewayCtx(), scriptedRun([ @@ -261,7 +273,7 @@ test('generates an image from an additional_tools declaration without moving the ]); delete invocation.payload.tools; - const result = await shim(invocation, gatewayCtx(), scriptedRun([ + const result = await dispatcher(invocation, gatewayCtx(), scriptedRun([ callTurn(0, 'call_1', 'a cat'), messageTurn('done'), ])); @@ -281,7 +293,7 @@ test('invalid image generation in tool_search_output reports its input path', as }]); delete invocation.payload.tools; - const result = await shim(invocation, gatewayCtx(), async () => { + const result = await dispatcher(invocation, gatewayCtx(), async () => { throw new Error('Invalid request reached upstream'); }); assert(result.type === 'api-error'); @@ -295,7 +307,7 @@ test('restores a forced hosted choice when the terminal upstream echo omits it', const invocation = makeCtx([], 'generate', { quality: 'high' }, hostedChoice); const hostedTool = invocation.payload.tools![0]; const replacement = { type: 'function', name: 'image_generation', parameters: {}, strict: false } as OpenAIResponsesTool; - const result = await shim(invocation, gatewayCtx(), scriptedRun([ + const result = await dispatcher(invocation, gatewayCtx(), scriptedRun([ withResponseEcho(callTurn(0, 'call_1', 'a cat'), [replacement], { type: 'function', name: 'image_generation' }), withResponseEcho(messageTurn('done'), [replacement]), ])); @@ -314,7 +326,7 @@ test('keeps the top-level hosted echo when a later input declaration changes the ], 'generate', { quality: 'low' }); const topLevelTool = invocation.payload.tools![0]; const replacement = { type: 'function', name: 'image_generation', parameters: {}, strict: false } as OpenAIResponsesTool; - const result = await shim(invocation, gatewayCtx(), scriptedRun([ + const result = await dispatcher(invocation, gatewayCtx(), scriptedRun([ withResponseEcho(callTurn(0, 'call_1', 'a cat'), [replacement]), withResponseEcho(messageTurn('done'), [replacement]), ])); @@ -332,7 +344,7 @@ test('relays real partial_image frames when partial_images > 0', async () => { JSON.stringify({ type: 'image_generation.partial_image', partial_image_index: 1, b64_json: 'UDE=' }), JSON.stringify({ type: 'image_generation.completed', b64_json: 'RklO' }), ])]; - const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw a cat' }], 'auto', { partial_images: 2 }), gatewayCtx(), scriptedRun([ + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw a cat' }], 'auto', { partial_images: 2 }), gatewayCtx(), scriptedRun([ callTurn(0, 'call_1', 'a cat'), messageTurn('done'), ])); @@ -348,13 +360,13 @@ test('relays real partial_image frames when partial_images > 0', async () => { }); test('an image generated in turn 1 is re-collected as an edit source in turn 2', async () => { - // Turn 1 generates "AAAA"; the shim feeds it back as an input_image. Turn 2's + // Turn 1 generates "AAAA"; the dispatcher feeds it back as an input_image. Turn 2's // call must therefore resolve to an EDIT whose image[] part carries those // exact bytes — proving the dispatcher re-collects the live input rather than // a frozen registration-time snapshot. stub.nextGenerations = [jsonResponse('QUFBQQ==')]; // "AAAA" stub.nextEdits = [jsonResponse('QkJCQg==')]; // "BBBB" - const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw then border it' }]), gatewayCtx(), scriptedRun([ + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw then border it' }]), gatewayCtx(), scriptedRun([ callTurn(0, 'call_1', 'a cat'), callTurn(0, 'call_2', 'add a black border'), messageTurn('done'), @@ -399,7 +411,7 @@ test('a prefetched remote edit source remains visible to orchestration and is re return await baseRun(); }; - await drain(await shim(invocation, gatewayCtx(), run)); + await drain(await dispatcher(invocation, gatewayCtx(), run)); assertEquals(fetched, ['https://example.com/source.png']); assertEquals(orchestratorImageUrl, 'https://example.com/source.png'); @@ -420,7 +432,7 @@ test('mask-only GIF edit transcodes one shared image and mask to WebP', async () }); const gif = 'R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw=='; stub.nextEdits = [jsonResponse('RURJVA==')]; - const result = await shim(makeCtx([], 'edit', { + const result = await dispatcher(makeCtx([], 'edit', { input_image_mask: { image_url: `data:image/gif;base64,${gif}` }, }), gatewayCtx(), scriptedRun([ callTurn(0, 'call_1', 'edit from the mask'), @@ -451,7 +463,7 @@ test('identical GIF source and mask share one transcode', async () => { }); const gif = 'R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw=='; stub.nextEdits = [jsonResponse('RURJVA==')]; - const result = await shim(makeCtx([{ + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: [{ type: 'input_image', image_url: `data:image/gif;base64,${gif}`, detail: 'auto' }], }], 'edit', { @@ -476,7 +488,7 @@ test('image transcoding failure becomes a terminal image tool failure', async () compressToWebp: () => Promise.reject(new Error('codec down')), }); const gif = 'R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw=='; - const result = await shim(makeCtx([{ + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: [{ type: 'input_image', image_url: `data:image/gif;base64,${gif}`, detail: 'auto' }], }], 'edit'), gatewayCtx(), scriptedRun([ @@ -495,25 +507,27 @@ test('image transcoding failure becomes a terminal image tool failure', async () assertStringIncludes(item.error.message, 'codec down'); }); -test('retries on 429 and surfaces the eventual success', async () => { - // Two 429s then success; the orchestrator should see one completed image_generation_call. +for (const stream of [false, true]) test(`retries on 429 and settles every actual call before its ${stream ? 'streamed' : 'single-body'} success`, async () => { + const usage = { input_tokens: 9, output_tokens: 4 }; stub.nextGenerations = [ - rateLimitResponse(1), - rateLimitResponse(1), - jsonResponse('T0s='), + rateLimitResponse(1), rateLimitResponse(1), + stream ? sseResponse([JSON.stringify({ type: 'image_generation.completed', b64_json: 'T0s=', usage })]) : jsonResponse('T0s=', usage), ]; - const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ - callTurn(0, 'call_1', 'a cat'), - messageTurn('done'), + const pending: Promise[] = []; + const ctx = mockChatGatewayCtx({ wantsStream: true, backgroundScheduler: work => { pending.push(work); } }); + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw' }], 'auto', stream ? { partial_images: 1 } : {}), ctx, scriptedRun([ + callTurn(0, 'call_1', 'a cat'), messageTurn('done'), ])); const events = await drain(result); - + await Promise.all(pending); assertEquals(stub.generationsCalls.length, 3); const igcDone = events.find(e => e.type === 'response.output_item.done' && (e as { item: { type: string } }).item.type === 'image_generation_call'); assert(igcDone !== undefined); const item = (igcDone as { item: { status: string; result: string } }).item; assertEquals(item.status, 'completed'); assertEquals(item.result, 'T0s='); + const calls = (await repo.usage.listAll()).filter(row => row.model === 'gpt-image-2'); + assertEquals(calls.reduce((total, row) => total + row.requests, 0), 3); }); test('gives up after MAX_RATE_LIMIT_RETRIES on persistent 429 and surfaces a failed item', async () => { @@ -523,7 +537,7 @@ test('gives up after MAX_RATE_LIMIT_RETRIES on persistent 429 and surfaces a fai rateLimitResponse(1), rateLimitResponse(1), ]; - const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ callTurn(0, 'call_1', 'a cat'), messageTurn('sorry'), ])); @@ -545,7 +559,7 @@ test('does not retry non-rate-limit upstream failures', async () => { { status: 400, headers: new Headers({ 'content-type': 'application/json' }) }, ), ]; - const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ callTurn(0, 'call_1', 'a cat'), messageTurn('sorry'), ])); @@ -559,14 +573,14 @@ test('does not retry non-rate-limit upstream failures', async () => { assertEquals(item.error.code, 'invalid_value'); }); -// resolveImageCandidate failure branches: each variant is normalized into +// Shared image resolution failure branches: each variant is normalized into // a terminal `image_generation_call` item with `status: 'failed'` and a // specific error.code so the orchestrator can distinguish unknown-model // from existing-but-unsupported. -test('resolveImageCandidate renders model_not_found when no upstream knows the model id', async () => { +test('shared image resolution renders model_not_found when no upstream knows the model id', async () => { stub.nextResolutionOverride = { candidates: [], sawModel: false, failedUpstreams: [] }; - const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ callTurn(0, 'call_1', 'a cat'), messageTurn('sorry'), ])); @@ -579,14 +593,14 @@ test('resolveImageCandidate renders model_not_found when no upstream knows the m const item = (igcDone as { item: { status: string; error: { code: string; message: string } } }).item; assertEquals(item.status, 'failed'); assertEquals(item.error.code, 'model_not_found'); - assert(item.error.message.includes("No upstream provides model 'gpt-image-2'"), `unexpected message: ${item.error.message}`); + assert(item.error.message.includes('Model gpt-image-2 is not available on any configured upstream.'), `unexpected message: ${item.error.message}`); }); -test('resolveImageCandidate renders model_not_supported when sawModel=true but no candidate is an image kind', async () => { +test('shared image resolution renders model_not_supported when sawModel=true but no candidate is an image kind', async () => { // Mirrors the resolver's "id exists in some catalog but the kind filter // dropped it" signal — sawModel=true, candidates=[]. stub.nextResolutionOverride = { candidates: [], sawModel: true, failedUpstreams: [] }; - const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ callTurn(0, 'call_1', 'a cat'), messageTurn('sorry'), ])); @@ -598,10 +612,10 @@ test('resolveImageCandidate renders model_not_supported when sawModel=true but n const item = (igcDone as { item: { status: string; error: { code: string; message: string } } }).item; assertEquals(item.status, 'failed'); assertEquals(item.error.code, 'model_not_supported'); - assert(item.error.message.includes("Model 'gpt-image-2' is not an image model."), `unexpected message: ${item.error.message}`); + assert(item.error.message.includes('Model gpt-image-2 does not support the /images/generations endpoint.'), `unexpected message: ${item.error.message}`); }); -test('resolveImageCandidate renders model_not_supported when image-kind candidates exist but none expose the openaiImagesGenerations endpoint', async () => { +test('shared image resolution renders model_not_supported when image-kind candidates exist but none expose the openaiImagesGenerations endpoint', async () => { // The resolver produced an image-kind candidate but its `endpoints` does // not include the per-endpoint key the request needs (openaiImagesGenerations). stub.nextResolutionOverride = { @@ -615,7 +629,7 @@ test('resolveImageCandidate renders model_not_supported when image-kind candidat sawModel: true, failedUpstreams: [], }; - const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw' }]), gatewayCtx(), scriptedRun([ callTurn(0, 'call_1', 'a cat'), messageTurn('sorry'), ])); @@ -631,8 +645,8 @@ test('resolveImageCandidate renders model_not_supported when image-kind candidat }); test('an image sub-call records its own perf row attributed to the image backend, leaving the outer attempt untouched', async () => { - stub.nextGenerations = [jsonResponse('R0VO')]; - // Real scheduler: capture the promises the shim fires so the test can + stub.nextGenerations = [jsonResponse('R0VO', { input_tokens: 9, output_tokens: 4, total_tokens: 13 })]; + // Real scheduler: capture the promises the dispatcher fires so the test can // await them before querying the repo (the default no-op scheduler in // `gatewayCtx()` would drop the recordSample write). const pending: Promise[] = []; @@ -640,20 +654,29 @@ test('an image sub-call records its own perf row attributed to the image backend wantsStream: true, backgroundScheduler: p => { pending.push(p); }, }); - const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw a cat' }]), ctx, scriptedRun([ + const result = await dispatcher(makeCtx([{ type: 'message', role: 'user', content: 'draw a cat' }]), ctx, scriptedRun([ callTurn(0, 'call_1', 'a cat'), messageTurn('here it is'), ])); await drain(result); await Promise.all(pending); + // Through settlement, like every other upstream call: the row is written from the reading the + // sub-run hands up rather than by a telemetry call beside the dial. const perfRows = await repo.performance.listAll(); const imageRows = perfRows.filter(r => r.operation === 'image_generation'); assertEquals(imageRows.length, 1); assertEquals(imageRows[0].upstream, 'u'); assertEquals(imageRows[0].keyId, 'key_test'); assertEquals(imageRows[0].model, 'gpt-image-2'); - // The image shim runs on a local AttemptState distinct from the outer + + // And the usage row beside it, from the same reading — one settlement writes both, which is + // what a sub-request being a run rather than a dial with telemetry beside it buys. + const usageRows = await repo.usage.listAll(); + const imageUsage = usageRows.filter(row => row.model === 'gpt-image-2'); + assertEquals(imageUsage.length, 1); + assertEquals(imageUsage[0]!.upstream, 'u'); + // The image dispatcher runs on a local AttemptState distinct from the outer // OpenAI Responses turn's — no image-call stamps may leak onto ctx.attempt. assertEquals(ctx.attempt.timing.upstreamCallStartedAt, null); assertEquals(ctx.attempt.timing.firstOutputTokenAt, null); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tools/image-generation_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation_test.ts similarity index 90% rename from packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tools/image-generation_test.ts rename to packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation_test.ts index 63bf4721cc..40fed5f22f 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tools/image-generation_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-generation_test.ts @@ -6,24 +6,22 @@ import { inspectImageSources, DEFAULT_IMAGE_MODEL, type ImageGenerationConfig, - type ImageOutcome, - imageGenerationServerTool, - imageTerminal, + imageGenerationHostedTool, isHostedImageGenerationTool, - parseImageStreamEvent, - parseRetryAfterMs, prepareImageGenerationConfig, resolveImageOperation, - SHIM_TOOL_NAME, + FUNCTION_TOOL_NAME, synthesizeImageGenerationCallId, transformInputItemsForImageGeneration, -} from '../../../../../../src/data-plane/chat/openai-responses/interceptors/server-tools/image-generation.ts'; -import type { OpenAIResponsesInvocation } from '../../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; -import { initRepo } from '../../../../../../src/repo/index.ts'; -import { InMemoryRepo } from '../../../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../../../test-utils/gateway-ctx.ts'; +} from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/image-generation.ts'; +import { imageTerminal, projectImageStreamEvent, type ImageOutcome } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/result.ts'; +import { parseRetryAfterMs } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/retry.ts'; +import { initRepo } from '../../../../../src/repo/index.ts'; +import { InMemoryRepo } from '../../../../repo/memory.ts'; +import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; import { initExternalResourceFetcher } from '@floway-dev/platform'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesInputImage, OpenAIResponsesInputItem, OpenAIResponsesPayload, OpenAIResponsesTool } from '@floway-dev/protocols/openai-responses'; +import type { OpenAIResponsesInvocation } from '@floway-dev/provider'; import { assert, assertEquals, assertFalse, assertStringIncludes, assertThrows, stubModelCandidate } from '@floway-dev/test-utils'; const PNG_B64 = 'aGVsbG8='; // "hello" — any decodable base64 works for source tests. @@ -239,9 +237,9 @@ test('prepareImageGenerationConfig decodes image_url masks and reports file_id a // ── buildImageGenerationFunctionTool ── test('buildImageGenerationFunctionTool exposes only an optional prompt and is non-strict', () => { - const tool = buildImageGenerationFunctionTool({ type: 'image_generation' }, SHIM_TOOL_NAME); + const tool = buildImageGenerationFunctionTool({ type: 'image_generation' }, FUNCTION_TOOL_NAME); assertEquals(tool.type, 'function'); - assertEquals(tool.name, SHIM_TOOL_NAME); + assertEquals(tool.name, FUNCTION_TOOL_NAME); assertEquals(tool.strict, false); const params = tool.parameters as { properties: Record; required: unknown[]; additionalProperties: unknown }; assertEquals(Object.keys(params.properties), ['prompt']); @@ -383,7 +381,7 @@ test('a request inspector reuses decoded bytes after generated results become re output_format: 'jpeg', }; const initial = inspect([generated]); - const replay = inspect(transformInputItemsForImageGeneration([generated], SHIM_TOOL_NAME)); + const replay = inspect(transformInputItemsForImageGeneration([generated], FUNCTION_TOOL_NAME)); assert(initial.sources[0] === replay.sources[0]); }); @@ -396,7 +394,7 @@ test('auto generation pivots to edit when a generated image is fed back', () => const replayInput = transformInputItemsForImageGeneration([ { type: 'image_generation_call', id: 'ig_replay', status: 'completed', result: PNG_B64 }, - ], SHIM_TOOL_NAME); + ], FUNCTION_TOOL_NAME); const replay = resolveImageOperation(config, inspect(replayInput)); assert(replay.ok); assertEquals(replay.action, 'edit'); @@ -483,33 +481,30 @@ test('imageTerminal on failure emits a failed item and no closing events', () => assertEquals(endEvents.length, 0); }); -// ── parseImageStreamEvent ── - -test('parseImageStreamEvent maps generations and edits partial/completed/error with backend echo', () => { - const genPartial = parseImageStreamEvent(JSON.stringify({ type: 'image_generation.partial_image', partial_image_index: 1, b64_json: PNG_B64, background: 'opaque', output_format: 'png', quality: 'low', size: '1024x1024' })); +test('native image events project generations and edits partial/completed/error with backend echo', () => { + const genPartial = projectImageStreamEvent({ type: 'image_generation.partial_image', partial_image_index: 1, b64_json: PNG_B64, background: 'opaque', output_format: 'png', quality: 'low', size: '1024x1024' }); assert(genPartial?.kind === 'partial'); assertEquals(genPartial.index, 1); assertEquals(genPartial.b64, PNG_B64); assertEquals(genPartial.echo, { background: 'opaque', output_format: 'png', quality: 'low', size: '1024x1024' }); - const editPartial = parseImageStreamEvent(JSON.stringify({ type: 'image_edit.partial_image', partial_image_index: 0, b64_json: PNG_B64 })); + const editPartial = projectImageStreamEvent({ type: 'image_edit.partial_image', partial_image_index: 0, b64_json: PNG_B64 }); assert(editPartial?.kind === 'partial'); assertEquals(editPartial.echo, {}); - const completed = parseImageStreamEvent(JSON.stringify({ type: 'image_generation.completed', b64_json: PNG_B64, usage: { total_tokens: 1 }, quality: 'high' })); + const completed = projectImageStreamEvent({ type: 'image_generation.completed', b64_json: PNG_B64, usage: { total_tokens: 1 }, quality: 'high' }); assert(completed?.kind === 'completed'); assertEquals(completed.b64, PNG_B64); assertEquals(completed.echo.quality, 'high'); - const err = parseImageStreamEvent(JSON.stringify({ type: 'error', error: { type: 'image_generation_server_error', code: 'image_generation_failed', message: 'boom' } })); + const err = projectImageStreamEvent({ type: 'error', error: { type: 'image_generation_server_error', code: 'image_generation_failed', message: 'boom' } }); assert(err?.kind === 'error'); assertEquals(err.error.code, 'image_generation_failed'); assertEquals(err.error.retryable, true); }); -test('parseImageStreamEvent returns null for non-JSON or unrelated events', () => { - assertEquals(parseImageStreamEvent('[DONE]'), null); - assertEquals(parseImageStreamEvent(JSON.stringify({ type: 'image_generation.queued' })), null); +test('native unrelated image events do not become hosted lifecycle events', () => { + assertEquals(projectImageStreamEvent({ type: 'image_generation.queued' }), null); }); test('prepareImageGenerationConfig rejects a present-but-invalid model', () => { @@ -625,17 +620,17 @@ test('synthesizeImageGenerationCallId produces a canonical image-generation id', assert(/^ig_[0-9a-f]{32}$/.test(id)); }); -// ── imageGenerationServerTool: unsupported input format (C) ── +// ── imageGenerationHostedTool: unsupported input format (C) ── -test('imageGenerationServerTool accepts GIF edit input for local WebP transcoding', async () => { - const result = await imageGenerationServerTool( +test('imageGenerationHostedTool accepts GIF edit input for local WebP transcoding', async () => { + const result = await imageGenerationHostedTool( makeCtx({ tools: [{ type: 'image_generation' }], input: [imageMessage('image/gif')] }), gatewayCtx(), ); assert(result.type === 'active'); }); -test('imageGenerationServerTool fetches repeated remote edit sources once', async () => { +test('imageGenerationHostedTool fetches repeated remote edit sources once', async () => { const urls: string[] = []; initExternalResourceFetcher(url => { urls.push(url.href); @@ -647,21 +642,21 @@ test('imageGenerationServerTool fetches repeated remote edit sources once', asyn imageInputContainers.function_output(remote), ]; - const result = await imageGenerationServerTool( + const result = await imageGenerationHostedTool( makeCtx({ tools: [{ type: 'image_generation', action: 'edit' }], input }), gatewayCtx(), ); assert(result.type === 'active'); assertEquals(urls, ['https://example.com/source.png']); - const transformed = result.transformItems?.(input, SHIM_TOOL_NAME); + const transformed = result.transformItems?.(input, FUNCTION_TOOL_NAME); assert(transformed?.[0].type === 'message' && Array.isArray(transformed[0].content)); const first = transformed[0].content[0]; assert(first.type === 'input_image'); assertEquals(first.image_url, 'https://example.com/source.png'); }); -test('imageGenerationServerTool counts and fetches one URL once when shared by a source and mask', async () => { +test('imageGenerationHostedTool counts and fetches one URL once when shared by a source and mask', async () => { let requests = 0; initExternalResourceFetcher(() => { requests += 1; @@ -669,7 +664,7 @@ test('imageGenerationServerTool counts and fetches one URL once when shared by a }); const imageUrl = 'https://example.com/shared.png'; - const result = await imageGenerationServerTool( + const result = await imageGenerationHostedTool( makeCtx({ tools: [{ type: 'image_generation', action: 'edit', input_image_mask: { image_url: imageUrl } }], input: [imageInputContainers.message({ type: 'input_image', image_url: imageUrl, detail: 'auto' })], @@ -681,14 +676,14 @@ test('imageGenerationServerTool counts and fetches one URL once when shared by a assertEquals(requests, 1); }); -test('imageGenerationServerTool enforces the aggregate limit across distinct successful downloads', async () => { +test('imageGenerationHostedTool enforces the aggregate limit across distinct successful downloads', async () => { let requests = 0; initExternalResourceFetcher(() => { requests += 1; return Promise.resolve(paddedPngResponse(26 * 1024 * 1024)); }); - const result = await imageGenerationServerTool( + const result = await imageGenerationHostedTool( makeCtx({ tools: [{ type: 'image_generation', action: 'edit' }], input: [ @@ -724,9 +719,9 @@ test.each([ })), message: 'Error while downloading file.', }, -])('imageGenerationServerTool mirrors the native remote-source error for $name', async ({ fetcher, message }) => { +])('imageGenerationHostedTool mirrors the native remote-source error for $name', async ({ fetcher, message }) => { initExternalResourceFetcher(fetcher); - const result = await imageGenerationServerTool( + const result = await imageGenerationHostedTool( makeCtx({ tools: [{ type: 'image_generation', action: 'edit' }], input: [imageInputContainers.message({ type: 'input_image', image_url: 'https://example.invalid/source.png', detail: 'auto' })], @@ -741,11 +736,11 @@ test.each([ assertEquals(result.code, 'invalid_value'); }); -test('imageGenerationServerTool mirrors the native error for a fetched non-image', async () => { +test('imageGenerationHostedTool mirrors the native error for a fetched non-image', async () => { initExternalResourceFetcher(() => Promise.resolve(new Response('not an image', { headers: { 'content-type': 'text/plain' }, }))); - const result = await imageGenerationServerTool( + const result = await imageGenerationHostedTool( makeCtx({ tools: [{ type: 'image_generation', action: 'edit' }], input: [imageInputContainers.message({ type: 'input_image', image_url: 'https://example.com/readme.txt', detail: 'auto' })], @@ -760,7 +755,7 @@ test('imageGenerationServerTool mirrors the native error for a fetched non-image assertEquals(result.code, 'invalid_value'); }); -test('imageGenerationServerTool follows redirects before materializing a remote source', async () => { +test('imageGenerationHostedTool follows redirects before materializing a remote source', async () => { const urls: string[] = []; initExternalResourceFetcher(url => { urls.push(url.href); @@ -769,7 +764,7 @@ test('imageGenerationServerTool follows redirects before materializing a remote : validPngResponse()); }); - const result = await imageGenerationServerTool( + const result = await imageGenerationHostedTool( makeCtx({ tools: [{ type: 'image_generation', action: 'edit' }], input: [imageInputContainers.message({ type: 'input_image', image_url: 'https://example.com/redirect', detail: 'auto' })], @@ -781,16 +776,16 @@ test('imageGenerationServerTool follows redirects before materializing a remote assertEquals(urls, ['https://example.com/redirect', 'https://example.com/image.png']); }); -test('imageGenerationServerTool materializes a remote mask and mirrors its download error', async () => { +test('imageGenerationHostedTool materializes a remote mask and mirrors its download error', async () => { initExternalResourceFetcher(() => Promise.resolve(validPngResponse())); - const accepted = await imageGenerationServerTool( + const accepted = await imageGenerationHostedTool( makeCtx({ tools: [{ type: 'image_generation', action: 'generate', input_image_mask: { image_url: 'https://example.com/mask.png' } }] }), gatewayCtx(), ); assert(accepted.type === 'active'); initExternalResourceFetcher(() => Promise.resolve(new Response(null, { status: 404 }))); - const rejected = await imageGenerationServerTool( + const rejected = await imageGenerationHostedTool( makeCtx({ tools: [{ type: 'image_generation', action: 'edit', input_image_mask: { image_url: 'https://example.com/missing.png' } }] }), gatewayCtx(), ); @@ -801,13 +796,13 @@ test('imageGenerationServerTool materializes a remote mask and mirrors its downl assertEquals(rejected.code, null); }); -test('imageGenerationServerTool validates a remote mask before applying the Floway file-ID guard', async () => { +test('imageGenerationHostedTool validates a remote mask before applying the Floway file-ID guard', async () => { let requests = 0; initExternalResourceFetcher(() => { requests += 1; return Promise.resolve(validPngResponse()); }); - const result = await imageGenerationServerTool( + const result = await imageGenerationHostedTool( makeCtx({ tools: [{ type: 'image_generation', @@ -825,8 +820,8 @@ test('imageGenerationServerTool validates a remote mask before applying the Flow assertEquals(requests, 1); }); -test('imageGenerationServerTool reports a malformed remote mask at its native path', async () => { - const result = await imageGenerationServerTool( +test('imageGenerationHostedTool reports a malformed remote mask at its native path', async () => { + const result = await imageGenerationHostedTool( makeCtx({ tools: [{ type: 'image_generation', input_image_mask: { file_id: 'file_123', image_url: 'https://' } }] }), gatewayCtx(), ); @@ -837,9 +832,9 @@ test('imageGenerationServerTool reports a malformed remote mask at its native pa }); test.each(unresolvableSourceCases)( - 'imageGenerationServerTool handles $source images in $container for action $action', + 'imageGenerationHostedTool handles $source images in $container for action $action', async ({ source, action, input, path }) => { - const result = await imageGenerationServerTool( + const result = await imageGenerationHostedTool( makeCtx({ tools: [{ type: 'image_generation', action }], input: [input] }), gatewayCtx(), ); @@ -863,8 +858,8 @@ test.each(unresolvableSourceCases)( }, ); -test.each(['generate', 'auto', 'edit'] as const)('imageGenerationServerTool accepts a mask without a separate image for action %s', async action => { - const result = await imageGenerationServerTool( +test.each(['generate', 'auto', 'edit'] as const)('imageGenerationHostedTool accepts a mask without a separate image for action %s', async action => { + const result = await imageGenerationHostedTool( makeCtx({ tools: [{ type: 'image_generation', @@ -894,8 +889,8 @@ test.each([ assertEquals(operation.sources.length, sourceCount); }); -test('imageGenerationServerTool accepts an auto-action mask with an edit source', async () => { - const result = await imageGenerationServerTool( +test('imageGenerationHostedTool accepts an auto-action mask with an edit source', async () => { + const result = await imageGenerationHostedTool( makeCtx({ tools: [{ type: 'image_generation', @@ -909,8 +904,8 @@ test('imageGenerationServerTool accepts an auto-action mask with an edit source' assert(result.type === 'active'); }); -test('imageGenerationServerTool rejects explicit edit without an input image', async () => { - const result = await imageGenerationServerTool( +test('imageGenerationHostedTool rejects explicit edit without an input image', async () => { + const result = await imageGenerationHostedTool( makeCtx({ tools: [{ type: 'image_generation', action: 'edit' }] }), gatewayCtx(), ); @@ -921,18 +916,18 @@ test('imageGenerationServerTool rejects explicit edit without an input image', a assertEquals(result.param, 'input'); }); -test('imageGenerationServerTool accepts webp input for editing', async () => { - const result = await imageGenerationServerTool( +test('imageGenerationHostedTool accepts webp input for editing', async () => { + const result = await imageGenerationHostedTool( makeCtx({ tools: [{ type: 'image_generation' }], input: [imageMessage('image/webp')] }), gatewayCtx(), ); assert(result.type === 'active'); }); -test('imageGenerationServerTool ignores input format when action is generate', async () => { +test('imageGenerationHostedTool ignores input format when action is generate', async () => { // A gif is fine as pure vision context; action:"generate" never forwards it // to the edit backend, so the format is not validated. - const result = await imageGenerationServerTool( + const result = await imageGenerationHostedTool( makeCtx({ tools: [{ type: 'image_generation', action: 'generate' }], input: [imageMessage('image/gif')] }), gatewayCtx(), ); @@ -941,7 +936,7 @@ test('imageGenerationServerTool ignores input format when action is generate', a test('image dispatcher exposes a newly introduced invalid edit source as an invariant failure', async () => { const invocation = makeCtx({ tools: [{ type: 'image_generation', action: 'auto' }] }); - const result = await imageGenerationServerTool(invocation, gatewayCtx()); + const result = await imageGenerationHostedTool(invocation, gatewayCtx()); assert(result.type === 'active' && result.hosted !== undefined); invocation.payload = { @@ -952,7 +947,7 @@ test('image dispatcher exposes a newly introduced invalid edit source as an inva }; assertThrows( () => result.hosted?.dispatcher({ - intercepted: { callId: 'call_live_invalid', name: SHIM_TOOL_NAME, arguments: { prompt: 'edit it' } }, + intercepted: { callId: 'call_live_invalid', name: FUNCTION_TOOL_NAME, arguments: { prompt: 'edit it' } }, loopState: { iterationCount: 2, remainingToolCalls: undefined }, }), Error, @@ -960,13 +955,13 @@ test('image dispatcher exposes a newly introduced invalid edit source as an inva ); }); -// ── imageGenerationServerTool: per-response dispatch budget (B) ── +// ── imageGenerationHostedTool: per-response dispatch budget (B) ── test('image dispatch budget caps real backend calls per response, not ReAct turns', async () => { - const result = await imageGenerationServerTool(makeCtx({ tools: [{ type: 'image_generation', action: 'generate' }] }), gatewayCtx()); + const result = await imageGenerationHostedTool(makeCtx({ tools: [{ type: 'image_generation', action: 'generate' }] }), gatewayCtx()); assert(result.type === 'active' && result.hosted !== undefined); const dispatch = result.hosted.dispatcher; - const intercepted = { callId: 'c', name: SHIM_TOOL_NAME, argumentsJson: '{}', arguments: { prompt: 'x' } }; + const intercepted = { callId: 'c', name: FUNCTION_TOOL_NAME, argumentsJson: '{}', arguments: { prompt: 'x' } }; const loopState = { iterationCount: 1, remainingToolCalls: undefined }; for (let i = 0; i < 10; i++) { diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-sub-request_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-sub-request_test.ts new file mode 100644 index 0000000000..750472bd36 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/image-sub-request_test.ts @@ -0,0 +1,169 @@ +import { afterEach, expect, test, vi } from 'vitest'; + +import * as imageRequest from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/request.ts'; +import type { ImageGenerationRequest } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/request.ts'; +import * as imageResult from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/result.ts'; +import { runImageGenerationSubRequest } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts'; +import * as resolution from '../../../../../src/data-plane/providers/resolution.ts'; +import { initDumpBroker, initDumpStore } from '../../../../../src/dump/registry.ts'; +import { openRunDump } from '../../../../../src/dump/run-sink.ts'; +import { initRepo } from '../../../../../src/repo/index.ts'; +import type { ApiKey } from '../../../../../src/repo/types.ts'; +import { eventsOf, installDumpStubs } from '../../../../dump/test-fixtures.ts'; +import { InMemoryRepo } from '../../../../repo/memory.ts'; +import { flushBackground, trackBackground } from '../../../../test-utils/background-tracker.ts'; +import { mockGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; +import { stubProviderPipeline } from '../../../../test-utils/provider-pipeline.ts'; +import { createRunReader, getFailureFacts, type DumpEvent } from '@floway-dev/pipeline'; +import { stubModelCandidate } from '@floway-dev/test-utils'; + +const apiKey: ApiKey = { id: 'image-key', userId: 1, name: 'Image key', key: 'floway-image-key', serverSecret: '00'.repeat(32), createdAt: '2026-01-01T00:00:00Z', upstreamIds: null, deletedAt: null, dumpRetentionSeconds: 3600, openaiResponsesRetentionSeconds: 0 }; +afterEach(() => { vi.restoreAllMocks(); }); + +for (const action of ['generate', 'edit'] as const) for (const consume of [false, true]) { + test(`a recorded hosted image ${action} carries source facts through the full model chain when ${consume ? 'consumed' : 'returned unread'}`, async () => { + const repo = new InMemoryRepo(); + initRepo(repo); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); + const dump = openRunDump(apiKey, { method: 'POST', path: '/v1/responses', body: { bytes: new Uint8Array(), streamError: null } }, trackBackground, true, { upstreamCallStartedAt: null, firstOutputTokenAt: null }); + const parent = mockGatewayCtx({ dump, backgroundScheduler: trackBackground }); + const plainCandidate = stubModelCandidate({ model: { id: 'image-model', kind: 'image', endpoints: { openaiImagesGenerations: {}, openaiImagesEdits: {} } } }); + const prefix = action === 'edit' ? 'image_edit' : 'image_generation'; + const response = () => new Response([ + { type: `${prefix}.partial_image`, partial_image_index: 0, b64_json: 'preview' }, + { type: `${prefix}.completed`, b64_json: 'pixels', usage: { input_tokens: 7, output_tokens: 3 } }, + ].map(event => `data: ${JSON.stringify(event)}\n\n`).join(''), { headers: { 'content-type': 'text/event-stream' } }); + const generate = vi.fn(async () => ({ response: response(), modelKey: 'backend-image' })); + const edit = vi.fn(async () => ({ response: response(), modelKey: 'backend-image' })); + const candidate = { + ...plainCandidate, provider: { + ...plainCandidate.provider, pipelines: { + openaiImagesGenerations: stubProviderPipeline('openaiImagesGenerations', generate), + openaiImagesEdits: stubProviderPipeline('openaiImagesEdits', edit), + }, + }, + }; + vi.spyOn(resolution, 'enumerateModelCandidates').mockResolvedValue({ candidates: [candidate], sawModel: true, failedUpstreams: [] }); + const request: ImageGenerationRequest = { + prompt: 'draw a tree', action, config: { model: 'image-model', action, quality: 'high', partial_images: 1 }, + sources: action === 'edit' ? [{ bytes: new Uint8Array([1, 2, 3]), mimeType: 'image/png' }] : [], + }; + const run = await runImageGenerationSubRequest(parent, request); + if (consume) { + expect(await run.lifecycle.next()).toMatchObject({ done: false, value: { type: 'response.image_generation_call.partial_image', partial_image_index: 0, partial_image_b64: 'preview' } }); + expect(await run.lifecycle.next()).toMatchObject({ done: true, value: { item: { type: 'image_generation_call', status: 'completed', action, revised_prompt: 'draw a tree', result: 'pixels' }, endEvents: [{ type: 'response.image_generation_call.completed' }] } }); + } else { + await run.lifecycle.return(undefined as never); + } + await run.drain(); + dump?.finalize(200, 0); + await flushBackground(); + expect(generate).toHaveBeenCalledTimes(action === 'generate' ? 1 : 0); + expect(edit).toHaveBeenCalledTimes(action === 'edit' ? 1 : 0); + expect(parent.attempt.telemetry).toBeUndefined(); + expect(dumps.stored).toHaveLength(2); + expect(dumps.stored.filter(item => item.record.meta.path === '/v1/responses')).toHaveLength(1); + const record = dumps.stored.find(item => item.record.meta.path !== '/v1/responses')!.record; + expect(record.meta.path).toBe(action === 'edit' ? '/images/edits' : '/images/generations'); + const read = createRunReader(); + const decoded = eventsOf(record).map(event => read(event as unknown as DumpEvent)); + expect(decoded.find(event => event?.facts && 'request.imageGeneration.canonical' in event.facts)?.facts?.['request.imageGeneration.canonical']).toEqual({ ...request, sources: action === 'edit' ? [{ bytes: { bytes: 'AQID' }, mimeType: 'image/png' }] : [] }); + const prepared = decoded.find(event => event?.facts && 'request.openaiImages.canonical' in event.facts)?.facts?.['request.openaiImages.canonical']; + expect(prepared).toMatchObject({ operation: action === 'edit' ? 'edits' : 'generations', parameters: { prompt: 'draw a tree', n: 1, quality: 'high', stream: true, partial_images: 1 } }); + if (action === 'edit') expect(prepared).toMatchObject({ images: [{ kind: 'file', file: { fileName: 'image_0.png', mediaType: 'image/png', bytes: { bytes: 'AQID' } } }] }); + const entered = eventsOf(record).filter(event => event.type === 'stage.entered').map(event => event.name); + expect(entered).toEqual(['writeSettlement', 'emitHostedImageGeneration', 'prepareHostedImageGeneration', 'resolveCandidates', 'failover', 'retryRateLimitedImages', 'callOpenAIImagesUpstream', 'stubHttp']); + const frames = decoded.flatMap(event => event?.frames ?? []); + if (consume) expect(frames).toContainEqual({ type: 'event', event: { item: { type: 'image_generation_call', status: 'completed', action, result: 'pixels', revised_prompt: 'draw a tree' }, endEvents: [{ type: 'response.image_generation_call.completed' }] } }); + else expect(frames).toEqual([]); + const observations = await repo.usage.listAll(); + expect(observations).toMatchObject([{ model: 'image-model', modelKey: 'backend-image', requests: 1 }]); + expect(await repo.performance.listAll()).toMatchObject([{ requests: 1, neutral: 1, errorsNoOutput: 0, errorsWithOutput: 0 }]); + expect(record.meta.status).toBe(200); + expect(record.meta.error).toBeNull(); + }); +} + +test('hosted image preparation preserves an internal TypeError and records the failed child boundary', async () => { + initRepo(new InMemoryRepo()); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); + const dump = openRunDump(apiKey, { method: 'POST', path: '/v1/responses', body: { bytes: new Uint8Array(), streamError: null } }, trackBackground, true, { upstreamCallStartedAt: null, firstOutputTokenAt: null }); + const parent = mockGatewayCtx({ dump, backgroundScheduler: trackBackground }); + const fault = new TypeError('image preparation state is invalid'); + vi.spyOn(imageRequest, 'prepareImageRequest').mockRejectedValue(fault); + const request: ImageGenerationRequest = { prompt: 'tree', action: 'generate', config: { model: 'image-model', action: 'generate' }, sources: [] }; + await expect(runImageGenerationSubRequest(parent, request)).rejects.toBe(fault); + expect(getFailureFacts(fault)?.['request.imageGeneration.canonical']).toBe(request); + dump?.finalize(200, 0); + await flushBackground(); + expect(dumps.stored).toHaveLength(2); + const record = dumps.stored.find(item => item.record.meta.path === '/images/generations')!.record; + expect(record.meta.status).toBe(500); + expect(record.meta.error).toEqual({ kind: 'failed', reason: fault.message }); + const events = eventsOf(record); + const entered = events.find(event => event.type === 'stage.entered' && event.name === 'prepareHostedImageGeneration'); + expect(entered).toBeDefined(); + expect(events).toContainEqual(expect.objectContaining({ type: 'stage.failed', stageId: entered!.stageId })); +}); + +const openGeneration = async (stream: boolean, signal?: AbortSignal) => { + const repo = new InMemoryRepo(); + initRepo(repo); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); + const dump = openRunDump(apiKey, { method: 'POST', path: '/v1/responses', body: { bytes: new Uint8Array(), streamError: null } }, trackBackground, true, { upstreamCallStartedAt: null, firstOutputTokenAt: null }); + const parent = mockGatewayCtx({ dump, backgroundScheduler: trackBackground, ...(signal === undefined ? {} : { abortSignal: signal }) }); + const body = stream ? new Response([ + { type: 'image_generation.partial_image', partial_image_index: 0, b64_json: 'preview' }, + { type: 'image_generation.completed', b64_json: 'pixels', usage: { input_tokens: 7, output_tokens: 3 } }, + ].map(event => `data: ${JSON.stringify(event)}\n\n`).join(''), { headers: { 'content-type': 'text/event-stream' } }) + : Response.json({ data: [{ b64_json: 'pixels' }], usage: { input_tokens: 7, output_tokens: 3 } }); + const dial = vi.fn(async () => ({ response: body, modelKey: 'backend-image' })); + const plain = stubModelCandidate({ model: { id: 'image-model', kind: 'image', endpoints: { openaiImagesGenerations: {} } } }); + const candidate = { ...plain, provider: { ...plain.provider, pipelines: { openaiImagesGenerations: stubProviderPipeline('openaiImagesGenerations', dial) } } }; + vi.spyOn(resolution, 'enumerateModelCandidates').mockResolvedValue({ candidates: [candidate], sawModel: true, failedUpstreams: [] }); + const call = await runImageGenerationSubRequest(parent, { prompt: 'tree', action: 'generate', config: { model: 'image-model', action: 'generate', ...(stream ? { partial_images: 1 } : {}) }, sources: [] }); + return { repo, dumps, dump, call, dial }; +}; + +for (const stream of [false, true]) test(`a ${stream ? 'streamed' : 'JSON'} child emission exception keeps measured usage and fails through drain`, async () => { + const { repo, dumps, dump, call, dial } = await openGeneration(stream); + const fault = new TypeError('image lifecycle formatter broke', { cause: new Error('invalid rendering state') }); + vi.spyOn(imageResult, 'imageTerminal').mockImplementation(() => { throw fault; }); + if (stream) expect(await call.lifecycle.next()).toMatchObject({ done: false }); + await expect(call.lifecycle.next()).rejects.toBe(fault); + await expect(call.drain()).rejects.toBe(fault); + dump?.finalize(200, 0); + await flushBackground(); + expect(dial).toHaveBeenCalledOnce(); + expect(await repo.usage.listAll()).toMatchObject([{ + requests: 1, metrics: [ + { metric: 'input_tokens', quantity: '7' }, { metric: 'output_tokens', quantity: '3' }, + ], + }]); + expect(await repo.performance.listAll()).toMatchObject([{ requests: 1, neutral: 0, errorsNoOutput: 1 }]); + const record = dumps.stored.find(item => item.record.meta.path === '/images/generations')!.record; + expect(record.meta).toMatchObject({ status: 500, inputTokens: 7, outputTokens: 3, error: { kind: 'failed', reason: fault.message } }); + const read = createRunReader(); + const logs: unknown[] = []; + for (const item of eventsOf(record)) { + const event = item as unknown as DumpEvent; + read(event); + if (event.type === 'stage.log' && event.fields !== undefined) logs.push(read.decode(event.fields)); + } + expect(logs).toContainEqual(expect.objectContaining({ error: expect.objectContaining({ error: expect.objectContaining({ message: fault.message, cause: expect.objectContaining({ error: expect.objectContaining({ message: 'invalid rendering state' }) }) }) }) })); +}); + +test('an actual parent abort keeps the source failure verdict and original streamed HTTP status', async () => { + const controller = new AbortController(); + const { repo, dumps, dump, call } = await openGeneration(true, controller.signal); + controller.abort(new Error('parent canceled')); + expect(controller.signal.aborted).toBe(true); + await call.lifecycle.return(undefined as never); + await expect(call.drain()).rejects.toThrow('OpenAI Images stream ended without a completed event.'); + dump?.finalize(200, 0); + await flushBackground(); + expect(await repo.usage.listAll()).toMatchObject([{ requests: 1, metrics: [] }]); + expect(await repo.performance.listAll()).toMatchObject([{ requests: 1, errorsNoOutput: 1, neutral: 0 }]); + const record = dumps.stored.find(item => item.record.meta.path === '/images/generations')!.record; + expect(record.meta).toMatchObject({ status: 200, error: { kind: 'failed', reason: 'OpenAI Images stream ended without a completed event.' } }); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/lifecycle_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/lifecycle_test.ts new file mode 100644 index 0000000000..cbbcdd4ae6 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/lifecycle_test.ts @@ -0,0 +1,86 @@ +import { expect, test } from 'vitest'; + +import { emitOpenAIResponses } from '../../../../../src/data-plane/chat/openai-responses/emit.ts'; +import type { HostedToolRegistration } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/types.ts'; +import { hostedTools } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools.ts'; +import { openaiResponsesWire } from '../../../../../src/data-plane/chat/openai-responses/wire.ts'; +import { prologueFor, type StreamOutcome } from '../../../../../src/data-plane/pipeline/serve.ts'; +import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; +import { isReplayableBody } from '@floway-dev/http'; +import { compose, move, run, type Deferred } from '@floway-dev/pipeline'; +import type { SseFrame } from '@floway-dev/protocols/common'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import type { ModelCandidate } from '@floway-dev/provider'; +import { createCustomPipelines } from '@floway-dev/provider-custom'; +import { stubModelCandidate } from '@floway-dev/test-utils'; + +const key = 'response.chat.openaiResponses.streamedUsage'; +const payload: CanonicalOpenAIResponsesPayload = { model: 'model', input: [], tools: [{ type: 'web_search' }] }; +const shell = (status: OpenAIResponsesResult['status'], input: number, output: number): OpenAIResponsesResult => ({ id: 'upstream', object: 'response', model: 'model', output: [], status, error: null, incomplete_details: null, usage: { input_tokens: input, output_tokens: output, total_tokens: input + output } }); +const first: OpenAIResponsesStreamEvent[] = [ + { type: 'response.created', response: shell('in_progress', 0, 0) }, + { type: 'response.output_item.added', output_index: 0, item: { type: 'function_call', id: 'tool', call_id: 'call', name: 'test_search', arguments: '', status: 'in_progress' } }, + { type: 'response.output_item.done', output_index: 0, item: { type: 'function_call', id: 'tool', call_id: 'call', name: 'test_search', arguments: '{}', status: 'completed' } }, + { type: 'response.completed', response: shell('completed', 3, 2) }, +]; +const second: OpenAIResponsesStreamEvent[] = [ + { type: 'response.created', response: shell('in_progress', 0, 0) }, + { type: 'response.output_text.delta', output_index: 0, item_id: 'message', content_index: 0, delta: 'partial' }, + { type: 'response.completed', response: shell('completed', 4, 3) }, +]; + +for (const client of ['unread', 'partial-before-tool', 'partial-second', 'complete'] as const) test(`hosted ReAct settles actual model calls after ${client} output`, async () => { + let calls = 0; + let tools = 0; + const registration: HostedToolRegistration = () => ({ + type: 'active', baseToolName: 'test_search', hosted: { + hostedTypes: ['web_search'], + canonicalize: tool => tool.type === 'web_search' ? tool : undefined, + buildFunctionTool: (_tool, name) => ({ type: 'function', name, description: 'Search', parameters: { type: 'object' }, strict: false }), + dispatcher: () => { + tools += 1; + return [{ id: 'search-item', startItem: { type: 'web_search_call', status: 'in_progress' }, startEvents: [], async *run() { return { item: { type: 'web_search_call', status: 'completed' }, endEvents: [] }; } }]; + }, + }, + }); + const gateway = mockChatGatewayCtx({ wantsStream: true }); + const base = prologueFor(gateway, { body: { bytes: new Uint8Array(), streamError: null }, headers: [] }); + const plain = stubModelCandidate(); + const candidate: ModelCandidate = { + ...plain, + provider: { ...plain.provider, pipelines: createCustomPipelines({ baseUrl: 'https://custom.example', authStyle: 'none', endpoints: {}, ingressHeadersRules: [], modelsFetch: { enabled: false }, models: [] }) }, + fetcher: async (_url, init) => { + if (!isReplayableBody(init.body)) throw new Error('Expected JSON HTTP request'); + expect(await new Response(init.body.open()).json()).toMatchObject({ stream: true }); + calls += 1; + return new Response((calls === 1 ? first : second).map(event => `data: ${JSON.stringify(event)}\n\n`).join(''), { headers: { 'content-type': 'text/event-stream' } }); + }, + }; + const [selector] = base.services.rememberCandidates([candidate]); + const executed = await run(compose, Record>('hostedLifecycle', [ + emitOpenAIResponses('sse'), hostedTools([registration], { streamedUsage: key, targetOf: () => 'openaiResponses' }), ...openaiResponsesWire(key), + ]), move({ 'request.chat.openaiResponses': payload, 'route.attempt': selector, 'ingress.http.headers': [], 'ingress.chat.sourceProtocol': 'openaiResponses', 'ingress.chat.openaiResponses.wantsStream': true, 'serve.usage.prior': [] }), { + ...base.services, gateway, recordProtocolFrames: (frames: AsyncIterable) => frames, selectAffinity: () => {}, + }); + const rendered = executed.facts['response.chat.openaiResponses.rendered'] as AsyncIterable; + if (client !== 'unread') { + const iterator = rendered[Symbol.asyncIterator](); + if (client === 'complete') { + let step = await iterator.next(); + while (!step.done) step = await iterator.next(); + } else if (client === 'partial-before-tool') await iterator.next(); + else { + let step = await iterator.next(); + while (!step.done && !(calls === 2 && step.value.data.includes('partial'))) step = await iterator.next(); + expect(step.done).toBe(false); + } + if (client !== 'complete') await iterator.return?.(); + } + await executed.drain(); + const outcome = await (executed.facts[key] as Deferred); + const repeated = client === 'partial-second' || client === 'complete'; + expect(calls).toBe(repeated ? 2 : 1); + expect(tools).toBe(repeated ? 1 : 0); + expect(outcome.failed).toBe(client !== 'complete'); + expect(outcome.billable.map(entity => [entity.quantities.input_tokens, entity.quantities.output_tokens])).toEqual(repeated ? [['3', '2'], ['4', '3']] : [['3', '2']]); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts new file mode 100644 index 0000000000..61f6adcc6a --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/search-sub-request_test.ts @@ -0,0 +1,81 @@ +import { expect, test } from 'vitest'; + +import type { WebSearchRequest } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/facts.ts'; +import { runWebSearchSubRequest } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/search-sub-request.ts'; +import { initDumpBroker, initDumpStore } from '../../../../../src/dump/registry.ts'; +import { openRunDump } from '../../../../../src/dump/run-sink.ts'; +import { initRepo } from '../../../../../src/repo/index.ts'; +import type { ApiKey } from '../../../../../src/repo/types.ts'; +import { eventsOf, installDumpStubs } from '../../../../dump/test-fixtures.ts'; +import { InMemoryRepo } from '../../../../repo/memory.ts'; +import { stubAlphaSearchCandidate } from '../../../../test-utils/alpha-search-provider.ts'; +import { flushBackground, trackBackground } from '../../../../test-utils/background-tracker.ts'; +import { mockGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; +import { createRunReader, type DumpEvent } from '@floway-dev/pipeline'; + +const apiKey: ApiKey = { id: 'search-key', userId: 1, name: 'Search key', key: 'floway-search-key', serverSecret: '00'.repeat(32), createdAt: '2026-01-01T00:00:00Z', upstreamIds: null, deletedAt: null, dumpRetentionSeconds: 3600, openaiResponsesRetentionSeconds: 0 }; +const request: WebSearchRequest = { + commands: { search_query: [{ q: 'pipeline streams' }] }, toolName: 'web_search', iterationCount: 1, + filters: { allowedDomains: ['example.com'], maxResults: 20 }, includeSearchActionSources: true, + settings: { search_context_size: 'medium' }, input: [{ type: 'message', role: 'user', content: [{ type: 'input_text', text: 'Find the documentation' }] }], +}; + +for (const outcome of ['success', 'transportFailure', 'bodyReadFailure'] as const) { + test(`web search records backend content before dispatch and closes its ${outcome} child run`, async () => { + const failed = outcome !== 'success'; + const repo = new InMemoryRepo(); + initRepo(repo); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); + const dump = openRunDump(apiKey, { method: 'POST', path: '/v1/responses', body: { bytes: new Uint8Array(), streamError: null } }, trackBackground, true, { upstreamCallStartedAt: null, firstOutputTokenAt: null }); + const parent = mockGatewayCtx({ dump, backgroundScheduler: trackBackground }); + const fault = new Error(outcome === 'bodyReadFailure' ? 'search body read failed' : 'search transport failed', { cause: new Error('socket reset') }); + let dispatched = false; + let replyProduced = false; + const invocation = runWebSearchSubRequest(parent, request, { + session: { pageCache: new Map(), getProvider: () => { throw new Error('local provider must not dispatch in alpha mode'); }, apiKeyId: apiKey.id }, + alpha: { + sessionId: 'search-session', resolveCandidate: async () => stubAlphaSearchCandidate(async body => { + dispatched = true; + expect(body).toEqual({ id: 'search-session', commands: request.commands, settings: request.settings, input: request.input }); + expect(Object.isFrozen(body.commands)).toBe(true); + if (outcome === 'transportFailure') throw fault; + replyProduced = true; + if (outcome === 'bodyReadFailure') { + return new Response(new ReadableStream({ pull: controller => controller.error(fault) }), { headers: { 'content-type': 'application/json' } }); + } + return new Response(JSON.stringify({ output: 'documentation' }), { headers: { 'content-type': 'application/json' } }); + }), + }, + }); + if (failed) await expect(invocation).rejects.toBe(fault); + else expect(await invocation).toMatchObject({ outputText: 'documentation' }); + expect(dispatched).toBe(true); + expect(replyProduced).toBe(outcome !== 'transportFailure'); + dump?.finalize(200, 0); + await flushBackground(); + expect(dumps.stored).toHaveLength(2); + expect(dumps.stored.filter(item => item.record.meta.path === '/v1/responses')).toHaveLength(1); + const record = dumps.stored.find(item => item.record.meta.path !== '/v1/responses')!.record; + expect(record.meta.path).toBe('/alpha/search'); + expect(record.meta.status).toBe(failed ? 500 : 200); + expect(record.meta.error).toEqual(failed ? { kind: 'failed', reason: fault.message } : null); + const usage = await repo.usage.listAll(); + if (outcome === 'transportFailure') expect(usage).toEqual([]); + else { + expect(usage).toHaveLength(1); + expect(usage[0]).toMatchObject({ requests: 1, modelKey: 'search-model', metrics: [] }); + } + const read = createRunReader(); + const decoded = eventsOf(record).map(event => read(event as unknown as DumpEvent)); + const entered = decoded.find(event => event?.facts && 'request.webSearch.canonical' in event.facts); + expect(entered?.facts?.['request.webSearch.canonical']).toEqual(request); + expect(decoded).toContainEqual(expect.objectContaining({ facts: expect.objectContaining({ 'request.provider.payload': { id: 'search-session', commands: request.commands, settings: request.settings, input: request.input } }) })); + expect(new TextDecoder().decode(record.events)).not.toContain('searchCall'); + if (failed) { + const events = eventsOf(record); + const stage = events.find(event => event.type === 'stage.entered' && event.name === 'runWebSearchCall'); + expect(stage).toBeDefined(); + expect(events).toContainEqual(expect.objectContaining({ type: 'stage.failed', stageId: stage!.stageId })); + } + }); +} diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/shared_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/shared_test.ts new file mode 100644 index 0000000000..29f7395f1c --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/shared_test.ts @@ -0,0 +1,30 @@ +import { expect, test } from 'vitest'; + +import { createMergeState, materializeHostedToolItems } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/shared.ts'; +import type { HostedToolLifecycleEvent, HostedToolTerminal } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/types.ts'; +import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; + +test('returning hosted materialization closes the active child lifecycle after its delivered partial frame', async () => { + let closed = false; + const frames = materializeHostedToolItems([{ + slots: [{ + intercepted: { callId: 'call', name: 'image_generation', arguments: { prompt: 'tree' } }, + outputIndex: 0, + slot: { + id: 'image', startItem: { type: 'image_generation_call', status: 'in_progress' }, startEvents: [], + run: () => (async function* (): AsyncGenerator { + try { + yield { type: 'response.image_generation_call.partial_image', partial_image_index: 0, partial_image_b64: 'preview' }; + throw new Error('a cancelled lifecycle must not request another event'); + } finally { + closed = true; + } + })(), + }, + }], + }], createMergeState(), mockChatGatewayCtx().store); + expect(await frames.next()).toMatchObject({ done: false, value: { type: 'event', event: { type: 'response.image_generation_call.partial_image', partial_image_b64: 'preview', item_id: 'image', output_index: 0 } } }); + expect(closed).toBe(false); + await frames.return(); + expect(closed).toBe(true); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tools/web-search_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/web-search_test.ts similarity index 92% rename from packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tools/web-search_test.ts rename to packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/web-search_test.ts index cf64eea9fc..70720561a2 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tools/web-search_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools/web-search_test.ts @@ -1,22 +1,22 @@ import { test } from 'vitest'; -import { resolveServerToolName } from '../../../../../../src/data-plane/chat/openai-responses/interceptors/server-tool-shim.ts'; +import { resolveHostedToolName } from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/shared.ts'; import { alphaSearchSettingsFromHosted, - buildShimFunctionTool, + buildFunctionTool, isHostedWebSearchTool, - prepareToolsForShim, - SHIM_TOOL_NAME, + prepareTools, + FUNCTION_TOOL_NAME, synthesizeWebSearchCallId, transformInputItemsForWebSearch, WEB_SEARCH_HOSTED_TYPES, type WebSearchCallPrivatePayload, -} from '../../../../../../src/data-plane/chat/openai-responses/interceptors/server-tools/web-search.ts'; -import { assertLocalWebSearchSupport, findMatches, formatMatches, isUrlAllowed, parseWebSearchOperations, type WebSearchOperation } from '../../../../../../src/data-plane/tools/web-search/operations.ts'; +} from '../../../../../src/data-plane/chat/openai-responses/hosted-tools/web-search.ts'; +import { assertLocalWebSearchSupport, findMatches, formatMatches, isUrlAllowed, parseWebSearchOperations, type WebSearchOperation } from '../../../../../src/data-plane/tools/web-search/operations.ts'; import type { OpenAIResponsesHostedTool, OpenAIResponsesTool, OpenAIResponsesWebSearchAction, OpenAIResponsesWebSearchResult } from '@floway-dev/protocols/openai-responses'; import { assert, assertEquals, assertThrows } from '@floway-dev/test-utils'; -// ── Shim call argument parsing (parseWebSearchOperations) ── +// ── Dispatcher call argument parsing (parseWebSearchOperations) ── const opsOf = (args: Record | null): WebSearchOperation[] => { const parsed = parseWebSearchOperations(args); @@ -25,7 +25,7 @@ const opsOf = (args: Record | null): WebSearchOperation[] => { }; test('injected web_search function exposes the complete OpenAI alpha-search command shape', () => { - const functionTool = buildShimFunctionTool({ type: 'web_search' }, SHIM_TOOL_NAME); + const functionTool = buildFunctionTool({ type: 'web_search' }, FUNCTION_TOOL_NAME); const properties = (functionTool.parameters as { properties: Record }).properties; assertEquals(Object.keys(properties), [ 'search_query', @@ -382,11 +382,11 @@ test('formatMatches: multi-match output uses Match N: headers', () => { // ── Tool detection, filter prep, and name resolution ── -const SHIM_TOOL = SHIM_TOOL_NAME; +const FUNCTION_NAME = FUNCTION_TOOL_NAME; const hostedVariants = ['web_search', 'web_search_2025_08_26', 'web_search_preview', 'web_search_preview_2025_03_11'] as const; const prepare = (tools: OpenAIResponsesTool[]) => { - const result = prepareToolsForShim(tools); + const result = prepareTools(tools); assert(result.ok); return { filters: result.filters }; }; @@ -399,12 +399,12 @@ test('isHostedWebSearchTool recognizes every hosted variant', () => { }); for (const type of hostedVariants) { - test(`prepareToolsForShim accepts ${type} and extracts default filters`, () => { + test(`prepareTools accepts ${type} and extracts default filters`, () => { assertEquals(prepare([{ type } as OpenAIResponsesTool]).filters, { maxResults: 20 }); }); } -test('prepareToolsForShim extracts filters, user_location, and context size', () => { +test('prepareTools extracts filters, user_location, and context size', () => { const { filters } = prepare([{ type: 'web_search', filters: { allowed_domains: ['a.com'], blocked_domains: ['b.com'] }, @@ -417,7 +417,7 @@ test('prepareToolsForShim extracts filters, user_location, and context size', () assertEquals(filters.maxResults, 40); }); -test('prepareToolsForShim selects the last web_search declaration as one configuration', () => { +test('prepareTools selects the last web_search declaration as one configuration', () => { const { filters } = prepare([ { type: 'web_search', @@ -439,22 +439,22 @@ test('prepareToolsForShim selects the last web_search declaration as one configu }); }); -test('prepareToolsForShim passes through with empty filters when no hosted web_search exists', () => { +test('prepareTools passes through with empty filters when no hosted web_search exists', () => { const fn: OpenAIResponsesTool = { type: 'function', name: 'foo', parameters: {}, strict: false }; assertEquals(prepare([fn]).filters, {}); }); -test('resolveServerToolName returns the first free sequential name', () => { - assertEquals(resolveServerToolName(SHIM_TOOL, []), SHIM_TOOL); - assertEquals(resolveServerToolName(SHIM_TOOL, [{ type: 'function', name: SHIM_TOOL, parameters: {}, strict: false }]), `${SHIM_TOOL}_2`); - assertEquals(resolveServerToolName(SHIM_TOOL, [ - { type: 'function', name: SHIM_TOOL, parameters: {}, strict: false }, - { type: 'custom', name: `${SHIM_TOOL}_2` }, - ]), `${SHIM_TOOL}_3`); +test('resolveHostedToolName returns the first free sequential name', () => { + assertEquals(resolveHostedToolName(FUNCTION_NAME, []), FUNCTION_NAME); + assertEquals(resolveHostedToolName(FUNCTION_NAME, [{ type: 'function', name: FUNCTION_NAME, parameters: {}, strict: false }]), `${FUNCTION_NAME}_2`); + assertEquals(resolveHostedToolName(FUNCTION_NAME, [ + { type: 'function', name: FUNCTION_NAME, parameters: {}, strict: false }, + { type: 'custom', name: `${FUNCTION_NAME}_2` }, + ]), `${FUNCTION_NAME}_3`); }); -test('prepareToolsForShim rejects invalid hosted fields', () => { - const result = prepareToolsForShim([{ type: 'web_search', search_context_size: 'huge' } as unknown as OpenAIResponsesTool]); +test('prepareTools rejects invalid hosted fields', () => { + const result = prepareTools([{ type: 'web_search', search_context_size: 'huge' } as unknown as OpenAIResponsesTool]); assertEquals(result.ok, false); }); @@ -478,7 +478,7 @@ const makePrivatePayload = ( }); test('transformInputItemsForWebSearch replays the upstream function_call verbatim when a private payload exists', () => { - // One wsc maps 1:1 to one shim call. The shim's + // One wsc maps 1:1 to one dispatcher call. The dispatcher's // jsonrepair-canonical args and the per-op output are persisted on // this single row. const payload = makePrivatePayload( @@ -506,7 +506,7 @@ test('transformInputItemsForWebSearch replays the upstream function_call verbati }); test('transformInputItemsForWebSearch replays each echoed wsc independently (one pair per wsc)', () => { - // Two distinct shim calls (different upstream call_ids) → two replay + // Two distinct dispatcher calls (different upstream call_ids) → two replay // pairs, one per wsc. const p1 = makePrivatePayload('call_u1', '{"search_query":[{"q":"q1"}]}', { type: 'search', queries: ['q1'] }, [{ type: 'text_result', url: 'u1', title: 't1', snippet: 'body1' }]); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tool-shim_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools_test.ts similarity index 84% rename from packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tool-shim_test.ts rename to packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools_test.ts index ae7013a996..55556d7117 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tool-shim_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/hosted-tools_test.ts @@ -1,26 +1,15 @@ import { beforeEach, test, vi } from 'vitest'; -import { withOpenAIResponsesServerToolShim } from '../../../../../src/data-plane/chat/openai-responses/interceptors/server-tool-shim.ts'; -import { - consumeTurnStreaming, - createMergeState, - materializeAccumulatedOutput, - parseServerToolArguments, - resolveServerToolName, - sumUsage, - type InterceptedFunctionCall, - type ServerToolResultSlot, - type TurnSummary, - type UpstreamTerminal, -} from '../../../../../src/data-plane/chat/openai-responses/interceptors/server-tool-shim.ts'; -import { imageGenerationServerTool } from '../../../../../src/data-plane/chat/openai-responses/interceptors/server-tools/image-generation.ts'; -import { SHIM_TOOL_NAME, webSearchServerTool } from '../../../../../src/data-plane/chat/openai-responses/interceptors/server-tools/web-search.ts'; -import type { OpenAIResponsesInterceptor, OpenAIResponsesInvocation } from '../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; -import { createNonOpenAIResponsesSourceStore } from '../../../../../src/data-plane/chat/openai-responses/items/store.ts'; -import type { ChatGatewayCtx } from '../../../../../src/data-plane/chat/shared/gateway-ctx.ts'; -import { resolveAlphaSearchDispatcher } from '../../../../../src/data-plane/tools/web-search/alpha-search/upstream.ts'; -import type { AlphaSearchDispatcher } from '../../../../../src/data-plane/tools/web-search/alpha-search/upstream.ts'; -import { resolveConfiguredWebSearchProvider } from '../../../../../src/data-plane/tools/web-search/provider.ts'; +import { driveHostedToolStage } from './hosted-tools/drive.ts'; +import { imageGenerationHostedTool } from '../../../../src/data-plane/chat/openai-responses/hosted-tools/image-generation.ts'; +import { consumeTurnStreaming, createMergeState, materializeAccumulatedOutput, parseHostedToolArguments, resolveHostedToolName, sumUsage } from '../../../../src/data-plane/chat/openai-responses/hosted-tools/shared.ts'; +import { type InterceptedFunctionCall, type HostedToolResultSlot, type TurnSummary, type UpstreamTerminal } from '../../../../src/data-plane/chat/openai-responses/hosted-tools/types.ts'; +import { FUNCTION_TOOL_NAME, webSearchHostedTool } from '../../../../src/data-plane/chat/openai-responses/hosted-tools/web-search.ts'; +import { createNonOpenAIResponsesSourceStore } from '../../../../src/data-plane/chat/openai-responses/items/store.ts'; +import type { ChatGatewayCtx } from '../../../../src/data-plane/chat/shared/gateway-ctx.ts'; +import { resolveAlphaSearchCandidate } from '../../../../src/data-plane/tools/web-search/alpha-search/upstream.ts'; +import type { AlphaSearchDispatcher } from '../../../../src/data-plane/tools/web-search/alpha-search/upstream.ts'; +import { resolveConfiguredWebSearchProvider } from '../../../../src/data-plane/tools/web-search/provider.ts'; import type { ConfiguredWebSearchProvider, WebSearchConfig, @@ -29,10 +18,15 @@ import type { WebSearchProvider, WebSearchProviderRequest, WebSearchProviderResult, -} from '../../../../../src/data-plane/tools/web-search/types.ts'; -import { getRepo, initRepo } from '../../../../../src/repo/index.ts'; -import { InMemoryRepo } from '../../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; +} from '../../../../src/data-plane/tools/web-search/types.ts'; +import { initDumpBroker, initDumpStore } from '../../../../src/dump/registry.ts'; +import { openRunDump } from '../../../../src/dump/run-sink.ts'; +import { getRepo, initRepo } from '../../../../src/repo/index.ts'; +import { eventsOf, installDumpStubs } from '../../../dump/test-fixtures.ts'; +import { InMemoryRepo } from '../../../repo/memory.ts'; +import { stubAlphaSearchCandidate } from '../../../test-utils/alpha-search-provider.ts'; +import { flushBackground } from '../../../test-utils/background-tracker.ts'; +import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { eventFrame } from '@floway-dev/protocols/common'; import type { BillableUsage, ProtocolFrame } from '@floway-dev/protocols/common'; @@ -50,11 +44,11 @@ import type { OpenAIResponsesToolChoice, OpenAIResponsesWebSearchAction, } from '@floway-dev/protocols/openai-responses'; -import { eventResult, type EventResult, type ExecuteResult, type FlagId } from '@floway-dev/provider'; -import { assert, assertEquals, assertFalse, assertRejects, stubModelCandidate } from '@floway-dev/test-utils'; +import { type FlagId, type OpenAIResponsesInvocation } from '@floway-dev/provider'; +import { assert, assertEquals, assertFalse, assertRejects, stubModelCandidate, eventResult, type EventResult, type ExecuteResult } from '@floway-dev/test-utils'; import { translateOpenAIResponsesViaAnthropicMessages, translateOpenAIResponsesViaOpenAIChatCompletions } from '@floway-dev/translate'; -const withOpenAIResponsesWebSearchShim = withOpenAIResponsesServerToolShim([webSearchServerTool]); +const runHostedWebSearch = driveHostedToolStage([webSearchHostedTool]); const emptyResult = (id: string, status: OpenAIResponsesResult['status']): OpenAIResponsesResult => ({ id, @@ -196,8 +190,8 @@ const mkReasoningDone = (outputIndex: number, reasoningId: string): ProtocolFram // non-default values; the mock then ignores the config and returns a // test stub. Tests that need a specific configured state set // `mockResolveConfigured.mockReturnValue(...)` per call. -vi.mock('../../../../../src/data-plane/tools/web-search/provider.ts'); -vi.mock('../../../../../src/data-plane/tools/web-search/alpha-search/upstream.ts'); +vi.mock('../../../../src/data-plane/tools/web-search/provider.ts'); +vi.mock('../../../../src/data-plane/tools/web-search/alpha-search/upstream.ts'); const mkResponseCompleted = ( usage?: OpenAIResponsesResult['usage'], @@ -223,8 +217,8 @@ const mkResponseIncomplete = ( }, }); -// Integration tests for the OpenAI Responses web_search shim. Internal helpers -// are unit-tested through focused web-search server-tool suites; this file covers +// Integration tests for the OpenAI Responses web_search dispatcher. Internal helpers +// are unit-tested through focused web-search hosted-tool suites; this file covers // activation gating, tool rewrite, multi-turn loop, downstream stream // merge invariants, and error propagation. @@ -289,7 +283,7 @@ interface DepsOverrides { } const mockResolveConfigured = vi.mocked(resolveConfiguredWebSearchProvider); -const mockResolveAlpha = vi.mocked(resolveAlphaSearchDispatcher); +const mockResolveAlpha = vi.mocked(resolveAlphaSearchCandidate); // Seed a per-test InMemoryRepo and a default tavily search config so // `loadWebSearchConfig()` returns a non-default value. The actual provider @@ -359,11 +353,11 @@ const collectFrames = async (iter: AsyncIterable): Promise => { return out; }; -// The shim streams lazily: backend calls, ctx.payload.input mutation, and -// subsequent run() calls all happen inside the events generator. Tests -// asserting on side-effects MUST drain the events stream first. -const runShimAndDrain = async ( - shim: OpenAIResponsesInterceptor, +// The stage streams lazily: backend calls, the payload each descent sends, and the descents +// themselves all happen inside the events generator. Tests asserting on side-effects MUST drain +// the events stream first. +const runHostedToolsAndDrain = async ( + dispatcher: ReturnType, inv: OpenAIResponsesInvocation, gatewayCtx: ChatGatewayCtx, run: () => Promise>>, @@ -371,7 +365,7 @@ const runShimAndDrain = async ( result: ExecuteResult>; frames: ProtocolFrame[]; }> => { - const result = await shim(inv, gatewayCtx, run); + const result = await dispatcher(inv, gatewayCtx, run); const frames: ProtocolFrame[] = result.type === 'events' ? await collectFrames(result.events) : []; @@ -408,7 +402,7 @@ const scriptedRun = (turns: ScriptedTurn[]): ScriptedRun => { }; // `outputIndex` is the UPSTREAM index (which resets to 0 every run()); -// the shim assigns its own downstream index. +// the dispatcher assigns its own downstream index. const messageTurn = (text: string, outputIndex = 0): ScriptedTurn => [ mkResponseCreated(), mkResponseInProgress(), @@ -432,13 +426,13 @@ const fcTurn = ( ]; const searchCallTurn = (outputIndex: number, callId: string, query: string): ScriptedTurn => - fcTurn(outputIndex, callId, SHIM_TOOL_NAME, JSON.stringify({ search_query: [{ q: query }] })); + fcTurn(outputIndex, callId, FUNCTION_TOOL_NAME, JSON.stringify({ search_query: [{ q: query }] })); const openCallTurn = (outputIndex: number, callId: string, url: string): ScriptedTurn => - fcTurn(outputIndex, callId, SHIM_TOOL_NAME, JSON.stringify({ open: [{ ref_id: url }] })); + fcTurn(outputIndex, callId, FUNCTION_TOOL_NAME, JSON.stringify({ open: [{ ref_id: url }] })); const findCallTurn = (outputIndex: number, callId: string, url: string, pattern: string): ScriptedTurn => - fcTurn(outputIndex, callId, SHIM_TOOL_NAME, JSON.stringify({ find: [{ ref_id: url, pattern }] })); + fcTurn(outputIndex, callId, FUNCTION_TOOL_NAME, JSON.stringify({ find: [{ ref_id: url, pattern }] })); const eventPayloads = (frames: ProtocolFrame[]): OpenAIResponsesStreamEvent[] => frames.filter(f => f.type === 'event').map(f => (f as { type: 'event'; event: OpenAIResponsesStreamEvent }).event); @@ -449,9 +443,9 @@ const outputItemDoneEvents = (frames: ProtocolFrame[ // ── Activation gating ────────────────────────────────────────────────── -test('shim no-ops when targetApi=responses and flag is off', async () => { +test('dispatcher no-ops when targetApi=openaiResponses and flag is off', async () => { const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ targetApi: 'openaiResponses', enabledFlags: new Set(), @@ -459,92 +453,56 @@ test('shim no-ops when targetApi=responses and flag is off', async () => { const originalPayload = inv.payload; const script = scriptedRun([messageTurn('hi back')]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assertEquals(script.callCount(), 1); - // Payload reference unchanged: the shim did not rewrite tools. + // Payload reference unchanged: the dispatcher did not rewrite tools. assertEquals(inv.payload, originalPayload); assertEquals(backend.calls.length, 0); assertEquals(result.type, 'events'); }); -test('shim activates when targetApi=responses and flag is on', async () => { +test('dispatcher activates when targetApi=openaiResponses and flag is on', async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ targetApi: 'openaiResponses', enabledFlags: new Set(['openai-responses-web-search-shim']), }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(inv.payload.tools?.length, 1); assertEquals(inv.payload.tools?.[0].type, 'function'); - assertEquals((inv.payload.tools?.[0] as { name: string }).name, SHIM_TOOL_NAME); + assertEquals((inv.payload.tools?.[0] as { name: string }).name, FUNCTION_TOOL_NAME); }); -for (const item of [ - { type: 'additional_tools' as const, role: 'developer' as const, id: 'at_1', tools: [{ type: 'web_search' as const }] }, - { type: 'tool_search_output' as const, call_id: 'search_1', execution: 'client' as const, tools: [{ type: 'web_search' as const }] }, -]) { - test(`shim rewrites hosted tools inside ${item.type} at their input position`, async () => { - const { backend } = makeStubDeps(); - const inv = makeInvocation({ - targetApi: 'openaiResponses', - enabledFlags: new Set(['openai-responses-web-search-shim']), - payload: { - tools: undefined, - tool_choice: { type: 'web_search' }, - input: [ - { type: 'message', role: 'user', content: 'before' }, - item, - { type: 'message', role: 'user', content: 'after' }, - ], - }, - }); - const args = JSON.stringify({ search_query: [{ q: 'test' }] }); - const script = scriptedRun([fcTurn(0, 'call_1', SHIM_TOOL_NAME, args), messageTurn('done')]); - - await runShimAndDrain(withOpenAIResponsesWebSearchShim, inv, makeGatewayCtx(), script.run); - - assertEquals(inv.payload.tools, undefined); - assertEquals(inv.payload.tool_choice, 'auto'); - assertEquals(inv.payload.input[0], { type: 'message', role: 'user', content: 'before' }); - const rewritten = inv.payload.input[1]; - assert(rewritten.type === item.type); - assertEquals(rewritten.tools.map(tool => tool.type), ['function']); - assertEquals((rewritten.tools[0] as { name: string }).name, SHIM_TOOL_NAME); - assertEquals(rewritten.type === 'additional_tools' ? rewritten.id : rewritten.call_id, item.type === 'additional_tools' ? 'at_1' : 'search_1'); - assertEquals(backend.calls.length, 1); - }); -} - -test('shim activates when targetApi=messages and flag is off', async () => { +test('dispatcher activates when targetApi=anthropicMessages and flag is off', async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ targetApi: 'anthropicMessages', enabledFlags: new Set() }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(inv.payload.tools?.length, 1); }); -test('shim activates when targetApi=openai-chat-completions and flag is off', async () => { +test('dispatcher activates when targetApi=openai-chat-completions and flag is off', async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ targetApi: 'openaiChatCompletions', enabledFlags: new Set() }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(inv.payload.tools?.length, 1); }); -test('shim no-ops when no hosted web_search tool is present', async () => { +test('dispatcher no-ops when no hosted web_search tool is present', async () => { const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [{ type: 'function', name: 'other', parameters: { type: 'object' }, strict: false }], @@ -552,7 +510,7 @@ test('shim no-ops when no hosted web_search tool is present', async () => { }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(script.callCount(), 1); assertEquals(backend.calls.length, 0); @@ -564,27 +522,27 @@ test('shim no-ops when no hosted web_search tool is present', async () => { const hostedAliasTypes = ['web_search', 'web_search_2025_08_26', 'web_search_preview', 'web_search_preview_2025_03_11'] as const; for (const type of hostedAliasTypes) { - test(`shim rewrites hosted ${type} alias into the shim function tool`, async () => { + test(`dispatcher rewrites hosted ${type} alias into the dispatcher function tool`, async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [{ type }] }, }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(inv.payload.tools?.length, 1); - assertEquals((inv.payload.tools?.[0] as { name?: string }).name, SHIM_TOOL_NAME); + assertEquals((inv.payload.tools?.[0] as { name?: string }).name, FUNCTION_TOOL_NAME); }); } // ── tool_choice rewrite × 4 hosted-type values ───────────────────────── for (const type of hostedAliasTypes) { - test(`shim rewrites tool_choice {type: ${type}} to forced shim's function tool`, async () => { + test(`dispatcher rewrites tool_choice {type: ${type}} to forced dispatcher's function tool`, async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [{ type }], @@ -593,17 +551,17 @@ for (const type of hostedAliasTypes) { }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); - assertEquals(inv.payload.tool_choice, { type: 'function', name: SHIM_TOOL_NAME }); + assertEquals(inv.payload.tool_choice, { type: 'function', name: FUNCTION_TOOL_NAME }); }); } // ── Per-tool fields propagate ────────────────────────────────────────── -test('shim propagates filters / user_location / search_context_size to backend', async () => { +test('dispatcher propagates filters / user_location / search_context_size to backend', async () => { const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -621,7 +579,7 @@ test('shim propagates filters / user_location / search_context_size to backend', messageTurn('summary', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(backend.calls.length, 1); const searchCall = backend.calls[0].request as WebSearchProviderRequest; @@ -634,13 +592,13 @@ test('shim propagates filters / user_location / search_context_size to backend', // ── Single-iteration message (no function calls) ─────────────────────── -test('shim forwards a single-turn message without backend calls', async () => { +test('dispatcher forwards a single-turn message without backend calls', async () => { const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([messageTurn('hello back')]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assertEquals(script.callCount(), 1); assertEquals(backend.calls.length, 0); @@ -655,16 +613,16 @@ test('shim forwards a single-turn message without backend calls', async () => { // ── One search then message ──────────────────────────────────────────── -test('shim drives one search then a final message in two upstream turns', async () => { +test('dispatcher drives one search then a final message in two upstream turns', async () => { const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ searchCallTurn(0, 'call_1', 'q1'), messageTurn('summary', 0), ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const frames = await collectFrames(result.events); @@ -676,7 +634,7 @@ test('shim drives one search then a final message in two upstream turns', async assertEquals(tail[0].type, 'function_call'); assertEquals(tail[1].type, 'function_call_output'); assert(tail[0].type === 'function_call'); - // Shim replay preserves the upstream's original shim call + // Dispatcher replay preserves the upstream's original dispatcher call // verbatim (call_id, name, jsonrepair-canonical args) so the upstream // model on turn 2 sees its prior assistant turn unchanged. assertEquals(tail[0].call_id, 'call_1'); @@ -688,10 +646,10 @@ test('shim drives one search then a final message in two upstream turns', async test('translated source with a no-backing store still replays private search state across turns', async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); // A non-OpenAI-Responses source (Anthropic Messages/Gemini generateContent/OpenAI Chat Completions) translated into an OpenAI Responses - // upstream carries a no-backing store, not a persisting one; the shim's + // upstream carries a no-backing store, not a persisting one; the dispatcher's // request-private replay must still round-trip within the single request. const ctx = mockChatGatewayCtx({ apiKeyId: 'k1', wantsStream: true, store: createNonOpenAIResponsesSourceStore('k1') }); const script = scriptedRun([ @@ -699,7 +657,7 @@ test('translated source with a no-backing store still replays private search sta messageTurn('summary', 0), ]); - const result = await shim(inv, ctx, script.run); + const result = await dispatcher(inv, ctx, script.run); assert(result.type === 'events'); await collectFrames(result.events); @@ -715,7 +673,7 @@ test('translated source with a no-backing store still replays private search sta test('synthesized web_search_call ids retain request-private replay state', async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const ctx = makeGatewayCtx(); const script = scriptedRun([ @@ -723,7 +681,7 @@ test('synthesized web_search_call ids retain request-private replay state', asyn messageTurn('summary', 0), ]); - const { frames } = await runShimAndDrain(shim, inv, ctx, script.run); + const { frames } = await runHostedToolsAndDrain(dispatcher, inv, ctx, script.run); const doneEvents = outputItemDoneEvents(frames); const wsCallDoneIds = doneEvents.filter(e => e.item.type === 'web_search_call').map(e => e.item.id!); @@ -758,7 +716,7 @@ test('find_in_page reuses cache when same URL was opened first', async () => { }, }, }); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ openCallTurn(0, 'call_o', 'https://example.com/p'), @@ -766,7 +724,7 @@ test('find_in_page reuses cache when same URL was opened first', async () => { messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); // open issues fetchPage; find reuses cache → still 1 fetchPage call. const fetchCalls = backend.calls.filter(c => c.kind === 'fetchPage'); @@ -777,14 +735,14 @@ test('find_in_page reuses cache when same URL was opened first', async () => { test('find_in_page triggers implicit fetchPage when URL not cached', async () => { const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ findCallTurn(0, 'call_f', 'https://example.com/fresh', 'body'), messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const fetchCalls = backend.calls.filter(c => c.kind === 'fetchPage'); assertEquals(fetchCalls.length, 1); @@ -794,7 +752,7 @@ test('find_in_page triggers implicit fetchPage when URL not cached', async () => test('iteration cap returns the iteration-cap notice without backend call on cap+1', async () => { const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const searchTurns: ScriptedTurn[] = []; for (let i = 0; i < 30; i++) { @@ -804,7 +762,7 @@ test('iteration cap returns the iteration-cap notice without backend call on cap searchTurns.push(messageTurn('done', 0)); const script = scriptedRun(searchTurns); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); @@ -820,7 +778,7 @@ test('iteration cap returns the iteration-cap notice without backend call on cap // lifecycle: 31 done events = 30 successful + 1 capped. The capped // call surfaces as the schema-error shape (action.type='search' with // the cap diagnostic in queries[0]) rather than the original query — - // the shim call is rejected before backend dispatch. + // the dispatcher call is rejected before backend dispatch. const wsCallDone = events.filter((e): e is Extract => e.type === 'response.output_item.done' && (e as { item?: { type?: string } }).item?.type === 'web_search_call'); @@ -832,25 +790,25 @@ test('iteration cap returns the iteration-cap notice without backend call on cap // ── Ambiguous multi-op function_call rejection ───────────────────────────── -// `web_search_call.action` carries exactly one action type; a shim call +// `web_search_call.action` carries exactly one action type; a dispatcher call // that mixes kinds (search + open) or stacks multiple non-search ops // (multi `open`/`find`) cannot reduce to a single wsc and is rejected // before backend dispatch. Multi-`search_query` collapses into one wsc // with a multi-query search action — see the collapse tests below. -const assertAmbiguousShimRejection = async (args: string, backend: { calls: unknown[] }): Promise => { - const shim = withOpenAIResponsesWebSearchShim; +const assertAmbiguousCallRejection = async (args: string, backend: { calls: unknown[] }): Promise => { + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const turn: ScriptedTurn = [ mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'call_ambig', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_ambig', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, args), - mkFunctionCallDone(0, 'call_ambig', SHIM_TOOL_NAME, args), + mkFunctionCallDone(0, 'call_ambig', FUNCTION_TOOL_NAME, args), mkResponseCompleted(), ]; const script = scriptedRun([turn, messageTurn('done', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); assertEquals(backend.calls.length, 0); @@ -865,25 +823,25 @@ const assertAmbiguousShimRejection = async (args: string, backend: { calls: unkn assert(item.results![0].snippet.includes('one web_search_call')); }; -test('multi-kind shim call (search_query + open) is rejected as ambiguous before any backend dispatch', async () => { +test('multi-kind dispatcher call (search_query + open) is rejected as ambiguous before any backend dispatch', async () => { const { backend } = makeStubDeps(); - await assertAmbiguousShimRejection( + await assertAmbiguousCallRejection( JSON.stringify({ search_query: [{ q: 'q' }], open: [{ ref_id: 'https://example.com/' }] }), backend, ); }); -test('multi-instance same-kind shim call (two open entries) is rejected as ambiguous', async () => { +test('multi-instance same-kind dispatcher call (two open entries) is rejected as ambiguous', async () => { const { backend } = makeStubDeps(); - await assertAmbiguousShimRejection( + await assertAmbiguousCallRejection( JSON.stringify({ open: [{ ref_id: 'https://example.com/a' }, { ref_id: 'https://example.com/b' }] }), backend, ); }); -test('multi-instance same-kind shim call (two find entries) is rejected as ambiguous', async () => { +test('multi-instance same-kind dispatcher call (two find entries) is rejected as ambiguous', async () => { const { backend } = makeStubDeps(); - await assertAmbiguousShimRejection( + await assertAmbiguousCallRejection( JSON.stringify({ find: [{ ref_id: 'https://example.com/', pattern: 'a' }, { ref_id: 'https://example.com/', pattern: 'b' }] }), backend, ); @@ -905,19 +863,19 @@ test('multi-`search_query` entries collapse into one web_search_call with a mult }, }, }); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const args = JSON.stringify({ search_query: [{ q: 'q1' }, { q: 'q2' }, { q: 'q3' }] }); const turn: ScriptedTurn = [ mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'call_multi', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_multi', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, args), - mkFunctionCallDone(0, 'call_multi', SHIM_TOOL_NAME, args), + mkFunctionCallDone(0, 'call_multi', FUNCTION_TOOL_NAME, args), mkResponseCompleted(), ]; const script = scriptedRun([turn, messageTurn('done', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); @@ -942,7 +900,7 @@ test('multi-`search_query` entries collapse into one web_search_call with a mult test('multi-`search_query` with one malformed entry (missing q) is rejected as ambiguous', async () => { const { backend } = makeStubDeps(); - await assertAmbiguousShimRejection( + await assertAmbiguousCallRejection( JSON.stringify({ search_query: [{ q: 'q1' }, {}] }), backend, ); @@ -958,14 +916,14 @@ test('search returning zero results surfaces the "(no results)" template', async }, }, }); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ searchCallTurn(0, 'call_1', 'empty-query'), messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const input = inv.payload.input as OpenAIResponsesInputItem[]; const lastOutput = input[input.length - 1]; @@ -986,14 +944,14 @@ test('backend search failure surfaces "Search failed: "', async () => { }, }, }); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ searchCallTurn(0, 'call_1', 'q1'), messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const input = inv.payload.input as OpenAIResponsesInputItem[]; const lastOutput = input[input.length - 1]; @@ -1013,14 +971,14 @@ test('fetchPage whole-batch failure surfaces the open-page error text', async () }, }, }); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ openCallTurn(0, 'call_o', 'https://example.com/x'), messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const input = inv.payload.input as OpenAIResponsesInputItem[]; const lastOutput = input[input.length - 1]; @@ -1033,14 +991,14 @@ test('fetchPage whole-batch failure surfaces the open-page error text', async () // ── Domain filter input validation ──────────────────────────────────── test('invalid request registration preserves an upstream error type and null code', async () => { - const shim = withOpenAIResponsesServerToolShim([() => ({ + const dispatcher = driveHostedToolStage([() => ({ type: 'invalid-request', message: 'native image error', param: 'input', errorType: 'image_generation_user_error', code: null, })]); - const result = await shim( + const result = await dispatcher( makeInvocation(), makeGatewayCtx(), () => Promise.reject(new Error('run should not be called')), @@ -1068,7 +1026,7 @@ test('invalid hosted web search in additional_tools reports its input path', asy }], }, }); - const result = await withOpenAIResponsesWebSearchShim(inv, makeGatewayCtx(), async () => { + const result = await runHostedWebSearch(inv, makeGatewayCtx(), async () => { throw new Error('Invalid request reached upstream'); }); assert(result.type === 'api-error'); @@ -1092,9 +1050,9 @@ test('later hosted validation keeps original paths before another tool rewrites }, }); const originalInput = structuredClone(inv.payload.input); - const shim = withOpenAIResponsesServerToolShim([webSearchServerTool, imageGenerationServerTool]); + const dispatcher = driveHostedToolStage([webSearchHostedTool, imageGenerationHostedTool]); - const result = await shim(inv, makeGatewayCtx(), async () => { + const result = await dispatcher(inv, makeGatewayCtx(), async () => { throw new Error('Invalid request reached upstream'); }); @@ -1110,7 +1068,7 @@ test('non-empty allowed_domains with every entry malformed is rejected as 400 in // loud 400 because the client believed they had a restrictive // allow-list. const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -1123,7 +1081,7 @@ test('non-empty allowed_domains with every entry malformed is rejected as 400 in }); const script = scriptedRun([messageTurn('never reached', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assertEquals(result.type, 'api-error'); assert(result.type === 'api-error'); assertEquals(result.status, 400); @@ -1142,7 +1100,7 @@ test('non-empty blocked_domains with every entry malformed is rejected as 400 in // every malformed entry would turn "block these sites" into "block // nothing", letting traffic the client intended to block through. const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -1155,7 +1113,7 @@ test('non-empty blocked_domains with every entry malformed is rejected as 400 in }); const script = scriptedRun([messageTurn('never reached', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assertEquals(result.type, 'api-error'); assert(result.type === 'api-error'); assertEquals(result.status, 400); @@ -1172,7 +1130,7 @@ test('domain lists with any malformed entry alongside valid entries are rejected // believed was blocked / outside the allow-list through. The // diagnostic must name the offending index so the client can fix it. const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -1188,7 +1146,7 @@ test('domain lists with any malformed entry alongside valid entries are rejected }); const script = scriptedRun([messageTurn('never reached', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'api-error'); assertEquals(result.status, 400); assertEquals(script.callCount(), 0); @@ -1200,12 +1158,12 @@ test('domain lists with any malformed entry alongside valid entries are rejected }); test('multiple hosted web_search entries: filter CONTENT is validated on each (not just last-wins)', async () => { - // `rewriteToolsForShim` last-wins on filter extraction, so a content + // `rewriteHostedTools` last-wins on filter extraction, so a content // check against `rewritten.filters` alone would miss malformed // entries on earlier hosted tools that get discarded. Per-entry // validation catches them — first failure wins. const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -1218,7 +1176,7 @@ test('multiple hosted web_search entries: filter CONTENT is validated on each (n }, }); const script = scriptedRun([messageTurn('never reached', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'api-error'); assertEquals(result.status, 400); assertEquals(script.callCount(), 0); @@ -1230,14 +1188,14 @@ test('multiple hosted web_search entries: filter CONTENT is validated on each (n test('omitted filters do not trigger the validation reject (no false positive on the default case)', async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [{ type: 'web_search' }], }, }); const script = scriptedRun([messageTurn('done', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); await collectFrames(result.events); }); @@ -1249,7 +1207,7 @@ test('allowed_domains containing a non-string entry rejects with 400 (no 502 cra // 400 invalid_request_error keeps clients on the diagnostic-rich error // shape SDKs already speak. const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -1263,7 +1221,7 @@ test('allowed_domains containing a non-string entry rejects with 400 (no 502 cra }, }); const script = scriptedRun([messageTurn('never reached', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'api-error'); assertEquals(result.status, 400); assertEquals(script.callCount(), 0); @@ -1275,7 +1233,7 @@ test('allowed_domains containing a non-string entry rejects with 400 (no 502 cra test('blocked_domains containing an object entry rejects with 400', async () => { const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -1287,7 +1245,7 @@ test('blocked_domains containing an object entry rejects with 400', async () => }, }); const script = scriptedRun([messageTurn('never reached', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'api-error'); assertEquals(result.status, 400); assertEquals(script.callCount(), 0); @@ -1301,17 +1259,17 @@ test('blocked_domains containing an object entry rejects with 400', async () => // Each entry must be a bare hostname per OpenAI's web_search docs // (https://developers.openai.com/api/docs/guides/tools-web-search.md): // "omit the HTTP or HTTPS prefix" and use a domain like `openai.com`. -// The shim rejects every deviation at the boundary so silent surface +// The dispatcher rejects every deviation at the boundary so silent surface // expansion is impossible — `blocked_domains: ['reddit.com', // 'https://quora.com/']` used to silently leave quora unblocked // because the second entry dropped during normalization. -const runShimWithDomainEntry = async ( +const runWithDomainEntry = async ( field: 'allowed_domains' | 'blocked_domains', raw: unknown, ) => { const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -1323,13 +1281,13 @@ const runShimWithDomainEntry = async ( }, }); const script = scriptedRun([messageTurn('never reached', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); return { result, backend, script }; }; for (const field of ['allowed_domains', 'blocked_domains'] as const) { test(`${field} entry with https:// prefix rejects with 400 invalid domain`, async () => { - const { result, backend, script } = await runShimWithDomainEntry(field, 'https://quora.com/'); + const { result, backend, script } = await runWithDomainEntry(field, 'https://quora.com/'); assert(result.type === 'api-error'); assertEquals(result.status, 400); assertEquals(script.callCount(), 0); @@ -1341,7 +1299,7 @@ for (const field of ['allowed_domains', 'blocked_domains'] as const) { }); test(`${field} entry with http:// prefix rejects with 400`, async () => { - const { result } = await runShimWithDomainEntry(field, 'http://example.com'); + const { result } = await runWithDomainEntry(field, 'http://example.com'); assert(result.type === 'api-error'); assertEquals(result.status, 400); const body = JSON.parse(new TextDecoder().decode(result.body)) as { error: { param: string; message: string } }; @@ -1350,7 +1308,7 @@ for (const field of ['allowed_domains', 'blocked_domains'] as const) { }); test(`${field} entry with a path rejects with 400`, async () => { - const { result } = await runShimWithDomainEntry(field, 'example.com/some/path'); + const { result } = await runWithDomainEntry(field, 'example.com/some/path'); assert(result.type === 'api-error'); assertEquals(result.status, 400); const body = JSON.parse(new TextDecoder().decode(result.body)) as { error: { param: string; message: string } }; @@ -1359,7 +1317,7 @@ for (const field of ['allowed_domains', 'blocked_domains'] as const) { }); test(`${field} entry with a port rejects with 400`, async () => { - const { result } = await runShimWithDomainEntry(field, 'example.com:8080'); + const { result } = await runWithDomainEntry(field, 'example.com:8080'); assert(result.type === 'api-error'); assertEquals(result.status, 400); const body = JSON.parse(new TextDecoder().decode(result.body)) as { error: { param: string; message: string } }; @@ -1367,13 +1325,13 @@ for (const field of ['allowed_domains', 'blocked_domains'] as const) { }); test(`${field} entry with a query string rejects with 400`, async () => { - const { result } = await runShimWithDomainEntry(field, 'example.com?q=1'); + const { result } = await runWithDomainEntry(field, 'example.com?q=1'); assert(result.type === 'api-error'); assertEquals(result.status, 400); }); test(`${field} empty-string entry rejects with 400`, async () => { - const { result } = await runShimWithDomainEntry(field, ''); + const { result } = await runWithDomainEntry(field, ''); assert(result.type === 'api-error'); assertEquals(result.status, 400); const body = JSON.parse(new TextDecoder().decode(result.body)) as { error: { param: string; message: string } }; @@ -1381,13 +1339,13 @@ for (const field of ['allowed_domains', 'blocked_domains'] as const) { }); test(`${field} whitespace-only entry rejects with 400 (no surface expansion via .trim())`, async () => { - const { result } = await runShimWithDomainEntry(field, ' '); + const { result } = await runWithDomainEntry(field, ' '); assert(result.type === 'api-error'); assertEquals(result.status, 400); }); test(`${field} non-string entry rejects with 400 (typed-but-not-string)`, async () => { - const { result } = await runShimWithDomainEntry(field, null); + const { result } = await runWithDomainEntry(field, null); assert(result.type === 'api-error'); assertEquals(result.status, 400); const body = JSON.parse(new TextDecoder().decode(result.body)) as { error: { param: string; message: string } }; @@ -1400,14 +1358,14 @@ for (const field of ['allowed_domains', 'blocked_domains'] as const) { test(`${field} exceeding 100 entries rejects with 400 (matches OpenAI documented cap)`, async () => { const oversized = Array.from({ length: 101 }, (_, i) => `host${i}.example`); const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [{ type: 'web_search', filters: { [field]: oversized } }], }, }); const script = scriptedRun([messageTurn('never reached', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'api-error'); assertEquals(result.status, 400); assertEquals(script.callCount(), 0); @@ -1422,14 +1380,14 @@ for (const field of ['allowed_domains', 'blocked_domains'] as const) { test(`${field} exactly 100 entries is accepted (boundary inclusive)`, async () => { const exactly100 = Array.from({ length: 100 }, (_, i) => `host${i}.example`); makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [{ type: 'web_search', filters: { [field]: exactly100 } }], }, }); const script = scriptedRun([messageTurn('done', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); await collectFrames(result.events); }); @@ -1440,7 +1398,7 @@ test('mixed list with one prefixed entry names the offending index (no silent dr // ['reddit.com', 'https://quora.com/'] previously had `quora.com` // silently NOT blocked. Reject at the boundary so the violating // index is named. - const { result } = await runShimMixedList('blocked_domains', ['reddit.com', 'https://quora.com/']); + const { result } = await runWithMixedList('blocked_domains', ['reddit.com', 'https://quora.com/']); assert(result.type === 'api-error'); assertEquals(result.status, 400); const body = JSON.parse(new TextDecoder().decode(result.body)) as { error: { param: string; message: string } }; @@ -1448,12 +1406,12 @@ test('mixed list with one prefixed entry names the offending index (no silent dr assert(body.error.message.includes('quora.com')); }); -const runShimMixedList = async ( +const runWithMixedList = async ( field: 'allowed_domains' | 'blocked_domains', entries: unknown[], ) => { const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -1465,7 +1423,7 @@ const runShimMixedList = async ( }, }); const script = scriptedRun([messageTurn('never reached', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); return { result, backend, script }; }; @@ -1476,7 +1434,7 @@ test('invalid search_context_size value rejects with 400 (no silent fall-through // default — silently shrinking the result set. Reject at the // boundary so the misuse surfaces. const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -1485,7 +1443,7 @@ test('invalid search_context_size value rejects with 400 (no silent fall-through }, }); const script = scriptedRun([messageTurn('never reached', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'api-error'); assertEquals(result.status, 400); assertEquals(script.callCount(), 0); @@ -1497,14 +1455,14 @@ test('invalid search_context_size value rejects with 400 (no silent fall-through }); for (const field of ['external_web_access', 'search_content_types', 'return_token_budget'] as const) { - test(`explicitly-set hosted ${field} is silently stripped (the function tool the shim forwards never carries it)`, async () => { - // The shim replaces the hosted entry with its function tool; any - // hosted-only field — including ones the shim has no opinion on — + test(`explicitly-set hosted ${field} is silently stripped (the function tool the dispatcher forwards never carries it)`, async () => { + // The dispatcher replaces the hosted entry with its function tool; any + // hosted-only field — including ones the dispatcher has no opinion on — // drops out with the entry. Mirrors native: silently stripped. // Tests the request completes normally instead of being rejected // as a 400. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const fieldValue: Record = { external_web_access: false, search_content_types: ['image'], @@ -1516,7 +1474,7 @@ for (const field of ['external_web_access', 'search_content_types', 'return_toke }, }); const script = scriptedRun([messageTurn('hello', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); assertEquals(events[events.length - 1].type, 'response.completed'); @@ -1531,7 +1489,7 @@ test('array-shaped filters rejects with 400 (typeof null/[] === "object" guards // arrays at the boundary so the client sees the misuse instead of // a downstream filter behaving as if nothing was configured. const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -1543,7 +1501,7 @@ test('array-shaped filters rejects with 400 (typeof null/[] === "object" guards }, }); const script = scriptedRun([messageTurn('never reached', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'api-error'); assertEquals(result.status, 400); assertEquals(script.callCount(), 0); @@ -1561,14 +1519,14 @@ test('empty allowed_domains array is a no-op (not a misuse signal)', async () => // client input. Only a NON-empty list with all-malformed entries is // a misuse worth rejecting. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [{ type: 'web_search', filters: { allowed_domains: [] } }], }, }); const script = scriptedRun([messageTurn('done', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); await collectFrames(result.events); }); @@ -1577,7 +1535,7 @@ test('null allowed_domains is a no-op (treated the same as omitted)', async () = // Tool authors sometimes use `null` and `undefined` interchangeably // as the "field absent" signal; neither should reject. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -1589,7 +1547,7 @@ test('null allowed_domains is a no-op (treated the same as omitted)', async () = }, }); const script = scriptedRun([messageTurn('done', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); await collectFrames(result.events); }); @@ -1598,7 +1556,7 @@ test('null allowed_domains is a no-op (treated the same as omitted)', async () = test('open blocked by domain filter never calls backend', async () => { const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [{ type: 'web_search', filters: { blocked_domains: ['blocked.com'] } }], @@ -1609,7 +1567,7 @@ test('open blocked by domain filter never calls backend', async () => { messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(backend.calls.length, 0); const input = inv.payload.input as OpenAIResponsesInputItem[]; @@ -1628,7 +1586,7 @@ test('find blocked by domain filter emits wire-side find_in_page (not open_page) // model's intent (e.g. a UI that draws a "searched inside // for " badge for find ops only). const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [{ type: 'web_search', filters: { blocked_domains: ['blocked.com'] } }], @@ -1639,7 +1597,7 @@ test('find blocked by domain filter emits wire-side find_in_page (not open_page) messageTurn('done', 0), ]); - const { frames } = await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + const { frames } = await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(backend.calls.length, 0); // Wire-side: the synthesized output_item.done for the web_search_call @@ -1677,14 +1635,14 @@ test('find with no matches returns the no-matches text', async () => { }, }, }); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ findCallTurn(0, 'call_f', 'https://example.com/p', 'nonexistent'), messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const input = inv.payload.input as OpenAIResponsesInputItem[]; const lastOutput = input[input.length - 1]; @@ -1713,14 +1671,14 @@ test('find with matches returns bracketed context', async () => { }, }, }); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ findCallTurn(0, 'call_f', 'https://example.com/p', 'needle'), messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const input = inv.payload.input as OpenAIResponsesInputItem[]; const lastOutput = input[input.length - 1]; @@ -1750,14 +1708,14 @@ test('truncated page contents append the truncation sentinel', async () => { }, }, }); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ openCallTurn(0, 'call_o', 'https://example.com/x'), messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const input = inv.payload.input as OpenAIResponsesInputItem[]; const lastOutput = input[input.length - 1]; @@ -1772,7 +1730,7 @@ test('truncated page contents append the truncation sentinel', async () => { test('multi-turn merge: output_index unique, sequence_number monotonic, response.created once', async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ searchCallTurn(0, 'call_1', 'q1'), @@ -1780,7 +1738,7 @@ test('multi-turn merge: output_index unique, sequence_number monotonic, response messageTurn('done', 0), ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); @@ -1814,7 +1772,7 @@ test('multi-turn merge: output_index unique, sequence_number monotonic, response // Wire mock for upstream `response.created` that announces a specific // served model — distinct from the client's payload.model so we can -// prove the shim quotes upstream's served identity rather than the +// prove the dispatcher quotes upstream's served identity rather than the // requested literal. const mkResponseCreatedWithModel = (model: string, responseId = 'upstream_test'): ProtocolFrame => eventFrame({ @@ -1843,16 +1801,16 @@ test('synthesized response.created / completed quote the upstream-reported model // 'gpt-5.4-2025-01-20'. Every synthesized frame must mirror upstream. const SERVED_MODEL = 'gpt-5.4-2025-01-20'; makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { model: 'gpt-5' } }); const script = scriptedRun([ [ mkResponseCreatedWithModel(SERVED_MODEL), mkResponseInProgressWithModel(SERVED_MODEL), - mkFunctionCallAdded(0, 'call_1', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_1', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, JSON.stringify({ search_query: [{ q: 'hi' }] })), - mkFunctionCallDone(0, 'call_1', SHIM_TOOL_NAME, JSON.stringify({ search_query: [{ q: 'hi' }] })), + mkFunctionCallDone(0, 'call_1', FUNCTION_TOOL_NAME, JSON.stringify({ search_query: [{ q: 'hi' }] })), mkResponseCompletedWithModel(SERVED_MODEL), ], [ @@ -1864,7 +1822,7 @@ test('synthesized response.created / completed quote the upstream-reported model ], ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); @@ -1879,7 +1837,7 @@ test('synthesized response.created / completed quote the upstream-reported model test('synthesized response.failed (upstream error mid-stream) quotes the upstream-reported model', async () => { const SERVED_MODEL = 'gpt-5.4-2025-01-20'; makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { model: 'gpt-5' } }); let runCalls = 0; @@ -1889,9 +1847,9 @@ test('synthesized response.failed (upstream error mid-stream) quotes the upstrea const frames: ProtocolFrame[] = [ mkResponseCreatedWithModel(SERVED_MODEL), mkResponseInProgressWithModel(SERVED_MODEL), - mkFunctionCallAdded(0, 'call_1', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_1', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, JSON.stringify({ search_query: [{ q: 'hi' }] })), - mkFunctionCallDone(0, 'call_1', SHIM_TOOL_NAME, JSON.stringify({ search_query: [{ q: 'hi' }] })), + mkFunctionCallDone(0, 'call_1', FUNCTION_TOOL_NAME, JSON.stringify({ search_query: [{ q: 'hi' }] })), mkResponseCompletedWithModel(SERVED_MODEL), ]; const iterable: AsyncIterable> = (async function* () { @@ -1908,7 +1866,7 @@ test('synthesized response.failed (upstream error mid-stream) quotes the upstrea }; }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const failed = events[events.length - 1] as Extract; @@ -1916,7 +1874,7 @@ test('synthesized response.failed (upstream error mid-stream) quotes the upstrea assertEquals(failed.response.model, SERVED_MODEL); }); -test('shim refuses to synthesize a response envelope when upstream response.created has no model', async () => { +test('dispatcher refuses to synthesize a response envelope when upstream response.created has no model', async () => { // No-fallback contract: a missing model on upstream's first // response.created is a protocol violation. The `ensureModel()` // invariant throws from inside `consumeTurnStreaming`; the throw @@ -1924,7 +1882,7 @@ test('shim refuses to synthesize a response envelope when upstream response.crea // (which reports it upward) rather than being silently swallowed // or papered over with `ctx.payload.model`. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { model: 'gpt-5' } }); const modelless = eventFrame({ @@ -1933,7 +1891,7 @@ test('shim refuses to synthesize a response envelope when upstream response.crea }); const script = scriptedRun([[modelless, mkResponseCompleted()]]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); let thrown: unknown = undefined; try { @@ -1947,13 +1905,13 @@ test('shim refuses to synthesize a response envelope when upstream response.crea assert(/did not report a `model`|never reported a `model`/.test(thrown.message)); }); -test('upstream response.created with no `id` field is tolerated (downstream uses the shim-synthesized id regardless)', async () => { - // The shim never quotes upstream's `id` downstream — every - // synthesized envelope carries the shim's own `resp_shim_` +test('upstream response.created with no `id` field is tolerated (downstream uses the gateway-synthesized id regardless)', async () => { + // The dispatcher never quotes upstream's `id` downstream — every + // synthesized envelope carries the dispatcher's own `resp_hosted_` // identity. So a missing id on upstream's `response.created` is // harmless; the downstream response still gets a valid id. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { model: 'gpt-5' } }); const idless = eventFrame({ @@ -1962,22 +1920,22 @@ test('upstream response.created with no `id` field is tolerated (downstream uses }); const script = scriptedRun([[idless, mkResponseCompleted()]]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const created = events.find(e => e.type === 'response.created') as Extract; - assert(created.response.id.startsWith('resp_shim_')); + assert(created.response.id.startsWith('resp_hosted_')); }); test('synthesized response.created / completed quote the once-per-request synthesized id (not the upstream id, not a fresh per-event id)', async () => { - // Upstream's id is irrelevant downstream — the shim generates a - // single `resp_shim_` at activation and quotes it on every + // Upstream's id is irrelevant downstream — the dispatcher generates a + // single `resp_hosted_` at activation and quotes it on every // synthesized envelope (created, in_progress, completed). One - // stable id is what clients correlate against across the shim's + // stable id is what clients correlate against across the dispatcher's // multi-turn response. const UPSTREAM_ID = 'resp_abc123def456'; makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ [ @@ -1989,14 +1947,14 @@ test('synthesized response.created / completed quote the once-per-request synthe ], ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const created = events.find(e => e.type === 'response.created') as Extract; const inProgress = events.find(e => e.type === 'response.in_progress') as Extract; const completed = events.find(e => e.type === 'response.completed') as Extract; - assert(created.response.id.startsWith('resp_shim_')); + assert(created.response.id.startsWith('resp_hosted_')); assertEquals(inProgress.response.id, created.response.id); assertEquals(completed.response.id, created.response.id); // Upstream's id is not what the wire carries. @@ -2006,21 +1964,21 @@ test('synthesized response.created / completed quote the once-per-request synthe test('synthesized terminal id stays constant across multi-turn upstream id rotation', async () => { // Upstream re-encrypts response.id per turn (turn-N's // response.created and terminal frames can carry a different id - // than turn-1's). The shim's synthesized id is generated ONCE per + // than turn-1's). The dispatcher's synthesized id is generated ONCE per // request, so cross-turn synthesis quotes the same value end to // end regardless of what upstream rotates to. const TURN1_ID = 'resp_turn1_aaa'; const TURN2_ID = 'resp_turn2_bbb_rotated'; makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ [ mkResponseCreated(TURN1_ID), mkResponseInProgress(TURN1_ID), - mkFunctionCallAdded(0, 'call_1', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_1', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, JSON.stringify({ search_query: [{ q: 'q' }] })), - mkFunctionCallDone(0, 'call_1', SHIM_TOOL_NAME, JSON.stringify({ search_query: [{ q: 'q' }] })), + mkFunctionCallDone(0, 'call_1', FUNCTION_TOOL_NAME, JSON.stringify({ search_query: [{ q: 'q' }] })), mkResponseCompleted(undefined, TURN1_ID), ], [ @@ -2032,13 +1990,13 @@ test('synthesized terminal id stays constant across multi-turn upstream id rotat ], ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const created = events.find(e => e.type === 'response.created') as Extract; const completed = events.find(e => e.type === 'response.completed') as Extract; - assert(created.response.id.startsWith('resp_shim_')); - // Same shim-synthesized id end-to-end; neither upstream turn id + assert(created.response.id.startsWith('resp_hosted_')); + // Same gateway-synthesized id end-to-end; neither upstream turn id // leaks downstream. assertEquals(completed.response.id, created.response.id); assertFalse(completed.response.id === TURN1_ID); @@ -2049,7 +2007,7 @@ test('synthesized terminal id stays constant across multi-turn upstream id rotat test('usage accumulates across three iterations', async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const turnWithUsage = ( @@ -2061,9 +2019,9 @@ test('usage accumulates across three iterations', async () => { return [ mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, callId, SHIM_TOOL_NAME), + mkFunctionCallAdded(0, callId, FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, args), - mkFunctionCallDone(0, callId, SHIM_TOOL_NAME, args), + mkFunctionCallDone(0, callId, FUNCTION_TOOL_NAME, args), mkResponseCompleted({ input_tokens: inTok, output_tokens: outTok, total_tokens: inTok + outTok }), ]; }; @@ -2082,7 +2040,7 @@ test('usage accumulates across three iterations', async () => { finalTurn, ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const completed = events.find((e): e is Extract => e.type === 'response.completed'); @@ -2098,16 +2056,16 @@ test('usage cached_tokens reported on one turn carries through (last-turn omissi // turn 2 omits it. Final usage must still surface cached_tokens // (turn 1's value). makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const turn1Args = JSON.stringify({ search_query: [{ q: 'q1' }] }); const turn1: ScriptedTurn = [ mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'call_1', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_1', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, turn1Args), - mkFunctionCallDone(0, 'call_1', SHIM_TOOL_NAME, turn1Args), + mkFunctionCallDone(0, 'call_1', FUNCTION_TOOL_NAME, turn1Args), mkResponseCompleted({ input_tokens: 100, output_tokens: 50, @@ -2124,7 +2082,7 @@ test('usage cached_tokens reported on one turn carries through (last-turn omissi mkResponseCompleted({ input_tokens: 50, output_tokens: 10, total_tokens: 60 }), ]; - const result = await shim(inv, makeGatewayCtx(), scriptedRun([turn1, turn2]).run); + const result = await dispatcher(inv, makeGatewayCtx(), scriptedRun([turn1, turn2]).run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const completed = events.find((e): e is Extract => e.type === 'response.completed'); @@ -2142,16 +2100,16 @@ test('usage cached_tokens never reported on any turn is omitted from wire (no fa // as `cached_tokens: 0`. The wire shape matches what a native // upstream that doesn't track caching would produce. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const turn1Args = JSON.stringify({ search_query: [{ q: 'q1' }] }); const turn1: ScriptedTurn = [ mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'call_1', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_1', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, turn1Args), - mkFunctionCallDone(0, 'call_1', SHIM_TOOL_NAME, turn1Args), + mkFunctionCallDone(0, 'call_1', FUNCTION_TOOL_NAME, turn1Args), mkResponseCompleted({ input_tokens: 100, output_tokens: 50, total_tokens: 150 }), ]; const turn2: ScriptedTurn = [ @@ -2162,7 +2120,7 @@ test('usage cached_tokens never reported on any turn is omitted from wire (no fa mkResponseCompleted({ input_tokens: 50, output_tokens: 10, total_tokens: 60 }), ]; - const result = await shim(inv, makeGatewayCtx(), scriptedRun([turn1, turn2]).run); + const result = await dispatcher(inv, makeGatewayCtx(), scriptedRun([turn1, turn2]).run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const completed = events.find((e): e is Extract => e.type === 'response.completed'); @@ -2172,7 +2130,7 @@ test('usage cached_tokens never reported on any turn is omitted from wire (no fa assertEquals(completed.response.usage?.output_tokens_details, undefined); }); -test('next-turn function_call echo always carries the canonical re-stringified shim call args (single shape unified with client-roundtrip)', async () => { +test('next-turn function_call echo always carries the canonical re-stringified dispatcher call args (single shape unified with client-roundtrip)', async () => { // The dispatcher always overwrites the intercepted call's // arguments with the canonical re-stringified form, regardless of // whether the upstream string was already valid JSON. This @@ -2185,21 +2143,21 @@ test('next-turn function_call echo always carries the canonical re-stringified s // upstream would 400 on the broken raw string — but the same // canonical form is also used for already-valid input. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); // Trailing-comma JSON survives jsonrepair but is not valid strict JSON. const brokenArgs = '{"search_query":[{"q":"q"}],}'; const brokenTurn: ScriptedTurn = [ mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'call_broken', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_broken', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, brokenArgs), - mkFunctionCallDone(0, 'call_broken', SHIM_TOOL_NAME, brokenArgs), + mkFunctionCallDone(0, 'call_broken', FUNCTION_TOOL_NAME, brokenArgs), mkResponseCompleted(), ]; const script = scriptedRun([brokenTurn, messageTurn('done', 0)]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const input = inv.payload.input as OpenAIResponsesInputItem[]; const fc = input.find(i => i.type === 'function_call') as @@ -2211,7 +2169,7 @@ test('next-turn function_call echo always carries the canonical re-stringified s assertEquals(fc.arguments, '{"search_query":[{"q":"q"}]}'); }); -test('upstream sends bare `error` frame BEFORE any response.created: shim throws from the events iterator (no synthetic response.failed with empty identity)', async () => { +test('upstream sends bare `error` frame BEFORE any response.created: dispatcher throws from the events iterator (no synthetic response.failed with empty identity)', async () => { // A bare `{type:'error'}` arriving before any `response.created` // produces no captured identity (id / model). Synthesizing a // `response.failed` envelope would require those fields. The @@ -2221,7 +2179,7 @@ test('upstream sends bare `error` frame BEFORE any response.created: shim throws // honest about the upstream protocol violation rather than // silently lying about the served identity. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const errorOnlyTurn: ScriptedTurn = [ eventFrame({ @@ -2231,7 +2189,7 @@ test('upstream sends bare `error` frame BEFORE any response.created: shim throws ]; const script = scriptedRun([errorOnlyTurn]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); let thrown: unknown = undefined; try { @@ -2242,13 +2200,13 @@ test('upstream sends bare `error` frame BEFORE any response.created: shim throws assert(thrown instanceof Error); }); -test('upstream sends bare `error` frame AFTER response.created: shim emits response.failed with synthesized id + last-seen model', async () => { - // When model is captured before the bare error arrives, the shim +test('upstream sends bare `error` frame AFTER response.created: dispatcher emits response.failed with synthesized id + last-seen model', async () => { + // When model is captured before the bare error arrives, the dispatcher // can synthesize a wire-valid `response.failed` envelope. The - // synthesized id is the shim's own (`resp_shim_`), and the + // synthesized id is the dispatcher's own (`resp_hosted_`), and the // model is the last-seen upstream model. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const createdWith = eventFrame({ type: 'response.created', @@ -2271,12 +2229,12 @@ test('upstream sends bare `error` frame AFTER response.created: shim emits respo }), ]]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const failed = events[events.length - 1] as Extract; assertEquals(failed.type, 'response.failed'); - assert(failed.response.id.startsWith('resp_shim_')); + assert(failed.response.id.startsWith('resp_hosted_')); assertEquals(failed.response.model, 'gpt-5.4-2025-01-20'); assertEquals(failed.response.status, 'failed'); assertEquals(failed.response.error?.message, 'mid-stream upstream blew up'); @@ -2288,7 +2246,7 @@ test('upstream sends bare `error` frame AFTER response.created: shim emits respo assertFalse('type' in (failed.response.error as object)); }); -test('upstream iterator rejects before yielding any frame: shim surfaces the throw through the events iterator', async () => { +test('upstream iterator rejects before yielding any frame: dispatcher surfaces the throw through the events iterator', async () => { // An iterator that throws synchronously on `.next()` (malformed // SSE JSON, reader rejection, network reset mid-handshake) yields // no identity capture. The synthesizer can't build a wire-valid @@ -2296,7 +2254,7 @@ test('upstream iterator rejects before yielding any frame: shim surfaces the thr // the source responder reports it upward — same channel as any // other mid-stream failure with no captured identity. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const failingIterator: AsyncIterable> = (async function* () { throw new Error('malformed SSE JSON at byte 42'); @@ -2308,7 +2266,7 @@ test('upstream iterator rejects before yielding any frame: shim surfaces the thr modelIdentity: testTelemetryModelIdentity, }); - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); let thrown: unknown = undefined; try { @@ -2319,7 +2277,7 @@ test('upstream iterator rejects before yielding any frame: shim surfaces the thr assert(thrown instanceof Error); }); -test('pathological upstream emitting frames without response.created: shim eventually completes without identity capture (no infinite-loop)', async () => { +test('pathological upstream emitting frames without response.created: dispatcher eventually completes without identity capture (no infinite-loop)', async () => { // The previous preflight enforced a 100-frame budget here. With // preflight removed, indexed-unknown frames flow through // consume-turn directly. The protective behavior now relies on @@ -2330,7 +2288,7 @@ test('pathological upstream emitting frames without response.created: shim event // (matches a malformed-but-finite upstream that drops without // ever sending a shell). makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const noisyIterator: AsyncIterable> = (async function* () { for (let i = 0; i < 5; i++) { @@ -2347,7 +2305,7 @@ test('pathological upstream emitting frames without response.created: shim event modelIdentity: testTelemetryModelIdentity, }); - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); let thrown: unknown = undefined; try { @@ -2364,7 +2322,7 @@ test('turn-1 iterator throws AFTER response.created: synthesizes response.failed // downstream wire mirrors a native upstream's mid-stream drop // instead of escaping uncaught to a gateway internal_error. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const failingMidStream: AsyncIterable> = (async function* () { yield mkResponseCreated('upstream_mid'); @@ -2376,14 +2334,14 @@ test('turn-1 iterator throws AFTER response.created: synthesizes response.failed modelIdentity: testTelemetryModelIdentity, }); - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const terminal = events[events.length - 1]; assertEquals(terminal.type, 'response.failed'); const failed = terminal as Extract; assertEquals(failed.response.status, 'failed'); - assert(failed.response.id.startsWith('resp_shim_')); + assert(failed.response.id.startsWith('resp_hosted_')); assertEquals(failed.response.model, 'test-model'); assertEquals(failed.response.error?.code, 'server_error'); assert(failed.response.error?.message.includes('connection reset by peer')); @@ -2398,7 +2356,7 @@ test('turn-2 iterator throws: synthesizes response.failed with captured id/model // A throw after the multi-turn loop has crossed run() boundaries // must still funnel through the `response.failed` synthesizer. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); let runCalls = 0; @@ -2418,7 +2376,7 @@ test('turn-2 iterator throws: synthesizes response.failed with captured id/model return { type: 'events', events: failingTurn2, modelIdentity: testTelemetryModelIdentity }; }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const terminal = events[events.length - 1]; @@ -2430,15 +2388,15 @@ test('turn-2 iterator throws: synthesizes response.failed with captured id/model assertFalse('type' in (failed.response.error as object)); }); -test('consume-turn finishes without identity AND without bare-error-pre-shell: shim throws from synthesizer (no silent invented identity)', async () => { +test('consume-turn finishes without identity AND without bare-error-pre-shell: dispatcher throws from synthesizer (no silent invented identity)', async () => { // A `response.completed` arriving without a preceding // `response.created` reaches the terminal completed state without // capturing identity. Synthesizing a completed envelope would - // require id + model the shim never observed — `ensureModel` / + // require id + model the dispatcher never observed — `ensureModel` / // `ensureResponseId` throw instead of inventing values, and the // throw escapes the events generator. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const malformed: AsyncIterable> = (async function* () { yield mkResponseCompleted(); @@ -2449,7 +2407,7 @@ test('consume-turn finishes without identity AND without bare-error-pre-shell: s modelIdentity: testTelemetryModelIdentity, }); - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); let thrown: unknown = undefined; try { @@ -2463,7 +2421,7 @@ test('consume-turn finishes without identity AND without bare-error-pre-shell: s // ── Snapshot pass-through preserves upstream-owned envelope fields ── test('snapshot pass-through: upstream tools/tool_choice/temperature/parallel_tool_calls/reasoning/service_tier survive synthesized response.created + .completed', async () => { - // The shim used to build fresh OpenAIResponsesResult shells with hard- + // The dispatcher used to build fresh OpenAIResponsesResult shells with hard- // coded {id, object, model, status, output, output_text, [usage]} // and drop everything else upstream sent. Real upstream wire frames // carry parallel_tool_calls, tool_choice, tools, temperature, top_p, @@ -2508,11 +2466,11 @@ test('snapshot pass-through: upstream tools/tool_choice/temperature/parallel_too } as OpenAIResponsesResult, }); makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([[createdWith, mkMessageAdded(0), mkMessageDone(0, 'done'), completedWith]]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const created = events.find(e => e.type === 'response.created') as Extract; @@ -2542,7 +2500,7 @@ test('snapshot pass-through: upstream tools/tool_choice/temperature/parallel_too test('snapshot pass-through: synthesized response.failed (mid-stream upstream error) preserves upstream-owned envelope fields from turn 1 snapshot', async () => { // The error path builds its `response.failed` envelope from the - // shim's accumulated state but still spreads the captured snapshot + // dispatcher's accumulated state but still spreads the captured snapshot // so client tool_choice / tools / reasoning visibility survives even // when a later turn fails. const upstreamSnapshot = { @@ -2561,7 +2519,7 @@ test('snapshot pass-through: synthesized response.failed (mid-stream upstream er response: upstreamSnapshot as OpenAIResponsesResult, }); makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); let runCalls = 0; @@ -2571,9 +2529,9 @@ test('snapshot pass-through: synthesized response.failed (mid-stream upstream er const wsArgs = JSON.stringify({ search_query: [{ q: 'q' }] }); const frames: AsyncIterable> = (async function* () { yield createdWith; - yield mkFunctionCallAdded(0, 'call_1', SHIM_TOOL_NAME); + yield mkFunctionCallAdded(0, 'call_1', FUNCTION_TOOL_NAME); yield mkFunctionCallArgsDone(0, wsArgs); - yield mkFunctionCallDone(0, 'call_1', SHIM_TOOL_NAME, wsArgs); + yield mkFunctionCallDone(0, 'call_1', FUNCTION_TOOL_NAME, wsArgs); yield eventFrame({ type: 'response.completed', response: { ...upstreamSnapshot, status: 'completed' } as OpenAIResponsesResult, @@ -2590,7 +2548,7 @@ test('snapshot pass-through: synthesized response.failed (mid-stream upstream er }; }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const failed = events.find(e => e.type === 'response.failed') as Extract; @@ -2600,9 +2558,9 @@ test('snapshot pass-through: synthesized response.failed (mid-stream upstream er assertEquals(r.reasoning, { effort: 'high' }); }); -test('snapshot pass-through: snapshot fields like completed_at flow through verbatim onto every synthesized envelope (the shim only overrides id/model/status/output/usage)', async () => { +test('snapshot pass-through: snapshot fields like completed_at flow through verbatim onto every synthesized envelope (the dispatcher only overrides id/model/status/output/usage)', async () => { // Synthesizers spread upstream's snapshot in full and override only - // the shim-owned fields. Snapshot pass-through is the contract — + // the gateway-owned fields. Snapshot pass-through is the contract — // a turn-2 synthesizer reads turn-1's snapshot until refreshed by // a later upstream frame, so `completed_at`-style fields can ride // through onto a multi-turn failed envelope. Clients reading @@ -2631,7 +2589,7 @@ test('snapshot pass-through: snapshot fields like completed_at flow through verb } as unknown as OpenAIResponsesResult, }); makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); let runCalls = 0; @@ -2641,9 +2599,9 @@ test('snapshot pass-through: snapshot fields like completed_at flow through verb const wsArgs = JSON.stringify({ search_query: [{ q: 'q' }] }); const frames: AsyncIterable> = (async function* () { yield turn1Created; - yield mkFunctionCallAdded(0, 'call_1', SHIM_TOOL_NAME); + yield mkFunctionCallAdded(0, 'call_1', FUNCTION_TOOL_NAME); yield mkFunctionCallArgsDone(0, wsArgs); - yield mkFunctionCallDone(0, 'call_1', SHIM_TOOL_NAME, wsArgs); + yield mkFunctionCallDone(0, 'call_1', FUNCTION_TOOL_NAME, wsArgs); yield turn1Completed; })(); return { type: 'events', events: frames, modelIdentity: testTelemetryModelIdentity }; @@ -2657,7 +2615,7 @@ test('snapshot pass-through: snapshot fields like completed_at flow through verb }; }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const failed = events.find(e => e.type === 'response.failed') as Extract; @@ -2696,7 +2654,7 @@ test('snapshot strip: emitFinalCompleted re-adds completed_at when upstream supp } as unknown as OpenAIResponsesResult, }); makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([[ turn1Created, @@ -2705,7 +2663,7 @@ test('snapshot strip: emitFinalCompleted re-adds completed_at when upstream supp turn1Completed, ]]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const completed = events.find(e => e.type === 'response.completed') as Extract; @@ -2717,7 +2675,7 @@ test('snapshot strip: emitFinalCompleted re-adds completed_at when upstream supp test('snapshot pass-through: incomplete_details: null on the captured snapshot propagates verbatim to the synthesized response.completed', async () => { // Native upstreams sometimes emit `incomplete_details: null` on a // `response.completed` (Pydantic round-trips `None` for explicit - // null vs. omitted-field distinctions). The shim must preserve the + // null vs. omitted-field distinctions). The dispatcher must preserve the // exact field state — coercing `null` to omitted (or vice versa) // would break clients that probe for the field's presence rather // than its truthiness. @@ -2743,7 +2701,7 @@ test('snapshot pass-through: incomplete_details: null on the captured snapshot p } as unknown as OpenAIResponsesResult, }); makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([[ turn1Created, @@ -2752,7 +2710,7 @@ test('snapshot pass-through: incomplete_details: null on the captured snapshot p turn1Completed, ]]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const completed = events.find(e => e.type === 'response.completed') as Extract; @@ -2769,18 +2727,18 @@ test('success-path synth envelopes carry spec-required `error: null` and `incomp // REQUIRED on every Response (both nullable). Reference: // https://github.com/openai/openai-openapi/blob/master/openapi.yaml // `Response.required` lists both. Typed-SDK clients parse against - // that contract and reject envelopes missing the keys; the shim's + // that contract and reject envelopes missing the keys; the dispatcher's // `overlayOnSnapshot` therefore defaults both to null after stripping // terminal-only snapshot fields, then lets explicit overlays (e.g. // `response.failed.error`, real terminal `incomplete_details`) win. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); // Single turn, pure message, no tool call — exercises the // emitFinalCompleted success path through overlayOnSnapshot. const script = scriptedRun([messageTurn('hi')]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); @@ -2799,11 +2757,11 @@ test('success-path synth envelopes carry spec-required `error: null` and `incomp // ── output_text is never synthesized ───────────────────────────────── -test('shim rebuilds `output_text` on terminal envelopes from the accumulated message items (matches openai-python Response.output_text)', async () => { +test('dispatcher rebuilds `output_text` on terminal envelopes from the accumulated message items (matches openai-python Response.output_text)', async () => { // SDKs derive `output_text` from the `output` array. Per-turn // upstream `output_text` on a terminal frame only describes that - // one turn, so on multi-turn shim responses the snapshot value - // would desync from the cross-turn aggregated `output`. The shim + // one turn, so on multi-turn dispatcher responses the snapshot value + // would desync from the cross-turn aggregated `output`. The dispatcher // rebuilds the alias from `accumulatedOutput` and overrides the // snapshot's value on the terminal envelope. const omitOutputText = (responseId = 'upstream_test'): ProtocolFrame => @@ -2833,7 +2791,7 @@ test('shim rebuilds `output_text` on terminal envelopes from the accumulated mes }, }); makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const turn: ScriptedTurn = [ omitOutputTextCreated(), @@ -2843,7 +2801,7 @@ test('shim rebuilds `output_text` on terminal envelopes from the accumulated mes ]; const script = scriptedRun([turn]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const completed = events.find((e): e is Extract => e.type === 'response.completed'); @@ -2853,7 +2811,7 @@ test('shim rebuilds `output_text` on terminal envelopes from the accumulated mes test('upstream-emitted `output_text` on in-progress envelopes flows through verbatim from the snapshot', async () => { // Snapshot pass-through contract: every upstream field flows - // through unchanged unless the shim explicitly overrides it. + // through unchanged unless the dispatcher explicitly overrides it. // `output_text` on in-progress envelopes is just the snapshot's // value — terminal envelopes get a separately-rebuilt // `output_text` aggregated across turns (covered by the dedicated @@ -2899,7 +2857,7 @@ test('upstream-emitted `output_text` on in-progress envelopes flows through verb } as OpenAIResponsesResult, }); makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([[ createdWithStaleText, @@ -2909,7 +2867,7 @@ test('upstream-emitted `output_text` on in-progress envelopes flows through verb completedFrame, ]]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const created = events.find((e): e is Extract => e.type === 'response.created'); @@ -2917,7 +2875,7 @@ test('upstream-emitted `output_text` on in-progress envelopes flows through verb assert(created !== undefined); assert(inProgress !== undefined); // Snapshot's output_text flows through verbatim on in-progress - // envelopes — the shim only overrides id/model/status/output. + // envelopes — the dispatcher only overrides id/model/status/output. assertEquals((created.response as unknown as { output_text?: string }).output_text, 'snapshot output_text'); assertEquals((inProgress.response as unknown as { output_text?: string }).output_text, 'snapshot output_text'); }); @@ -2932,7 +2890,7 @@ test('finalMetadata resolves with the LATEST turn modelIdentity, not turn 1', as // through firstResult.finalMetadata would freeze it to turn 1's // modelKey. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); let runCalls = 0; @@ -2958,13 +2916,15 @@ test('finalMetadata resolves with the LATEST turn modelIdentity, not turn 1', as }; }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); - // Drain so the multi-turn loop completes and finalMetadata resolves. + // Drain so the multi-turn loop completes and the reading it hands up settles. await collectFrames(result.events); - assert(result.finalMetadata !== undefined); - const meta = await result.finalMetadata; - assertEquals(meta.modelIdentity.modelKey, 'turn-2-key'); + assert(result.reading !== null); + const reading = await result.reading; + // Every call the loop made is its own billable entity, in the order it made them, so the + // variant a later turn was served by is named rather than frozen to the first one's. + assertEquals(reading.billable.map(entity => entity.identity.modelKey), ['turn-1-key', 'turn-2-key']); }); // ── Results are always included on web_search_call items ───────────── @@ -2975,14 +2935,14 @@ test('synthesized web_search_call (search action) carries both `query` (singular // `queries: list[str]`. Emit BOTH so every typed SDK reads a // populated value regardless of which field its model declares. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ searchCallTurn(0, 'call_1', 'who invented graphql'), messageTurn('done', 0), ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const doneEvents = outputItemDoneEvents(await collectFrames(result.events)); const wsCallDone = doneEvents.find(e => e.item.type === 'web_search_call'); @@ -2996,18 +2956,18 @@ test('synthesized web_search_call (search action) carries both `query` (singular test('response.output_item.added for web_search_call omits action (mirrors native — action populated only on .done)', async () => { // Native upstreams omit `action` on the `.added` half and populate - // it only on `.done` once the operation completes. The shim + // it only on `.done` once the operation completes. The dispatcher // follows suit; clients that render action.* read from the done // frame. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ searchCallTurn(0, 'call_1', 'who invented graphql'), messageTurn('done', 0), ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const wsAdded = events.find( @@ -3037,21 +2997,21 @@ test('open with invalid ref_id: done frame carries action.type="search" with the // typed-SDK clients render a coherent intent). The added frame // omits `action` regardless, matching native. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const invalidRef = 'turn_4_ws_0'; const argsJson = JSON.stringify({ open: [{ ref_id: invalidRef }] }); const turn1: ScriptedTurn = [ mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'call_open', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_open', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, argsJson), - mkFunctionCallDone(0, 'call_open', SHIM_TOOL_NAME, argsJson), + mkFunctionCallDone(0, 'call_open', FUNCTION_TOOL_NAME, argsJson), mkResponseCompleted(), ]; const script = scriptedRun([turn1, messageTurn('done', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const wsDone = events.find( @@ -3083,12 +3043,12 @@ test('open with valid URL whose fetch fails: done frame carries action.type="ope }, }, }); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const turn1 = openCallTurn(0, 'call_open', failingUrl); const script = scriptedRun([turn1, messageTurn('done', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const wsDone = events.find( @@ -3107,8 +3067,8 @@ test('open with valid URL whose fetch fails: done frame carries action.type="ope test('include: ["web_search_call.action.sources"] populates action.sources with search-result URLs', async () => { // Native OpenAI Responses gates `action.sources` on this exact include // token. Clients reading `web_search_call.action.sources` against - // a native upstream see the URLs of every search hit; the shim - // mirrors that opt-in shape so a switch from native to shim is + // a native upstream see the URLs of every search hit; the dispatcher + // mirrors that opt-in shape so a switch from native to dispatcher is // observably identical. makeStubDeps({ providerOverrides: { @@ -3123,7 +3083,7 @@ test('include: ["web_search_call.action.sources"] populates action.sources with }, }, }); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { include: ['web_search_call.action.sources'] }, }); @@ -3132,7 +3092,7 @@ test('include: ["web_search_call.action.sources"] populates action.sources with messageTurn('done', 0), ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const doneEvents = outputItemDoneEvents(await collectFrames(result.events)); const wsCallDone = doneEvents.find(e => e.item.type === 'web_search_call'); @@ -3147,14 +3107,14 @@ test('include: ["web_search_call.action.sources"] populates action.sources with test('without include: ["web_search_call.action.sources"], action.sources is absent (opt-in shape matches native)', async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ searchCallTurn(0, 'call_1', 'q1'), messageTurn('done', 0), ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const doneEvents = outputItemDoneEvents(await collectFrames(result.events)); const wsCallDone = doneEvents.find(e => e.item.type === 'web_search_call'); @@ -3166,14 +3126,14 @@ test('without include: ["web_search_call.action.sources"], action.sources is abs test('web_search_call results field is populated on the wire when the client opted in via include: ["web_search_call.results"]', async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); // Default already opts in. const script = scriptedRun([ searchCallTurn(0, 'call_1', 'q1'), messageTurn('done', 0), ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const doneEvents = outputItemDoneEvents(await collectFrames(result.events)); const wsCallDone = doneEvents.find(e => e.item.type === 'web_search_call'); @@ -3189,14 +3149,14 @@ test('web_search_call results field is omitted from the wire when the client did // therefore the persisted `payload.private`) still holds the full // results so a later-turn echo can be hydrated. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { include: [] } }); const script = scriptedRun([ searchCallTurn(0, 'call_1', 'q1'), messageTurn('done', 0), ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const doneEvents = outputItemDoneEvents(await collectFrames(result.events)); const wsCallDone = doneEvents.find(e => e.item.type === 'web_search_call'); @@ -3205,23 +3165,23 @@ test('web_search_call results field is omitted from the wire when the client did assertEquals(item.results, undefined); }); -// ── Mixed-tool turn (shim call + client tool present) ───────────────── +// ── Mixed-tool turn (dispatcher call + client tool present) ───────────────── -test('mixed-tool: shim call + client function_call exits to client after one turn, with both sets of items downstream', async () => { - // GPT-5.x emits both kinds of tool calls in one turn. The shim executes - // the shim's searches server-side (so the client sees completed +test('mixed-tool: dispatcher call + client function_call exits to client after one turn, with both sets of items downstream', async () => { + // GPT-5.x emits both kinds of tool calls in one turn. The dispatcher executes + // the dispatcher's searches server-side (so the client sees completed // web_search_call lifecycles) and lets the client round-trip its own // function_call. No internal rerun, no rejection injection. const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const wsArgs = JSON.stringify({ search_query: [{ q: 'q1' }] }); const mixedTurn: ScriptedTurn = [ mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'call_ws', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_ws', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, wsArgs), - mkFunctionCallDone(0, 'call_ws', SHIM_TOOL_NAME, wsArgs), + mkFunctionCallDone(0, 'call_ws', FUNCTION_TOOL_NAME, wsArgs), mkFunctionCallAdded(1, 'call_other', 'lookup'), mkFunctionCallArgsDone(1, '{"q":"x"}', 'fc_1'), mkFunctionCallDone(1, 'call_other', 'lookup', '{"q":"x"}'), @@ -3230,7 +3190,7 @@ test('mixed-tool: shim call + client function_call exits to client after one tur const script = scriptedRun([mixedTurn]); const originalInputLen = (inv.payload.input as OpenAIResponsesInputItem[]).length; - const { frames } = await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + const { frames } = await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(backend.calls.length, 1); assertEquals(script.callCount(), 1); @@ -3246,17 +3206,17 @@ test('mixed-tool: shim call + client function_call exits to client after one tur assertEquals(wsLifecycleCount, 3); }); -test('mixed-tool: shim call + custom_tool_call exits to client; custom_tool_call frames flush downstream', async () => { +test('mixed-tool: dispatcher call + custom_tool_call exits to client; custom_tool_call frames flush downstream', async () => { const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const wsArgs = JSON.stringify({ search_query: [{ q: 'q' }] }); const mixedTurn: ScriptedTurn = [ mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'call_ws', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_ws', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, wsArgs), - mkFunctionCallDone(0, 'call_ws', SHIM_TOOL_NAME, wsArgs), + mkFunctionCallDone(0, 'call_ws', FUNCTION_TOOL_NAME, wsArgs), mkCustomToolCallAdded(1, 'call_ct', 'my_freeform_tool'), mkCustomToolCallInputDone(1, 'raw input'), mkCustomToolCallDone(1, 'call_ct', 'my_freeform_tool', 'raw input'), @@ -3264,7 +3224,7 @@ test('mixed-tool: shim call + custom_tool_call exits to client; custom_tool_call ]; const script = scriptedRun([mixedTurn]); - const { frames } = await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + const { frames } = await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(backend.calls.length, 1); assertEquals(script.callCount(), 1); @@ -3276,9 +3236,9 @@ test('mixed-tool: shim call + custom_tool_call exits to client; custom_tool_call assert(customAdded !== undefined); }); -test('client-only tool turn (no shim call): pass-through function_call frames flush downstream and close the loop', async () => { +test('client-only tool turn (no dispatcher call): pass-through function_call frames flush downstream and close the loop', async () => { const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const clientOnlyTurn: ScriptedTurn = [ mkResponseCreated(), @@ -3291,7 +3251,7 @@ test('client-only tool turn (no shim call): pass-through function_call frames fl const script = scriptedRun([clientOnlyTurn]); const originalInputLen = (inv.payload.input as OpenAIResponsesInputItem[]).length; - const { frames } = await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + const { frames } = await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); // Loop closes after the single turn — the client drives the next round. assertEquals(script.callCount(), 1); assertEquals(backend.calls.length, 0); @@ -3307,20 +3267,20 @@ test('client-only tool turn (no shim call): pass-through function_call frames fl // ── Pass-through of replayed web_search_call from input ──────────────── -// ── Input preprocessor: web_search_call items → shim call pair ───────── +// ── Input preprocessor: web_search_call items → dispatcher call pair ───────── -test('input preprocessor: each web_search_call item becomes one shim call + function_call_output pair', async () => { - // Upstream knows the shim call only as a function tool; a hosted +test('input preprocessor: each web_search_call item becomes one dispatcher call + function_call_output pair', async () => { + // Upstream knows the dispatcher call only as a function tool; a hosted // `web_search_call` item type in its input would be unrecognized. We // translate each echoed item into a function_call + function_call_output // pair that the upstream model can reason over. With no per-item - // private payload (this test runs raw through the shim), the + // private payload (this test runs raw through the dispatcher), the // function_call mirrors the wire action shape and the - // function_call_output is the placeholder — the shim deliberately + // function_call_output is the placeholder — the dispatcher deliberately // ignores the wire `results` field because gateway-side state is the // only source of truth. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const replayedSearch: OpenAIResponsesInputWebSearchCall = { type: 'web_search_call', id: 'ws_old_search', @@ -3347,7 +3307,7 @@ test('input preprocessor: each web_search_call item becomes one shim call + func }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const input = inv.payload.input as OpenAIResponsesInputItem[]; // Original 2 messages + 2 expanded pairs (2 items each) = 6 items. @@ -3360,7 +3320,7 @@ test('input preprocessor: each web_search_call item becomes one shim call + func assertFalse(input.some(i => i.type === 'web_search_call')); // The pair from the search reflects its action. const searchFc = input[1] as { name: string; arguments: string }; - assertEquals(searchFc.name, SHIM_TOOL_NAME); + assertEquals(searchFc.name, FUNCTION_TOOL_NAME); assert(searchFc.arguments.includes('hello world')); // No payload → output is the not-preserved placeholder, not the // wire snippet. A re-search prompt is emitted when data is needed. @@ -3374,7 +3334,7 @@ test('input preprocessor: each web_search_call item becomes one shim call + func test('input preprocessor: replay-only activation leaves hosted tool_choice unchanged when no hosted tool is declared', async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [], @@ -3393,7 +3353,7 @@ test('input preprocessor: replay-only activation leaves hosted tool_choice uncha }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(inv.payload.tool_choice, { type: 'web_search_preview' }); const input = inv.payload.input as OpenAIResponsesInputItem[]; @@ -3405,12 +3365,12 @@ test('input preprocessor: web_search_call without an action is replaced by a pla // item entirely silently shortens conversation history, which can // mislead the model (e.g. its reasoning about "your last 3 tool // calls" no longer matches reality). Replace with a placeholder - // shim call (empty args, no logical ops) + a + // dispatcher call (empty args, no logical ops) + a // function_call_output telling the model the prior contents // weren't preserved. Same idea as the no-payload-with-action path // for partial echoes. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { input: [ @@ -3424,7 +3384,7 @@ test('input preprocessor: web_search_call without an action is replaced by a pla }, }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const input = inv.payload.input as OpenAIResponsesInputItem[]; // user message + placeholder function_call + placeholder function_call_output. @@ -3433,7 +3393,7 @@ test('input preprocessor: web_search_call without an action is replaced by a pla assertEquals(input[1].type, 'function_call'); assertEquals(input[2].type, 'function_call_output'); const fc = input[1] as { type: 'function_call'; name: string; arguments: string }; - assertEquals(fc.name, SHIM_TOOL_NAME); + assertEquals(fc.name, FUNCTION_TOOL_NAME); // Empty args → upstream model sees no logical operations, // matching the intent of "we don't know what this call did". assertEquals(fc.arguments, '{}'); @@ -3447,7 +3407,7 @@ test('input preprocessor: web_search_call without an action is replaced by a pla test('input preprocessor: web_search_call with empty id has its id synthesized and produces an upstream pair', async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { input: [ @@ -3462,7 +3422,7 @@ test('input preprocessor: web_search_call with empty id has its id synthesized a }, }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const input = inv.payload.input as OpenAIResponsesInputItem[]; // user message + function_call + function_call_output (paired from the @@ -3478,7 +3438,7 @@ test('input preprocessor: web_search_call without results emits the not-preserve // synthesizing a phantom zero-hit response that would mislead the // model into thinking the prior search returned nothing. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { input: [ @@ -3492,7 +3452,7 @@ test('input preprocessor: web_search_call without results emits the not-preserve }, }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const input = inv.payload.input as OpenAIResponsesInputItem[]; assertEquals(input.length, 2); @@ -3500,11 +3460,11 @@ test('input preprocessor: web_search_call without results emits the not-preserve assertEquals(fco.output, 'Prior search results were not preserved in the conversation history. Call web_search again if you need them.'); }); -// ── Shim tool name resolution / collision fallback ──────────────── +// ── Dispatcher tool name resolution / collision fallback ──────────────── -test('client declaring web_search + hosted web_search: shim falls back to web_search_2', async () => { +test('client declaring web_search + hosted web_search: dispatcher falls back to web_search_2', async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -3519,15 +3479,15 @@ test('client declaring web_search + hosted web_search: shim falls back to web_se messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const names = new Set((inv.payload.tools ?? []).map(t => (t as { name: string }).name)); assertEquals(names, new Set(['web_search_2', 'web_search'])); }); -test('client declaring web_search AND web_search_2 + hosted web_search: shim falls back to web_search_3', async () => { +test('client declaring web_search AND web_search_2 + hosted web_search: dispatcher falls back to web_search_3', async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -3543,7 +3503,7 @@ test('client declaring web_search AND web_search_2 + hosted web_search: shim fal messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const names = new Set((inv.payload.tools ?? []).map(t => (t as { name: string }).name)); assertEquals(names, new Set(['web_search', 'web_search_2', 'web_search_3'])); @@ -3552,11 +3512,11 @@ test('client declaring web_search AND web_search_2 + hosted web_search: shim fal for (const [source, item] of [ ['additional_tools', { type: 'additional_tools', role: 'developer', - tools: [{ type: 'function', name: SHIM_TOOL_NAME, parameters: { type: 'object' } }], + tools: [{ type: 'function', name: FUNCTION_TOOL_NAME, parameters: { type: 'object' } }], }], ['tool_search_output', { type: 'tool_search_output', execution: 'client', call_id: 'call_search', - tools: [{ type: 'custom', name: SHIM_TOOL_NAME }], + tools: [{ type: 'custom', name: FUNCTION_TOOL_NAME }], }], ] as const) { test(`hosted function reserves a distinct alias beside a flat client tool from historical ${source}`, async () => { @@ -3568,12 +3528,12 @@ for (const [source, item] of [ }, }); const script = scriptedRun([messageTurn('done')]); - const { result } = await runShimAndDrain(withOpenAIResponsesWebSearchShim, inv, makeGatewayCtx(), script.run); + const { result } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); assertEquals(result.type, 'events'); assertEquals(script.callCount(), 1); const injected = inv.payload.tools?.[0]; assert(injected?.type === 'function'); - assertEquals(injected.name, `${SHIM_TOOL_NAME}_2`); + assertEquals(injected.name, `${FUNCTION_TOOL_NAME}_2`); assertEquals(inv.payload.input[1], item); assertEquals(backend.calls, []); }); @@ -3589,7 +3549,7 @@ test('a historical namespace child keeps its identity beside an aliased hosted f type: 'additional_tools', role: 'developer', tools: [{ type: 'namespace', name: 'client', description: 'Client tools', - tools: [{ type: 'function', name: SHIM_TOOL_NAME, parameters: { type: 'object' } }], + tools: [{ type: 'function', name: FUNCTION_TOOL_NAME, parameters: { type: 'object' } }], }], }, ], @@ -3597,11 +3557,11 @@ test('a historical namespace child keeps its identity beside an aliased hosted f }); const history = structuredClone(inv.payload.input); const script = scriptedRun([messageTurn('done')]); - const { result } = await runShimAndDrain(withOpenAIResponsesWebSearchShim, inv, makeGatewayCtx(), script.run); + const { result } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); assertEquals(result.type, 'events'); const injected = inv.payload.tools?.[0]; assert(injected?.type === 'function'); - assertEquals(injected.name, `${SHIM_TOOL_NAME}_2`); + assertEquals(injected.name, `${FUNCTION_TOOL_NAME}_2`); assertEquals(inv.payload.input, history); }); @@ -3609,7 +3569,7 @@ test('a historical namespace child keeps its identity beside an aliased hosted f test('upstream returning no actionable events closes the response cleanly', async () => { const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); // Turn with only response.created / in_progress / completed. const emptyTurn: ScriptedTurn = [ @@ -3619,7 +3579,7 @@ test('upstream returning no actionable events closes the response cleanly', asyn ]; const script = scriptedRun([emptyTurn]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assertEquals(script.callCount(), 1); assertEquals(backend.calls.length, 0); @@ -3634,11 +3594,11 @@ test('disabled search provider: dispatched op surfaces explanation snippet (no 5 makeStubDeps({ configured: { type: 'disabled' }, }); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([searchCallTurn(0, 'call_1', 'q'), messageTurn('done', 0)]); - const { result, frames } = await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + const { result, frames } = await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(result.type, 'events'); // Two upstream turns still ran — replay-only / mid-conversation @@ -3657,11 +3617,11 @@ test('missing-credential search provider: dispatched op surfaces explanation sni makeStubDeps({ configured: { type: 'missing-credential', provider: 'tavily' }, }); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([searchCallTurn(0, 'call_1', 'q'), messageTurn('done', 0)]); - const { result, frames } = await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + const { result, frames } = await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); assertEquals(result.type, 'events'); assertEquals(script.callCount(), 2); @@ -3675,12 +3635,12 @@ test('missing-credential search provider: dispatched op surfaces explanation sni // ── Streaming-specific behavior ─────────────────────────────────────────── -test('shim yields first turn frames BEFORE later turns resolve', async () => { +test('dispatcher yields first turn frames BEFORE later turns resolve', async () => { // Turn 2's run() never resolves until the test releases its gate. Turn // 1's frames must be drainable from the iterator before that resolves; // otherwise the iterator would block on the gate. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); let releaseTurn2: (() => void) | undefined; @@ -3715,7 +3675,7 @@ test('shim yields first turn frames BEFORE later turns resolve', async () => { throw new Error(`unexpected run() call ${runCalls}`); }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const iter = result.events[Symbol.asyncIterator](); @@ -3751,11 +3711,11 @@ test('shim yields first turn frames BEFORE later turns resolve', async () => { test('turn 1 pure-text response streams BEFORE upstream terminal (no TTFT regression)', async () => { // A turn-1 pure-text response (no tool call) must not block downstream // until `response.completed` arrives — that would regress TTFT on - // every shim-routed request. The preflight stops the moment + // every hosted-tool request. The preflight stops the moment // `response.created` is captured; everything else (output_text deltas, // terminal) streams live through the same iterator. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); // Gate the upstream terminal frame so a buffered implementation would // be stuck waiting on it before yielding any byte downstream. @@ -3782,7 +3742,7 @@ test('turn 1 pure-text response streams BEFORE upstream terminal (no TTFT regres return { type: 'events', events: frames, modelIdentity: testTelemetryModelIdentity }; }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const iter = result.events[Symbol.asyncIterator](); @@ -3805,12 +3765,12 @@ test('turn 1 pure-text response streams BEFORE upstream terminal (no TTFT regres test('mid-stream upstream error yields response.failed and closes the SSE stream', async () => { // After turn 1 emits a web_search call, turn 2's run() returns an - // upstream-error envelope. The shim must yield a single terminal + // upstream-error envelope. The dispatcher must yield a single terminal // response.failed frame and end the iterator — switching the outer // envelope to upstream-error is impossible: the outer envelope shape // is locked once the first frame is yielded. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); let runCalls = 0; @@ -3835,7 +3795,7 @@ test('mid-stream upstream error yields response.failed and closes the SSE stream }; }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); @@ -3844,7 +3804,7 @@ test('mid-stream upstream error yields response.failed and closes the SSE stream const failedEv = terminal as Extract; assertEquals(failedEv.response.status, 'failed'); assert(failedEv.response.error !== undefined); - // Pass-through code: the shim quotes upstream's HTTP status + // Pass-through code: the dispatcher quotes upstream's HTTP status // (`upstream_`) when the body has no OpenAI-shaped error // envelope. No normalization to a spec enum value. assertEquals(failedEv.response.error?.code, 'upstream_503'); @@ -3854,13 +3814,13 @@ test('mid-stream upstream error yields response.failed and closes the SSE stream }); test('mid-stream 429 pass-through: code reflects upstream HTTP status (no spec-enum normalization)', async () => { - // The shim no longer normalizes upstream error codes to the + // The dispatcher no longer normalizes upstream error codes to the // OpenAPI `OpenAIResponsesErrorCode` enum. A 429 with no OpenAI-shaped // body falls back to `upstream_429` — the HTTP status is the most // honest signal, and downstream clients pattern-matching on // upstream's actual code see it directly. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); let runCalls = 0; @@ -3881,7 +3841,7 @@ test('mid-stream 429 pass-through: code reflects upstream HTTP status (no spec-e }; }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const failed = events[events.length - 1] as Extract; @@ -3889,12 +3849,12 @@ test('mid-stream 429 pass-through: code reflects upstream HTTP status (no spec-e }); test('mid-stream 400 with non-OpenAI body falls back to upstream_400 code (no spec-enum normalization)', async () => { - // Without an OpenAI-shaped `error.code` in the body, the shim + // Without an OpenAI-shaped `error.code` in the body, the dispatcher // synthesizes `upstream_` rather than mapping to a spec // enum value. The body excerpt and status live in `error.message` // so clients still see the upstream detail. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); let runCalls = 0; @@ -3915,7 +3875,7 @@ test('mid-stream 400 with non-OpenAI body falls back to upstream_400 code (no sp }; }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const failed = events[events.length - 1] as Extract; @@ -3932,7 +3892,7 @@ test('mid-stream upstream error with OpenAI-shaped JSON body forwards code/type/ // envelope, pass it through unchanged so SDKs see the same vocabulary // they'd see hitting upstream directly. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); let runCalls = 0; @@ -3960,7 +3920,7 @@ test('mid-stream upstream error with OpenAI-shaped JSON body forwards code/type/ }; }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const failed = events[events.length - 1] as Extract; @@ -3972,12 +3932,12 @@ test('mid-stream upstream error with OpenAI-shaped JSON body forwards code/type/ test('upstream response.incomplete forwards as response.incomplete with the same incomplete_details.reason', async () => { // Native upstreams emit response.incomplete when the model stopped // before producing a terminal message (max_output_tokens, - // content_filter, ...). The shim used to rewrite it as + // content_filter, ...). The dispatcher used to rewrite it as // response.completed, silently destroying the reason field that // clients branch on. Forward the frame as-is with upstream's reason // preserved. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const incompleteTurn: ScriptedTurn = [ @@ -3989,7 +3949,7 @@ test('upstream response.incomplete forwards as response.incomplete with the same ]; const script = scriptedRun([incompleteTurn]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); @@ -4003,13 +3963,13 @@ test('upstream response.incomplete forwards as response.incomplete with the same }); test('upstream response.incomplete WITHOUT incomplete_details forwards as response.incomplete with whatever upstream emitted (no synthetic fold to response.failed)', async () => { - // Pass-through contract for `response.incomplete`: the shim + // Pass-through contract for `response.incomplete`: the dispatcher // forwards upstream's emission verbatim. If upstream sent // `incomplete_details: null`, the downstream wire keeps null; // synthesizing a different terminal kind would lie about what // upstream said. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const incompleteTurn: ScriptedTurn = [ @@ -4021,7 +3981,7 @@ test('upstream response.incomplete WITHOUT incomplete_details forwards as respon ]; const script = scriptedRun([incompleteTurn]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); @@ -4032,9 +3992,9 @@ test('upstream response.incomplete WITHOUT incomplete_details forwards as respon assertEquals(incomplete.response.output.map(item => item.type), ['message']); }); -test('upstream response.incomplete after a server tool call keeps synthesized tool output in the terminal response', async () => { +test('upstream response.incomplete after a hosted tool call keeps synthesized tool output in the terminal response', async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const argsJson = JSON.stringify({ search_query: [{ q: 'q1' }] }); const incompleteTurn: ScriptedTurn = [ @@ -4042,14 +4002,14 @@ test('upstream response.incomplete after a server tool call keeps synthesized to mkResponseInProgress(), mkMessageAdded(0), mkMessageDone(0, 'partial answer'), - mkFunctionCallAdded(1, 'call_ws', SHIM_TOOL_NAME), + mkFunctionCallAdded(1, 'call_ws', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(1, argsJson), - mkFunctionCallDone(1, 'call_ws', SHIM_TOOL_NAME, argsJson), + mkFunctionCallDone(1, 'call_ws', FUNCTION_TOOL_NAME, argsJson), mkResponseIncomplete({ reason: 'max_output_tokens' }), ]; const script = scriptedRun([incompleteTurn]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const terminal = events.at(-1) as Extract; @@ -4070,7 +4030,7 @@ const lastFunctionCallOutput = (input: OpenAIResponsesInputItem[]): string => { test('responses target with flag on: function_call_output is plain-text formatted search results', async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ targetApi: 'openaiResponses', enabledFlags: new Set(['openai-responses-web-search-shim']), @@ -4080,7 +4040,7 @@ test('responses target with flag on: function_call_output is plain-text formatte messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const text = lastFunctionCallOutput(inv.payload.input as OpenAIResponsesInputItem[]); assert(text.startsWith('Search results for "q1":')); @@ -4100,7 +4060,7 @@ test('responses target with OpenAI passthrough forwards the complete alpha-searc output: 'alpha output', results: [{ type: 'text_result', url: 'https://example.com', title: 'Example', snippet: 'alpha snippet' }], }), { status: 200, headers: { 'content-type': 'application/json' } })); - mockResolveAlpha.mockResolvedValue(call); + mockResolveAlpha.mockResolvedValue(stubAlphaSearchCandidate(call)); const inv = makeInvocation({ targetApi: 'openaiResponses', enabledFlags: new Set(['openai-responses-web-search-shim']), @@ -4120,11 +4080,11 @@ test('responses target with OpenAI passthrough forwards the complete alpha-searc response_length: 'long', }; const script = scriptedRun([ - fcTurn(0, 'call_1', SHIM_TOOL_NAME, JSON.stringify(commands)), + fcTurn(0, 'call_1', FUNCTION_TOOL_NAME, JSON.stringify(commands)), messageTurn('done', 0), ]); - await runShimAndDrain(withOpenAIResponsesWebSearchShim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); assertEquals(lastFunctionCallOutput(inv.payload.input as OpenAIResponsesInputItem[]), 'alpha output'); assertEquals(call.mock.calls[0]?.[0].commands, commands); @@ -4146,10 +4106,10 @@ test('local and cascaded Floway unsupported commands produce the same agent-visi enabledFlags: new Set(['openai-responses-web-search-shim']), }); const script = scriptedRun([ - fcTurn(0, 'call_unsupported', SHIM_TOOL_NAME, JSON.stringify(commands)), + fcTurn(0, 'call_unsupported', FUNCTION_TOOL_NAME, JSON.stringify(commands)), messageTurn('model repeated the tool error', 0), ]); - await runShimAndDrain(withOpenAIResponsesWebSearchShim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); return lastFunctionCallOutput(inv.payload.input as OpenAIResponsesInputItem[]); }; @@ -4162,13 +4122,13 @@ test('local and cascaded Floway unsupported commands produce the same agent-visi jina: { apiKey: '' }, passthroughOpenAiSearch: { enabled: true, upstreamId: 'up_floway', model: 'gpt-search' }, } satisfies WebSearchConfig); - mockResolveAlpha.mockResolvedValue(async () => new Response(JSON.stringify({ + mockResolveAlpha.mockResolvedValue(stubAlphaSearchCandidate(async () => new Response(JSON.stringify({ error: { type: 'internal_error', name: 'UnsupportedLocalWebSearchFeatureError', message: 'The configured web search provider does not implement OpenAI search feature `commands.image_query`.', }, - }), { status: 500, headers: { 'content-type': 'application/json' } })); + }), { status: 500, headers: { 'content-type': 'application/json' } }))); const cascadedMessage = await runUnsupported(); assertEquals(localMessage, expected); @@ -4177,14 +4137,14 @@ test('local and cascaded Floway unsupported commands produce the same agent-visi test('openai-chat-completions target: function_call_output is plain-text formatted search results', async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ targetApi: 'openaiChatCompletions' }); const script = scriptedRun([ searchCallTurn(0, 'call_1', 'q1'), messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const text = lastFunctionCallOutput(inv.payload.input as OpenAIResponsesInputItem[]); assert(text.startsWith('Search results for "q1":')); @@ -4192,14 +4152,14 @@ test('openai-chat-completions target: function_call_output is plain-text formatt test('messages target: function_call_output is plain-text formatted search results', async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ targetApi: 'anthropicMessages' }); const script = scriptedRun([ searchCallTurn(0, 'call_1', 'q1'), messageTurn('done', 0), ]); - await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const text = lastFunctionCallOutput(inv.payload.input as OpenAIResponsesInputItem[]); assert(text.startsWith('Search results for "q1":')); @@ -4211,7 +4171,7 @@ test('tool_choice "required" demotes to "auto" after first intercepted turn', as // Without demote: the model is required to call a tool every turn, // never produces a terminal message, loops until the iteration cap. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tool_choice: 'required' } }); // Capture tool_choice on each run() call so we can verify the payload // mutation before draining events. @@ -4225,17 +4185,17 @@ test('tool_choice "required" demotes to "auto" after first intercepted turn', as return await baseScript.run(); }; - await runShimAndDrain(shim, inv, makeGatewayCtx(), wrappedRun); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), wrappedRun); assertEquals(seenToolChoices[0], 'required'); assertEquals(seenToolChoices[1], 'auto'); }); -test('tool_choice {type:"function", name:} demotes to "auto" after first intercepted turn', async () => { +test('tool_choice {type:"function", name:} demotes to "auto" after first intercepted turn', async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ - payload: { tool_choice: { type: 'function', name: SHIM_TOOL_NAME } }, + payload: { tool_choice: { type: 'function', name: FUNCTION_TOOL_NAME } }, }); const seenToolChoices: unknown[] = []; const baseScript = scriptedRun([ @@ -4247,15 +4207,15 @@ test('tool_choice {type:"function", name:} demotes to "auto" aft return await baseScript.run(); }; - await runShimAndDrain(shim, inv, makeGatewayCtx(), wrappedRun); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), wrappedRun); - assertEquals(seenToolChoices[0], { type: 'function', name: SHIM_TOOL_NAME }); + assertEquals(seenToolChoices[0], { type: 'function', name: FUNCTION_TOOL_NAME }); assertEquals(seenToolChoices[1], 'auto'); }); test('hosted {type:"web_search_preview"} tool_choice gets rewritten and then demoted', async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tool_choice: { type: 'web_search_preview' }, @@ -4271,15 +4231,15 @@ test('hosted {type:"web_search_preview"} tool_choice gets rewritten and then dem return await baseScript.run(); }; - await runShimAndDrain(shim, inv, makeGatewayCtx(), wrappedRun); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), wrappedRun); - assertEquals(seenToolChoices[0], { type: 'function', name: SHIM_TOOL_NAME }); + assertEquals(seenToolChoices[0], { type: 'function', name: FUNCTION_TOOL_NAME }); assertEquals(seenToolChoices[1], 'auto'); }); test('tool_choice "auto" stays "auto" — no demotion when never forced', async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tool_choice: 'auto' } }); const seenToolChoices: unknown[] = []; const baseScript = scriptedRun([ @@ -4291,15 +4251,15 @@ test('tool_choice "auto" stays "auto" — no demotion when never forced', async return await baseScript.run(); }; - await runShimAndDrain(shim, inv, makeGatewayCtx(), wrappedRun); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), wrappedRun); assertEquals(seenToolChoices[0], 'auto'); assertEquals(seenToolChoices[1], 'auto'); }); -test.each([false, true])('forced namespaced client tool choice is not mistaken for the hosted shim function (declared %s)', async declared => { +test.each([false, true])('forced namespaced client tool choice is not mistaken for the hosted dispatcher function (declared %s)', async declared => { const { backend } = makeStubDeps(); - const choice = { type: 'function' as const, namespace: 'client', name: SHIM_TOOL_NAME }; + const choice = { type: 'function' as const, namespace: 'client', name: FUNCTION_TOOL_NAME }; const inv = makeInvocation({ payload: { tool_choice: choice, @@ -4307,25 +4267,25 @@ test.each([false, true])('forced namespaced client tool choice is not mistaken f { type: 'web_search' }, ...(declared ? [{ type: 'namespace', name: 'client', description: 'Client tools', - tools: [{ type: 'function', name: SHIM_TOOL_NAME, parameters: { type: 'object' } }], + tools: [{ type: 'function', name: FUNCTION_TOOL_NAME, parameters: { type: 'object' } }], } satisfies OpenAIResponsesTool] : []), ], }, }); const seenToolChoices: unknown[] = []; const script = scriptedRun([ - fcTurn(0, 'call_search', `${SHIM_TOOL_NAME}_2`, JSON.stringify({ search_query: [{ q: 'q1' }] })), + fcTurn(0, 'call_search', `${FUNCTION_TOOL_NAME}_2`, JSON.stringify({ search_query: [{ q: 'q1' }] })), messageTurn('done'), ]); const run = async () => { const injected = inv.payload.tools?.[0]; assert(injected?.type === 'function'); - assertEquals(injected.name, `${SHIM_TOOL_NAME}_2`); + assertEquals(injected.name, `${FUNCTION_TOOL_NAME}_2`); seenToolChoices.push(inv.payload.tool_choice); return await script.run(); }; - await runShimAndDrain(withOpenAIResponsesWebSearchShim, inv, makeGatewayCtx(), run); + await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), run); assertEquals(seenToolChoices, [choice, choice]); assertEquals(backend.calls.length, 1); @@ -4338,7 +4298,7 @@ test('cap-exceeded does NOT set tool_choice="none" — the cap snippet alone nud // path relies solely on the exhausted-budget snippet to nudge the model // to switch tools or settle on a terminal message. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tool_choice: 'auto' } }); // 30 search turns, then turn 31 (cap-exceeded), then a final message. const searchTurns: ScriptedTurn[] = []; @@ -4353,16 +4313,16 @@ test('cap-exceeded does NOT set tool_choice="none" — the cap snippet alone nud return await baseScript.run(); }; - await runShimAndDrain(shim, inv, makeGatewayCtx(), wrappedRun); + await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), wrappedRun); // Every turn observed `'auto'` — no demotion to 'none' after the cap. assertEquals(seenToolChoices[30], 'auto'); assertEquals(seenToolChoices[31], 'auto'); }); -test('max_tool_calls is forwarded to upstream turns but does not locally bypass server tools', async () => { +test('max_tool_calls is forwarded to upstream turns but does not locally bypass hosted tools', async () => { const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { max_tool_calls: 1 } }); const seenMaxToolCalls: unknown[] = []; const baseScript = scriptedRun([ @@ -4375,7 +4335,7 @@ test('max_tool_calls is forwarded to upstream turns but does not locally bypass return await baseScript.run(); }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); await collectFrames(result.events); assertEquals(seenMaxToolCalls, [1, 0, 0]); @@ -4384,11 +4344,11 @@ test('max_tool_calls is forwarded to upstream turns but does not locally bypass test('max_tool_calls invalid values pass through to upstream and do not engage local accounting', async () => { makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { max_tool_calls: 'one' as unknown as number } }); const script = scriptedRun([messageTurn('hello', 0)]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); assertEquals(events[events.length - 1].type, 'response.completed'); @@ -4402,7 +4362,7 @@ test('final-turn text deltas stream as they arrive, not buffered until response. // first delta is drainable while the gate is still closed proves true // byte-by-byte streaming. makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); let releaseRestOfTurn2: (() => void) | undefined; @@ -4450,7 +4410,7 @@ test('final-turn text deltas stream as they arrive, not buffered until response. throw new Error(`unexpected run() call ${runCalls}`); }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const iter = result.events[Symbol.asyncIterator](); @@ -4477,7 +4437,7 @@ test('mixed turn (reasoning + message_partial + function_call) preserves the mes // [reasoning, message, function_call]. The message forwards live and // the synthesized web_search_call follows it. const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const mixedTurn: ScriptedTurn = [ @@ -4494,9 +4454,9 @@ test('mixed turn (reasoning + message_partial + function_call) preserves the mes delta: 'thinking out loud', }), mkMessageDone(1, 'thinking out loud'), - mkFunctionCallAdded(2, 'call_ws', SHIM_TOOL_NAME), + mkFunctionCallAdded(2, 'call_ws', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(2, JSON.stringify({ search_query: [{ q: 'q1' }] })), - mkFunctionCallDone(2, 'call_ws', SHIM_TOOL_NAME, JSON.stringify({ search_query: [{ q: 'q1' }] })), + mkFunctionCallDone(2, 'call_ws', FUNCTION_TOOL_NAME, JSON.stringify({ search_query: [{ q: 'q1' }] })), mkResponseCompleted(), ]; const script = scriptedRun([mixedTurn, messageTurn('final answer', 0)]); @@ -4506,7 +4466,7 @@ test('mixed turn (reasoning + message_partial + function_call) preserves the mes return await script.run(); }; - const result = await shim(inv, makeGatewayCtx(), run); + const result = await dispatcher(inv, makeGatewayCtx(), run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); @@ -4534,7 +4494,7 @@ test('two consecutive tool-call turns each with a thinking-out-loud message pres // before dispatching its tool call. Both intermediate messages forward // live and survive in the downstream output. const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const thinkingTurn = ( @@ -4550,9 +4510,9 @@ test('two consecutive tool-call turns each with a thinking-out-loud message pres mkResponseInProgress(), mkMessageAdded(msgUpstreamIdx), mkMessageDone(msgUpstreamIdx, msgText), - mkFunctionCallAdded(fcUpstreamIdx, callId, SHIM_TOOL_NAME), + mkFunctionCallAdded(fcUpstreamIdx, callId, FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(fcUpstreamIdx, args), - mkFunctionCallDone(fcUpstreamIdx, callId, SHIM_TOOL_NAME, args), + mkFunctionCallDone(fcUpstreamIdx, callId, FUNCTION_TOOL_NAME, args), mkResponseCompleted(), ]; }; @@ -4563,7 +4523,7 @@ test('two consecutive tool-call turns each with a thinking-out-loud message pres messageTurn('final answer', 0), ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); @@ -4591,7 +4551,7 @@ test('two consecutive tool-call turns each with a thinking-out-loud message pres test('lifecycle start frames yield BEFORE backend resolves, giving searching real wall-clock duration', async () => { // Gate provider.search() so the test can observe ordering: lifecycle // start frames must be drainable WHILE the backend is pending. If the - // shim awaits the backend before yielding start frames, next() after + // dispatcher awaits the backend before yielding start frames, next() after // response.created will hang on the gate. let releaseSearch: (() => void) | null = null; const searchGate = new Promise(resolve => { releaseSearch = resolve; }); @@ -4610,14 +4570,14 @@ test('lifecycle start frames yield BEFORE backend resolves, giving searching rea }, }, }); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const script = scriptedRun([ searchCallTurn(0, 'call_1', 'gated-q'), messageTurn('done', 0), ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const iter = result.events[Symbol.asyncIterator](); @@ -4636,7 +4596,7 @@ test('lifecycle start frames yield BEFORE backend resolves, giving searching rea if (f.event.type === 'response.web_search_call.completed') sawCompleted = true; } assertEquals(sawSearching, true); - // Critical: searching arrived but completed did NOT — the shim hasn't + // Critical: searching arrived but completed did NOT — the dispatcher hasn't // awaited the backend yet, but the wire already shows searching. assertEquals(sawCompleted, false); // The backend call IS in flight (awaiting the gate); the call was placed @@ -4655,13 +4615,13 @@ test('lifecycle start frames yield BEFORE backend resolves, giving searching rea assert(types.includes('response.output_item.done')); }); -test('terminal response.completed.output is in output_index order, not completion order (shim backend resolves AFTER a later live item)', async () => { - // The shim call reserves downstream index 0 at output_item.added time; +test('terminal response.completed.output is in output_index order, not completion order (dispatcher backend resolves AFTER a later live item)', async () => { + // The dispatcher call reserves downstream index 0 at output_item.added time; // the mixed-tool client function_call gets downstream index 1. - // We gate the backend so the shim's output_item.done fires AFTER + // We gate the backend so the dispatcher's output_item.done fires AFTER // the client function_call has already finalized into accumulatedOutput. // The sparse-index materialization at terminal time must still place - // the shim call at output[0] and the client tool at output[1]. + // the dispatcher call at output[0] and the client tool at output[1]. let releaseSearch: (() => void) | null = null; const searchGate = new Promise(resolve => { releaseSearch = resolve; }); makeStubDeps({ @@ -4679,67 +4639,67 @@ test('terminal response.completed.output is in output_index order, not completio }, }, }); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const wsArgs = JSON.stringify({ search_query: [{ q: 'q' }] }); - // Shim call at upstream index 0, client tool at upstream index 1. The + // Dispatcher call at upstream index 0, client tool at upstream index 1. The // upstream's response.completed arrives BEFORE the gated backend - // resolves; the shim will release end frames lazily as the consumer + // resolves; the dispatcher will release end frames lazily as the consumer // pulls (so we must release the gate to finish the stream). const mixedTurn: ScriptedTurn = [ mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'call_ws', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_ws', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, wsArgs), - mkFunctionCallDone(0, 'call_ws', SHIM_TOOL_NAME, wsArgs), + mkFunctionCallDone(0, 'call_ws', FUNCTION_TOOL_NAME, wsArgs), mkFunctionCallAdded(1, 'call_other', 'lookup'), mkFunctionCallArgsDone(1, '{"q":"x"}', 'fc_1'), mkFunctionCallDone(1, 'call_other', 'lookup', '{"q":"x"}'), mkResponseCompleted(), ]; const script = scriptedRun([mixedTurn]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); - // Release the gate so the shim call end frames + terminal can resolve. + // Release the gate so the dispatcher call end frames + terminal can resolve. releaseSearch!(); const events = eventPayloads(await collectFrames(result.events)); const terminal = events[events.length - 1]; assertEquals(terminal.type, 'response.completed'); const output = (terminal as { response: { output: Array<{ type: string }> } }).response.output; - // Two items: shim web_search_call at slot 0, client function_call at slot 1. + // Two items: dispatcher web_search_call at slot 0, client function_call at slot 1. // Reserved-slot ordering wins even though the client tool finalized - // first (no await on the .done path) and the shim call finalized last. + // first (no await on the .done path) and the dispatcher call finalized last. assertEquals(output.map(o => o.type), ['web_search_call', 'function_call']); }); // ── End-to-end protocol-violation paths ────────────────────────────── -test('shim call without output_item.done synthesizes response.failed (no backend dispatch)', async () => { - // Protocol violation: upstream emits the shim's added + - // arguments deltas but no `.done`. The unmatched-shim-call detector +test('dispatcher call without output_item.done synthesizes response.failed (no backend dispatch)', async () => { + // Protocol violation: upstream emits the dispatcher's added + + // arguments deltas but no `.done`. The unmatched-hosted-call detector // in consume-turn must promote the turn to `response.failed`; no - // backend search / fetchPage should fire (the shim call never + // backend search / fetchPage should fire (the dispatcher call never // dispatched). const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const argsJson = JSON.stringify({ search_query: [{ q: 'truncated' }] }); const script = scriptedRun([[ mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'call_truncated', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'call_truncated', FUNCTION_TOOL_NAME), mkFunctionCallArgsDone(0, argsJson), // No `mkFunctionCallDone` — upstream protocol violation. mkResponseCompleted(), ]]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const events = eventPayloads(await collectFrames(result.events)); const terminal = events[events.length - 1] as Extract; assertEquals(terminal.type, 'response.failed'); assertEquals(terminal.response.error?.code, 'server_error'); - assert(terminal.response.error?.message.includes('without closing shim call items')); + assert(terminal.response.error?.message.includes('without closing dispatcher call items')); // Backend never invoked — dispatch only fires from output_item.done. assertEquals(backend.calls.length, 0); }); @@ -4751,16 +4711,16 @@ test('wrong-typed supported sub-property (search_query as object) synthesizes a // a "wrong-type sub-property" snippet) instead of crashing the // dispatcher. Backend search should NOT fire. const { backend } = makeStubDeps(); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); // Pass `search_query` as a single object — wrong type. const wrongArgs = JSON.stringify({ search_query: { q: 'x' } }); const script = scriptedRun([ - fcTurn(0, 'call_wrong', SHIM_TOOL_NAME, wrongArgs), + fcTurn(0, 'call_wrong', FUNCTION_TOOL_NAME, wrongArgs), messageTurn('giving up', 0), ]); - const result = await shim(inv, makeGatewayCtx(), script.run); + const result = await dispatcher(inv, makeGatewayCtx(), script.run); assert(result.type === 'events'); const frames = await collectFrames(result.events); const doneEvents = outputItemDoneEvents(frames); @@ -4778,14 +4738,14 @@ test('wrong-typed supported sub-property (search_query as object) synthesizes a }); test('downstream AbortSignal threads through to provider search / fetchPage and propagates aborts', async () => { - // Cancelled requests must stop generating upstream load. The shim + // Cancelled requests must stop generating upstream load. The dispatcher // threads the request's `downstreamAbortSignal` into every backend // provider call so providers can observe and abort. let observedSignal: AbortSignal | undefined; const controller = new AbortController(); const { backend } = makeStubDeps({ providerOverrides: { - // Capture the signal the shim hands us; resolve only when the + // Capture the signal the dispatcher hands us; resolve only when the // signal aborts so we can assert downstream propagation. async search(request) { observedSignal = request.signal; @@ -4801,7 +4761,7 @@ test('downstream AbortSignal threads through to provider search / fetchPage and }, }, }); - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation(); const gatewayCtx = mockChatGatewayCtx({ apiKeyId: 'k1', @@ -4812,7 +4772,7 @@ test('downstream AbortSignal threads through to provider search / fetchPage and searchCallTurn(0, 'call_1', 'will-be-aborted'), ]); - const result = await shim(inv, gatewayCtx, script.run); + const result = await dispatcher(inv, gatewayCtx, script.run); assert(result.type === 'events'); // Drain the events stream in the background. The backend search // will hang until we abort, so the drain promise won't resolve @@ -4864,11 +4824,11 @@ interface DispatchRecord { intercepted: InterceptedFunctionCall; } -// Records every shim call without producing IRs or start frames. +// Records every dispatcher call without producing IRs or start frames. // Tests that care about dispatcher behavior pass a custom dispatcher. const recordingDispatcher = (records: DispatchRecord[]) => ({ intercepted }: { intercepted: InterceptedFunctionCall }) => { records.push({ intercepted }); - return [] as ServerToolResultSlot[]; + return [] as HostedToolResultSlot[]; }; type DrainResult = { @@ -4896,19 +4856,19 @@ const drain = async ( const loopState = () => ({ iterationCount: 1, remainingToolCalls: undefined }); -test('parseServerToolArguments repairs malformed JSON and returns an object', () => { - assertEquals(parseServerToolArguments('{not json'), { 'not json': null }); +test('parseHostedToolArguments repairs malformed JSON and returns an object', () => { + assertEquals(parseHostedToolArguments('{not json'), { 'not json': null }); }); -test('parseServerToolArguments returns null for non-object values', () => { - assertEquals(parseServerToolArguments('[1,2,3]'), null); - assertEquals(parseServerToolArguments('"hello"'), null); - assertEquals(parseServerToolArguments('42'), null); - assertEquals(parseServerToolArguments('null'), null); +test('parseHostedToolArguments returns null for non-object values', () => { + assertEquals(parseHostedToolArguments('[1,2,3]'), null); + assertEquals(parseHostedToolArguments('"hello"'), null); + assertEquals(parseHostedToolArguments('42'), null); + assertEquals(parseHostedToolArguments('null'), null); }); -test('parseServerToolArguments returns empty object for empty args', () => { - assertEquals(parseServerToolArguments(''), {}); +test('parseHostedToolArguments returns empty object for empty args', () => { + assertEquals(parseHostedToolArguments(''), {}); }); const consumeTurn = async ( @@ -4918,15 +4878,15 @@ const consumeTurn = async ( ): Promise => { const records: DispatchRecord[] = []; return await drain( - consumeTurnStreaming(frames, state, isFirstTurn, new Map([[SHIM_TOOL_NAME, recordingDispatcher(records)]]), loopState(), []), + consumeTurnStreaming(frames, state, isFirstTurn, new Map([[FUNCTION_TOOL_NAME, recordingDispatcher(records)]]), loopState(), []), records, ); }; -test('consumeTurn forwards a namespaced client call sharing the hosted shim name', async () => { +test('consumeTurn forwards a namespaced client call sharing the hosted dispatcher name', async () => { const call = { type: 'function_call' as const, - id: 'fc_client', call_id: 'call_client', namespace: 'client', name: SHIM_TOOL_NAME, + id: 'fc_client', call_id: 'call_client', namespace: 'client', name: FUNCTION_TOOL_NAME, arguments: '{"query":"client data"}', status: 'completed', }; const result = await consumeTurn(framesOf( @@ -4949,7 +4909,7 @@ test('consumeTurn forwards a namespaced client call sharing the hosted shim name test('consumeTurn uses the completed call namespace before executing a hosted tool', async () => { const completed = { type: 'function_call' as const, - id: 'fc_client', call_id: 'call_client', namespace: 'client', name: SHIM_TOOL_NAME, + id: 'fc_client', call_id: 'call_client', namespace: 'client', name: FUNCTION_TOOL_NAME, arguments: '{"client":true}', status: 'completed', }; const result = await consumeTurn(framesOf( @@ -4980,20 +4940,20 @@ test('consumeTurn dispatches a hosted call whose added item carried a namespace' type: 'response.output_item.added', output_index: 0, item: { type: 'function_call', id: 'fc_search', call_id: 'call_search', namespace: 'client', - name: SHIM_TOOL_NAME, arguments: '', status: 'in_progress', + name: FUNCTION_TOOL_NAME, arguments: '', status: 'in_progress', }, }), eventFrame({ type: 'response.output_item.done', output_index: 0, item: { type: 'function_call', id: 'fc_search', call_id: 'call_search', - name: SHIM_TOOL_NAME, arguments: '{"search_query":[]}', status: 'completed', + name: FUNCTION_TOOL_NAME, arguments: '{"search_query":[]}', status: 'completed', }, }), mkResponseCompleted(), ), createMergeState(), true); - assertEquals(result.records.map(record => record.intercepted.name), [SHIM_TOOL_NAME]); + assertEquals(result.records.map(record => record.intercepted.name), [FUNCTION_TOOL_NAME]); assertEquals(result.summary.sawClientToolCall, false); assertEquals(outputItemDoneEvents(result.downstreamFrames), []); }); @@ -5031,10 +4991,10 @@ test('consumeTurn first turn synthesizes response.created with the once-per-requ const created = result.downstreamFrames[0]; assert(created.type === 'event'); const createdEv = created.event as Extract; - // Downstream id is the shim-synthesized value (stable cross-turn); + // Downstream id is the gateway-synthesized value (stable cross-turn); // upstream's id is captured nowhere and never exposed downstream. assertEquals(createdEv.response.id, state.synthesizedResponseId); - assert(state.synthesizedResponseId.startsWith('resp_shim_')); + assert(state.synthesizedResponseId.startsWith('resp_hosted_')); }); test('consumeTurn synthesizes response.created with the upstream-reported model (no client fallback)', async () => { @@ -5075,7 +5035,7 @@ test('consumeTurn throws when upstream response.created has no model field (no c ), state, true, - new Map([[SHIM_TOOL_NAME, recordingDispatcher([])]]), + new Map([[FUNCTION_TOOL_NAME, recordingDispatcher([])]]), loopState(), [], ); @@ -5132,8 +5092,8 @@ test('consumeTurn re-captures upstream-reported model when later turns change it assertEquals(state.lastSeenModel, 'gpt-5.6-2025-12-01'); }); -test('consumeTurn does NOT capture upstream response.id (downstream uses the shim-synthesized id only)', async () => { - // Upstream's id rotates per turn and the shim never exposes it +test('consumeTurn does NOT capture upstream response.id (downstream uses the gateway-synthesized id only)', async () => { + // Upstream's id rotates per turn and the dispatcher never exposes it // downstream — `synthesizedResponseId` is the single cross-turn // identity the client correlates against. Verify upstream's id // doesn't slip into any MergeState field. @@ -5191,17 +5151,17 @@ test('consumeTurn second turn swallows upstream response.created and in_progress assertEquals(eventTypesOf(result.downstreamFrames), []); }); -test('consumeTurn intercepts the shim tool and does NOT forward its 4 events', async () => { +test('consumeTurn intercepts the dispatcher tool and does NOT forward its 4 events', async () => { const state = createMergeState(); const result = await consumeTurn( framesOf( mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'cc_1', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'cc_1', FUNCTION_TOOL_NAME), mkFunctionCallArgsDelta(0, '{"search_q'), mkFunctionCallArgsDelta(0, 'uery":[{"q":"hello"}]}'), mkFunctionCallArgsDone(0, '{"search_query":[{"q":"hello"}]}'), - mkFunctionCallDone(0, 'cc_1', SHIM_TOOL_NAME, '{"search_query":[{"q":"hello"}]}'), + mkFunctionCallDone(0, 'cc_1', FUNCTION_TOOL_NAME, '{"search_query":[{"q":"hello"}]}'), mkResponseCompleted(), ), state, @@ -5211,7 +5171,7 @@ test('consumeTurn intercepts the shim tool and does NOT forward its 4 events', a assertEquals(result.records.length, 1); assertEquals(result.records[0].intercepted, { callId: 'cc_1', - name: SHIM_TOOL_NAME, + name: FUNCTION_TOOL_NAME, arguments: { search_query: [{ q: 'hello' }] }, }); assertEquals(result.summary.dispatched.length, 1); @@ -5227,15 +5187,15 @@ test('consumeTurn intercepts the shim tool and does NOT forward its 4 events', a assertFalse(result.summary.sawClientToolCall); }); -test('consumeTurn intercepts two shim calls within one turn', async () => { +test('consumeTurn intercepts two dispatcher calls within one turn', async () => { const state = createMergeState(); const result = await consumeTurn( framesOf( mkResponseCreated(), - mkFunctionCallAdded(0, 'cc_o', SHIM_TOOL_NAME), - mkFunctionCallDone(0, 'cc_o', SHIM_TOOL_NAME, '{"open":[{"ref_id":"https://x"}]}'), - mkFunctionCallAdded(1, 'cc_f', SHIM_TOOL_NAME), - mkFunctionCallDone(1, 'cc_f', SHIM_TOOL_NAME, '{"find":[{"ref_id":"https://x","pattern":"p"}]}'), + mkFunctionCallAdded(0, 'cc_o', FUNCTION_TOOL_NAME), + mkFunctionCallDone(0, 'cc_o', FUNCTION_TOOL_NAME, '{"open":[{"ref_id":"https://x"}]}'), + mkFunctionCallAdded(1, 'cc_f', FUNCTION_TOOL_NAME), + mkFunctionCallDone(1, 'cc_f', FUNCTION_TOOL_NAME, '{"find":[{"ref_id":"https://x","pattern":"p"}]}'), mkResponseCompleted(), ), state, @@ -5246,8 +5206,8 @@ test('consumeTurn intercepts two shim calls within one turn', async () => { assertEquals(result.records[1].intercepted.arguments, { find: [{ ref_id: 'https://x', pattern: 'p' }] }); }); -test('consumeTurn synthesizes response.failed when upstream terminates without closing a shim call', async () => { - // A shim call reservation that never receives `output_item.done` is +test('consumeTurn synthesizes response.failed when upstream terminates without closing a dispatcher call', async () => { + // A dispatcher call reservation that never receives `output_item.done` is // an upstream protocol violation: the model intended a tool call, // the gateway accepted the reservation, but the close frame never // arrived. Without explicit detection here the reservation is @@ -5261,7 +5221,7 @@ test('consumeTurn synthesizes response.failed when upstream terminates without c consumeTurnStreaming( framesOf( mkResponseCreated(), - mkFunctionCallAdded(0, 'cc_1', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'cc_1', FUNCTION_TOOL_NAME), mkFunctionCallArgsDelta(0, '{"x":'), mkFunctionCallArgsDelta(0, '1}'), // No function_call.done — dispatcher never fires. @@ -5269,7 +5229,7 @@ test('consumeTurn synthesizes response.failed when upstream terminates without c ), state, true, - new Map([[SHIM_TOOL_NAME, recordingDispatcher(records)]]), + new Map([[FUNCTION_TOOL_NAME, recordingDispatcher(records)]]), loopState(), [], ), @@ -5278,7 +5238,7 @@ test('consumeTurn synthesizes response.failed when upstream terminates without c assertEquals(result.summary.dispatched.length, 0); assertEquals(result.summary.terminalStatus.kind, 'failed'); const ts = result.summary.terminalStatus as Extract; - assert(ts.response.error!.message.includes('without closing shim call items')); + assert(ts.response.error!.message.includes('without closing dispatcher call items')); assert(ts.response.error!.message.includes('response.completed')); }); @@ -5287,11 +5247,11 @@ test('consumeTurn dispatches at function_call.done with .done args canonical ove const result = await consumeTurn( framesOf( mkResponseCreated(), - mkFunctionCallAdded(0, 'cc_1', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'cc_1', FUNCTION_TOOL_NAME), mkFunctionCallArgsDelta(0, '{"stale":'), mkFunctionCallArgsDelta(0, '1}'), mkFunctionCallArgsDone(0, '{"search_query":[{"q":"x"}]}'), - mkFunctionCallDone(0, 'cc_1', SHIM_TOOL_NAME, '{"search_query":[{"q":"x"}]}'), + mkFunctionCallDone(0, 'cc_1', FUNCTION_TOOL_NAME, '{"search_query":[{"q":"x"}]}'), mkResponseCompleted(), ), state, @@ -5300,7 +5260,7 @@ test('consumeTurn dispatches at function_call.done with .done args canonical ove assertEquals(result.records[0].intercepted.arguments, { search_query: [{ q: 'x' }] }); }); -test('consumeTurn live-forwards non-shim function_calls and sets sawClientToolCall', async () => { +test('consumeTurn live-forwards client-owned function_calls and sets sawClientToolCall', async () => { const state = createMergeState(); const result = await consumeTurn( framesOf( @@ -5422,14 +5382,14 @@ test('consumeTurn single iteration ending in message: forwards full message life ]); }); -test('consumeTurn one shim call then message in same turn: FORWARDS the message live (shim call is consumed)', async () => { +test('consumeTurn one dispatcher call then message in same turn: FORWARDS the message live (dispatcher call is consumed)', async () => { const state = createMergeState(); const result = await consumeTurn( framesOf( mkResponseCreated(), mkResponseInProgress(), - mkFunctionCallAdded(0, 'cc_1', SHIM_TOOL_NAME), - mkFunctionCallDone(0, 'cc_1', SHIM_TOOL_NAME, '{"search_query":[{"q":"hi"}]}'), + mkFunctionCallAdded(0, 'cc_1', FUNCTION_TOOL_NAME), + mkFunctionCallDone(0, 'cc_1', FUNCTION_TOOL_NAME, '{"search_query":[{"q":"hi"}]}'), mkMessageAdded(1), mkMessageDone(1, 'intermediate text'), mkResponseCompleted(), @@ -5440,7 +5400,7 @@ test('consumeTurn one shim call then message in same turn: FORWARDS the message assertEquals(result.records.length, 1); assertEquals(state.accumulatedOutput.size, 1); - // Recording dispatcher doesn't emit lifecycle frames so the shim's + // Recording dispatcher doesn't emit lifecycle frames so the dispatcher's // reserved slot stays empty in accumulatedOutput, but outputIndex was // still bumped. The message therefore lands at index 1. assertEquals(state.accumulatedOutput.get(1)?.type, 'message'); @@ -5567,19 +5527,19 @@ test('consumeTurn rewrites any structurally indexed child event without an event assertEquals((futureFrame.event as { item_id: string }).item_id, messageId); }); -test('consumeTurn swallows future indexed events attached to an intercepted server tool call', async () => { +test('consumeTurn swallows future indexed events attached to an intercepted hosted tool call', async () => { const state = createMergeState(); const result = await consumeTurn( framesOf( mkResponseCreated(), - mkFunctionCallAdded(0, 'cc_1', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'cc_1', FUNCTION_TOOL_NAME), eventFrame({ type: 'response.future_function_call_arguments.delta', item_id: 'fc_hidden', output_index: 0, delta: 'hidden', } as unknown as OpenAIResponsesStreamEvent), - mkFunctionCallDone(0, 'cc_1', SHIM_TOOL_NAME, '{"search_query":[{"q":"q"}]}'), + mkFunctionCallDone(0, 'cc_1', FUNCTION_TOOL_NAME, '{"search_query":[{"q":"q"}]}'), mkResponseCompleted(), ), state, @@ -5643,8 +5603,8 @@ test('consumeTurn forwards message text events live even when mixed with an inte const result = await consumeTurn( framesOf( mkResponseCreated(), - mkFunctionCallAdded(0, 'cc_1', SHIM_TOOL_NAME), - mkFunctionCallDone(0, 'cc_1', SHIM_TOOL_NAME, '{"search_query":[{"q":"q"}]}'), + mkFunctionCallAdded(0, 'cc_1', FUNCTION_TOOL_NAME), + mkFunctionCallDone(0, 'cc_1', FUNCTION_TOOL_NAME, '{"search_query":[{"q":"q"}]}'), mkMessageAdded(1), eventFrame({ type: 'response.output_text.delta', @@ -5841,7 +5801,7 @@ test('consumeTurn surfaces bare `error` event as terminalStatus.failed with a sy assertEquals(ts.response.status, 'failed'); assertEquals(ts.response.error?.message, 'upstream blew up'); assertEquals(ts.response.error?.code, 'server_error'); - // Synthesized envelope's id is the shim-synthesized response id + // Synthesized envelope's id is the gateway-synthesized response id // — upstream's id is not exposed downstream. assertEquals(ts.response.id, state.synthesizedResponseId); }); @@ -5907,7 +5867,7 @@ test('consumeTurn surfaces bare `error` event arriving BEFORE response.created a // identity is captured (truncated TLS, transport drop, intermediate // proxy injection), we cannot synthesize a wire-valid // `OpenAIResponsesResult` (id and model are required, not nullable). Use - // a distinct terminal status `bare-error-pre-shell` so the shim's + // a distinct terminal status `bare-error-pre-shell` so the dispatcher's // outer loop can short-circuit to a non-events `upstream-error` // result instead of fabricating empty-string identity fields. const state = createMergeState(); @@ -5952,7 +5912,7 @@ test('consumeTurnStreaming yields forwarded frames before upstream completes', a countedFrames, state, true, - new Map([[SHIM_TOOL_NAME, recordingDispatcher(records)]]), + new Map([[FUNCTION_TOOL_NAME, recordingDispatcher(records)]]), loopState(), [], ); @@ -5966,7 +5926,7 @@ test('consumeTurnStreaming yields forwarded frames before upstream completes', a while (!(await iter.next()).done) { /* drain */ } }); -test('dispatcher start frames yield IN-LINE at function_call.done (shim call slot precedes later items)', async () => { +test('dispatcher start frames yield IN-LINE at function_call.done (dispatcher call slot precedes later items)', async () => { const state = createMergeState(); let dispatchOrder = 0; const records: DispatchRecord[] = []; @@ -5983,22 +5943,22 @@ test('dispatcher start frames yield IN-LINE at function_call.done (shim call slo endEvents: [{ type: 'response.web_search_call.completed' } as never], }; }, - } satisfies ServerToolResultSlot]; + } satisfies HostedToolResultSlot]; }; const result = await drain( consumeTurnStreaming( framesOf( mkResponseCreated(), - mkFunctionCallAdded(0, 'cc_1', SHIM_TOOL_NAME), - mkFunctionCallDone(0, 'cc_1', SHIM_TOOL_NAME, '{"search_query":[{"q":"hi"}]}'), + mkFunctionCallAdded(0, 'cc_1', FUNCTION_TOOL_NAME), + mkFunctionCallDone(0, 'cc_1', FUNCTION_TOOL_NAME, '{"search_query":[{"q":"hi"}]}'), mkMessageAdded(1), mkMessageDone(1, 'after'), mkResponseCompleted(), ), state, true, - new Map([[SHIM_TOOL_NAME, dispatcher]]), + new Map([[FUNCTION_TOOL_NAME, dispatcher]]), loopState(), [], ), @@ -6016,12 +5976,12 @@ test('dispatcher start frames yield IN-LINE at function_call.done (shim call slo assert(syntheticIdx < messageAddedIdx, `expected dispatcher start frame BEFORE later live items (synth=${syntheticIdx}, msgAdded=${messageAddedIdx})`); }); -test('shim call output_index is reserved at output_item.added so interleaved items get later indices', async () => { - // Reserving at `.added` (rather than `.done`) keeps a non-shim call - // item arriving between added and done from stealing the shim's +test('dispatcher call output_index is reserved at output_item.added so interleaved items get later indices', async () => { + // Reserving at `.added` (rather than `.done`) keeps a client-owned call + // item arriving between added and done from stealing the dispatcher's // would-be downstream index. const state = createMergeState(); - const dispatcher = () => [] as ServerToolResultSlot[]; + const dispatcher = () => [] as HostedToolResultSlot[]; const interleaved = eventFrame({ type: 'response.image_generation_call.in_progress', @@ -6034,14 +5994,14 @@ test('shim call output_index is reserved at output_item.added so interleaved ite consumeTurnStreaming( framesOf( mkResponseCreated(), - mkFunctionCallAdded(0, 'cc_1', SHIM_TOOL_NAME), + mkFunctionCallAdded(0, 'cc_1', FUNCTION_TOOL_NAME), interleaved, - mkFunctionCallDone(0, 'cc_1', SHIM_TOOL_NAME, '{"search_query":[{"q":"x"}]}'), + mkFunctionCallDone(0, 'cc_1', FUNCTION_TOOL_NAME, '{"search_query":[{"q":"x"}]}'), mkResponseCompleted(), ), state, true, - new Map([[SHIM_TOOL_NAME, dispatcher]]), + new Map([[FUNCTION_TOOL_NAME, dispatcher]]), loopState(), [], ), @@ -6119,7 +6079,7 @@ test('consumeTurn synthesizes terminalStatus.failed when upstream stream ends wi test('createMergeState starts with empty sparse usage accumulator and a synthesized response id', () => { const s = createMergeState(); - assert(s.synthesizedResponseId.startsWith('resp_shim_')); + assert(s.synthesizedResponseId.startsWith('resp_hosted_')); assertEquals(s.lastSeenModel, null); assertEquals(s.sequenceNumber, 0); assertEquals(s.outputIndex, 0); @@ -6210,8 +6170,8 @@ test('sumUsage with one-sided details preserves the field (treats missing side a // ── Slot run() contract ──────────────────────────────────────────────── -test('ServerToolResultSlot run() yields mid events in order then returns the terminal', async () => { - const slot: ServerToolResultSlot = { +test('HostedToolResultSlot run() yields mid events in order then returns the terminal', async () => { + const slot: HostedToolResultSlot = { id: 'ig_1', startItem: { type: 'image_generation_call', status: 'in_progress' }, startEvents: [{ type: 'response.image_generation_call.in_progress' }], @@ -6237,8 +6197,8 @@ test('ServerToolResultSlot run() yields mid events in order then returns the ter assertEquals(step.value.endEvents.map(e => e.type), ['response.image_generation_call.completed']); }); -test('ServerToolResultSlot run() yields nothing and returns the terminal', async () => { - const slot: ServerToolResultSlot = { +test('HostedToolResultSlot run() yields nothing and returns the terminal', async () => { + const slot: HostedToolResultSlot = { id: 'ws_1', startItem: { type: 'web_search_call', status: 'in_progress' }, startEvents: [], @@ -6258,7 +6218,7 @@ test('ServerToolResultSlot run() yields nothing and returns the terminal', async // ── Echo restore on `response.tools` / `response.tool_choice` ───────────── // -// The shim rewrites the request's hosted `web_search` to a function tool +// The dispatcher rewrites the request's hosted `web_search` to a function tool // before upstream; on echo (per OpenAI spec, `Response` composes // `ResponseProperties`, so `tools` and `tool_choice` are echoed) the // synthesized envelope must restore the canonical hosted form so the @@ -6303,15 +6263,15 @@ const findResponseCompleted = ( test('echo restore swaps the injected function tool back to the canonical hosted web_search', async () => { makeStubDeps(); - // Upstream's echoed `tools` includes the function tool the shim + // Upstream's echoed `tools` includes the function tool the dispatcher // injected plus an ordinary client function tool — restore should // touch only the injected one. (Spec defaults like // `additionalProperties:false` that Copilot injects on function - // tools must pass through verbatim on the non-shim tool.) + // tools must pass through verbatim on the client-owned tool.) const upstreamEchoedTools: OpenAIResponsesTool[] = [ { type: 'function', - name: SHIM_TOOL_NAME, + name: FUNCTION_TOOL_NAME, parameters: { type: 'object', properties: {}, additionalProperties: false }, strict: false, description: 'gateway-internal', @@ -6324,7 +6284,7 @@ test('echo restore swaps the injected function tool back to the canonical hosted description: null as unknown as string, }, ]; - const shim = withOpenAIResponsesWebSearchShim; + const dispatcher = runHostedWebSearch; const inv = makeInvocation({ payload: { tools: [ @@ -6341,7 +6301,7 @@ test('echo restore swaps the injected function tool back to the canonical hosted mkResponseCompletedWithTools(upstreamEchoedTools), ]]); - const { frames } = await runShimAndDrain(shim, inv, makeGatewayCtx(), script.run); + const { frames } = await runHostedToolsAndDrain(dispatcher, inv, makeGatewayCtx(), script.run); const completed = findResponseCompleted(frames); assert(Array.isArray(completed.response.tools)); @@ -6359,7 +6319,7 @@ test('echo restore preserves client-supplied filters / user_location on the cano const userLoc = { city: 'Tokyo', country: 'JP' }; const filters = { allowed_domains: ['weather.gov'] }; const upstreamEchoedTools: OpenAIResponsesTool[] = [ - { type: 'function', name: SHIM_TOOL_NAME, parameters: { type: 'object', properties: {}, additionalProperties: false }, strict: false }, + { type: 'function', name: FUNCTION_TOOL_NAME, parameters: { type: 'object', properties: {}, additionalProperties: false }, strict: false }, ]; const inv = makeInvocation({ payload: { tools: [{ type: 'web_search', filters, user_location: userLoc }] } }); const script = scriptedRun([[ @@ -6370,7 +6330,7 @@ test('echo restore preserves client-supplied filters / user_location on the cano mkResponseCompletedWithTools(upstreamEchoedTools), ]]); - const { frames } = await runShimAndDrain(withOpenAIResponsesWebSearchShim, inv, makeGatewayCtx(), script.run); + const { frames } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); const completed = findResponseCompleted(frames); assertEquals(completed.response.tools![0].type, 'web_search'); @@ -6393,7 +6353,7 @@ test('duplicate hosted web_search declarations collapse to the last complete dec // Upstream sees exactly one function tool because the framework // dedupes before run(). const upstreamEchoedTools: OpenAIResponsesTool[] = [ - { type: 'function', name: SHIM_TOOL_NAME, parameters: { type: 'object', properties: {}, additionalProperties: false }, strict: false }, + { type: 'function', name: FUNCTION_TOOL_NAME, parameters: { type: 'object', properties: {}, additionalProperties: false }, strict: false }, ]; const script = scriptedRun([[ mkResponseCreatedWithTools(upstreamEchoedTools), @@ -6403,7 +6363,7 @@ test('duplicate hosted web_search declarations collapse to the last complete dec mkResponseCompletedWithTools(upstreamEchoedTools), ]]); - const { frames } = await runShimAndDrain(withOpenAIResponsesWebSearchShim, inv, makeGatewayCtx(), script.run); + const { frames } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); // Request side: only one function tool was sent upstream. assertEquals(inv.payload.tools?.length, 1); @@ -6420,7 +6380,7 @@ test('duplicate hosted web_search declarations collapse to the last complete dec test('echo restore swaps the function-typed tool_choice back to a hosted tool_choice', async () => { makeStubDeps(); const upstreamEchoedTools: OpenAIResponsesTool[] = [ - { type: 'function', name: SHIM_TOOL_NAME, parameters: { type: 'object', properties: {}, additionalProperties: false }, strict: false }, + { type: 'function', name: FUNCTION_TOOL_NAME, parameters: { type: 'object', properties: {}, additionalProperties: false }, strict: false }, ]; const inv = makeInvocation({ payload: { @@ -6429,17 +6389,17 @@ test('echo restore swaps the function-typed tool_choice back to a hosted tool_ch }, }); const script = scriptedRun([[ - mkResponseCreatedWithTools(upstreamEchoedTools, { type: 'function', name: SHIM_TOOL_NAME }), + mkResponseCreatedWithTools(upstreamEchoedTools, { type: 'function', name: FUNCTION_TOOL_NAME }), mkResponseInProgress(), mkMessageAdded(0), mkMessageDone(0, 'done'), - mkResponseCompletedWithTools(upstreamEchoedTools, { type: 'function', name: SHIM_TOOL_NAME }), + mkResponseCompletedWithTools(upstreamEchoedTools, { type: 'function', name: FUNCTION_TOOL_NAME }), ]]); - const { frames } = await runShimAndDrain(withOpenAIResponsesWebSearchShim, inv, makeGatewayCtx(), script.run); + const { frames } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); // Request side: tool_choice was rewritten to the function form. - assertEquals(inv.payload.tool_choice, { type: 'function', name: SHIM_TOOL_NAME }); + assertEquals(inv.payload.tool_choice, { type: 'function', name: FUNCTION_TOOL_NAME }); const completed = findResponseCompleted(frames); assertEquals(completed.response.tool_choice, { type: 'web_search' }); }); @@ -6447,9 +6407,9 @@ test('echo restore swaps the function-typed tool_choice back to a hosted tool_ch test('echo restore leaves a non-injected function-typed tool_choice untouched', async () => { makeStubDeps(); // Upstream echoes a tool_choice naming a CLIENT-supplied function - // tool, not the shim's. Restore must not rewrite that. + // tool, not the dispatcher's. Restore must not rewrite that. const upstreamEchoedTools: OpenAIResponsesTool[] = [ - { type: 'function', name: SHIM_TOOL_NAME, parameters: { type: 'object', properties: {}, additionalProperties: false }, strict: false }, + { type: 'function', name: FUNCTION_TOOL_NAME, parameters: { type: 'object', properties: {}, additionalProperties: false }, strict: false }, { type: 'function', name: 'get_weather', parameters: { type: 'object', properties: {}, additionalProperties: false }, strict: false }, ]; const inv = makeInvocation({ @@ -6469,7 +6429,7 @@ test('echo restore leaves a non-injected function-typed tool_choice untouched', mkResponseCompletedWithTools(upstreamEchoedTools, { type: 'function', name: 'get_weather' }), ]]); - const { frames } = await runShimAndDrain(withOpenAIResponsesWebSearchShim, inv, makeGatewayCtx(), script.run); + const { frames } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); const completed = findResponseCompleted(frames); assertEquals(completed.response.tool_choice, { type: 'function', name: 'get_weather' }); @@ -6482,7 +6442,7 @@ test('upstream that does not echo `tools` produces a synthesized envelope withou // upstream that simply omits the echo. const script = scriptedRun([messageTurn('done')]); - const { frames } = await runShimAndDrain(withOpenAIResponsesWebSearchShim, inv, makeGatewayCtx(), script.run); + const { frames } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); const completed = findResponseCompleted(frames); assertEquals(completed.response.tools, undefined); @@ -6491,15 +6451,15 @@ test('upstream that does not echo `tools` produces a synthesized envelope withou // ── Billable usage ──────────────────────────────────────────────────────── -test('the shim carries the upstream turn cost onto the metadata pricing reads', async () => { - // The shim publishes its own finalMetadata, and that is what pricing reads. - // Failing to carry the cost onto it loses billing for the whole response - // silently, because the client-facing usage is no longer consulted. +test('the dispatcher carries the upstream turn cost onto the reading settlement writes from', async () => { + // The stage publishes its own reading, and that is what settlement writes the row from. + // Failing to carry the cost onto it loses billing for the whole response silently, because + // the client-facing usage is no longer consulted. const billableUsage: BillableUsage = { input: 10, cacheRead: 3, cacheWrite: 0, cacheWrite1h: 0, output: 2 }; makeStubDeps(); const inv = makeInvocation({ enabledFlags: new Set(['openai-responses-web-search-shim'] as const) }); - const result = await withOpenAIResponsesWebSearchShim(inv, makeGatewayCtx(), async () => ({ + const result = await runHostedWebSearch(inv, makeGatewayCtx(), async () => ({ type: 'events', events: (async function* () { for (const frame of messageTurn('no search needed')) yield frame; @@ -6510,11 +6470,16 @@ test('the shim carries the upstream turn cost onto the metadata pricing reads', assert(result.type === 'events'); await collectFrames(result.events); - assertEquals((await result.finalMetadata!).billableUsage, billableUsage); + assert(result.reading !== null); + assertEquals((await result.reading).billable, [{ + identity: testTelemetryModelIdentity, + quantities: { input_tokens: '10', input_cache_read_tokens: '3', output_tokens: '2' }, + pricingFacts: { inputTokens: 13, serviceTier: undefined }, + }]); }); for (const carrier of ['tools', 'additional_tools', 'tool_search_output'] as const) { - test(`the shim drains ${carrier} continuation streams before awaiting billable metadata`, async () => { + test(`the dispatcher drains ${carrier} continuation streams before awaiting billable metadata`, async () => { const { backend } = makeStubDeps(); const tools: OpenAIResponsesTool[] = [{ type: 'web_search' }]; const inv = makeInvocation({ @@ -6529,7 +6494,7 @@ for (const carrier of ['tools', 'additional_tools', 'tool_search_output'] as con }); const turns = [searchCallTurn(0, 'first', 'first search'), searchCallTurn(0, 'second', 'second search'), messageTurn('done')]; let runCalls = 0; - const result = await withOpenAIResponsesWebSearchShim(inv, makeGatewayCtx(), async () => { + const result = await runHostedWebSearch(inv, makeGatewayCtx(), async () => { const frames = turns[runCalls++]; const billableUsage: BillableUsage = { input: runCalls, cacheRead: 0, cacheWrite: 0, cacheWrite1h: 0, output: runCalls }; let resolveFinal!: (metadata: { modelIdentity: typeof testTelemetryModelIdentity; billableUsage: BillableUsage }) => void; @@ -6545,16 +6510,17 @@ for (const carrier of ['tools', 'additional_tools', 'tool_search_output'] as con assertEquals(findResponseCompleted(frames).response.status, 'completed'); assertEquals(runCalls, 3); assertEquals(backend.calls.length, 2); - assertEquals((await result.finalMetadata!).billableUsage, { input: 6, cacheRead: 0, cacheWrite: 0, cacheWrite1h: 0, output: 6 }); + assert(result.reading !== null); + assertEquals((await result.reading).billable.map(entity => entity.quantities), [1, 2, 3].map(tokens => ({ input_tokens: String(tokens), output_tokens: String(tokens) }))); }, 1_000); } for (const failedTurn of [1, 2]) { - test(`the shim retains observed billing when turn ${failedTurn} throws after its usage`, async () => { + test(`the dispatcher retains observed billing when turn ${failedTurn} throws after its usage`, async () => { makeStubDeps(); let runCalls = 0; const streamError = new Error('stream interrupted after usage'); - const result = await withOpenAIResponsesWebSearchShim(makeInvocation(), makeGatewayCtx(), async () => { + const result = await runHostedWebSearch(makeInvocation(), makeGatewayCtx(), async () => { const fails = ++runCalls === failedTurn; let resolveFinal!: (metadata: { modelIdentity: typeof testTelemetryModelIdentity; billableUsage: BillableUsage }) => void; const finalMetadata = new Promise<{ modelIdentity: typeof testTelemetryModelIdentity; billableUsage: BillableUsage }>(resolve => { resolveFinal = resolve; }); @@ -6576,7 +6542,10 @@ for (const failedTurn of [1, 2]) { assert(failed?.type === 'response.failed'); assertEquals(failed.response.error?.message, `Upstream stream failed mid-response: ${streamError.message}`); assertEquals(runCalls, failedTurn); - assertEquals((await result.finalMetadata!).billableUsage, { input: 10 * failedTurn, cacheRead: 0, cacheWrite: 0, cacheWrite1h: 0, output: 2 * failedTurn }); + assert(result.reading !== null); + const outcome = await result.reading; + assertEquals(outcome.failed, true); + assertEquals(outcome.billable.map(entity => entity.quantities), Array.from({ length: failedTurn }, () => ({ input_tokens: '10', output_tokens: '2' }))); }, 1_000); } @@ -6598,6 +6567,39 @@ test('consumeTurn forwards an event carrying no output_index instead of dropping assertEquals((forwarded.event as unknown as { detail: string }).detail, 'carried through'); }); +// ── A search is a run of its own ────────────────────────────────────────── + +// Ruling 2-and-6: a hosted tool's backend call is not part of the turn that asked for it. What +// makes that observable is the record — a search bills no entity and names no performance +// operation, so the run is the whole of what it buys, and its stages number from 1 like any +// run's, which is why the record has to be its own rather than merged into the turn's. +test('a search runs with its own prologue and its own record', async () => { + const { backend: _backend } = makeStubDeps(); + const stubs = installDumpStubs(initDumpStore, initDumpBroker); + const turn = openRunDump( + { id: 'k1', dumpRetentionSeconds: 3600 } as never, + { method: 'POST', path: '/v1/responses', body: { bytes: new Uint8Array(), streamError: null } }, + promise => { void promise; }, + true, + { upstreamCallStartedAt: null, firstOutputTokenAt: null }, + ); + assert(turn !== null); + const inv = makeInvocation({ payload: { tools: [{ type: 'web_search' }] } }); + const script = scriptedRun([searchCallTurn(0, 'call_1', 'floway'), messageTurn('done', 0)]); + + await runHostedToolsAndDrain(runHostedWebSearch, inv, mockChatGatewayCtx({ apiKeyId: 'k1', wantsStream: true, dump: turn }), script.run); + await flushBackground(); + + // One record, and it is the search's rather than the turn's: the turn's own is finalized by + // the seam that served it, which no test harness here reaches. + assertEquals(stubs.stored.length, 1); + assertEquals(stubs.stored[0]!.record.meta.path, '/alpha/search'); + const entered = eventsOf(stubs.stored[0]!.record) + .filter(event => event.type === 'stage.entered') + .map(event => event.name); + assertEquals(entered, ['writeSettlement', 'runWebSearchCall']); +}); + test('helper allocation reserves names across callable scopes, history and search-loaded tools without moving declarations', () => { const tools: OpenAIResponsesTool[] = [ { type: 'namespace', name: 'functions', description: '', tools: [{ type: 'function', name: 'web_search' }] }, @@ -6611,22 +6613,22 @@ test('helper allocation reserves names across callable scopes, history and searc { type: 'additional_tools', role: 'developer', tools: [{ type: 'custom', name: 'web_search_6' }] }, ]; const original = structuredClone({ tools, input }); - assertEquals(resolveServerToolName('web_search', tools, input, { type: 'allowed_tools', mode: 'auto', tools: [{ type: 'function', name: 'web_search_7', namespace: 'client' }] }), 'web_search_8'); - assertEquals(resolveServerToolName('web_search', [], [], { type: 'custom', name: 'web_search' }), 'web_search_2'); + assertEquals(resolveHostedToolName('web_search', tools, input, { type: 'allowed_tools', mode: 'auto', tools: [{ type: 'function', name: 'web_search_7', namespace: 'client' }] }), 'web_search_8'); + assertEquals(resolveHostedToolName('web_search', [], [], { type: 'custom', name: 'web_search' }), 'web_search_2'); assertEquals({ tools, input }, original); }); const undeclaredSelectorChoices: { name: string; choice: OpenAIResponsesToolChoice }[] = [ - { name: 'forced', choice: { type: 'function', name: `${SHIM_TOOL_NAME}_2` } }, - { name: 'allowed auto', choice: { type: 'allowed_tools', mode: 'auto', tools: [{ type: 'function', name: `${SHIM_TOOL_NAME}_2` }] } }, - { name: 'allowed required', choice: { type: 'allowed_tools', mode: 'required', tools: [{ type: 'function', name: `${SHIM_TOOL_NAME}_2` }] } }, + { name: 'forced', choice: { type: 'function', name: `${FUNCTION_TOOL_NAME}_2` } }, + { name: 'allowed auto', choice: { type: 'allowed_tools', mode: 'auto', tools: [{ type: 'function', name: `${FUNCTION_TOOL_NAME}_2` }] } }, + { name: 'allowed required', choice: { type: 'allowed_tools', mode: 'required', tools: [{ type: 'function', name: `${FUNCTION_TOOL_NAME}_2` }] } }, ]; test.each(undeclaredSelectorChoices)('helper allocation does not capture an undeclared $name selector', async ({ choice }) => { const { backend } = makeStubDeps(); const tools: OpenAIResponsesTool[] = [ { type: 'web_search' }, - { type: 'namespace', name: 'aux', description: '', tools: [{ type: 'function', name: SHIM_TOOL_NAME }] }, + { type: 'namespace', name: 'aux', description: '', tools: [{ type: 'function', name: FUNCTION_TOOL_NAME }] }, ]; const original = structuredClone({ tools, choice }); const inv = makeInvocation({ @@ -6635,10 +6637,10 @@ test.each(undeclaredSelectorChoices)('helper allocation does not capture an unde payload: { tools, tool_choice: choice }, }); const script = scriptedRun([messageTurn('done')]); - await runShimAndDrain(withOpenAIResponsesWebSearchShim, inv, makeGatewayCtx(), async () => { + await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), async () => { const helper = inv.payload.tools?.[0]; assert(helper?.type === 'function'); - assertEquals(helper.name, `${SHIM_TOOL_NAME}_3`); + assertEquals(helper.name, `${FUNCTION_TOOL_NAME}_3`); assertEquals(inv.payload.tool_choice, choice); return await script.run(); }); @@ -6654,23 +6656,23 @@ test.each([ const records: DispatchRecord[] = []; const iter = consumeTurnStreaming(framesOf( mkResponseCreated(), - eventFrame({ type: 'response.output_item.added', output_index: 0, item: { type: 'function_call', name: SHIM_TOOL_NAME, id: 'item', call_id: 'call', arguments: '', status: 'in_progress' } }), - eventFrame({ type: 'response.output_item.done', output_index: 0, item: { type: 'function_call', name: SHIM_TOOL_NAME, id: 'item', call_id: 'call', arguments: '{}', status: 'completed', ...changed } }), + eventFrame({ type: 'response.output_item.added', output_index: 0, item: { type: 'function_call', name: FUNCTION_TOOL_NAME, id: 'item', call_id: 'call', arguments: '', status: 'in_progress' } }), + eventFrame({ type: 'response.output_item.done', output_index: 0, item: { type: 'function_call', name: FUNCTION_TOOL_NAME, id: 'item', call_id: 'call', arguments: '{}', status: 'completed', ...changed } }), mkResponseCompleted(), - ), createMergeState(), true, new Map([[SHIM_TOOL_NAME, recordingDispatcher(records)]]), loopState(), []); - await assertRejects(() => drain(iter, records), Error, 'changed a server-tool function identity'); + ), createMergeState(), true, new Map([[FUNCTION_TOOL_NAME, recordingDispatcher(records)]]), loopState(), []); + await assertRejects(() => drain(iter, records), Error, 'changed a hosted-tool function identity'); assertEquals(records, []); }); test.each([undefined, 'item'])('hosted dispatch accepts a matching call instance with item ID %s', async id => { const records: DispatchRecord[] = []; - const item = { type: 'function_call' as const, name: SHIM_TOOL_NAME, id, call_id: 'call', arguments: '{"q":"matched"}', status: 'completed' as const }; + const item = { type: 'function_call' as const, name: FUNCTION_TOOL_NAME, id, call_id: 'call', arguments: '{"q":"matched"}', status: 'completed' as const }; const iter = consumeTurnStreaming(framesOf( mkResponseCreated(), eventFrame({ type: 'response.output_item.added', output_index: 0, item: { ...item, arguments: '', status: 'in_progress' } }), eventFrame({ type: 'response.output_item.done', output_index: 0, item }), mkResponseCompleted(), - ), createMergeState(), true, new Map([[SHIM_TOOL_NAME, recordingDispatcher(records)]]), loopState(), []); + ), createMergeState(), true, new Map([[FUNCTION_TOOL_NAME, recordingDispatcher(records)]]), loopState(), []); await drain(iter, records); assertEquals(records.length, 1); }); @@ -6680,16 +6682,16 @@ for (const targetApi of ['openaiChatCompletions', 'anthropicMessages', 'openaiRe const { backend } = makeStubDeps(); const inv = makeInvocation({ targetApi, enabledFlags: new Set(['openai-responses-web-search-shim']), payload: { - tools: [{ type: 'web_search' }, { type: 'namespace', name: 'functions', description: '', tools: [{ type: 'function', name: SHIM_TOOL_NAME, parameters: { type: 'object' } }] }], + tools: [{ type: 'web_search' }, { type: 'namespace', name: 'functions', description: '', tools: [{ type: 'function', name: FUNCTION_TOOL_NAME, parameters: { type: 'object' } }] }], }, }); const ctx = makeGatewayCtx(); let turns = 0; - const { frames } = await runShimAndDrain(withOpenAIResponsesWebSearchShim, inv, ctx, async () => { + const { frames } = await runHostedToolsAndDrain(runHostedWebSearch, inv, ctx, async () => { turns++; - assertEquals(inv.payload.tools?.[0], { ...inv.payload.tools?.[0], name: `${SHIM_TOOL_NAME}_2` }); + assertEquals(inv.payload.tools?.[0], { ...inv.payload.tools?.[0], name: `${FUNCTION_TOOL_NAME}_2` }); if (targetApi === 'openaiResponses') { - const item = { type: 'function_call' as const, name: SHIM_TOOL_NAME, namespace: 'functions', call_id: 'client', arguments: '{}', status: 'completed' as const }; + const item = { type: 'function_call' as const, name: FUNCTION_TOOL_NAME, namespace: 'functions', call_id: 'client', arguments: '{}', status: 'completed' as const }; return await scriptedRun([[ mkResponseCreated(), eventFrame({ type: 'response.output_item.added', output_index: 0, item: { ...item, status: 'in_progress' } }), @@ -6722,7 +6724,7 @@ for (const targetApi of ['openaiChatCompletions', 'anthropicMessages', 'openaiRe }); assertEquals(turns, 1); assertEquals(backend.calls, []); - assertEquals(findResponseCompleted(frames).response.output.map(item => item.type === 'function_call' ? [item.type, item.name, item.namespace] : [item.type]), [['function_call', SHIM_TOOL_NAME, 'functions']]); + assertEquals(findResponseCompleted(frames).response.output.map(item => item.type === 'function_call' ? [item.type, item.name, item.namespace] : [item.type]), [['function_call', FUNCTION_TOOL_NAME, 'functions']]); }); } @@ -6730,13 +6732,13 @@ test('native helper injection allocates past namespace children while retaining const { backend } = makeStubDeps(); const inv = makeInvocation({ targetApi: 'openaiResponses', enabledFlags: new Set(['openai-responses-web-search-shim']), payload: { - tools: [{ type: 'namespace', name: 'functions', description: '', tools: [{ type: 'function', name: SHIM_TOOL_NAME }] }, { type: 'web_search' }], + tools: [{ type: 'namespace', name: 'functions', description: '', tools: [{ type: 'function', name: FUNCTION_TOOL_NAME }] }, { type: 'web_search' }], }, }); - const script = scriptedRun([fcTurn(0, 'hosted', `${SHIM_TOOL_NAME}_2`, '{"search_query":[{"q":"hosted"}]}'), messageTurn('done')]); - const { frames } = await runShimAndDrain(withOpenAIResponsesWebSearchShim, inv, makeGatewayCtx(), script.run); + const script = scriptedRun([fcTurn(0, 'hosted', `${FUNCTION_TOOL_NAME}_2`, '{"search_query":[{"q":"hosted"}]}'), messageTurn('done')]); + const { frames } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), script.run); assertEquals(inv.payload.tools?.[1].type, 'function'); - assertEquals((inv.payload.tools?.[1] as { name: string }).name, `${SHIM_TOOL_NAME}_2`); + assertEquals((inv.payload.tools?.[1] as { name: string }).name, `${FUNCTION_TOOL_NAME}_2`); assertEquals(backend.calls.length, 1); assertEquals(script.callCount(), 2); assertEquals(findResponseCompleted(frames).response.status, 'completed'); @@ -6746,20 +6748,20 @@ for (const target of ['openaiChatCompletions', 'anthropicMessages'] as const) { for (const mode of ['forced', 'auto', 'required'] as const) { test(`${target} ${mode} helper choice follows a namespace-induced alias and restores its original echo`, async () => { const { backend } = makeStubDeps(); - const selector = { type: 'function' as const, name: SHIM_TOOL_NAME }; + const selector = { type: 'function' as const, name: FUNCTION_TOOL_NAME }; const choice: OpenAIResponsesToolChoice = mode === 'forced' ? selector : { type: 'allowed_tools', mode, tools: [selector] }; const original = structuredClone(choice); const inv = makeInvocation({ targetApi: target, payload: { - tools: [{ type: 'web_search' }, { type: 'namespace', name: 'client', description: '', tools: [{ type: 'function', name: SHIM_TOOL_NAME }] }], + tools: [{ type: 'web_search' }, { type: 'namespace', name: 'client', description: '', tools: [{ type: 'function', name: FUNCTION_TOOL_NAME }] }], tool_choice: choice, }, }); - const alias = `${SHIM_TOOL_NAME}_2`; + const alias = `${FUNCTION_TOOL_NAME}_2`; const script = scriptedRun([fcTurn(0, 'hosted', alias, '{"search_query":[{"q":"hosted"}]}'), messageTurn('done')]); const choices: unknown[] = []; - const { frames } = await runShimAndDrain(withOpenAIResponsesWebSearchShim, inv, makeGatewayCtx(), async () => { + const { frames } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), async () => { choices.push(structuredClone(inv.payload.tool_choice)); const before = structuredClone(inv.payload); const trip = target === 'openaiChatCompletions' @@ -6789,27 +6791,27 @@ for (const owner of ['function history', 'custom history', 'additional_tools', ' const { backend } = makeStubDeps(); const tools: OpenAIResponsesTool[] = [ { type: 'web_search' }, - { type: 'namespace', name: 'client', description: '', tools: [{ type: 'function', name: SHIM_TOOL_NAME }] }, + { type: 'namespace', name: 'client', description: '', tools: [{ type: 'function', name: FUNCTION_TOOL_NAME }] }, ]; const input: OpenAIResponsesInputItem[] = [{ type: 'message', role: 'user', content: 'Continue.' }]; if (owner === 'function history') { - input.push({ type: 'function_call', name: SHIM_TOOL_NAME, call_id: 'past', arguments: '{}', status: 'completed' }); + input.push({ type: 'function_call', name: FUNCTION_TOOL_NAME, call_id: 'past', arguments: '{}', status: 'completed' }); } else if (owner === 'custom history') { - input.push({ type: 'custom_tool_call', name: SHIM_TOOL_NAME, call_id: 'past', input: 'client input' }); + input.push({ type: 'custom_tool_call', name: FUNCTION_TOOL_NAME, call_id: 'past', input: 'client input' }); } else if (owner === 'additional_tools') { - input.push({ type: 'additional_tools', role: 'developer', tools: [{ type: 'function', name: SHIM_TOOL_NAME }] }); + input.push({ type: 'additional_tools', role: 'developer', tools: [{ type: 'function', name: FUNCTION_TOOL_NAME }] }); } else { - input.push({ type: 'tool_search_output', tools: [{ type: 'custom', name: SHIM_TOOL_NAME }] }); + input.push({ type: 'tool_search_output', tools: [{ type: 'custom', name: FUNCTION_TOOL_NAME }] }); } - const selector = { type: 'function' as const, name: SHIM_TOOL_NAME }; + const selector = { type: 'function' as const, name: FUNCTION_TOOL_NAME }; const choice: OpenAIResponsesToolChoice = selection === 'forced' ? selector : { type: 'allowed_tools', mode: 'auto', tools: [selector] }; const inv = makeInvocation({ payload: { tools, input, tool_choice: choice } }); - const script = scriptedRun([fcTurn(0, 'client', SHIM_TOOL_NAME, '{}')]); - const { frames } = await runShimAndDrain(withOpenAIResponsesWebSearchShim, inv, makeGatewayCtx(), async () => { + const script = scriptedRun([fcTurn(0, 'client', FUNCTION_TOOL_NAME, '{}')]); + const { frames } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), async () => { assertEquals(inv.payload.tool_choice, choice); const helper = inv.payload.tools?.[0]; assert(helper?.type === 'function'); - assertEquals(helper.name, `${SHIM_TOOL_NAME}_2`); + assertEquals(helper.name, `${FUNCTION_TOOL_NAME}_2`); return await script.run(); }); assertEquals(script.callCount(), 1); @@ -6825,7 +6827,7 @@ for (const selectorType of ['web_search', 'function'] as const) { const { backend } = makeStubDeps(); const choice: OpenAIResponsesToolChoice = { type: 'allowed_tools', mode: 'required', tools: [ - { type: selectorType, ...(selectorType === 'function' ? { name: SHIM_TOOL_NAME } : {}) }, + { type: selectorType, ...(selectorType === 'function' ? { name: FUNCTION_TOOL_NAME } : {}) }, ...(includeClient ? [{ type: 'function', name: 'client_allowed' }] : []), ], }; @@ -6834,14 +6836,14 @@ for (const selectorType of ['web_search', 'function'] as const) { const script = scriptedRun([searchCallTurn(0, 'hosted', 'query'), messageTurn('done')]); const choices: unknown[] = []; const declarations: Array = []; - const { frames } = await runShimAndDrain(withOpenAIResponsesWebSearchShim, inv, makeGatewayCtx(), async () => { + const { frames } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), async () => { choices.push(inv.payload.tool_choice); declarations.push(inv.payload.tools); return await script.run(); }); - const allowed = [{ type: 'function', name: SHIM_TOOL_NAME }, ...(includeClient ? [{ type: 'function', name: 'client_allowed' }] : [])]; + const allowed = [{ type: 'function', name: FUNCTION_TOOL_NAME }, ...(includeClient ? [{ type: 'function', name: 'client_allowed' }] : [])]; assertEquals(choices, [{ type: 'allowed_tools', mode: 'required', tools: allowed }, { type: 'allowed_tools', mode: 'auto', tools: allowed }]); - assertEquals(declarations[0]?.map(tool => 'name' in tool ? tool.name : undefined), [SHIM_TOOL_NAME, 'client_allowed', 'excluded']); + assertEquals(declarations[0]?.map(tool => 'name' in tool ? tool.name : undefined), [FUNCTION_TOOL_NAME, 'client_allowed', 'excluded']); assert(declarations[0] === declarations[1], 'mode demotion must retain the same declaration inventory'); assertEquals(script.callCount(), 2); assertEquals(backend.calls.length, 1); @@ -6859,7 +6861,7 @@ test('required client-only allowed_tools is not relaxed by an unrelated hosted i // allowed subset, so the test observes that client policy is not demoted. const script = scriptedRun([searchCallTurn(0, 'hosted', 'query'), messageTurn('done')]); const choices: unknown[] = []; - await runShimAndDrain(withOpenAIResponsesWebSearchShim, inv, makeGatewayCtx(), async () => { + await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), async () => { choices.push(structuredClone(inv.payload.tool_choice)); return await script.run(); }); @@ -6873,12 +6875,12 @@ test('helper echoes distinguish namespaced functions and rewrite hosted allowed_ makeStubDeps(); const choice: OpenAIResponsesToolChoice = { type: 'allowed_tools', mode: 'required', tools: [{ type: 'web_search' }] }; const inv = makeInvocation({ payload: { tool_choice: structuredClone(choice) } }); - const qualified = { type: 'function' as const, name: SHIM_TOOL_NAME, namespace: 'client' }; - const { frames } = await runShimAndDrain(withOpenAIResponsesWebSearchShim, inv, makeGatewayCtx(), async () => { - assertEquals(inv.payload.tool_choice, { type: 'allowed_tools', mode: 'required', tools: [{ type: 'function', name: SHIM_TOOL_NAME }] }); + const qualified = { type: 'function' as const, name: FUNCTION_TOOL_NAME, namespace: 'client' }; + const { frames } = await runHostedToolsAndDrain(runHostedWebSearch, inv, makeGatewayCtx(), async () => { + assertEquals(inv.payload.tool_choice, { type: 'allowed_tools', mode: 'required', tools: [{ type: 'function', name: FUNCTION_TOOL_NAME }] }); return await scriptedRun([[ mkResponseCreated(), - eventFrame({ type: 'response.completed', response: { ...emptyResult('upstream', 'completed'), tools: [qualified, { type: 'function', name: SHIM_TOOL_NAME }], tool_choice: inv.payload.tool_choice } }), + eventFrame({ type: 'response.completed', response: { ...emptyResult('upstream', 'completed'), tools: [qualified, { type: 'function', name: FUNCTION_TOOL_NAME }], tool_choice: inv.payload.tool_choice } }), ]]).run(); }); const tools = findResponseCompleted(frames).response.tools; @@ -6887,3 +6889,18 @@ test('helper echoes distinguish namespaced functions and rewrite hosted allowed_ assertEquals(tools?.[1].type, 'web_search'); assertEquals(findResponseCompleted(frames).response.tool_choice, choice); }); + +test('an unused hosted web-search declaration starts no Alpha selection work', async () => { + makeStubDeps(); + await getRepo().webSearchConfig.save({ + provider: 'tavily', tavily: { apiKey: 'test-key' }, microsoftWebIq: { apiKey: '' }, jina: { apiKey: '' }, + passthroughOpenAiSearch: { enabled: true, upstreamId: 'outside-key-scope', model: 'gpt-search' }, + } satisfies WebSearchConfig); + mockResolveAlpha.mockRejectedValue(new Error('Selected OpenAI search upstream is outside this API key scope')); + const invocation = makeInvocation({ targetApi: 'openaiResponses', enabledFlags: new Set(['openai-responses-web-search-shim']) }); + const script = scriptedRun([messageTurn('no search requested', 0)]); + const result = await runHostedWebSearch(invocation, mockChatGatewayCtx(), script.run); + if (result.type !== 'events') throw new Error('Expected an ordinary model response'); + await collectFrames(result.events); + assertEquals(mockResolveAlpha.mock.calls.length, 0); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts index c3ba4dbd04..ed5043286b 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts @@ -7,12 +7,13 @@ import type { AuthVars } from '../../../../src/middleware/auth.ts'; import { initRepo } from '../../../../src/repo/index.ts'; import type { ApiKey, User } from '../../../../src/repo/types.ts'; import { InMemoryRepo } from '../../../repo/memory.ts'; +import { stubChatProviderPipelines, type ChatFixtureCalls } from '../../../test-utils/chat-provider-pipelines.ts'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { type AliasRules, doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import { openaiResponsesResultToEvents, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesResult, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { type FlagId, type ModelCandidate, directFetcher, type ProviderOpenAIResponsesResult, type OpenAIResponsesAction, type UpstreamCallOptions } from '@floway-dev/provider'; -import { assert, assertEquals, stubProvider, stubInternalModel, stubProviderModel } from '@floway-dev/test-utils'; +import { type FlagId, type ModelCandidate, directFetcher, type OpenAIResponsesAction, type UpstreamCallOptions } from '@floway-dev/provider'; +import { assert, assertEquals, stubProvider, stubInternalModel, stubProviderModel, type ProviderOpenAIResponsesResult } from '@floway-dev/test-utils'; // Mock the resolver seam so each test hands the http entry exactly the // provider candidates it wants, optionally with an alias-rules overlay @@ -129,9 +130,10 @@ const makeCandidate = (overrides: { } = {}): ModelCandidate => { const upstream = overrides.upstream ?? 'up_test'; const endpoints = overrides.endpoints ?? { openaiChatCompletions: {}, openaiResponses: {}, anthropicMessages: {} }; - const provider = stubProvider({ + const calls = { callOpenAIResponses: overrides.callOpenAIResponses, - }); + }; + const provider = stubProvider(); return { provider: { upstreamId: upstream, @@ -140,7 +142,7 @@ const makeCandidate = (overrides: { inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, - modelsCache: null, pipelines: {}, + modelsCache: null, pipelines: stubChatProviderPipelines(calls), instance: provider, }, model: stubInternalModel({ @@ -544,6 +546,85 @@ test('POST /v1/responses with an unresolvable previous_response_id renders the v assertEquals(body.error.code, 'previous_response_not_found'); }); +test('POST /v1/responses replays a stored turn when the next one names it as previous_response_id', async () => { + installRepo(); + const observedBodies: Array> = []; + const callOpenAIResponses = vi.fn(async (_model, body): Promise => { + observedBodies.push(body as Omit); + return { + action: 'generate', ok: true, + events: makeProviderEvents(completedEvents(`resp_upstream_${observedBodies.length}`)), + modelKey: 'test-model-key', + headers: new Headers(), + }; + }); + const candidate = makeCandidate({ callOpenAIResponses }); + queueResolution([candidate]); + queueResolution([candidate]); + const app = makeApp(); + + const first = await app.request('/v1/responses', { + method: 'POST', + headers: new Headers({ 'content-type': 'application/json' }), + body: JSON.stringify({ model: 'test-model', input: 'hello', store: true }), + }); + assertEquals(first.status, 200); + const stored = await first.json() as OpenAIResponsesResult; + + const second = await app.request('/v1/responses', { + method: 'POST', + headers: new Headers({ 'content-type': 'application/json' }), + body: JSON.stringify({ + model: 'test-model', + store: true, + previous_response_id: stored.id, + input: 'continue', + }), + }); + assertEquals(second.status, 200); + await second.json(); + + // The prior turn is replayed as the rows the store holds — this turn's own input last — + // and the snapshot id names something only this gateway has, so it never reaches the wire. + const continued = observedBodies[1]; + if (continued === undefined) throw new Error('expected a second upstream call'); + assertEquals(continued.previous_response_id, undefined); + assertEquals(continued.input, [ + { type: 'message', role: 'user', content: 'hello' }, + { + type: 'message', + id: 'msg_1', + role: 'assistant', + status: 'completed', + content: [{ type: 'output_text', text: 'hi', annotations: [] }], + }, + { type: 'message', role: 'user', content: 'continue' }, + ]); +}); + +test('POST /v1/responses rejects an item_reference the store cannot resolve', async () => { + installRepo(); + + // No candidates need to be queued — hydration answers before routing runs. + const response = await makeApp().request('/v1/responses', { + method: 'POST', + headers: new Headers({ 'content-type': 'application/json' }), + body: JSON.stringify({ + model: 'test-model', + input: [{ type: 'item_reference', id: 'msg_missing' }], + }), + }); + + assertEquals(response.status, 404); + const body = await response.json() as { error: { message: string; type: string; param: string; code: string | null } }; + assertEquals(body.error, { + message: "Item with id 'msg_missing' not found.", + type: 'invalid_request_error', + param: 'input', + code: null, + }); +}); + test('POST /v1/responses and /v1/responses/compact reject a body without `model` with the OpenAI missing-parameter 400', async () => { installRepo(); @@ -724,7 +805,7 @@ const translatedCustomCandidate = ( callExec = false, ): ModelCandidate => { const candidate = makeCandidate({ upstream: `up_${target}`, endpoints: { [target]: {} } }); - const instance = stubProvider({ + const calls: ChatFixtureCalls = { callOpenAIChatCompletions: async (_model, body) => { observe(body as unknown as Record); const chunk = (choices: OpenAIChatCompletionsStreamEvent['choices']): OpenAIChatCompletionsStreamEvent => ({ id: 'chat_exec', object: 'chat.completion.chunk', created: 0, model: 'test-model', choices }); @@ -750,8 +831,8 @@ const translatedCustomCandidate = ( })(), }; }, - }); - return { ...candidate, provider: { ...candidate.provider, instance } }; + }; + return { ...candidate, provider: { ...candidate.provider, pipelines: stubChatProviderPipelines(calls) } }; }; for (const target of ['openaiChatCompletions', 'anthropicMessages'] as const) { diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/answer-websocket-warmup_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/answer-websocket-warmup_test.ts deleted file mode 100644 index 9a06d175d5..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/answer-websocket-warmup_test.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { test } from 'vitest'; - -import { answerWebSocketWarmup } from '../../../../../src/data-plane/chat/openai-responses/interceptors/answer-websocket-warmup.ts'; -import type { OpenAIResponsesInvocation } from '../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { doneFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { eventResult } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesInvocation => ({ - payload, - candidate: stubModelCandidate(), - targetApi: 'openaiResponses', - headers: new Headers(), - action: 'generate', -}); - -const payload = (generate?: boolean): CanonicalOpenAIResponsesPayload => ({ - model: 'gpt-test', - input: [{ type: 'message', role: 'developer', content: 'Base instructions' }], - ...(generate === undefined ? {} : { generate }), -}); - -test('answers a generate:false prewarm with an empty completed response and no upstream call', async () => { - let upstreamCalls = 0; - const result = await answerWebSocketWarmup(invocation(payload(false)), stubCtx, () => { - upstreamCalls++; - return Promise.resolve(eventResult((async function* () { yield doneFrame(); })(), testTelemetryModelIdentity)); - }); - - assertEquals(upstreamCalls, 0); - if (result.type !== 'events') throw new Error(`expected events, got ${result.type}`); - const frames: ProtocolFrame[] = []; - for await (const frame of result.events) frames.push(frame); - const types = frames.map(frame => frame.type === 'event' ? frame.event.type : frame.type); - assertEquals(types, ['response.created', 'response.in_progress', 'response.completed', 'done']); - const terminal = frames[2]; - if (terminal?.type !== 'event' || terminal.event.type !== 'response.completed') throw new Error('expected a completed terminal'); - assertEquals(terminal.event.response.status, 'completed'); - assertEquals(terminal.event.response.output, []); - assertEquals(terminal.event.response.model, 'gpt-test'); - assertEquals(terminal.event.response.usage, { input_tokens: 0, output_tokens: 0, total_tokens: 0 }); -}); - -for (const generate of [undefined, true]) { - test(`passes a generate:${generate} request through to the upstream`, async () => { - let upstreamCalls = 0; - const upstream = eventResult((async function* () { yield doneFrame(); })(), testTelemetryModelIdentity); - const result = await answerWebSocketWarmup(invocation(payload(generate)), stubCtx, () => { - upstreamCalls++; - return Promise.resolve(upstream); - }); - - assertEquals(upstreamCalls, 1); - assertEquals(result, upstream); - }); -} diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/apply-role-compatibility_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/apply-role-compatibility_test.ts deleted file mode 100644 index e5d5e48334..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/apply-role-compatibility_test.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { test } from 'vitest'; - -import { withRoleCompatibilityApplied } from '../../../../../src/data-plane/chat/openai-responses/interceptors/apply-role-compatibility.ts'; -import type { OpenAIResponsesInvocation } from '../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { doneFrame } from '@floway-dev/protocols/common'; -import type { OpenAIResponsesInputItem } from '@floway-dev/protocols/openai-responses'; -import { eventResult, type FlagId } from '@floway-dev/provider'; -import { assert, assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const gatewayCtx = mockChatGatewayCtx(); -const okEvents = () => Promise.resolve(eventResult((async function* () { yield doneFrame(); })(), testTelemetryModelIdentity)); - -const applyRoles = async ( - input: OpenAIResponsesInputItem[], - enabledFlags: ReadonlySet, - targetApi: OpenAIResponsesInvocation['targetApi'] = 'openaiResponses', -): Promise => { - const invocation: OpenAIResponsesInvocation = { - payload: { model: 'test-model', input }, - candidate: stubModelCandidate({ enabledFlags }), - targetApi, - headers: new Headers(), - action: 'generate', - }; - await withRoleCompatibilityApplied(invocation, gatewayCtx, okEvents); - return invocation.payload.input; -}; - -test('leaves roles unchanged without flags or at a translated target', async () => { - const input: OpenAIResponsesInputItem[] = [ - { type: 'message', role: 'system', content: 'rules' }, - { type: 'message', role: 'developer', content: 'developer rules' }, - ]; - - assertEquals(await applyRoles(input, new Set()), input); - assertEquals(await applyRoles(input, new Set(['rewrite-system-to-developer']), 'openaiChatCompletions'), input); -}); - -test('applies the system and developer rewrites independently', async () => { - assertEquals( - await applyRoles( - [{ type: 'message', role: 'system', content: 'rules' }], - new Set(['rewrite-system-to-developer']), - ), - [{ type: 'message', role: 'developer', content: 'rules' }], - ); - assertEquals( - await applyRoles( - [{ type: 'message', role: 'developer', content: 'rules' }], - new Set(['rewrite-developer-to-system']), - ), - [{ type: 'message', role: 'system', content: 'rules' }], - ); -}); - -test('uses non-message items as the boundary before rewriting later system', async () => { - assertEquals( - await applyRoles( - [ - { type: 'message', role: 'system', content: 'base rules' }, - { type: 'reasoning', id: 'rs_1', summary: [] }, - { type: 'message', role: 'system', content: 'inline rules' }, - ], - new Set(['rewrite-mid-conv-system-to-user']), - ), - [ - { type: 'message', role: 'system', content: 'base rules' }, - { type: 'reasoning', id: 'rs_1', summary: [] }, - { type: 'message', role: 'user', content: 'inline rules' }, - ], - ); -}); - -test('keeps a leading-only system run and an empty input unchanged', async () => { - const leading: OpenAIResponsesInputItem[] = [ - { type: 'message', role: 'system', content: 'base A' }, - { type: 'message', role: 'system', content: 'base B' }, - ]; - assertEquals(await applyRoles(leading, new Set(['rewrite-mid-conv-system-to-user'])), leading); - assertEquals(await applyRoles([], new Set(['rewrite-mid-conv-system-to-user'])), []); -}); - -test('preserves multipart content identity when rewriting a mid-conversation system message', async () => { - const content = [ - { type: 'input_text' as const, text: 'one' }, - { type: 'input_text' as const, text: 'two' }, - ]; - const result = await applyRoles( - [ - { type: 'message', role: 'user', content: 'hello' }, - { type: 'message', role: 'system', content }, - ], - new Set(['rewrite-mid-conv-system-to-user']), - ); - assertEquals(result, [ - { type: 'message', role: 'user', content: 'hello' }, - { type: 'message', role: 'user', content }, - ]); - const rewritten = result[1]; - assert(rewritten?.type === 'message' && rewritten.content === content); -}); - -test('applies overlapping flags in system-to-developer then developer-to-system order', async () => { - assertEquals( - await applyRoles( - [ - { type: 'message', role: 'system', content: 'base rules' }, - { type: 'message', role: 'user', content: 'hello' }, - { type: 'message', role: 'system', content: 'inline rules' }, - ], - new Set([ - 'rewrite-system-to-developer', - 'rewrite-developer-to-system', - 'rewrite-mid-conv-system-to-user', - ]), - ), - [ - { type: 'message', role: 'system', content: 'base rules' }, - { type: 'message', role: 'user', content: 'hello' }, - { type: 'message', role: 'user', content: 'inline rules' }, - ], - ); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/compact-shim_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/compact-shim_test.ts deleted file mode 100644 index 45fe5435cc..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/compact-shim_test.ts +++ /dev/null @@ -1,914 +0,0 @@ -import { test } from 'vitest'; - -import { SUMMARY_PREFIX, expandShimCompactionItems, isOpenAIResponsesCompactShimItem, withOpenAIResponsesCompactShim } from '../../../../../src/data-plane/chat/openai-responses/interceptors/compact-shim.ts'; -import type { OpenAIResponsesInvocation } from '../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; -import { encodeBase64UrlJson } from '../../../../../src/shared/base64url-json.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import { collectOpenAIResponsesProtocolEventsToResult, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesInputItem, type OpenAIResponsesOutputItem, type OpenAIResponsesPayload, type OpenAIResponsesResult, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { eventResult, type ExecuteResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const makeInvocation = ( - payload: Partial = {}, - options: { action?: 'generate' | 'compact'; flagOn?: boolean; decryptFlagOn?: boolean; targetApi?: 'openaiResponses' | 'anthropicMessages' | 'openaiChatCompletions' } = {}, -): OpenAIResponsesInvocation => { - const enabledFlags = new Set(); - if (options.flagOn !== false) enabledFlags.add('openai-responses-compact-shim'); - if (options.decryptFlagOn === true) enabledFlags.add('openai-responses-compact-decrypt'); - return { - payload: { model: 'test-model', input: [], ...payload } as CanonicalOpenAIResponsesPayload, - action: options.action ?? 'generate', - candidate: stubModelCandidate({ enabledFlags }), - targetApi: options.targetApi ?? 'openaiResponses', - headers: new Headers(), - }; -}; - -const fakeUpstreamRun = (summaryText: string): () => Promise>> => { - const response: OpenAIResponsesResult = { - id: 'resp_fake_upstream', - object: 'response', - model: 'test-upstream-model', - status: 'completed', - output: [{ - type: 'message', - id: 'msg_1', - role: 'assistant', - status: 'completed', - content: [{ type: 'output_text', text: summaryText, annotations: [] }], - }], - error: null, - incomplete_details: null, - usage: { input_tokens: 10, output_tokens: 20, total_tokens: 30 }, - }; - return () => Promise.resolve(eventResult( - (async function* (): AsyncGenerator> { - yield eventFrame({ type: 'response.output_item.added', sequence_number: 0, output_index: 0, item: response.output[0] }); - yield eventFrame({ type: 'response.output_item.done', sequence_number: 1, output_index: 0, item: response.output[0] }); - yield eventFrame({ type: 'response.completed', sequence_number: 2, response }); - yield doneFrame(); - })(), - testTelemetryModelIdentity, - )); -}; - -// ── Inbound expansion (expandShimCompactionItems) ──────────────────────────── - -test('inbound: compaction and compaction_summary items with shim-encoded payloads expand inline', () => { - const userItem = { type: 'message' as const, role: 'user' as const, content: 'history one' }; - const encoded = encodeBase64UrlJson([userItem]); - - const expanded = expandShimCompactionItems({ - model: 'm', - input: [ - { type: 'compaction', id: 'cmp_1', encrypted_content: encoded }, - { type: 'message', role: 'user', content: 'new turn' }, - { type: 'compaction_summary', id: 'cmp_2', encrypted_content: encoded }, - ], - }); - - assertEquals(expanded.input.length, 3); - assertEquals(expanded.input[0], userItem); - assertEquals(expanded.input[1], { type: 'message', role: 'user', content: 'new turn' }); - assertEquals(expanded.input[2], userItem); -}); - -test('identifies only compaction items carrying the gateway-owned payload shape', () => { - const encoded = encodeBase64UrlJson([{ - type: 'message', - role: 'user', - content: [{ type: 'input_text', text: 'summary' }], - }]); - assertEquals(isOpenAIResponsesCompactShimItem({ type: 'compaction', encrypted_content: encoded }), true); - assertEquals(isOpenAIResponsesCompactShimItem({ type: 'compaction_summary', encrypted_content: encoded }), true); - assertEquals(isOpenAIResponsesCompactShimItem({ type: 'compaction', encrypted_content: 'OPAQUE_NATIVE_BLOB' }), false); - assertEquals(isOpenAIResponsesCompactShimItem({ type: 'compaction_summary', encrypted_content: 'OPAQUE_NATIVE_BLOB' }), false); - assertEquals(isOpenAIResponsesCompactShimItem({ type: 'compaction', encrypted_content: encodeBase64UrlJson({ type: 'message' }) }), false); - assertEquals(isOpenAIResponsesCompactShimItem({ type: 'reasoning', encrypted_content: encoded }), false); -}); - -test('inbound: foreign compaction blob (non-base64url-JSON) round-trips untouched', () => { - const original = { - model: 'm', - input: [ - { type: 'compaction', id: 'cmp_native', encrypted_content: 'OPAQUE_NATIVE_BLOB' } as unknown as never, - ], - }; - const expanded = expandShimCompactionItems(original); - // No items expanded — the foreign blob fails decode and the item passes - // through as-is. - assertEquals(expanded, original); -}); - -test('inbound: foreign compaction blob (valid base64url but wrong shape) round-trips untouched', () => { - // base64url-encoded JSON of an object (not an array) — decode succeeds, - // but the schema check rejects it. - const wrongShape = encodeBase64UrlJson({ not: 'an array' }); - const original = { - model: 'm', - input: [ - { type: 'compaction', id: 'cmp_foreign', encrypted_content: wrongShape } as unknown as never, - ], - }; - const expanded = expandShimCompactionItems(original); - assertEquals(expanded, original); -}); - -// ── Outbound summarization (withOpenAIResponsesCompactShim) ──────────────────────── - -test('compact + flag on: pivots to generate, drives upstream summarization, returns compaction envelope', async () => { - const inv = makeInvocation( - { input: [{ type: 'message', role: 'user', content: 'long conversation history' }] }, - { action: 'compact' }, - ); - - let seenPayload: OpenAIResponsesPayload | undefined; - let seenAction: 'generate' | 'compact' | undefined; - const result = await withOpenAIResponsesCompactShim(inv, stubCtx, () => { - seenPayload = inv.payload; - seenAction = inv.action; - return fakeUpstreamRun('CONDENSED SUMMARY')(); - }); - - if (result.type !== 'events') throw new Error(`expected events branch, got ${result.type}`); - // Inner action seen by the upstream is 'generate'. - assertEquals(seenAction, 'generate'); - // The pivot is one-way per the project's interceptor convention: outer - // ctx.action stays 'generate' after the run. attempt.invoke keys - // envelope-drain on the caller's intent action (captured by value), so - // leaving invocation.action='generate' does not change the result shape. - assertEquals(inv.action, 'generate'); - // Payload pivoted: SUMMARIZATION_PROMPT injected as a role=system input - // item at the head of the history (mirroring native compact's system-role - // compactor prompt), store:false, the original history retained after it - // (compaction_trigger items would be stripped but there are none here). - if (!seenPayload) throw new Error('expected the upstream call to see the rewritten payload'); - const rewrittenInput = seenPayload.input as OpenAIResponsesInputItem[]; - const compactorSystem = rewrittenInput[0] as { type: string; role: string; content: Array<{ text: string }> }; - assertEquals(compactorSystem.type, 'message'); - assertEquals(compactorSystem.role, 'system'); - assertEquals(compactorSystem.content[0].text.includes('CONTEXT CHECKPOINT COMPACTION'), true); - assertEquals(seenPayload.store, false); - - const collected = await collectOpenAIResponsesProtocolEventsToResult(result.events); - assertEquals(collected.object, 'response.compaction'); - const compactionItem = collected.output[0] as { type: string; encrypted_content: string }; - assertEquals(compactionItem.type, 'compaction'); -}); - -test("compact + flag on: caller's `instructions` field flows through untouched (bug-for-bug parity with native compact)", async () => { - // Native `/responses/compact` puts SUMMARIZATION_PROMPT into the compactor - // context as a role=system message AND forwards the caller's `instructions` - // as a developer-role message alongside; both are in scope. Prompt- - // injection extraction against the live Copilot upstream confirmed a - // caller-supplied `instructions="always mention 'quokka'"` reaches the - // compactor as a developer message and shows up inside the produced - // summary. The shim must reproduce that shape — do NOT overwrite the - // caller's instructions with SUMMARIZATION_PROMPT (it now lives as a - // separate role=system input item). - const inv = makeInvocation( - { - input: [{ type: 'message', role: 'user', content: 'history' }], - instructions: 'You are a helpful assistant. Always mention the word quokka.', - }, - { action: 'compact' }, - ); - - let seenPayload: OpenAIResponsesPayload | undefined; - await withOpenAIResponsesCompactShim(inv, stubCtx, () => { - seenPayload = inv.payload; - return fakeUpstreamRun('irrelevant')(); - }); - - if (!seenPayload) throw new Error('expected the upstream call to see the rewritten payload'); - // Original instructions preserved verbatim; SUMMARIZATION_PROMPT lives - // separately in the role=system input item at index 0. - assertEquals(seenPayload.instructions, 'You are a helpful assistant. Always mention the word quokka.'); - const rewrittenInput = seenPayload.input as OpenAIResponsesInputItem[]; - const compactorSystem = rewrittenInput[0] as { type: string; role: string; content: Array<{ text: string }> }; - assertEquals(compactorSystem.role, 'system'); - assertEquals(compactorSystem.content[0].text.includes('CONTEXT CHECKPOINT COMPACTION'), true); -}); - -test('compact + flag on: caller with no `instructions` still gets SUMMARIZATION_PROMPT as a role=system input item and instructions stays absent', async () => { - // Baseline: when the caller sends no `instructions` field, the shim still - // injects SUMMARIZATION_PROMPT via the role=system input item — matching - // native compact, which always has its system-role compactor prompt in - // scope regardless of whether the caller supplies a developer message. - // The `instructions` slot stays unset so a downstream translator does not - // synthesize a phantom developer message from thin air. - const inv = makeInvocation( - { input: [{ type: 'message', role: 'user', content: 'history' }] }, - { action: 'compact' }, - ); - - let seenPayload: OpenAIResponsesPayload | undefined; - await withOpenAIResponsesCompactShim(inv, stubCtx, () => { - seenPayload = inv.payload; - return fakeUpstreamRun('irrelevant')(); - }); - - if (!seenPayload) throw new Error('expected the upstream call to see the rewritten payload'); - assertEquals(seenPayload.instructions, undefined); - const rewrittenInput = seenPayload.input as OpenAIResponsesInputItem[]; - const compactorSystem = rewrittenInput[0] as { type: string; role: string; content: Array<{ text: string }> }; - assertEquals(compactorSystem.type, 'message'); - assertEquals(compactorSystem.role, 'system'); - assertEquals(compactorSystem.content[0].text.includes('CONTEXT CHECKPOINT COMPACTION'), true); -}); - -test('compact + flag on: synthesized encrypted_content decodes to a user message containing the summary prefixed with the handoff prompt', async () => { - const inv = makeInvocation( - { input: [{ type: 'message', role: 'user', content: 'history' }] }, - { action: 'compact' }, - ); - const result = await withOpenAIResponsesCompactShim(inv, stubCtx, fakeUpstreamRun('THE SUMMARY')); - if (result.type !== 'events') throw new Error('expected events branch'); - const collected = await collectOpenAIResponsesProtocolEventsToResult(result.events); - const compactionItem = collected.output[0] as { type: string; encrypted_content: string }; - - // The encrypted_content decodes to our base64url-JSON marker: one - // user-role message carrying `${SUMMARY_PREFIX}\n${summary}` as - // input_text. The prefix rides inside the blob so a downstream LLM - // that echoes the compaction back reads the message as "another LLM's - // handoff summary", not as raw user speech. - const decoded = JSON.parse( - new TextDecoder().decode( - Uint8Array.from( - atob(compactionItem.encrypted_content.replace(/-/g, '+').replace(/_/g, '/')), - c => c.charCodeAt(0), - ), - ), - ); - assertEquals(decoded.length, 1); - assertEquals(decoded[0].type, 'message'); - assertEquals(decoded[0].role, 'user'); - assertEquals(decoded[0].content[0].type, 'input_text'); - assertEquals(decoded[0].content[0].text, `${SUMMARY_PREFIX}\nTHE SUMMARY`); -}); - -test('compact + flag on: upstream `output_text` SDK alias is dropped from the synthesized envelope', async () => { - const inv = makeInvocation( - { input: [{ type: 'message', role: 'user', content: 'history' }] }, - { action: 'compact' }, - ); - - // Some upstreams (and some OpenAPI implementations) emit the convenience - // `output_text` alias alongside `output`. The synthesized - // `response.compaction` envelope must not forward it — its value is the - // upstream's summary plaintext, which a downstream SDK reading - // `output_text` on a compaction envelope would surface in place of the - // opaque-blob contract `encrypted_content` is supposed to carry. - const runWithOutputText = (): Promise>> => { - const response: OpenAIResponsesResult = { - id: 'resp_fake_upstream', - object: 'response', - model: 'test-upstream-model', - status: 'completed', - output: [{ - type: 'message', - id: 'msg_1', - role: 'assistant', - status: 'completed', - content: [{ type: 'output_text', text: 'THE SUMMARY', annotations: [] }], - }], - output_text: 'THE SUMMARY', - error: null, - incomplete_details: null, - usage: { input_tokens: 10, output_tokens: 20, total_tokens: 30 }, - }; - return Promise.resolve(eventResult( - (async function* (): AsyncGenerator> { - yield eventFrame({ type: 'response.completed', sequence_number: 0, response }); - yield doneFrame(); - })(), - testTelemetryModelIdentity, - )); - }; - const result = await withOpenAIResponsesCompactShim(inv, stubCtx, runWithOutputText); - if (result.type !== 'events') throw new Error('expected events branch'); - const collected = await collectOpenAIResponsesProtocolEventsToResult(result.events); - assertEquals(collected.output_text, undefined); -}); - -test('compact + flag on: upstream incomplete status propagates onto the synthesized envelope', async () => { - const inv = makeInvocation( - { input: [{ type: 'message', role: 'user', content: 'history' }] }, - { action: 'compact' }, - ); - - // Simulate a summarization turn that hit `max_output_tokens` mid-stream: - // the upstream returns `status: 'incomplete'` with `incomplete_details` - // populated. The synthesized envelope must surface that — not pretend the - // turn ran to completion. - const runIncomplete = (): Promise>> => { - const response: OpenAIResponsesResult = { - id: 'resp_fake_upstream', - object: 'response', - model: 'test-upstream-model', - status: 'incomplete', - incomplete_details: { reason: 'max_output_tokens' }, - output: [{ - type: 'message', - id: 'msg_1', - role: 'assistant', - status: 'incomplete', - content: [{ type: 'output_text', text: 'partial summary', annotations: [] }], - }], - error: null, - usage: { input_tokens: 10, output_tokens: 20, total_tokens: 30 }, - }; - return Promise.resolve(eventResult( - (async function* (): AsyncGenerator> { - yield eventFrame({ type: 'response.completed', sequence_number: 0, response }); - yield doneFrame(); - })(), - testTelemetryModelIdentity, - )); - }; - const result = await withOpenAIResponsesCompactShim(inv, stubCtx, runIncomplete); - if (result.type !== 'events') throw new Error('expected events branch'); - const collected = await collectOpenAIResponsesProtocolEventsToResult(result.events); - assertEquals(collected.status, 'incomplete'); - assertEquals(collected.incomplete_details, { reason: 'max_output_tokens' }); -}); - -test('compact + flag on: compaction_trigger items are stripped before the upstream call', async () => { - const inv = makeInvocation( - { - input: [ - { type: 'message', role: 'user', content: 'real history' }, - { type: 'compaction_trigger' } as unknown as never, - ], - }, - { action: 'compact' }, - ); - - let seenPayload: OpenAIResponsesPayload | undefined; - await withOpenAIResponsesCompactShim(inv, stubCtx, () => { - seenPayload = inv.payload; - return fakeUpstreamRun('s')(); - }); - if (!seenPayload) throw new Error('expected the upstream call to fire'); - const items = seenPayload.input as Array<{ type: string }>; - assertEquals(items.every(i => i.type !== 'compaction_trigger'), true); -}); - -test('compact + flag on: history ending on an assistant message gets a synthetic terminal user prompt appended', async () => { - // Anthropic Messages rejects assistant prefill: a conversation that - // ends on an assistant turn returns 400 `This model does not support - // assistant message prefill`. The shim normalizes by appending a - // synthetic user-role nudge so the summarization call always ends on - // a user message — harmless on OpenAI-style upstreams and load-bearing - // for translated Anthropic ones. - const inv = makeInvocation( - { - input: [ - { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'hi' }] }, - { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'hello back', annotations: [] }] } as unknown as never, - ], - }, - { action: 'compact' }, - ); - - let seenPayload: OpenAIResponsesPayload | undefined; - await withOpenAIResponsesCompactShim(inv, stubCtx, () => { - seenPayload = inv.payload; - return fakeUpstreamRun('s')(); - }); - if (!seenPayload) throw new Error('expected the upstream call to fire'); - const items = seenPayload.input as Array<{ type: string; role?: string; content?: Array<{ type: string; text: string }> }>; - // [SUMMARIZATION_PROMPT system, user 'hi', assistant 'hello back', synthetic user nudge] - assertEquals(items.length, 4); - assertEquals(items[0].role, 'system'); - const tail = items[items.length - 1]; - assertEquals(tail.type, 'message'); - assertEquals(tail.role, 'user'); - assertEquals(tail.content?.[0].type, 'input_text'); -}); - -test('compact + flag off: passes through to run() unchanged', async () => { - const inv = makeInvocation( - { input: [{ type: 'message', role: 'user', content: 'compact me' }] }, - { action: 'compact', flagOn: false }, - ); - - let runCalled = false; - await withOpenAIResponsesCompactShim(inv, stubCtx, () => { - runCalled = true; - return fakeUpstreamRun('unused')(); - }); - // Flag off: shim early-returns without expansion or pivot. The inner - // run() is called directly, action stays 'compact', payload unchanged. - assertEquals(runCalled, true); - assertEquals(inv.action, 'compact'); -}); - -test('compact decrypt: replays mixed compaction items individually and preserves their output positions', async () => { - const inv = makeInvocation( - { - input: [{ type: 'message', role: 'user', content: 'compact me' }], - instructions: 'caller instructions must not enter the replay turn', - tools: [{ type: 'function', name: 'irrelevant', parameters: null, strict: null }], - }, - { action: 'compact', flagOn: false, decryptFlagOn: true }, - ); - - const nativeResponse: OpenAIResponsesResult = { - id: 'resp_native_compact', - object: 'response.compaction', - model: 'test-upstream-model', - status: 'completed', - output: [ - { type: 'message', id: 'msg_retained', role: 'user', status: 'completed', content: [{ type: 'input_text', text: 'retained tail' }] } as unknown as never, - { type: 'compaction', id: 'cmp_native_1', encrypted_content: 'OPAQUE_NATIVE_BLOB_1' }, - { type: 'message', id: 'msg_between', role: 'assistant', status: 'completed', content: [{ type: 'input_text', text: 'retained middle' }] } as unknown as never, - { type: 'compaction_summary', id: 'cmp_native_2', encrypted_content: 'OPAQUE_NATIVE_BLOB_2' }, - ], - error: null, - incomplete_details: null, - usage: { input_tokens: 100, output_tokens: 40, total_tokens: 140 }, - }; - - let calls = 0; - const result = await withOpenAIResponsesCompactShim(inv, stubCtx, async () => { - calls += 1; - if (calls === 1) { - assertEquals(inv.action, 'compact'); - return eventResult( - (async function* (): AsyncGenerator> { - yield eventFrame({ type: 'response.completed', sequence_number: 0, response: nativeResponse }); - yield doneFrame(); - })(), - testTelemetryModelIdentity, - { - finalMetadata: Promise.resolve({ - modelIdentity: testTelemetryModelIdentity, - billableUsage: { input: 100, cacheRead: 0, cacheWrite: 0, cacheWrite1h: 0, output: 40 }, - }), - }, - ); - } - - assertEquals(inv.action, 'generate'); - assertEquals(inv.payload.store, false); - assertEquals(inv.payload.instructions, undefined); - assertEquals(inv.payload.tools, undefined); - assertEquals(inv.payload.input.length, 3); - const [prefix, compaction, suffix] = inv.payload.input; - assertEquals(prefix.type, 'message'); - if (prefix.type !== 'message') throw new Error('expected prefix message'); - assertEquals(prefix.role, 'system'); - assertEquals(prefix.content, [{ - type: 'input_text', - text: 'Repeat the following text exactly, which may contain a compaction summary, character for character.', - }]); - assertEquals(compaction, nativeResponse.output[calls === 2 ? 1 : 3]); - assertEquals(suffix.type, 'message'); - if (suffix.type !== 'message') throw new Error('expected suffix message'); - assertEquals(suffix.role, 'system'); - assertEquals(suffix.content, [{ - type: 'input_text', - text: 'Output only the exact summary text, with no preface, explanation, markdown fence, or changes.', - }]); - const replay = await fakeUpstreamRun(`EXACT DECRYPTED SUMMARY ${calls - 1}`)(); - if (replay.type !== 'events') throw new Error('expected replay events'); - return { - ...replay, - finalMetadata: Promise.resolve({ - modelIdentity: testTelemetryModelIdentity, - billableUsage: { input: 10, cacheRead: 0, cacheWrite: 0, cacheWrite1h: 0, output: 20 }, - }), - }; - }); - - assertEquals(calls, 3); - if (result.type !== 'events') throw new Error(`expected events branch, got ${result.type}`); - const collected = await collectOpenAIResponsesProtocolEventsToResult(result.events); - assertEquals(collected.object, 'response.compaction'); - assertEquals(collected.output[0], nativeResponse.output[0]); - assertEquals(collected.output[2], nativeResponse.output[2]); - assertEquals(collected.output.map(item => item.type), ['message', 'compaction', 'message', 'compaction_summary']); - assertEquals(collected.output[1].id, 'cmp_native_1'); - assertEquals(collected.output[3].id, 'cmp_native_2'); - assertEquals(collected.usage, { input_tokens: 120, output_tokens: 80, total_tokens: 200 }); - assertEquals((await result.finalMetadata)?.billableUsage, { - input: 120, - cacheRead: 0, - cacheWrite: 0, - cacheWrite1h: 0, - output: 80, - }); - - const expanded = expandShimCompactionItems({ - model: 'm', - input: collected.output.slice(1) as OpenAIResponsesInputItem[], - }); - assertEquals(expanded.input, [ - { - type: 'message', - role: 'user', - content: [{ type: 'input_text', text: 'EXACT DECRYPTED SUMMARY 1' }], - }, - nativeResponse.output[2], - { - type: 'message', - role: 'user', - content: [{ type: 'input_text', text: 'EXACT DECRYPTED SUMMARY 2' }], - }, - ]); -}); - -test('gateway-owned compaction expands on an ordinary generate request even when both compact flags are off', async () => { - const encoded = encodeBase64UrlJson([{ - type: 'message', - role: 'user', - content: [{ type: 'input_text', text: 'RECOVERED SUMMARY' }], - }]); - const inv = makeInvocation( - { - input: [ - { type: 'compaction', id: 'cmp_decrypted', encrypted_content: encoded } as unknown as never, - { type: 'message', role: 'user', content: 'continue' }, - ], - }, - { flagOn: false }, - ); - - let seenInput: OpenAIResponsesInputItem[] | undefined; - await withOpenAIResponsesCompactShim(inv, stubCtx, () => { - seenInput = inv.payload.input; - return fakeUpstreamRun('done')(); - }); - - assertEquals(seenInput, [ - { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'RECOVERED SUMMARY' }] }, - { type: 'message', role: 'user', content: 'continue' }, - ]); -}); - -test('compact decrypt: preserves a generate response envelope for the compaction_trigger path', async () => { - const inv = makeInvocation( - { input: [{ type: 'message', role: 'user', content: 'history' }, { type: 'compaction_trigger' }] }, - { flagOn: false, decryptFlagOn: true }, - ); - let calls = 0; - const result = await withOpenAIResponsesCompactShim(inv, stubCtx, () => { - calls += 1; - if (calls === 1) { - const response: OpenAIResponsesResult = { - id: 'resp_trigger', - object: 'response', - model: 'test-upstream-model', - status: 'completed', - output: [{ type: 'compaction', id: 'cmp_trigger', encrypted_content: 'OPAQUE_TRIGGER_BLOB' } as unknown as never], - error: null, - incomplete_details: null, - usage: { input_tokens: 80, output_tokens: 30, total_tokens: 110 }, - }; - return Promise.resolve(eventResult( - (async function* (): AsyncGenerator> { - yield eventFrame({ type: 'response.completed', sequence_number: 0, response }); - yield doneFrame(); - })(), - testTelemetryModelIdentity, - )); - } - return fakeUpstreamRun('TRIGGER SUMMARY')(); - }); - - if (result.type !== 'events') throw new Error(`expected events branch, got ${result.type}`); - const collected = await collectOpenAIResponsesProtocolEventsToResult(result.events); - assertEquals(calls, 2); - assertEquals(collected.object, 'response'); - assertEquals(collected.output[0]?.type, 'compaction'); -}); - -test('compact decrypt: decrypts provider-restored native compaction output with intermediate compacting events', async () => { - const inv = makeInvocation( - { - input: [ - { type: 'message', role: 'user', content: 'prior turn' }, - { type: 'compaction_trigger' } as unknown as never, - ], - }, - { flagOn: false, decryptFlagOn: true }, - ); - - const nativeCompactionItem: OpenAIResponsesOutputItem = { - type: 'compaction', - id: 'cmp_native_capture', - encrypted_content: 'OPAQUE_CAPTURE_BLOB', - metadata: { turn_id: 'turn-123' }, - internal_chat_message_metadata_passthrough: { turn_id: 'turn-123' }, - }; - - const inProgressEnvelope: OpenAIResponsesResult = { - id: 'resp_capture', - object: 'response', - model: 'test-upstream-model', - status: 'in_progress', - output: [], - error: null, - incomplete_details: null, - }; - - const completedEnvelope: OpenAIResponsesResult = { - id: 'resp_capture', - object: 'response', - model: 'test-upstream-model', - status: 'completed', - output: [nativeCompactionItem], - error: null, - incomplete_details: null, - usage: { input_tokens: 150, output_tokens: 50, total_tokens: 200 }, - }; - - let calls = 0; - const result = await withOpenAIResponsesCompactShim(inv, stubCtx, async () => { - calls += 1; - if (calls === 1) { - return eventResult( - (async function* (): AsyncGenerator> { - yield eventFrame({ type: 'response.created', sequence_number: 0, response: inProgressEnvelope }); - yield eventFrame({ type: 'response.in_progress', sequence_number: 1, response: inProgressEnvelope }); - yield eventFrame({ - type: 'response.output_item.added', - sequence_number: 2, - output_index: 0, - item: nativeCompactionItem, - }); - yield eventFrame({ - type: 'response.compaction.compacting', - sequence_number: 3, - item_id: 'cmp_native_capture', - output_index: 0, - }); - yield eventFrame({ - type: 'response.compaction.compacting', - sequence_number: 4, - item_id: 'cmp_native_capture', - output_index: 0, - }); - yield eventFrame({ - type: 'response.output_item.done', - sequence_number: 5, - output_index: 0, - item: nativeCompactionItem, - }); - yield eventFrame({ type: 'response.completed', sequence_number: 6, response: completedEnvelope }); - yield doneFrame(); - })(), - testTelemetryModelIdentity, - { - finalMetadata: Promise.resolve({ - modelIdentity: testTelemetryModelIdentity, - billableUsage: { input: 150, cacheRead: 0, cacheWrite: 0, cacheWrite1h: 0, output: 50 }, - }), - }, - ); - } - - assertEquals(inv.action, 'generate'); - assertEquals(inv.payload.store, false); - assertEquals(inv.payload.input.length, 3); - const [, replayCompaction] = inv.payload.input; - assertEquals(replayCompaction, nativeCompactionItem); - - const replay = await fakeUpstreamRun('DECRYPTED CAPTURE SUMMARY')(); - if (replay.type !== 'events') throw new Error('expected replay events'); - return { - ...replay, - finalMetadata: Promise.resolve({ - modelIdentity: testTelemetryModelIdentity, - billableUsage: { input: 15, cacheRead: 0, cacheWrite: 0, cacheWrite1h: 0, output: 25 }, - }), - }; - }); - - assertEquals(calls, 2); - if (result.type !== 'events') throw new Error(`expected events branch, got ${result.type}`); - const collected = await collectOpenAIResponsesProtocolEventsToResult(result.events); - assertEquals(collected.output.length, 1); - const decryptedItem = collected.output[0] as unknown as { - type: string; - id: string; - encrypted_content: string; - metadata?: Record; - internal_chat_message_metadata_passthrough?: Record; - }; - assertEquals(decryptedItem.type, 'compaction'); - assertEquals(decryptedItem.id, 'cmp_native_capture'); - assertEquals(decryptedItem.metadata, { turn_id: 'turn-123' }); - assertEquals(decryptedItem.internal_chat_message_metadata_passthrough, { turn_id: 'turn-123' }); - - const expanded = expandShimCompactionItems({ - model: 'test-model', - input: [decryptedItem as unknown as OpenAIResponsesInputItem], - }); - const expandedUserMessage = expanded.input[0] as { - type: string; - role: string; - content: Array<{ type: string; text: string }>; - }; - assertEquals(expandedUserMessage.type, 'message'); - assertEquals(expandedUserMessage.role, 'user'); - assertEquals(expandedUserMessage.content[0].text, 'DECRYPTED CAPTURE SUMMARY'); -}); - -// ── Bug 1 — engagement gating ──────────────────────────────────────────────── -// -// The shim engages when EITHER the per-upstream flag is on OR the candidate's -// targetApi is not 'openaiResponses'. The compact-shape check inside (action === -// 'compact' OR input contains compaction_trigger) decides whether to simulate -// or just pass the request through. Together these gates make the shim -// structurally required on non-OpenAI-Responses upstreams (Anthropic Messages / -// OpenAI Chat Completions translation has no `compaction_trigger` variant) while -// keeping the flag as the operator opt-in for OpenAI-Responses-target upstreams. - -test('generate + compaction_trigger + flag off + messages target: shim simulates (structurally required)', async () => { - const inv = makeInvocation( - { - input: [ - { type: 'message', role: 'user', content: 'real history' }, - { type: 'compaction_trigger' } as unknown as never, - ], - }, - { action: 'generate', flagOn: false, targetApi: 'anthropicMessages' }, - ); - - let seenPayload: OpenAIResponsesPayload | undefined; - const result = await withOpenAIResponsesCompactShim(inv, stubCtx, () => { - seenPayload = inv.payload; - return fakeUpstreamRun('CONDENSED SUMMARY')(); - }); - - // The shim should have engaged: the upstream sees the summarization - // prompt (as a role=system input item at the head of the history) and a - // stripped (no compaction_trigger) history, and the result is the - // synthesized `response.compaction` envelope. - if (!seenPayload) throw new Error('expected the upstream call to fire'); - const innerItems = seenPayload.input as OpenAIResponsesInputItem[]; - const head = innerItems[0] as { type: string; role: string; content: Array<{ text: string }> }; - assertEquals(head.type, 'message'); - assertEquals(head.role, 'system'); - assertEquals(head.content[0].text.includes('CONTEXT CHECKPOINT COMPACTION'), true); - assertEquals(innerItems.every(i => i.type !== 'compaction_trigger'), true); - - if (result.type !== 'events') throw new Error(`expected events branch, got ${result.type}`); - const collected = await collectOpenAIResponsesProtocolEventsToResult(result.events); - assertEquals(collected.object, 'response.compaction'); -}); - -test('generate + compaction_trigger + flag off + responses target: shim passes through (flag opt-in not taken)', async () => { - const inv = makeInvocation( - { - input: [ - { type: 'message', role: 'user', content: 'real history' }, - { type: 'compaction_trigger' } as unknown as never, - ], - }, - { action: 'generate', flagOn: false, targetApi: 'openaiResponses' }, - ); - - let runCalled = false; - let seenAction: 'generate' | 'compact' | undefined; - let seenPayload: OpenAIResponsesPayload | undefined; - await withOpenAIResponsesCompactShim(inv, stubCtx, () => { - runCalled = true; - seenAction = inv.action; - seenPayload = inv.payload; - return fakeUpstreamRun('unused')(); - }); - - // Shim did not engage — request flowed through untouched. - assertEquals(runCalled, true); - assertEquals(seenAction, 'generate'); - // Payload still carries the trigger (no expansion, no strip, no pivot). - const items = seenPayload?.input as Array<{ type: string }>; - assertEquals(items.some(i => i.type === 'compaction_trigger'), true); -}); - -test('generate + compaction_trigger + flag on + messages target: shim simulates (same as flag-off path)', async () => { - const inv = makeInvocation( - { - input: [ - { type: 'message', role: 'user', content: 'real history' }, - { type: 'compaction_trigger' } as unknown as never, - ], - }, - { action: 'generate', targetApi: 'anthropicMessages' }, - ); - - let seenPayload: OpenAIResponsesPayload | undefined; - const result = await withOpenAIResponsesCompactShim(inv, stubCtx, () => { - seenPayload = inv.payload; - return fakeUpstreamRun('CONDENSED SUMMARY')(); - }); - - if (!seenPayload) throw new Error('expected the upstream call to fire'); - const innerItems = seenPayload.input as OpenAIResponsesInputItem[]; - const head = innerItems[0] as { type: string; role: string; content: Array<{ text: string }> }; - assertEquals(head.role, 'system'); - assertEquals(head.content[0].text.includes('CONTEXT CHECKPOINT COMPACTION'), true); - if (result.type !== 'events') throw new Error(`expected events branch, got ${result.type}`); - const collected = await collectOpenAIResponsesProtocolEventsToResult(result.events); - assertEquals(collected.object, 'response.compaction'); -}); - -test('generate + flag on: runs inbound expansion but does not pivot', async () => { - const userItem = { type: 'message' as const, role: 'user' as const, content: 'expanded' }; - const encoded = encodeBase64UrlJson([userItem]); - const inv = makeInvocation( - { - input: [ - { type: 'compaction', id: 'cmp_1', encrypted_content: encoded } as unknown as never, - { type: 'message', role: 'user', content: 'follow-up' }, - ], - }, - { action: 'generate' }, - ); - - let runCalled = false; - await withOpenAIResponsesCompactShim(inv, stubCtx, () => { - runCalled = true; - return fakeUpstreamRun('unused')(); - }); - assertEquals(runCalled, true); - // generate action stays as-is. - assertEquals(inv.action, 'generate'); - // Inbound expansion ran: the compaction item was replaced by `userItem`. - const items = inv.payload.input as Array<{ type: string; content?: unknown }>; - assertEquals(items.length, 2); - assertEquals(items[0], userItem); -}); - -test('compact + flag on: upstream api-error propagates', async () => { - const inv = makeInvocation( - { input: [{ type: 'message', role: 'user', content: 'history' }] }, - { action: 'compact' }, - ); - - const errorResult: ExecuteResult> = { - type: 'api-error', - source: 'upstream', - status: 502, - headers: new Headers({ 'content-type': 'application/json' }), - body: new TextEncoder().encode('{"error":"upstream blew up"}'), - }; - const result = await withOpenAIResponsesCompactShim(inv, stubCtx, () => Promise.resolve(errorResult)); - if (result.type !== 'api-error') throw new Error(`expected api-error, got ${result.type}`); - assertEquals(result.status, 502); -}); - -// ── Round-trip ──────────────────────────────────────────────────────────────── - -test('round-trip: outbound synthesis then inbound expansion recovers the summary message prefixed with the handoff prompt', async () => { - // Step 1: simulate compaction — returns the synthesized envelope with - // shim-encoded `encrypted_content`. - const inv = makeInvocation( - { input: [{ type: 'message', role: 'user', content: 'long convo' }] }, - { action: 'compact' }, - ); - const result = await withOpenAIResponsesCompactShim(inv, stubCtx, fakeUpstreamRun('SUMMARY TEXT')); - if (result.type !== 'events') throw new Error('expected events'); - const collected = await collectOpenAIResponsesProtocolEventsToResult(result.events); - const compactionItem = collected.output[0] as { type: string; id?: string; encrypted_content: string }; - - // Step 2: next turn echoes the compaction item back as an input item; - // inbound expansion replaces it with the summary message — carrying the - // handoff prefix that was baked into the blob at encode time. - const nextTurn: CanonicalOpenAIResponsesPayload = { - model: 'test-model', - input: [ - { type: 'compaction', id: compactionItem.id ?? 'cmp_rt', encrypted_content: compactionItem.encrypted_content } as unknown as never, - ], - }; - const expanded = expandShimCompactionItems(nextTurn); - const items = expanded.input as Array<{ type: string; role: string; content: Array<{ type: string; text: string }> }>; - assertEquals(items.length, 1); - assertEquals(items[0].type, 'message'); - assertEquals(items[0].role, 'user'); - assertEquals(items[0].content[0].text, `${SUMMARY_PREFIX}\nSUMMARY TEXT`); -}); - -test('compact + flag on: reads the summary from the provider-restored terminal snapshot', async () => { - const inv = makeInvocation( - { input: [{ type: 'message', role: 'user', content: 'long conversation history' }] }, - { action: 'compact' }, - ); - - const result = await withOpenAIResponsesCompactShim(inv, stubCtx, fakeUpstreamRun('CONDENSED SUMMARY')); - if (result.type !== 'events') throw new Error(`expected events branch, got ${result.type}`); - - const collected = await collectOpenAIResponsesProtocolEventsToResult(result.events); - const compaction = collected.output[0] as unknown as { type: string; encrypted_content: string }; - assertEquals(compaction.type, 'compaction'); - const decoded = JSON.parse(new TextDecoder().decode( - Uint8Array.from(atob(compaction.encrypted_content.replace(/-/g, '+').replace(/_/g, '/')), c => c.charCodeAt(0)), - )) as Array<{ content: Array<{ text: string }> }>; - assertEquals(decoded[0]?.content[0]?.text, `${SUMMARY_PREFIX}\nCONDENSED SUMMARY`); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/disable-reasoning-on-forced-tool-choice_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/disable-reasoning-on-forced-tool-choice_test.ts deleted file mode 100644 index 7e37e208a3..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/disable-reasoning-on-forced-tool-choice_test.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { test } from 'vitest'; - -import { withReasoningDisabledOnForcedToolChoice } from '../../../../../src/data-plane/chat/openai-responses/interceptors/disable-reasoning-on-forced-tool-choice.ts'; -import type { OpenAIResponsesInvocation } from '../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { doneFrame } from '@floway-dev/protocols/common'; -import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; -import { eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = () => - Promise.resolve( - eventResult( - (async function* () { - yield doneFrame(); - })(), - testTelemetryModelIdentity, - ), - ); - -const invocation = ( - payload: CanonicalOpenAIResponsesPayload, - enabledFlags: ReadonlySet = new Set(['disable-reasoning-on-forced-tool-choice']), -): OpenAIResponsesInvocation => ({ - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi: 'openaiResponses', - headers: new Headers(), - action: 'generate', -}); - -test('responses required tool_choice sets reasoning.effort to none', async () => { - const input = invocation({ - model: 'm', - input: [{ type: 'message', role: 'user', content: 'hi' }], - reasoning: { effort: 'high' }, - tool_choice: 'required', - }); - - await withReasoningDisabledOnForcedToolChoice(input, stubCtx, okEvents); - - assertEquals(input.payload.reasoning, { effort: 'none' }); - const out = input.payload as unknown as Record; - assertEquals(out.thinking, undefined); - assertEquals(out.enable_thinking, undefined); -}); - -test('responses object tool_choice is forced', async () => { - const input = invocation({ - model: 'm', - input: [{ type: 'message', role: 'user', content: 'hi' }], - reasoning: { effort: 'high' }, - tool_choice: { type: 'custom', name: 'x' }, - }); - - await withReasoningDisabledOnForcedToolChoice(input, stubCtx, okEvents); - - assertEquals(input.payload.reasoning, { effort: 'none' }); -}); - -test('responses non-forced tool_choice leaves reasoning untouched', async () => { - for (const tool_choice of ['auto', 'none'] as const) { - const input = invocation({ - model: 'm', - input: [{ type: 'message', role: 'user', content: 'hi' }], - reasoning: { effort: 'high' }, - tool_choice, - }); - - await withReasoningDisabledOnForcedToolChoice(input, stubCtx, okEvents); - - assertEquals(input.payload.reasoning, { effort: 'high' }); - } -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/normalize-empty-tools-tool-choice_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/normalize-empty-tools-tool-choice_test.ts deleted file mode 100644 index cc4de2180c..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/normalize-empty-tools-tool-choice_test.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { test } from 'vitest'; - -import { withEmptyToolsToolChoiceNormalized } from '../../../../../src/data-plane/chat/openai-responses/interceptors/normalize-empty-tools-tool-choice.ts'; -import type { OpenAIResponsesInvocation } from '../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; -import { eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); -const okEvents = () => Promise.resolve(eventResult((async function* () {})(), testTelemetryModelIdentity)); - -const invocation = ( - payload: CanonicalOpenAIResponsesPayload, - enabledFlags: ReadonlySet = new Set(['empty-tools-tool-choice-none']), -): OpenAIResponsesInvocation => ({ - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi: 'openaiResponses', - headers: new Headers(), - action: 'generate', -}); - -const base = { model: 'm', input: [{ type: 'message' as const, role: 'user' as const, content: 'hi' }] }; - -test('responses empty tools rewrites tool_choice to none', async () => { - const input = invocation({ ...base, tools: [], tool_choice: 'auto' }); - - await withEmptyToolsToolChoiceNormalized(input, stubCtx, okEvents); - - assertEquals(input.payload.tools, []); - assertEquals(input.payload.tool_choice, 'none'); -}); - -test('responses rewrite requires the flag and an explicit empty array', async () => { - const disabled = invocation({ ...base, tools: [], tool_choice: 'auto' }, new Set()); - const missing = invocation({ ...base, tool_choice: 'auto' }); - const present = invocation({ - ...base, - tools: [{ type: 'function', name: 'x', parameters: {} }], - tool_choice: 'auto', - }); - - await withEmptyToolsToolChoiceNormalized(disabled, stubCtx, okEvents); - await withEmptyToolsToolChoiceNormalized(missing, stubCtx, okEvents); - await withEmptyToolsToolChoiceNormalized(present, stubCtx, okEvents); - - assertEquals(disabled.payload.tool_choice, 'auto'); - assertEquals(missing.payload.tool_choice, 'auto'); - assertEquals(present.payload.tool_choice, 'auto'); -}); - -test('responses keeps tool choice when an input item supplies tools', async () => { - for (const item of [ - { type: 'additional_tools' as const, role: 'developer' as const, tools: [{ type: 'web_search' as const }] }, - { type: 'tool_search_output' as const, tools: [{ type: 'web_search' as const }] }, - ]) { - const input = invocation({ ...base, tools: [], input: [item], tool_choice: 'required' }); - await withEmptyToolsToolChoiceNormalized(input, stubCtx, okEvents); - assertEquals(input.payload.tool_choice, 'required'); - } -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/normalize-exclusive-cached-tokens_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/normalize-exclusive-cached-tokens_test.ts deleted file mode 100644 index 3f6ab6ce05..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/normalize-exclusive-cached-tokens_test.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { expect, test } from 'vitest'; - -import { withExclusiveCachedTokensNormalized } from '../../../../../src/data-plane/chat/openai-responses/interceptors/normalize-exclusive-cached-tokens.ts'; -import type { OpenAIResponsesInvocation } from '../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; -import { billableUsageFromOpenAIResponsesResult } from '../../../../../src/data-plane/chat/openai-responses/usage.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { type ExecuteResult, eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const invocation = ( - enabledFlags: ReadonlySet = new Set(['usage-exclusive-cached-tokens']), - targetApi: OpenAIResponsesInvocation['targetApi'] = 'openaiResponses', -): OpenAIResponsesInvocation => ({ - payload: { model: 'test-model', input: [{ type: 'message', role: 'user', content: 'hi' }] }, - candidate: stubModelCandidate({ enabledFlags }), - targetApi, - headers: new Headers(), - action: 'generate', -}); - -// Same accounting as the Charm Hyper OpenAI Chat Completions capture, in OpenAI Responses -// field names: 479 + 13312 + 373 = 14164, so total_tokens witnesses it. -const exclusiveUsage = (): NonNullable => ({ - input_tokens: 479, - output_tokens: 373, - total_tokens: 14164, - input_tokens_details: { cached_tokens: 13312 }, -}); - -const completedEvent = (usage: NonNullable): OpenAIResponsesStreamEvent => ({ - type: 'response.completed', - sequence_number: 0, - response: { - id: 'resp_test', - object: 'response', - model: 'test-model', - status: 'completed', - output: [], - output_text: '', - error: null, - incomplete_details: null, - usage, - }, -}); - -const run = async (ctx: OpenAIResponsesInvocation, usage: NonNullable): Promise => { - const result: ExecuteResult> = await withExclusiveCachedTokensNormalized(ctx, stubCtx, () => - Promise.resolve(eventResult( - (async function* () { - yield eventFrame(completedEvent(usage)); - })(), - testTelemetryModelIdentity, - ))); - if (result.type !== 'events') throw new Error('expected events result'); - const frames: ProtocolFrame[] = []; - for await (const frame of result.events) frames.push(frame); - assertEquals(frames.length, 1); - const frame = frames[0]; - if (frame.type !== 'event') throw new Error('expected event frame'); - if (!('response' in frame.event)) throw new Error('expected a response-carrying event'); - return frame.event.response; -}; - -const exclusiveUsageWithoutTotal = (): NonNullable => { - const { total_tokens: _withheld, ...rest } = exclusiveUsage(); - return rest as NonNullable; -}; - -const inclusiveUsage = (): NonNullable => ({ - input_tokens: 1000, - output_tokens: 50, - total_tokens: 1050, - input_tokens_details: { cached_tokens: 400 }, -}); - -test('folds on the totals alone, with no flag set', async () => { - const response = await run(invocation(new Set()), exclusiveUsage()); - assertEquals(response.usage?.input_tokens, 13791); - assertEquals(response.usage?.total_tokens, 14164); - assertEquals(response.usage?.input_tokens_details, { cached_tokens: 13312 }); -}); - -test('folds on the flag when the totals witness nothing', async () => { - const response = await run(invocation(), exclusiveUsageWithoutTotal()); - assertEquals(response.usage?.input_tokens, 13791); -}); - -test('leaves a normalized response billable instead of underflowing', async () => { - expect(() => billableUsageFromOpenAIResponsesResult({ usage: exclusiveUsage() })).toThrowError(RangeError); - - const response = await run(invocation(new Set()), exclusiveUsage()); - assertEquals( - billableUsageFromOpenAIResponsesResult(response), - { input: 479, cacheRead: 13312, cacheWrite: 0, cacheWrite1h: 0, output: 373 }, - ); -}); - -test('folds cache writes back as well', async () => { - const response = await run(invocation(), { - input_tokens: 100, - output_tokens: 10, - input_tokens_details: { cached_tokens: 120, cache_write_tokens: 80 }, - } as unknown as NonNullable); - assertEquals(response.usage?.input_tokens, 300); -}); - -test('leaves an inclusive response alone when no flag claims otherwise', async () => { - const response = await run(invocation(new Set()), inclusiveUsage()); - assertEquals(response.usage?.input_tokens, 1000); -}); - -test('raises when the flag claims exclusive and the totals say inclusive', async () => { - await expect(run(invocation(), inclusiveUsage())).rejects.toThrowError(/usage-exclusive-cached-tokens is enabled/); -}); - -test('raises naming the flag when the cache counts underflow with no verdict', async () => { - await expect(run(invocation(new Set()), exclusiveUsageWithoutTotal())) - .rejects.toThrowError(/enable usage-exclusive-cached-tokens/); -}); - -test('stands down entirely when the wire it speaks about is elsewhere', async () => { - const response = await run(invocation(undefined, 'openaiChatCompletions'), exclusiveUsageWithoutTotal()); - assertEquals(response.usage?.input_tokens, 479); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/strip-prompt-cache-key_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/strip-prompt-cache-key_test.ts deleted file mode 100644 index 962fc40dbf..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/strip-prompt-cache-key_test.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { test } from 'vitest'; - -import { withPromptCacheKeyStripped } from '../../../../../src/data-plane/chat/openai-responses/interceptors/strip-prompt-cache-key.ts'; -import type { OpenAIResponsesInvocation } from '../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { doneFrame } from '@floway-dev/protocols/common'; -import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; -import { eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = () => - Promise.resolve( - eventResult( - (async function* () { - yield doneFrame(); - })(), - testTelemetryModelIdentity, - ), - ); - -const invocation = ( - payload: CanonicalOpenAIResponsesPayload, - enabledFlags: ReadonlySet = new Set(['strip-prompt-cache-key']), -): OpenAIResponsesInvocation => ({ - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi: 'openaiResponses', - headers: new Headers(), - action: 'generate', -}); - -test('drops top-level prompt_cache_key when the flag is on', async () => { - const input = invocation({ - model: 'm', - input: [{ type: 'message', role: 'user', content: 'hi' }], - prompt_cache_key: 'thread-42', - }); - - await withPromptCacheKeyStripped(input, stubCtx, okEvents); - - assertEquals(Object.hasOwn(input.payload, 'prompt_cache_key'), false); -}); - -test('leaves the payload untouched when prompt_cache_key is absent', async () => { - const input = invocation({ - model: 'm', - input: [{ type: 'message', role: 'user', content: 'hi' }], - }); - const before = input.payload; - - await withPromptCacheKeyStripped(input, stubCtx, okEvents); - - assertEquals(input.payload, before); -}); - -test('is a no-op when the flag is not set on the candidate', async () => { - const input = invocation( - { - model: 'm', - input: [{ type: 'message', role: 'user', content: 'hi' }], - prompt_cache_key: 'thread-42', - }, - new Set(), - ); - - await withPromptCacheKeyStripped(input, stubCtx, okEvents); - - assertEquals(input.payload.prompt_cache_key, 'thread-42'); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/vendor-deepseek-normalize_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/vendor-deepseek-normalize_test.ts deleted file mode 100644 index 86fff3f91c..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/vendor-deepseek-normalize_test.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { test } from 'vitest'; - -import type { OpenAIResponsesInvocation } from '../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; -import { withVendorDeepSeekOpenAIResponsesNormalize } from '../../../../../src/data-plane/chat/openai-responses/interceptors/vendor-deepseek-normalize.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { doneFrame } from '@floway-dev/protocols/common'; -import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; -import { eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = () => - Promise.resolve( - eventResult( - (async function* () { - yield doneFrame(); - })(), - testTelemetryModelIdentity, - ), - ); - -const invocation = (payload: CanonicalOpenAIResponsesPayload, enabledFlags: ReadonlySet = new Set(['vendor-deepseek'])): OpenAIResponsesInvocation => ({ - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi: 'openaiResponses', - headers: new Headers(), - action: 'generate', -}); - -test("vendor-deepseek translates canonical reasoning.effort: 'none' into top-level thinking:{type:'disabled'}", async () => { - const input = invocation({ - model: 'deepseek-reasoner', - input: [{ type: 'message', role: 'user', content: 'hi' }], - reasoning: { effort: 'none' }, - }); - - await withVendorDeepSeekOpenAIResponsesNormalize(input, stubCtx, okEvents); - - const out = input.payload as unknown as Record; - assertEquals(out.reasoning, undefined); - assertEquals(out.thinking, { type: 'disabled' }); -}); - -test('vendor-deepseek leaves a real reasoning.effort value untouched (only the none sentinel triggers the rewrite)', async () => { - const input = invocation({ - model: 'deepseek-reasoner', - input: [{ type: 'message', role: 'user', content: 'hi' }], - reasoning: { effort: 'high' }, - }); - - await withVendorDeepSeekOpenAIResponsesNormalize(input, stubCtx, okEvents); - - assertEquals(input.payload.reasoning, { effort: 'high' }); - const out = input.payload as unknown as Record; - assertEquals(out.thinking, undefined); -}); - -test('vendor-deepseek early-returns when its flag is not set on the candidate', async () => { - const input = invocation({ model: 'deepseek-reasoner', input: [{ type: 'message', role: 'user', content: 'hi' }], reasoning: { effort: 'none' } }, new Set()); - - await withVendorDeepSeekOpenAIResponsesNormalize(input, stubCtx, okEvents); - - assertEquals(input.payload.reasoning, { effort: 'none' }); - const out = input.payload as unknown as Record; - assertEquals(out.thinking, undefined); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/vendor-qwen-normalize_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/vendor-qwen-normalize_test.ts deleted file mode 100644 index d29eb72433..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/vendor-qwen-normalize_test.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { test } from 'vitest'; - -import type { OpenAIResponsesInvocation } from '../../../../../src/data-plane/chat/openai-responses/interceptors/types.ts'; -import { withVendorQwenOpenAIResponsesNormalize } from '../../../../../src/data-plane/chat/openai-responses/interceptors/vendor-qwen-normalize.ts'; -import { mockChatGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; -import { doneFrame } from '@floway-dev/protocols/common'; -import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; -import { eventResult, type FlagId } from '@floway-dev/provider'; -import { assertEquals, stubModelCandidate, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubCtx = mockChatGatewayCtx(); - -const okEvents = () => - Promise.resolve( - eventResult( - (async function* () { - yield doneFrame(); - })(), - testTelemetryModelIdentity, - ), - ); - -const invocation = (payload: CanonicalOpenAIResponsesPayload, enabledFlags: ReadonlySet = new Set(['vendor-qwen'])): OpenAIResponsesInvocation => ({ - payload, - candidate: stubModelCandidate({ enabledFlags }), - targetApi: 'openaiResponses', - headers: new Headers(), - action: 'generate', -}); - -test("vendor-qwen translates canonical reasoning.effort: 'none' into top-level enable_thinking:false", async () => { - const input = invocation({ - model: 'qwen-max', - input: [{ type: 'message', role: 'user', content: 'hi' }], - reasoning: { effort: 'none' }, - }); - - await withVendorQwenOpenAIResponsesNormalize(input, stubCtx, okEvents); - - const out = input.payload as unknown as Record; - assertEquals(out.reasoning, undefined); - assertEquals(out.enable_thinking, false); -}); - -test('vendor-qwen leaves a real reasoning.effort value untouched (only the none sentinel triggers the rewrite)', async () => { - const input = invocation({ model: 'qwen-max', input: [{ type: 'message', role: 'user', content: 'hi' }], reasoning: { effort: 'high' } }); - - await withVendorQwenOpenAIResponsesNormalize(input, stubCtx, okEvents); - - assertEquals(input.payload.reasoning, { effort: 'high' }); - const out = input.payload as unknown as Record; - assertEquals(out.enable_thinking, undefined); -}); - -test('vendor-qwen early-returns when its flag is not set on the candidate', async () => { - const input = invocation({ model: 'qwen-max', input: [{ type: 'message', role: 'user', content: 'hi' }], reasoning: { effort: 'none' } }, new Set()); - - await withVendorQwenOpenAIResponsesNormalize(input, stubCtx, okEvents); - - assertEquals(input.payload.reasoning, { effort: 'none' }); - const out = input.payload as unknown as Record; - assertEquals(out.enable_thinking, undefined); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/items/store_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/items/store_test.ts index 9afbc7402f..c9c203ade8 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/items/store_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/items/store_test.ts @@ -342,7 +342,7 @@ describe('OpenAIResponsesStatefulStore', () => { }); test('non-OpenAI-Responses-source store holds request-private tool state but persists and reads nothing', async () => { - // Translated sources (Anthropic Messages/Gemini generateContent/OpenAI Chat Completions) still run the server-tool shim, + // Translated sources (Anthropic Messages/Gemini generateContent/OpenAI Chat Completions) still run the hosted-tool shim, // whose per-attempt private-payload scratchpad lives on the store; the // no-backing store keeps that working without any durable state. const store = createNonOpenAIResponsesSourceStore('key-a'); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/response-resource_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/response-resource_test.ts index 42bcc25d5f..f05ec60955 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/response-resource_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/response-resource_test.ts @@ -199,7 +199,7 @@ describe('OpenAI Responses resource completion', () => { }); // The normalizers run on the resolved value, so a tool array the upstream - // echoed — or one the server-tool shim reconstructed — is completed like any + // echoed — or one the hosted-tool shim reconstructed — is completed like any // other. Resolving first and normalizing after is what closes that path. it('completes a minimal function tool the upstream echoed back', () => { const upstream: OpenAIResponsesResult = { diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/run-dump_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/run-dump_test.ts new file mode 100644 index 0000000000..938a71ef9d --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/run-dump_test.ts @@ -0,0 +1,69 @@ +// A pipelined chat turn over HTTP is recorded as its whole run, the same way the WebSocket +// transport records one. The two entries share `openChatPrologue`, and the sink it opens is +// the only thing that turns a recording into events — without it a key with retention +// configured pays for a record holding nothing but its metadata. + +import { test, vi } from 'vitest'; + +import { initDumpBroker, initDumpStore } from '../../../../src/dump/registry.ts'; +import { eventsOf, installDumpStubs, runRecordOf } from '../../../dump/test-fixtures.ts'; +import { copilotModels, flushAsyncWork, requestAppWithWarmModels as requestApp, setupAppTest, sseOpenAIResponsesResponse } from '../../../test-utils/app.ts'; +import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; + +const withOpenAIResponsesUpstream = async (run: () => Promise): Promise => + await withMockedFetch( + async request => { + const url = new URL(request.url); + if (url.hostname === 'update.code.visualstudio.com') return jsonResponse(['1.110.1']); + if (url.pathname === '/copilot_internal/v2/token') { + return jsonResponse({ token: 'copilot-access-token', expires_at: 4102444800, refresh_in: 3600, endpoints: { api: 'https://api.individual.githubcopilot.com' } }); + } + if (url.pathname === '/models') { + return jsonResponse(copilotModels([{ id: 'gpt-direct-responses', supported_endpoints: ['/responses'] }])); + } + if (url.pathname === '/responses') { + return sseOpenAIResponsesResponse({ + id: 'resp_run_dump', + object: 'response', + model: 'gpt-direct-responses', + status: 'completed', + output: [], + output_text: 'done', + usage: { input_tokens: 3, output_tokens: 5, total_tokens: 8 }, + }); + } + throw new Error(`Unhandled fetch ${request.url}`); + }, + run, + ); + +test('POST /v1/responses records the run it was, not an empty record', async () => { + const { apiKey, repo } = await setupAppTest(); + await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); + + await withOpenAIResponsesUpstream(async () => { + const response = await requestApp('/v1/responses', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + body: JSON.stringify({ model: 'gpt-direct-responses', input: 'hello', stream: false }), + }); + assertEquals(response.status, 200); + await response.json(); + }); + + await flushAsyncWork(); + await vi.waitFor(() => assertEquals(dumps.stored.length, 1)); + + const stored = dumps.stored[0]; + assertExists(stored); + const record = runRecordOf(stored.record); + assertEquals(record.meta.path, '/v1/responses'); + + // The events are the point: a recording whose sink was never handed to the runner still + // produces a record, and it holds nothing. Every run enters and leaves at least one stage. + const events = eventsOf(record); + assertEquals(events.length > 0, true); + assertEquals(events.some(event => event.type === 'stage.entered'), true); + assertEquals(events.some(event => event.type === 'stage.leaved'), true); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/serve_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/serve_test.ts deleted file mode 100644 index 7b3dc9c1f9..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/serve_test.ts +++ /dev/null @@ -1,652 +0,0 @@ -import { afterEach, test, vi } from 'vitest'; - -import { TEST_OPENAI_RESPONSES_RETENTION_SECONDS, testOpenAIResponsesStatePolicy } from './test-policy.ts'; -import { createOpenAIResponsesHttpStore, MemoryOpenAIResponsesStatefulBacking, LayeredOpenAIResponsesStatefulStore } from '../../../../src/data-plane/chat/openai-responses/items/store.ts'; -import type { ChatGatewayCtx } from '../../../../src/data-plane/chat/shared/gateway-ctx.ts'; -import { initRepo } from '../../../../src/repo/index.ts'; -import type { StoredOpenAIResponsesItem, StoredOpenAIResponsesSnapshot } from '../../../../src/repo/types.ts'; -import { InMemoryRepo } from '../../../repo/memory.ts'; -import { mockChatGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import { type AliasRules, doneFrame, eventFrame, type ModelEndpoints, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesPayload, OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { type ModelCandidate, directFetcher, type ProviderOpenAIResponsesResult, type ProviderStreamResult, type OpenAIResponsesAction, type UpstreamCallOptions } from '@floway-dev/provider'; -import { assert, assertEquals, stubProvider, stubInternalModel } from '@floway-dev/test-utils'; - -// Mock the resolver seam so each test hands the serve exactly the provider -// candidates it wants, optionally with an alias-rules overlay attached. -// `sawModel` defaults to true when at least one candidate was queued; the -// `model-missing` failure tests queue an empty list and expect `sawModel: -// false` so the serve renders 404 rather than 400. -interface QueuedResolution { - readonly candidates: readonly ModelCandidate[]; - readonly sawModel: boolean; - readonly failedUpstreams: readonly string[]; -} -const resolutionsQueue: QueuedResolution[] = []; -const lastResolveCall: { model?: string } = {}; -vi.mock('../../../../src/data-plane/providers/resolution.ts', async importOriginal => { - const original = await importOriginal(); - return { - ...original, - enumerateModelCandidates: vi.fn(async ({ model }: { model: string }) => { - lastResolveCall.model = model; - const next = resolutionsQueue.shift(); - if (next === undefined) throw new Error('serve_test: no resolution enqueued'); - return next; - }), - }; -}); - -const { openaiResponsesServe } = await import('../../../../src/data-plane/chat/openai-responses/serve.ts'); -const { expandPreviousResponseId } = await import('../../../../src/data-plane/chat/openai-responses/serve-prep.ts'); - -const API_KEY_ID = 'key_serve_test'; - -const queueResolution = ( - candidates: readonly ModelCandidate[], - extra: { sawModel?: boolean; aliasRules?: AliasRules } = {}, -): void => { - const rules = extra.aliasRules; - resolutionsQueue.push({ - candidates: rules !== undefined ? candidates.map(c => ({ ...c, rules })) : candidates, - sawModel: extra.sawModel ?? candidates.length > 0, - failedUpstreams: [], - }); -}; - -afterEach(() => { resolutionsQueue.length = 0; }); - -const installRepo = (): InMemoryRepo => { - const repo = new InMemoryRepo(); - initRepo(repo); - void repo.apiKeys.save({ - id: API_KEY_ID, userId: 1, name: 'OpenAI Responses test key', key: 'raw-responses-test', - serverSecret: '99'.repeat(32), createdAt: '2026-01-01T00:00:00.000Z', - upstreamIds: null, deletedAt: null, dumpRetentionSeconds: null, - openaiResponsesRetentionSeconds: TEST_OPENAI_RESPONSES_RETENTION_SECONDS, - }); - return repo; -}; - -const makeGatewayCtx = (store?: ChatGatewayCtx['store']) => - mockChatGatewayCtx({ - apiKeyId: API_KEY_ID, - wantsStream: true, - store: store ?? createOpenAIResponsesHttpStore(testOpenAIResponsesStatePolicy(API_KEY_ID), Date.now(), true), - }); - -const makePayload = (overrides: Partial = {}): CanonicalOpenAIResponsesPayload => ({ - model: 'test-model', - input: [{ type: 'message', role: 'user', content: 'hello' }], - ...overrides, -}); - -// Compact tests need a real input array (a bare string can't carry the -// compaction trigger or item_reference shapes the routing layer cares -// about). Default to the kept-user-message the existing happy-path test -// uses; override `input` when a test needs a different shape. -const compactPayload = (overrides: Partial = {}): CanonicalOpenAIResponsesPayload => - makePayload({ input: [{ type: 'message', role: 'user', content: 'kept' }], ...overrides }); - -const makeOpenAIResponsesResult = (id = 'resp_test'): OpenAIResponsesResult => ({ - id, - object: 'response', - model: 'test-model', - status: 'completed', - output: [{ - type: 'message', - id: 'msg_1', - role: 'assistant', - status: 'completed', - content: [{ type: 'output_text', text: 'hi', annotations: [] }], - }], - output_text: 'hi', - error: null, - incomplete_details: null, -}); - -const makeProtocolFrames = async function* (events: readonly E[]): AsyncGenerator> { - for (const event of events) yield eventFrame(event); - yield doneFrame(); -}; - -const makeCandidate = (overrides: { - upstream?: string; - modelId?: string; - endpoints?: ModelEndpoints; - callOpenAIResponses?: (model: unknown, body: unknown, action: OpenAIResponsesAction, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise; - callAnthropicMessages?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise>; - callOpenAIChatCompletions?: (model: unknown, body: unknown, signal?: AbortSignal, opts?: UpstreamCallOptions) => Promise>; -} = {}): ModelCandidate => { - const upstream = overrides.upstream ?? 'up_test'; - const provider = stubProvider({ - callOpenAIResponses: overrides.callOpenAIResponses, - callAnthropicMessages: overrides.callAnthropicMessages, - callOpenAIChatCompletions: overrides.callOpenAIChatCompletions, - }); - return { - provider: { - upstreamId: upstream, - kind: 'custom', - name: upstream, - inboundHeaderAllowlist: [], - disabledPublicModelIds: [], - modelPrefix: null, - modelsCache: null, pipelines: {}, - instance: provider, - }, - // Default keeps stubInternalModel's three-endpoint map intact; tests that - // need a rejected candidate pass an explicit `endpoints` override. - model: stubInternalModel({ - id: overrides.modelId ?? 'test-model', - ...(overrides.endpoints ? { endpoints: overrides.endpoints } : {}), - }, upstream), - fetcher: directFetcher, - }; -}; - -const collectEvents = async (events: AsyncIterable>): Promise => { - const out: OpenAIResponsesStreamEvent[] = []; - for await (const frame of events) { - if (frame.type === 'event') out.push(frame.event); - } - return out; -}; - -test('generate routes a native OpenAI Responses candidate end to end', async () => { - installRepo(); - const completed: OpenAIResponsesStreamEvent = { - type: 'response.completed', - sequence_number: 0, - response: makeOpenAIResponsesResult(), - }; - const callOpenAIResponses = vi.fn(async (): Promise => ({ - action: 'generate', ok: true, - events: makeProtocolFrames([completed]), - modelKey: 'test-model-key', - headers: new Headers(), - })); - const candidate = makeCandidate({ upstream: 'up_a', callOpenAIResponses }); - queueResolution([candidate]); - - const result = await openaiResponsesServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - const events = await collectEvents(result.events); - assert(events.length >= 1); - assertEquals(callOpenAIResponses.mock.calls.length, 1); -}); - -test('compact returns a result envelope from the wrapped attempt', async () => { - installRepo(); - const compactionItem = { type: 'compaction' as const, id: 'cmp_1', encrypted_content: 'ENC' }; - const compactionResult: OpenAIResponsesResult = { - ...makeOpenAIResponsesResult(), - object: 'response.compaction', - output: [compactionItem] as unknown as OpenAIResponsesResult['output'], - usage: { input_tokens: 12, output_tokens: 3, total_tokens: 15 }, - }; - const observedModelIds: string[] = []; - const callOpenAIResponses = vi.fn(async (model: unknown, _body: unknown, action: OpenAIResponsesAction): Promise => { - if (action !== 'compact') throw new Error(`expected compact, got ${action}`); - observedModelIds.push((model as { id: string }).id); - return { action: 'compact', ok: true, result: compactionResult, modelKey: 'test-model-key' }; - }); - const candidate = makeCandidate({ upstream: 'up_a', modelId: 'gpt-target', callOpenAIResponses }); - queueResolution([candidate]); - const payload = compactPayload({ model: 'gpt-alias' }); - - const result = await openaiResponsesServe.compact({ - payload, - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'result'); - if (result.type !== 'result') throw new Error('unreachable'); - assertEquals(result.result.object, 'response.compaction'); - assertEquals(callOpenAIResponses.mock.calls.length, 1); - assertEquals(callOpenAIResponses.mock.calls[0][2], 'compact'); - assertEquals(observedModelIds, ['gpt-target']); - assertEquals(payload.model, 'gpt-alias'); -}); - -test('generate falls through to the next candidate when the first yields an upstream error', async () => { - installRepo(); - const originalImageUrl = 'data:image/png;base64,AQID'; - const payload = makePayload({ - input: [{ - type: 'message', - role: 'user', - content: [{ type: 'input_image', image_url: originalImageUrl, detail: 'auto' }], - }], - }); - const firstError = new Response(JSON.stringify({ error: { message: 'nope' } }), { - status: 502, headers: new Headers({ 'content-type': 'application/json' }), - }); - const firstCall = vi.fn(async (_model: unknown, body: unknown): Promise => { - const item = (body as Omit).input[0]; - if (item.type !== 'message' || !Array.isArray(item.content) || item.content[0]?.type !== 'input_image') throw new Error('expected image content'); - item.content[0].image_url = 'data:image/webp;base64,COMPRESSED'; - return { action: 'generate', ok: false, response: firstError, modelKey: 'first-key' }; - }); - const completed: OpenAIResponsesStreamEvent = { - type: 'response.completed', - sequence_number: 0, - response: makeOpenAIResponsesResult('resp_second'), - }; - let fallbackImageUrl: string | null | undefined; - const secondCall = vi.fn(async (_model: unknown, body: unknown): Promise => { - const item = (body as Omit).input[0]; - if (item.type !== 'message' || !Array.isArray(item.content) || item.content[0]?.type !== 'input_image') throw new Error('expected image content'); - fallbackImageUrl = item.content[0].image_url; - return { action: 'generate', ok: true, events: makeProtocolFrames([completed]), modelKey: 'second-key', headers: new Headers() }; - }); - const first = makeCandidate({ upstream: 'up_a', callOpenAIResponses: firstCall }); - const second = makeCandidate({ upstream: 'up_b', callOpenAIResponses: secondCall }); - queueResolution([first, second]); - - const result = await openaiResponsesServe.generate({ - payload, - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - // The narrowed candidate list exists exactly so a transient upstream - // failure (5xx/429/network) on one entry rolls over to the next. The - // second candidate's success is the request's final answer. - assertEquals(result.type, 'events'); - assertEquals(firstCall.mock.calls.length, 1); - assertEquals(secondCall.mock.calls.length, 1); - assertEquals(fallbackImageUrl, originalImageUrl); - const sourceItem = payload.input[0]; - if (sourceItem.type !== 'message' || !Array.isArray(sourceItem.content) || sourceItem.content[0]?.type !== 'input_image') throw new Error('expected source image content'); - assertEquals(sourceItem.content[0].image_url, originalImageUrl); -}); - -// A mid-attempt throw (interceptor bug / translation error / provider-layer -// JS exception not represented as a ChatServeFailure) must attribute the perf -// error row to the throwing candidate, not the previous one that already -// failed cleanly with a 5xx. -test('mid-attempt throw stamps telemetry with the throwing candidate, not the previous one', async () => { - installRepo(); - const firstError = new Response(JSON.stringify({ error: { message: 'nope' } }), { - status: 502, headers: new Headers({ 'content-type': 'application/json' }), - }); - const firstCall = vi.fn(async (): Promise => ({ - action: 'generate', ok: false, response: firstError, modelKey: 'first-key', - })); - const secondCall = vi.fn(async (): Promise => { - throw new Error('simulated provider-layer JS exception'); - }); - queueResolution([ - makeCandidate({ upstream: 'up_a', callOpenAIResponses: firstCall }), - makeCandidate({ upstream: 'up_b', callOpenAIResponses: secondCall }), - ]); - - const ctx = makeGatewayCtx(); - await openaiResponsesServe.generate({ - payload: makePayload(), - ctx, - headers: new Headers(), - }).then( - () => { throw new Error('expected openaiResponsesServe.generate to throw'); }, - (error: unknown) => { - assertEquals((error as Error).message, 'simulated provider-layer JS exception'); - }, - ); - - assertEquals(firstCall.mock.calls.length, 1); - assertEquals(secondCall.mock.calls.length, 1); - assertEquals(ctx.attempt.telemetry?.upstream, 'up_b'); -}); - -test('generate renders model-missing when no candidates are available', async () => { - installRepo(); - queueResolution([]); - - const result = await openaiResponsesServe.generate({ - payload: makePayload({ model: 'unknown-model' }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 404); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assertEquals(body.error.type, 'invalid_request_error'); - assertEquals(body.error.message, 'Model unknown-model is not available on any configured upstream.'); -}); - -test('generate filters out candidates whose endpoints do not satisfy the responses preference and renders model-unsupported as a 400', async () => { - installRepo(); - const callOpenAIResponses = vi.fn(); - // openaiResponsesTarget prefers responses > messages > openai-chat-completions; an - // endpoints-only `openaiCompletions` candidate matches none and is filtered out. - queueResolution([makeCandidate({ upstream: 'up_x', endpoints: { openaiCompletions: {} }, callOpenAIResponses })]); - - const result = await openaiResponsesServe.generate({ - payload: makePayload({ model: 'wrong-endpoint-model' }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 400); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assertEquals(body.error.type, 'invalid_request_error'); - assert(typeof body.error.message === 'string' && body.error.message.includes('does not support')); - assertEquals(callOpenAIResponses.mock.calls.length, 0); -}); - -test('compact renders model-missing as a 404 when no candidates are available', async () => { - installRepo(); - queueResolution([]); - - const result = await openaiResponsesServe.compact({ - payload: compactPayload({ model: 'unknown-model' }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 404); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assertEquals(body.error.type, 'invalid_request_error'); - assert(typeof body.error.message === 'string' && body.error.message.includes('not available')); -}); - -test('compact renders model-unsupported as a 400 when the only candidate\'s endpoints don\'t satisfy responses target preferences', async () => { - installRepo(); - const callOpenAIResponses = vi.fn(); - // openaiResponsesTarget prefers responses > messages > openai-chat-completions; an - // endpoints-only `openaiCompletions` candidate matches none and is filtered out. - queueResolution([makeCandidate({ upstream: 'up_x', endpoints: { openaiCompletions: {} }, callOpenAIResponses })]); - - const result = await openaiResponsesServe.compact({ - payload: compactPayload({ model: 'wrong-endpoint-model' }), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 400); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assertEquals(body.error.type, 'invalid_request_error'); - assert(typeof body.error.message === 'string' && body.error.message.includes('does not support')); - assertEquals(callOpenAIResponses.mock.calls.length, 0); -}); - -test('expandPreviousResponseId prepends snapshot items and strips the previous_response_id field', async () => { - const repo = installRepo(); - const previousMessageId = 'msg_previous'; - await repo.openaiResponsesItems.insertMany([{ - id: previousMessageId, - apiKeyId: API_KEY_ID, - itemHash: 'previous-message-hash', - payload: { item: { type: 'message', id: previousMessageId, role: 'user', content: 'first turn' } }, - refreshedAt: Date.now(), - }], 0); - const snapshot: StoredOpenAIResponsesSnapshot = { - id: 'resp_prev', - apiKeyId: API_KEY_ID, - itemIds: [previousMessageId], - refreshedAt: Date.now(), - }; - await repo.openaiResponsesSnapshots.insert(snapshot); - - const store = createOpenAIResponsesHttpStore(testOpenAIResponsesStatePolicy(API_KEY_ID), Date.now(), true); - const expanded = await expandPreviousResponseId( - makePayload({ - previous_response_id: 'resp_prev', - input: [{ type: 'message', role: 'user', content: 'second turn' }], - }), - store, - ); - - assertEquals(expanded.previous_response_id, undefined); - assertEquals(expanded.input.length, 2); - assertEquals(expanded.input[0], { type: 'item_reference', id: previousMessageId }); - assertEquals(expanded.input[1], { type: 'message', role: 'user', content: 'second turn' }); -}); - -// In-memory store backed by the layered implementation but with no repo -// behind it, so an `expandPreviousResponseId` test can sit on a snapshot -// that lives nowhere else. -const memoryStore = async (snapshots: readonly StoredOpenAIResponsesSnapshot[], items: readonly StoredOpenAIResponsesItem[]) => { - const backing = new MemoryOpenAIResponsesStatefulBacking(); - for (const item of items) await backing.insertItems([item]); - for (const snapshot of snapshots) await backing.insertSnapshot(snapshot); - return new LayeredOpenAIResponsesStatefulStore({ - apiKeyId: API_KEY_ID, - reads: [backing], - writes: [backing], - }); -}; - -test('expandPreviousResponseId resolves snapshots from a non-repo-backed store', async () => { - installRepo(); // affinity lookups in the wider flow still need a repo, but here the helper only touches the store. - const id = 'msg_memory'; - const item: StoredOpenAIResponsesItem = { - id, - apiKeyId: API_KEY_ID, - itemHash: 'memory-message-hash', - payload: { item: { type: 'message', id, role: 'user', content: 'remembered' } }, - refreshedAt: 1_000, - }; - const snapshot: StoredOpenAIResponsesSnapshot = { - id: 'resp_mem', - apiKeyId: API_KEY_ID, - itemIds: [id], - refreshedAt: 1_000, - }; - const store = await memoryStore([snapshot], [item]); - - const expanded = await expandPreviousResponseId( - makePayload({ previous_response_id: 'resp_mem', input: [{ type: 'message', role: 'user', content: 'new turn' }] }), - store, - ); - - assertEquals(expanded.input.length, 2); - assertEquals(expanded.input[0], { type: 'item_reference', id }); -}); - -test('generate falls through translate-out to messages target', async () => { - installRepo(); - const callAnthropicMessages = vi.fn(async (): Promise> => ({ - ok: true, - events: makeProtocolFrames([ - { - type: 'message_start', - message: { - id: 'msg_translated', - type: 'message', - role: 'assistant', - content: [], - model: 'test-model', - stop_reason: null, - stop_sequence: null, - usage: { input_tokens: 4, output_tokens: 0 }, - }, - }, - { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, - { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'hi' } }, - { type: 'content_block_stop', index: 0 }, - { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 1 } }, - { type: 'message_stop' }, - ]), - modelKey: 'messages-key', - headers: new Headers(), - })); - const candidate = makeCandidate({ upstream: 'up_m', endpoints: { anthropicMessages: {} }, callAnthropicMessages }); - queueResolution([candidate]); - - const result = await openaiResponsesServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callAnthropicMessages.mock.calls.length, 1); -}); - -test('Anthropic Messages biology refusal becomes a non-retryable Codex OpenAI Responses policy failure', async () => { - installRepo(); - const callAnthropicMessages = vi.fn(async (): Promise> => ({ - ok: true, - events: makeProtocolFrames([ - { - type: 'message_start', - message: { - id: 'msg_bio_refusal', - type: 'message', - role: 'assistant', - content: [], - model: 'claude-fable-5', - stop_reason: null, - stop_sequence: null, - usage: { input_tokens: 4, output_tokens: 0 }, - }, - }, - { - type: 'message_delta', - delta: { - stop_reason: 'refusal', - stop_details: { - type: 'refusal', - category: 'bio', - explanation: 'This request could enable biological harm.', - }, - stop_sequence: null, - }, - usage: { output_tokens: 0 }, - }, - { type: 'message_stop' }, - ]), - modelKey: 'messages-key', - headers: new Headers(), - })); - const candidate = makeCandidate({ upstream: 'up_m', endpoints: { anthropicMessages: {} }, callAnthropicMessages }); - queueResolution([candidate]); - - const result = await openaiResponsesServe.generate({ payload: makePayload(), ctx: makeGatewayCtx(), headers: new Headers() }); - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - const events = await collectEvents(result.events); - const failed = events.find((event): event is Extract => event.type === 'response.failed'); - - assertEquals(failed?.response.status, 'failed'); - assertEquals(failed?.response.error, { - code: 'bio_policy', - message: 'This content was flagged for possible biological risk. This request could enable biological harm.', - }); - assertEquals(callAnthropicMessages.mock.calls.length, 1); -}); - -test('generate falls through translate-out to openai-chat-completions target', async () => { - installRepo(); - const callOpenAIChatCompletions = vi.fn(async (): Promise> => ({ - ok: true, - events: makeProtocolFrames([ - { - id: 'chatcmpl_translated', object: 'chat.completion.chunk', created: 0, model: 'test-model', - choices: [{ index: 0, delta: { role: 'assistant' }, finish_reason: null }], - }, - { - id: 'chatcmpl_translated', object: 'chat.completion.chunk', created: 0, model: 'test-model', - choices: [{ index: 0, delta: { content: 'hi' }, finish_reason: null }], - }, - { - id: 'chatcmpl_translated', object: 'chat.completion.chunk', created: 0, model: 'test-model', - choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], - }, - { - id: 'chatcmpl_translated', object: 'chat.completion.chunk', created: 0, model: 'test-model', - choices: [], usage: { prompt_tokens: 4, completion_tokens: 2, total_tokens: 6 }, - }, - ]), - modelKey: 'chat-completions-key', - headers: new Headers(), - })); - const candidate = makeCandidate({ upstream: 'up_c', endpoints: { openaiChatCompletions: {} }, callOpenAIChatCompletions }); - queueResolution([candidate]); - - const result = await openaiResponsesServe.generate({ - payload: makePayload(), - ctx: makeGatewayCtx(), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - assertEquals(callOpenAIChatCompletions.mock.calls.length, 1); -}); - -test('alias resolution swaps the inbound model id for the target and overlays rules onto the OpenAI Responses IR', async () => { - installRepo(); - const capturedBodies: OpenAIResponsesPayload[] = []; - const observedModelIds: string[] = []; - const callOpenAIResponses = vi.fn(async (model: unknown, body: unknown): Promise => { - observedModelIds.push((model as { id: string }).id); - capturedBodies.push(body as OpenAIResponsesPayload); - return { action: 'generate', ok: true, events: makeProtocolFrames([{ type: 'response.completed', sequence_number: 0, response: makeOpenAIResponsesResult() }]), modelKey: 'gpt-5.4', headers: new Headers() }; - }); - const candidate = makeCandidate({ upstream: 'up_a', modelId: 'gpt-5.4', callOpenAIResponses }); - queueResolution([candidate], { - aliasRules: { reasoning: { effort: 'high', summary: 'detailed' }, verbosity: 'medium', serviceTier: 'priority' }, - }); - - const payload = makePayload({ model: 'gpt-fast' }); - const result = await openaiResponsesServe.generate({ - payload, - ctx: makeGatewayCtx(createOpenAIResponsesHttpStore(testOpenAIResponsesStatePolicy(API_KEY_ID), Date.now(), true)), - headers: new Headers(), - }); - - assertEquals(result.type, 'events'); - if (result.type !== 'events') throw new Error('unreachable'); - await collectEvents(result.events); - - // Resolver and caller payload retain the inbound alias; the provider model - // argument carries the resolved target id while the body omits `model`. - assertEquals(lastResolveCall.model, 'gpt-fast'); - assertEquals(observedModelIds, ['gpt-5.4']); - assertEquals(payload.model, 'gpt-fast'); - const observed = capturedBodies[0]!; - assertEquals(observed.reasoning?.effort, 'high'); - assertEquals(observed.reasoning?.summary, 'detailed'); - assertEquals(observed.text?.verbosity, 'medium'); - assertEquals(observed.service_tier, 'priority'); -}); - -test('alias whose targets have no kind-matching binding surfaces as the regular model-missing 404', async () => { - installRepo(); - queueResolution([], { sawModel: false }); - - const result = await openaiResponsesServe.generate({ - payload: makePayload({ model: 'gpt-fast' }), - ctx: makeGatewayCtx(createOpenAIResponsesHttpStore(testOpenAIResponsesStatePolicy(API_KEY_ID), Date.now(), true)), - headers: new Headers(), - }); - - assertEquals(result.type, 'api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - assertEquals(result.status, 404); - const body = JSON.parse(new TextDecoder().decode(result.body)); - assertEquals(body.error.message, 'Model gpt-fast is not available on any configured upstream.'); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts index 60f496e5cd..aa5bad6e41 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/websocket_test.ts @@ -3,17 +3,16 @@ import { onTestFinished, test, vi } from 'vitest'; import { app } from '../../../../src/app.ts'; import { hashOpenAIResponsesItem } from '../../../../src/data-plane/chat/openai-responses/items/identity.ts'; -import { openaiResponsesServe } from '../../../../src/data-plane/chat/openai-responses/serve.ts'; import { KEEP_ALIVE_EVENT_TYPE } from '../../../../src/data-plane/chat/openai-responses/websocket.ts'; import { DOWNSTREAM_KEEP_ALIVE_INTERVAL_MS } from '../../../../src/data-plane/shared/sse.ts'; import { initDumpBroker, initDumpStore } from '../../../../src/dump/registry.ts'; -import { initBackgroundSchedulerResolver } from '../../../../src/runtime/background.ts'; -import { edgeRecordOf, installDumpStubs } from '../../../dump/test-fixtures.ts'; +import { tokenCountsFromUsage } from '../../../../src/repo/usage-metrics.ts'; +import { eventsOf, installDumpStubs, runRecordOf } from '../../../dump/test-fixtures.ts'; import { saveUpstreamForTest } from '../../../repo/upstreams.ts'; import { FakeTime } from '../../../test-time.ts'; import { buildCodexUpstreamRecord, codexModels, copilotModels, flushAsyncWork, setupAppTest, sseResponse, sseOpenAIResponsesResponse, warmModelsForTest } from '../../../test-utils/app.ts'; -import { trackBackground } from '../../../test-utils/background-tracker.ts'; import { installWorkerWebSocketRuntime, type TestWorkerWebSocket } from '../../../test-utils/worker-websocket.ts'; +import { createRunReader, type DumpEvent } from '@floway-dev/pipeline'; import { assert, assertEquals, assertExists, assertStringIncludes, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; const waitForMessages = async ( @@ -91,30 +90,6 @@ const connectOpenAIResponsesWebSocket = async (apiKey: string, upgradeHeaders: R return pair.client; }; -// The upgrade registers exactly one runtime background task: the session -// lifetime promise, which resolves once the socket is closed and every -// session-scoped write has drained. Capturing it lets a test observe the -// instant the runtime would be free to evict the isolate — on Cloudflare -// that is the deadline every session-scoped write has to beat. -const connectOpenAIResponsesWebSocketCapturingSessionLifetime = async ( - apiKey: string, -): Promise<{ client: TestWorkerWebSocket; sessionLifetime: Promise }> => { - const registered: Promise[] = []; - initBackgroundSchedulerResolver(_c => promise => { - registered.push(Promise.resolve(promise)); - trackBackground(promise); - }); - try { - const client = await connectOpenAIResponsesWebSocket(apiKey); - assertEquals(registered.length, 1, 'expected the upgrade to register exactly one background task'); - const sessionLifetime = registered[0]; - assertExists(sessionLifetime); - return { client, sessionLifetime }; - } finally { - initBackgroundSchedulerResolver(_c => trackBackground); - } -}; - let currentRuntime: ReturnType | undefined; const activeRuntime = (): ReturnType => { @@ -254,17 +229,26 @@ test('OpenAI Responses WebSocket starts capturing on the next turn when dump ret const stored = dumps.stored[0]; assertExists(stored); assertEquals(stored.keyId, apiKey.id); - const request = edgeRecordOf(stored.record).request; - assertEquals(request.method, 'WS'); - assertEquals(request.path, '/v1/responses'); - assertEquals(JSON.parse(new TextDecoder().decode(request.body)), { + // The shape follows the endpoint, and this turn was served through the pipeline: it is + // recorded as its whole run rather than as a pair of edges. One socket carries many + // turns, so each names itself — `WS /v1/responses`, not the `GET` upgrade that opened + // the connection — and the frame that opened it is that run's request, both as the + // bytes the record measured and as the payload the chain ran on. + const record = runRecordOf(stored.record); + assertEquals(record.meta.method, 'WS'); + assertEquals(record.meta.path, '/v1/responses'); + assertEquals(record.meta.requestBytes, new TextEncoder().encode(JSON.stringify({ type: 'response.create', event_id: 'capture-after-enable', response: { model: 'gpt-direct-responses', input: 'capture-after-enable', }, - }); + })).byteLength); + assert( + new TextDecoder().decode(record.events).includes('"capture-after-enable"'), + 'expected the run to record the payload the frame opened the turn with', + ); client.close(); }), ); @@ -318,6 +302,92 @@ test('OpenAI Responses WebSocket dump responseBytes equals the UTF-8 payload byt ); }); +const recordedClientEvents = (record: Parameters[0]): unknown[] => { + const events = eventsOf(runRecordOf(record)); + const read = createRunReader(); + const frames = new Map(); + let selected: number | undefined; + for (const event of events) { + const decoded = read(event as unknown as DumpEvent); + const client = decoded?.facts?.['response.chat.clientFrames']; + if (typeof client === 'object' && client !== null && 'stream' in client) selected = client.stream as number; + if (event.type === 'stream.frame') { + const values = frames.get(event.streamId as number) ?? []; + values.push(...(decoded?.frames ?? []).map(frame => (frame as { event: unknown }).event)); + frames.set(event.streamId as number, values); + } + } + assertExists(selected); + assertEquals(events.some(event => event.type === 'stream.end' && event.streamId === selected), true); + return frames.get(selected) ?? []; +}; + +test('OpenAI Responses WebSocket records one frame per event it sent', async () => { + const { apiKey, repo } = await setupAppTest(); + await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); + + await withSuccessfulOpenAIResponsesUpstream( + async () => await withWorkerWebSocketRuntime(async () => { + const client = await connectOpenAIResponsesWebSocket(apiKey.key); + const recorded = recordRawMessages(client); + try { + await completeOpenAIResponsesTurn(client, 'one-frame-per-event'); + await vi.waitFor(() => assertEquals(dumps.stored.length, 1)); + + // A keep-alive is the transport's own idle write rather than a frame of the turn, so + // it is not one of the events the record is counted against. + const sent = recorded.messages + .map(message => JSON.parse(message) as { type?: unknown }) + .filter(message => message.type !== KEEP_ALIVE_EVENT_TYPE); + assert(sent.length > 0, 'expected the turn to have sent events at all'); + assertEquals(recordedClientEvents(dumps.stored[0]?.record), sent); + } finally { + recorded.stop(); + client.close(); + } + }), + ); +}); + +// A turn on this transport is a run, and a run settles once: the chain reads what the +// upstream metered off its own events as they pass, and the transport settles that reading +// after the last frame has gone out. The rows are filtered to this turn's own key because +// every `setupAppTest` mints a fresh one, and background writes from earlier files in the +// suite resolve against whichever repo was installed last. +test('OpenAI Responses WebSocket settles a streamed turn once, with what the chain read off it', async () => { + const { apiKey, repo } = await setupAppTest(); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); + await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); + + await withSuccessfulOpenAIResponsesUpstream( + async () => await withWorkerWebSocketRuntime(async () => { + const client = await connectOpenAIResponsesWebSocket(apiKey.key); + await completeOpenAIResponsesTurn(client, 'settled-turn'); + client.close(); + }), + ); + await flushAsyncWork(); + + const usage = (await repo.usage.listAll()).filter(row => row.keyId === apiKey.id); + assertEquals(usage.length, 1); + assertEquals(usage[0]?.model, 'gpt-direct-responses'); + assertEquals(tokenCountsFromUsage(usage[0]!), { input: 3, output: 5 }); + + const performance = (await repo.performance.listAll()).filter(row => row.keyId === apiKey.id); + assertEquals(performance.length, 1); + assertEquals(performance[0]?.requests, 1); + assertEquals(performance[0]?.errorsNoOutput, 0); + + // The same reading, on the record: a turn that was settled is a turn the dump names the + // upstream's model and token counts on. + await vi.waitFor(() => assertEquals(dumps.stored.length, 1)); + const meta = runRecordOf(dumps.stored[0]?.record).meta; + assertEquals(meta.model, 'gpt-direct-responses'); + assertEquals(meta.inputTokens, 3); + assertEquals(meta.outputTokens, 5); +}); + test('OpenAI Responses WebSocket rejects the next turn after its API key is rotated', async () => { const { apiKey, repo } = await setupAppTest(); @@ -352,6 +422,8 @@ test('OpenAI Responses WebSocket rejects the next turn after its API key is rota test('OpenAI Responses WebSocket reports a failed turn when an output item cannot be persisted', async () => { const { apiKey, repo } = await setupAppTest(); + await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); const persistence = vi.spyOn(repo.openaiResponsesItems, 'insertMany').mockRejectedValue(new Error('simulated item persistence failure')); try { await withMockedFetch( @@ -403,6 +475,9 @@ test('OpenAI Responses WebSocket reports a failed turn when an output item canno assertEquals(error.error?.message, 'simulated item persistence failure'); assert(!messages.some(message => message.type === 'response.output_item.done')); assert(!messages.some(isTerminalResponseEvent)); + await vi.waitFor(() => assertEquals(dumps.stored.length, 1)); + assertEquals(recordedClientEvents(dumps.stored[0]?.record), messages); + assertEquals(dumps.stored[0]?.record.meta.error, { kind: 'failed', reason: 'simulated item persistence failure' }); }), ); } finally { @@ -411,7 +486,9 @@ test('OpenAI Responses WebSocket reports a failed turn when an output item canno }); test('OpenAI Responses WebSocket keep-alive waits for the first event and takes a slot in the stream sequence', async () => { - const { apiKey } = await setupAppTest(); + const { apiKey, repo } = await setupAppTest(); + await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); // Captured before the clock is faked: the turn's frames cross real event-loop // turns (upstream body reads, item persistence), which a faked `setTimeout` // cannot yield to. @@ -551,6 +628,8 @@ test('OpenAI Responses WebSocket keep-alive waits for the first event and takes ], 'expected the keep-alive to take a slot and shift every later event past it', ); + assert(await drainFramesUntil(() => dumps.stored.length === 1), 'expected the completed turn dump to persist'); + assertEquals(recordedClientEvents(dumps.stored[0]?.record), messages.filter(message => message.type !== KEEP_ALIVE_EVENT_TYPE)); } finally { client.removeEventListener('message', onMessage); } @@ -856,7 +935,9 @@ test('OpenAI Responses WebSocket store:false keeps session snapshots without dur }); test('OpenAI Responses WebSocket answers a Codex generate:false prewarm locally and continues from it', async () => { - const { apiKey } = await setupAppTest(); + const { apiKey, repo } = await setupAppTest(); + await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); const upstreamBodies: Record[] = []; await withMockedFetch( @@ -909,6 +990,13 @@ test('OpenAI Responses WebSocket answers a Codex generate:false prewarm locally // routed turn states. assertEquals(prewarmCompleted.response?.output?.map(item => item.type), ['reasoning']); assertEquals(upstreamBodies.length, 0); + await vi.waitFor(() => assertEquals(dumps.stored.length, 1)); + assertEquals(await repo.usage.listAll(), []); + assertEquals(await repo.performance.listAll(), []); + assertEquals(dumps.stored[0]?.record.meta.model, 'gpt-direct-responses'); + assertExists(dumps.stored[0]?.record.meta.upstream); + assertEquals(dumps.stored[0]?.record.meta.inputTokens, null); + assertEquals(dumps.stored[0]?.record.meta.outputTokens, null); const turnTerminal = waitForMessages(client, messages => messages.some(isTerminalResponseEvent)); client.send(JSON.stringify({ @@ -923,6 +1011,7 @@ test('OpenAI Responses WebSocket answers a Codex generate:false prewarm locally await turnTerminal; assertEquals(upstreamBodies.length, 1); + await vi.waitFor(async () => assertEquals((await repo.usage.listAll()).reduce((sum, row) => sum + row.requests, 0), 1)); const body = upstreamBodies[0] as { generate?: unknown; previous_response_id?: unknown; input: Array<{ type: string; role?: string; content?: unknown }> }; assertEquals(Object.hasOwn(body, 'generate'), false); assertEquals(body.previous_response_id, undefined); @@ -1474,22 +1563,16 @@ test('OpenAI Responses WebSocket aborts the in-flight OpenAI Responses request w ); }); -// A turn schedules its dump and usage writes from its terminal `finally`, so -// nothing of its own sits in the session's pending set while it streams. The -// session lifetime therefore has to hold the in-flight message chain itself: -// a client that closes mid-turn otherwise finds that set empty at the exact -// moment the close resolves `sessionClosed`, and the lifetime would resolve — -// freeing Cloudflare to evict the isolate — before the interrupted turn had -// recorded anything. -test('OpenAI Responses WebSocket holds the session lifetime open until a turn the client interrupted has recorded its dump', async () => { - const { apiKey, repo } = await setupAppTest(); - await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); - const dumps = installDumpStubs(initDumpStore, initDumpBroker); - const encoder = new TextEncoder(); - let resolveUpstreamReadStarted!: () => void; - const upstreamReadStarted = new Promise(resolve => { - resolveUpstreamReadStarted = resolve; - }); +// A throw that escapes the run never reached the settlement stage, so the sample +// that stage would have written is the WS transport's outer catch to write — +// alongside its sendError / dump.failed / dump.finalize it calls +// `recordFailedRequest(ctx, ctx.attempt.telemetry)`, and the row lands attributed +// to the candidate that was being dialled when it threw. The throw here is a real +// one: a refusal whose body cannot be read. `callOpenAIResponsesUpstream` stamps this +// candidate's telemetry before it dials and then reads the refusal's own words, so +// the read's rejection is a mid-attempt throw with attribution already in place. +test('OpenAI Responses WebSocket outer catch records a failed perf sample attributed to the throwing candidate', async () => { + const { apiKey, repo, copilotUpstream } = await setupAppTest(); await withMockedFetch( async request => { @@ -1502,132 +1585,46 @@ test('OpenAI Responses WebSocket holds the session lifetime open until a turn th return jsonResponse(copilotModels([{ id: 'gpt-direct-responses', supported_endpoints: ['/responses'] }])); } if (url.pathname === '/responses') { - return new Response(new ReadableStream({ - start(controller) { - controller.enqueue(encoder.encode(`event: response.created\ndata: ${JSON.stringify({ - type: 'response.created', - response: { - id: 'resp_ws_lifetime', - object: 'response', - model: 'gpt-direct-responses', - status: 'in_progress', - output: [], - output_text: '', - }, - sequence_number: 0, - })}\n\n`)); - // The turn never reaches a terminal event. The client's close - // reaches this body through the request signal, which is how a - // real streaming upstream is torn down mid-flight. - request.signal.addEventListener('abort', () => { - controller.error(request.signal.reason); - }, { once: true }); - }, - pull() { - resolveUpstreamReadStarted(); - }, - }), { - headers: { 'content-type': 'text/event-stream' }, - }); + return new Response( + new ReadableStream({ + start(controller) { controller.error(new Error('simulated mid-attempt provider throw')); }, + }), + { status: 400, headers: { 'content-type': 'application/json' } }, + ); } throw new Error(`Unhandled fetch ${request.url}`); }, async () => await withWorkerWebSocketRuntime(async () => { - const { client, sessionLifetime } = await connectOpenAIResponsesWebSocketCapturingSessionLifetime(apiKey.key); + const client = await connectOpenAIResponsesWebSocket(apiKey.key); + const received = waitForMessages(client, messages => messages.length === 1); client.send(JSON.stringify({ type: 'response.create', - event_id: 'evt_ws_lifetime', - response: { - model: 'gpt-direct-responses', - input: 'hello', - }, + event_id: 'evt_throw', + response: { model: 'gpt-direct-responses', input: 'hello' }, })); - await waitForMessages(client, messages => messages.length >= 1); - await upstreamReadStarted; - client.close(); - - await sessionLifetime; - // Sampled at the instant the runtime would be free to drop the isolate, - // deliberately without flushing background work first: the interrupted - // turn's dump has to be stored by then, not merely scheduled. - assertEquals(dumps.stored.length, 1); + const [errorMessage] = await received; + assertExists(errorMessage); + assertEquals(errorMessage.type, 'error'); + assertEquals(errorMessage.status, 500); + assertEquals(errorMessage.event_id, 'evt_throw'); }), ); -}); -// The four chat HTTP transports render a mid-attempt throw (interceptor -// bug, translation error, provider-layer JS exception not represented as a ChatServeFailure) through an -// `internalErrorResult(..., ctx.attempt.telemetry)` envelope, -// which internally reaches `recordFailedRequest` and lands an error row -// attributed to the throwing candidate. The WS transport's outer catch -// must do the same: alongside its sendError / dump.failed / dump.finalize, -// it calls `recordFailedRequest(ctx, ctx.attempt.telemetry)` so -// the failure shows up in performance_summary. -test('OpenAI Responses WebSocket outer catch records a failed perf sample attributed to the throwing candidate', async () => { - const { apiKey, repo } = await setupAppTest(); - - // Mirror what openaiResponsesServe.generate would have stamped before failing - // — telemetry set for the throwing candidate — then throw. - const generateSpy = vi.spyOn(openaiResponsesServe, 'generate').mockImplementation(async ({ ctx }) => { - ctx.attempt.telemetry = { - keyId: apiKey.id, - model: 'gpt-direct-responses', - upstream: 'up_throwing', - operation: 'chat', - runtimeLocation: 'TEST', - }; - throw new Error('simulated mid-attempt provider throw'); - }); - - try { - await withMockedFetch( - async request => { - const url = new URL(request.url); - if (url.hostname === 'update.code.visualstudio.com') return jsonResponse(['1.110.1']); - if (url.pathname === '/copilot_internal/v2/token') { - return jsonResponse({ token: 'copilot-access-token', expires_at: 4102444800, refresh_in: 3600, endpoints: { api: 'https://api.individual.githubcopilot.com' } }); - } - if (url.pathname === '/models') { - return jsonResponse(copilotModels([{ id: 'gpt-direct-responses', supported_endpoints: ['/responses'] }])); - } - throw new Error(`Unhandled fetch ${request.url}`); - }, - async () => await withWorkerWebSocketRuntime(async () => { - const client = await connectOpenAIResponsesWebSocket(apiKey.key); - const received = waitForMessages(client, messages => messages.length === 1); - client.send(JSON.stringify({ - type: 'response.create', - event_id: 'evt_throw', - response: { model: 'gpt-direct-responses', input: 'hello' }, - })); - - const [errorMessage] = await received; - assertExists(errorMessage); - assertEquals(errorMessage.type, 'error'); - assertEquals(errorMessage.status, 500); - assertEquals(errorMessage.event_id, 'evt_throw'); - }), - ); - - await flushAsyncWork(); - - // Filter to the throwing upstream: earlier WS tests in the same file - // schedule background recordFailedRequest calls through the session - // scheduler, and the shared `getRepo()` global resolves them against - // whichever repo `setupAppTest` last installed — so cross-test rows can - // land here. Only the row from the mocked generate is load-bearing for - // this fix. - const perfRows = (await repo.performance.listAll()).filter(row => row.upstream === 'up_throwing'); - assertEquals(perfRows.length, 1); - assertEquals(perfRows[0]?.upstream, 'up_throwing'); - assertEquals(perfRows[0]?.model, 'gpt-direct-responses'); - assertEquals(perfRows[0]?.operation, 'chat'); - assertEquals(perfRows[0]?.errorsNoOutput, 1); - assertEquals(perfRows[0]?.requests, 1); - } finally { - generateSpy.mockRestore(); - } + await flushAsyncWork(); + + // Filter to this turn's own key: earlier WS tests in the same file schedule + // background performance writes through the session scheduler, and the shared + // `getRepo()` global resolves them against whichever repo `setupAppTest` last + // installed — so cross-test rows for the same upstream can land here. Every + // `setupAppTest` mints a fresh api key, which is what tells them apart. + const perfRows = (await repo.performance.listAll()) + .filter(row => row.keyId === apiKey.id && row.upstream === copilotUpstream.id); + assertEquals(perfRows.length, 1); + assertEquals(perfRows[0]?.model, 'gpt-direct-responses'); + assertEquals(perfRows[0]?.operation, 'chat'); + assertEquals(perfRows[0]?.errorsNoOutput, 1); + assertEquals(perfRows[0]?.requests, 1); }); test('OpenAI Responses WebSocket dispatches each Codex turn with the metadata blob that turn carried', async () => { diff --git a/packages/gateway/__tests__/data-plane/chat/provider-handoffs_test.ts b/packages/gateway/__tests__/data-plane/chat/provider-handoffs_test.ts new file mode 100644 index 0000000000..841b476a0b --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/provider-handoffs_test.ts @@ -0,0 +1,106 @@ +import { expect, test } from 'vitest'; + +import { emitOpenAIChatCompletions } from '../../../src/data-plane/chat/openai-chat-completions/emit.ts'; +import { openaiChatCompletionsWire } from '../../../src/data-plane/chat/openai-chat-completions/wire.ts'; +import { prologueFor } from '../../../src/data-plane/pipeline/serve.ts'; +import type { StreamOutcome } from '../../../src/data-plane/pipeline/serve.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { isReplayableBody } from '@floway-dev/http'; +import { compose, move, run, type Deferred, type Event } from '@floway-dev/pipeline'; +import type { SseFrame } from '@floway-dev/protocols/common'; +import type { ModelCandidate } from '@floway-dev/provider'; +import { createCustomPipelines } from '@floway-dev/provider-custom'; +import { stubModelCandidate } from '@floway-dev/test-utils'; + +const config = { baseUrl: 'https://custom.example', authStyle: 'none' as const, endpoints: {}, ingressHeadersRules: [], modelsFetch: { enabled: false }, models: [] }; +const first = { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'wire-model', choices: [{ index: 0, delta: { content: 'partial' }, finish_reason: null }] }; +const usage = { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'wire-model', choices: [], usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 } }; + +const runWire = async (response: Response) => { + const gateway = mockChatGatewayCtx({ wantsStream: true }); + const base = prologueFor(gateway, { body: { bytes: new Uint8Array(), streamError: null }, headers: [] }); + const plain = stubModelCandidate({ model: { id: 'alias', endpoints: { openaiChatCompletions: {} } }, providerData: 'wire-model' }); + let calls = 0; + const candidate: ModelCandidate = { + ...plain, + provider: { ...plain.provider, pipelines: createCustomPipelines(config) }, + fetcher: async (url, init) => { + expect(url).toBe('https://custom.example/v1/chat/completions'); + if (!isReplayableBody(init.body)) throw new Error('JSON HTTP content must be replayable'); + calls += 1; + expect(await new Response(init.body.open()).json()).toMatchObject({ model: 'wire-model', stream: true, messages: [{ role: 'user', content: 'hello' }], stream_options: { include_usage: true } }); + return response; + }, + }; + const [selector] = base.services.rememberCandidates([candidate]); + const events: Event[] = []; + const executed = await run(compose, Record>('typedChat', [emitOpenAIChatCompletions, ...openaiChatCompletionsWire('response.chat.openaiChatCompletions.streamedUsage')]), move({ + 'ingress.chat.openaiChatCompletions.wantsStream': true, + 'ingress.chat.openaiChatCompletions.wantsUsageChunk': false, + 'ingress.chat.sourceProtocol': 'openaiChatCompletions', + 'ingress.http.headers': [], + 'request.chat.openaiChatCompletions': { model: 'alias', messages: [{ role: 'user', content: 'hello' }] }, + 'route.attempt': selector, + }), { + ...base.services, gateway, + recordProtocolFrames: (frames: AsyncIterable) => frames, + selectAffinity: () => {}, + dump: (event: Event) => { events.push(event); }, + }); + return { executed, events, calls: () => calls }; +}; + +for (const client of ['unread', 'partial', 'complete'] as const) test(`typed provider handoff preserves HTTP 201 and drains one reading after ${client} client output`, async () => { + const { executed, events, calls } = await runWire(new Response([first, usage].map(event => `data: ${JSON.stringify(event)}\n\n`).join(''), { status: 201, headers: { 'content-type': 'text/event-stream', 'x-upstream': 'kept' } })); + expect(executed.facts['response.http.status']).toBe(201); + expect(executed.facts['response.http.headers']).toContainEqual(['x-upstream', 'kept']); + const frames = executed.facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable; + if (client === 'partial') { + const iterator = frames[Symbol.asyncIterator](); + expect((await iterator.next()).value?.data).toContain('partial'); + await iterator.return?.(); + } else if (client === 'complete') { + const sent: SseFrame[] = []; + for await (const frame of frames) sent.push(frame); + expect(sent).toHaveLength(1); + } + await executed.drain(); + const outcome = await (executed.facts['response.chat.openaiChatCompletions.streamedUsage'] as Deferred); + expect(outcome.failed).toBe(client !== 'complete'); + expect(outcome.billable).toHaveLength(1); + expect(outcome.billable[0].quantities).toMatchObject({ input_tokens: '10', output_tokens: '5' }); + expect(calls()).toBe(1); + expect(events).toContainEqual(expect.objectContaining({ type: 'stage.entered', name: 'http' })); + expect(events).toContainEqual(expect.objectContaining({ type: 'stage.entered', facts: expect.objectContaining({ 'request.provider.payload': expect.objectContaining({ messages: [{ role: 'user', content: 'hello' }] }), 'request.provider.model': expect.objectContaining({ enabledFlags: [] }) }) })); +}); + +test('an actual upstream quantity-formatting fault rejects produced usage with the original error', async () => { + const raw = `data: ${JSON.stringify(first)}\n\ndata: {"id":"c1","object":"chat.completion.chunk","model":"wire-model","choices":[],"usage":{"prompt_tokens":1,"completion_tokens":1e999,"total_tokens":1}}\n\ndata: [DONE]\n\n`; + const { executed, calls } = await runWire(new Response(raw, { headers: { 'content-type': 'text/event-stream' } })); + const sent: SseFrame[] = []; + for await (const frame of executed.facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable) sent.push(frame); + const reading = executed.facts['response.chat.openaiChatCompletions.streamedUsage'] as Deferred; + let original: unknown; + try { await reading; } catch (error) { original = error; } + expect(original).toBeInstanceOf(TypeError); + expect((original as Error).message).toContain('Infinity'); + expect(JSON.parse(sent.at(-1)!.data)).toMatchObject({ error: { name: 'TypeError', message: (original as Error).message } }); + expect(executed.facts['response.usage.billable']).toMatchObject([{ quantities: {} }]); + expect(calls()).toBe(1); + let teardown: unknown; + try { await executed.drain(); } catch (error) { teardown = error; } + const causes = teardown instanceof AggregateError ? teardown.errors as unknown[] : [teardown]; + expect(causes.length).toBeGreaterThan(0); + expect(causes.every(error => error === original)).toBe(true); +}); + +test('actual native HTTP zero usage remains measured after the complete wire and client projection', async () => { + const zero = { ...usage, usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 } }; + const { executed } = await runWire(new Response([first, zero].map(event => `data: ${JSON.stringify(event)}\n\n`).join(''), { headers: { 'content-type': 'text/event-stream' } })); + const iterator = (executed.facts['response.chat.openaiChatCompletions.rendered'] as AsyncIterable)[Symbol.asyncIterator](); + let step = await iterator.next(); + while (!step.done) step = await iterator.next(); + await executed.drain(); + const outcome = await (executed.facts['response.chat.openaiChatCompletions.streamedUsage'] as Deferred); + expect(outcome).toMatchObject({ failed: false, billable: [{ quantities: { input_tokens: '0', output_tokens: '0' }, pricingFacts: { inputTokens: 0 } }] }); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/resolve-candidates_test.ts b/packages/gateway/__tests__/data-plane/chat/resolve-candidates_test.ts new file mode 100644 index 0000000000..2698d8ef75 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/resolve-candidates_test.ts @@ -0,0 +1,64 @@ +import { expect, test, vi } from 'vitest'; + +import type { ChatFacts } from '../../../src/data-plane/chat/facts.ts'; +import { resolveChatCandidates } from '../../../src/data-plane/chat/resolve-candidates.ts'; +import { createCandidateRegistry } from '../../../src/data-plane/pipeline/candidates.ts'; +import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { compose, defineStage, move, run } from '@floway-dev/pipeline'; +import { stubModelCandidate } from '@floway-dev/test-utils'; + +vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ + ...(await importOriginal()), + enumerateModelCandidates: vi.fn(), +})); + +test('resolution publishes pure candidate request projections without provider handles or dispatching an untried candidate', async () => { + const first = stubModelCandidate(); + const second = stubModelCandidate({ model: { id: 'second' } }); + const unsupported = stubModelCandidate({ model: { id: 'unsupported', endpoints: {} } }); + const payload = { model: 'model', messages: [{ role: 'user' as const, content: 'question' }] }; + const changed = { ...payload, messages: [{ role: 'user' as const, content: 'projected' }] }; + const projectFirst = vi.fn(() => payload); + const projectSecond = vi.fn(() => changed); + const evaluated: unknown[] = []; + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates: [first, second, unsupported], sawModel: true, failedUpstreams: [] }); + const resolver = resolveChatCandidates({ + requestKey: 'request.chat.openaiChatCompletions', + canServe: candidate => candidate !== unsupported, + affinity: async actual => { + expect(actual).toBe(payload); + return { + requiredTargets: [], + evaluateCandidate: candidate => { + evaluated.push(candidate); + return { kind: 'accepted', degrades: false, preferred: true, materialize: candidate === first ? projectFirst : projectSecond }; + }, + }; + }, + unsupported: model => model, + refuse: () => ({}), + refuses: [], + }); + const dispatch = vi.fn(); + const terminal = defineStage, Record>({ + name: 'observeResolution', + return: { provides: ['response.usage.billable', 'response.http.headers', 'response.http.status', 'response.http.body'] }, + execute: async facts => move({ ...facts, 'response.usage.billable': [], 'response.http.headers': [], 'response.http.status': 200, 'response.http.body': null }), + }); + const registry = createCandidateRegistry(); + const outcome = await run(compose<{ 'serve.model': string } & Pick, Record>('resolve', [resolver, terminal]), move({ + 'serve.model': 'model', 'request.chat.openaiChatCompletions': payload, + }), { gateway: mockChatGatewayCtx(), ...registry, httpCall: dispatch }); + const values = outcome.facts['request.chat.candidatePayloads'] as Readonly>; + expect(values[0]).toBe(payload); + expect(values[1]).toBe(changed); + expect(Object.keys(values)).toEqual(['0', '1']); + expect(Object.isFrozen(values)).toBe(true); + expect(Object.isFrozen(first.provider.instance)).toBe(false); + expect(evaluated).toEqual([first, second]); + expect(projectFirst).toHaveBeenCalledTimes(1); + expect(projectSecond).toHaveBeenCalledTimes(1); + expect(dispatch).not.toHaveBeenCalled(); + await outcome.drain(); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/routes-anthropic-messages_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-anthropic-messages_test.ts new file mode 100644 index 0000000000..16b421b4cc --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/routes-anthropic-messages_test.ts @@ -0,0 +1,231 @@ +// `/v1/messages`, from the client's socket to Copilot's and back. +// +// The same statement the OpenAI Chat Completions route file makes, for the protocol Claude Code +// speaks. Two things are this family's own and are said here rather than assumed: Anthropic +// ends a turn with an event rather than a transport sentinel, so `message_stop` is what the +// terminal assertion looks for; and the edge writes the turn's own affinity state into the +// answer as a leading `redacted_thinking` block, so the text is found by kind rather than at +// index zero. + +import { expect, test } from 'vitest'; + +import { tokenCountsFromUsage } from '../../../src/repo/usage-metrics.ts'; +import { copilotModels, parseSSEText, requestAppWithWarmModels as requestApp, setupAppTest, sseOpenAIChatCompletionsResponse, sseAnthropicMessagesResponse } from '../../test-utils/app.ts'; +import { flushBackground } from '../../test-utils/background-tracker.ts'; +import { jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; + +const ANSWER = 'hello from copilot'; + +/** A Copilot seat, as far as a turn can see it: the editor version probe, the token exchange, + * the catalog, and one data-plane endpoint whose answer the row decides. */ +const withCopilot = async ( + models: Parameters[0], + serve: (path: string, body: Record) => Response, + run: () => Promise, +): Promise => + await withMockedFetch( + async request => { + const url = new URL(request.url); + if (url.hostname === 'update.code.visualstudio.com') return jsonResponse(['1.110.1']); + if (url.pathname === '/copilot_internal/v2/token') { + return jsonResponse({ token: 'copilot-access-token', expires_at: 4102444800, refresh_in: 3600, endpoints: { api: 'https://api.individual.githubcopilot.com' } }); + } + if (url.pathname === '/models') return jsonResponse(copilotModels(models)); + return serve(url.pathname, await request.json() as Record); + }, + run, + ); + +/** The answer an Anthropic Messages upstream gives, as the SSE every chat endpoint really speaks. */ +const upstreamTurn = (model: string): Response => + sseAnthropicMessagesResponse({ + id: 'msg_route', + type: 'message', + role: 'assistant', + model, + content: [{ type: 'text', text: ANSWER }], + stop_reason: 'end_turn', + usage: { input_tokens: 17, output_tokens: 4 }, + }); + +const post = async (apiKey: string, body: Record): Promise => + await requestApp('/v1/messages', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey }, + body: JSON.stringify(body), + }); + +const turn = (model: string, extra: Record = {}): Record => + ({ model, max_tokens: 64, messages: [{ role: 'user', content: 'hi' }], ...extra }); + +interface AnthropicMessagesBody { + readonly type: string; + readonly role: string; + readonly stop_reason: string; + readonly content: readonly { readonly type: string; readonly text?: string }[]; + readonly usage: { readonly input_tokens: number; readonly output_tokens: number }; +} + +/** The answer's own text, past the carrier the edge writes back. */ +const textOf = (body: AnthropicMessagesBody): string | undefined => + body.content.find(block => block.type === 'text')?.text; + +test('a streaming turn reaches the client as Anthropic Messages SSE, message_stop included', async () => { + const { apiKey } = await setupAppTest(); + + const response = await withCopilot( + [{ id: 'anthropic-route', supported_endpoints: ['/messages'] }], + (path, body) => { + expect(path).toBe('/v1/messages'); + // The provider forces streaming on every chat endpoint, so what the client asked for is + // not what the wire carries. + expect(body.stream).toBe(true); + return upstreamTurn('anthropic-route'); + }, + async () => await post(apiKey.key, turn('anthropic-route', { stream: true })), + ); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')?.split(';')[0]).toBe('text/event-stream'); + + const frames = parseSSEText(await response.text()); + // Anthropic names its own SSE events, and every frame the client reads carries the name of + // the event its data states. + for (const frame of frames) { + expect(frame.event).toBe((JSON.parse(frame.data) as { type: string }).type); + } + expect(frames[0].event).toBe('message_start'); + // The turn ends with an event rather than a transport sentinel — there is no `[DONE]` here, + // and a stream that stopped before this one is an answer nobody can act on. + expect(frames.at(-1)?.event).toBe('message_stop'); + + const text = frames + .filter(frame => frame.event === 'content_block_delta') + .map(frame => (JSON.parse(frame.data) as { delta: { text?: string } }).delta.text ?? '') + .join(''); + expect(text).toBe(ANSWER); +}); + +test('a non-streaming turn reaches the client as one assembled Anthropic Messages body', async () => { + const { apiKey } = await setupAppTest(); + + const response = await withCopilot( + [{ id: 'anthropic-route', supported_endpoints: ['/messages'] }], + () => upstreamTurn('anthropic-route'), + async () => await post(apiKey.key, turn('anthropic-route')), + ); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')?.split(';')[0]).toBe('application/json'); + + const body = await response.json() as AnthropicMessagesBody; + expect(body.type).toBe('message'); + expect(body.role).toBe('assistant'); + expect(body.stop_reason).toBe('end_turn'); + expect(textOf(body)).toBe(ANSWER); + // What the upstream metered is stated to the client too, and it is the same reading the + // usage row below is written from. + expect(body.usage.input_tokens).toBe(17); + expect(body.usage.output_tokens).toBe(4); +}); + +test('an upstream refusal reaches the client in the upstream-s own words and status', async () => { + const { apiKey } = await setupAppTest(); + const refusal = { type: 'error', error: { type: 'permission_error', message: 'copilot said no' } }; + + const response = await withCopilot( + [{ id: 'anthropic-route', supported_endpoints: ['/messages'] }], + () => jsonResponse(refusal, 403), + async () => await post(apiKey.key, turn('anthropic-route')), + ); + + expect(response.status).toBe(403); + // Not the gateway's own `{ type: 'error', error: { type: 'api_error', … } }` — the answer to + // "why was I refused" is the refusing party's to give. + expect(await response.json()).toEqual(refusal); +}); + +test('a refusal on a turn that asked to stream is a body, not an opened stream', async () => { + const { apiKey } = await setupAppTest(); + const refusal = { type: 'error', error: { type: 'permission_error', message: 'copilot said no' } }; + + // The riskier half of the same statement: the client asked for SSE, so the seam has a + // stream to open and must not — nothing was ever generated, and Claude Code reads an error + // written into a 200 stream as a turn that succeeded and said nothing. + const response = await withCopilot( + [{ id: 'anthropic-route', supported_endpoints: ['/messages'] }], + () => jsonResponse(refusal, 403), + async () => await post(apiKey.key, turn('anthropic-route', { stream: true })), + ); + + expect(response.status).toBe(403); + expect(response.headers.get('content-type')?.split(';')[0]).toBe('application/json'); + expect(await response.json()).toEqual(refusal); +}); + +test('a served turn writes one usage row carrying the tokens the upstream reported', async () => { + const { apiKey, repo } = await setupAppTest(); + + const response = await withCopilot( + [{ id: 'anthropic-route', supported_endpoints: ['/messages'] }], + () => upstreamTurn('anthropic-route'), + async () => { + const answer = await post(apiKey.key, turn('anthropic-route', { stream: true })); + // A stream's numbers arrive with its last chunk, so the row is written after the client + // has read it — a test that never read the body would settle nothing. + await answer.text(); + return answer; + }, + ); + + expect(response.status).toBe(200); + await flushBackground(); + + const rows = await repo.usage.listAll(); + expect(rows).toHaveLength(1); + expect(rows[0].keyId).toBe(apiKey.id); + expect(rows[0].requests).toBe(1); + // The two regressions this fences: a row written with no metrics at all, and a row written + // with every quantity at zero. + expect(tokenCountsFromUsage(rows[0])).toEqual({ input: 17, output: 4 }); +}); + +test('a turn dialled over the OpenAI Chat Completions wire still answers the client in Anthropic Messages', async () => { + const { apiKey, repo } = await setupAppTest(); + + // The candidate serves `/chat/completions` and nothing else, so neither this protocol's own + // wire nor the OpenAI Responses one is available and the turn leaves through the handoff. A chain + // that dialled anything else would call a Copilot method with no upstream mocked here. + const response = await withCopilot( + [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], + (path, body) => { + expect(path).toBe('/chat/completions'); + // The usage chunk is asked for by the wire the turn landed on, not by the chain it + // arrived from — which is what keeps a translated turn metered at all. + expect(body.stream_options).toEqual({ include_usage: true, continuous_usage_stats: true }); + return sseOpenAIChatCompletionsResponse({ + id: 'chatcmpl_route', + object: 'chat.completion', + created: 1_772_000_000, + model: 'gpt-route', + choices: [{ index: 0, message: { role: 'assistant', content: ANSWER }, finish_reason: 'stop' }], + usage: { prompt_tokens: 9, completion_tokens: 6, total_tokens: 15 }, + }); + }, + async () => await post(apiKey.key, turn('gpt-route')), + ); + + expect(response.status).toBe(200); + const body = await response.json() as AnthropicMessagesBody; + expect(body.type).toBe('message'); + expect(body.role).toBe('assistant'); + expect(body.stop_reason).toBe('end_turn'); + expect(textOf(body)).toBe(ANSWER); + + // The reading is taken on the dialect the upstream actually spoke, so a translated turn is + // metered like any other rather than billing nothing. + await flushBackground(); + const rows = await repo.usage.listAll(); + expect(rows).toHaveLength(1); + expect(tokenCountsFromUsage(rows[0])).toEqual({ input: 9, output: 6 }); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/routes-gemini-generate-content_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-gemini-generate-content_test.ts new file mode 100644 index 0000000000..3279859118 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/routes-gemini-generate-content_test.ts @@ -0,0 +1,235 @@ +// `/v1beta/models/:model:generateContent`, from the client's socket to Copilot's and back. +// +// The same statement the other two route files make, for the one family with no wire of its +// own: the provider surface has no Gemini generateContent call, so every candidate is reached through a +// translation and each row here is already a translated turn. What the last row adds is the +// second pair — the same request answered over an Anthropic Messages-only candidate — because "the client +// gets its own protocol back" has to hold for whichever wire the picker landed on. +// +// Gemini generateContent also ends a turn differently from either OpenAI-shaped protocol: there is no +// sentinel, and the last frame is the one whose candidate states a `finishReason`. + +import { expect, test } from 'vitest'; + +import { tokenCountsFromUsage } from '../../../src/repo/usage-metrics.ts'; +import { copilotModels, parseSSEText, requestAppWithWarmModels as requestApp, setupAppTest, sseOpenAIChatCompletionsResponse, sseAnthropicMessagesResponse } from '../../test-utils/app.ts'; +import { flushBackground } from '../../test-utils/background-tracker.ts'; +import { jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; + +const ANSWER = 'hello from copilot'; + +/** A Copilot seat, as far as a turn can see it: the editor version probe, the token exchange, + * the catalog, and one data-plane endpoint whose answer the row decides. */ +const withCopilot = async ( + models: Parameters[0], + serve: (path: string, body: Record) => Response, + run: () => Promise, +): Promise => + await withMockedFetch( + async request => { + const url = new URL(request.url); + if (url.hostname === 'update.code.visualstudio.com') return jsonResponse(['1.110.1']); + if (url.pathname === '/copilot_internal/v2/token') { + return jsonResponse({ token: 'copilot-access-token', expires_at: 4102444800, refresh_in: 3600, endpoints: { api: 'https://api.individual.githubcopilot.com' } }); + } + if (url.pathname === '/models') return jsonResponse(copilotModels(models)); + return serve(url.pathname, await request.json() as Record); + }, + run, + ); + +/** The answer an OpenAI Chat Completions upstream gives, as the SSE every chat endpoint really speaks. + * Gemini generateContent's first-preference wire, so this is what most of these turns are dialled on. */ +const openaiChatCompletionsTurn = (): Response => + sseOpenAIChatCompletionsResponse({ + id: 'chatcmpl_route', + object: 'chat.completion', + created: 1_772_000_000, + model: 'gpt-route', + choices: [{ index: 0, message: { role: 'assistant', content: ANSWER }, finish_reason: 'stop' }], + usage: { prompt_tokens: 21, completion_tokens: 3, total_tokens: 24 }, + }); + +const post = async (apiKey: string, modelAction: string): Promise => + await requestApp(`/v1beta/models/${modelAction}`, { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey }, + body: JSON.stringify({ contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }), + }); + +interface GeminiGenerateContentEvent { + readonly candidates?: readonly { + readonly content: { readonly parts: readonly { readonly text?: string }[] }; + readonly finishReason?: string; + }[]; +} + +const geminiGenerateContentText = (event: GeminiGenerateContentEvent): string => + (event.candidates ?? []).flatMap(candidate => candidate.content.parts.map(part => part.text ?? '')).join(''); + +test('a streaming turn reaches the client as Gemini generateContent SSE, ending on the frame that states a finish reason', async () => { + const { apiKey } = await setupAppTest(); + + const response = await withCopilot( + [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], + (path, body) => { + expect(path).toBe('/chat/completions'); + // Gemini generateContent has no wire of its own, so what left the gateway is the + // translated turn — an OpenAI Chat Completions body, streaming because every chat + // endpoint here does. + expect(body.messages).toBeDefined(); + expect(body.stream).toBe(true); + return openaiChatCompletionsTurn(); + }, + async () => await post(apiKey.key, 'gpt-route:streamGenerateContent'), + ); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')?.split(';')[0]).toBe('text/event-stream'); + + const frames = parseSSEText(await response.text()); + expect(frames.length > 0).toBe(true); + // One JSON object per `data:` line, and no sentinel — Gemini generateContent has none to write, so a `[DONE]` + // here would be a frame no Google client can parse. + const events = frames.map(frame => JSON.parse(frame.data) as GeminiGenerateContentEvent); + for (const frame of frames) expect(frame.event).toBe('message'); + expect(events.map(geminiGenerateContentText).join('')).toBe(ANSWER); + // The turn's end is stated on the frames the client is handed, not on the wire below them. + expect(events.at(-1)?.candidates?.[0]?.finishReason).toBe('STOP'); + expect(events.slice(0, -1).every(event => event.candidates?.[0]?.finishReason === undefined)).toBe(true); +}); + +test('a non-streaming turn reaches the client as one assembled Gemini generateContent body', async () => { + const { apiKey } = await setupAppTest(); + + const response = await withCopilot( + [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], + () => openaiChatCompletionsTurn(), + async () => await post(apiKey.key, 'gpt-route:generateContent'), + ); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')?.split(';')[0]).toBe('application/json'); + + const body = await response.json() as GeminiGenerateContentEvent & { usageMetadata?: { promptTokenCount?: number; candidatesTokenCount?: number } }; + expect(body.candidates).toHaveLength(1); + expect(body.candidates?.[0].finishReason).toBe('STOP'); + expect(geminiGenerateContentText(body)).toBe(ANSWER); + // What the upstream metered is stated to the client too, in this protocol's own names. + expect(body.usageMetadata?.promptTokenCount).toBe(21); + expect(body.usageMetadata?.candidatesTokenCount).toBe(3); +}); + +test('an upstream refusal reaches the client in this protocol-s own envelope', async () => { + const { apiKey } = await setupAppTest(); + const refusal = { error: { message: 'copilot said no', type: 'insufficient_quota', code: 'quota_exceeded' } }; + + const response = await withCopilot( + [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], + () => jsonResponse(refusal, 402), + async () => await post(apiKey.key, 'gpt-route:generateContent'), + ); + + // Not the 402 the upstream sent. This envelope is minted rather than forwarded, and it + // states its code inside the body as well as naming it — a code this protocol has no name + // for reads `INTERNAL`, which belongs to 500. Answering 402 would put `code: 402` beside + // `status: "INTERNAL"`, so the status the envelope can express is the one that is sent. + expect(response.status).toBe(500); + + // Gemini generateContent has no wire of its own, so every refusal that gets here was written by whatever + // protocol the candidate was dialled on — here OpenAI's. Handing that object on would answer + // a Gemini generateContent client in another protocol's words: `error.status` is the field its SDKs read and + // an OpenAI envelope has none, while `error.code` would arrive as a string where a number + // belongs. So the upstream's words become the message and this protocol writes the shape. + const body = await response.json() as { error: { code: number; message: string; status: string } }; + expect(body.error.code).toBe(500); + // The Google-RPC name for the status, which is the field an SDK branches on — and the one + // the code above agrees with, because the two are decided together. + expect(body.error.status).toBe('INTERNAL'); + expect(body.error.message).toContain('copilot said no'); +}); + +test('a refusal on a turn that asked to stream is a body, not an opened stream', async () => { + const { apiKey } = await setupAppTest(); + const refusal = { error: { message: 'copilot said no', type: 'insufficient_quota' } }; + + // The riskier half of the same statement: the client asked for SSE, so the seam has a + // stream to open and must not — nothing was ever generated, and a Gemini generateContent stream that + // carried the refusal would be read as a turn that produced no candidates. + const response = await withCopilot( + [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], + () => jsonResponse(refusal, 402), + async () => await post(apiKey.key, 'gpt-route:streamGenerateContent'), + ); + + expect(response.status).toBe(500); + expect(response.headers.get('content-type')?.split(';')[0]).toBe('application/json'); + const body = await response.json() as { error: { code: number; message: string; status: string } }; + expect(body.error.code).toBe(500); + expect(body.error.message).toContain('copilot said no'); +}); + +test('a served turn writes one usage row carrying the tokens the upstream reported', async () => { + const { apiKey, repo } = await setupAppTest(); + + const response = await withCopilot( + [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], + () => openaiChatCompletionsTurn(), + async () => { + const answer = await post(apiKey.key, 'gpt-route:streamGenerateContent'); + // A stream's numbers arrive with its last chunk, so the row is written after the client + // has read it — a test that never read the body would settle nothing. + await answer.text(); + return answer; + }, + ); + + expect(response.status).toBe(200); + await flushBackground(); + + const rows = await repo.usage.listAll(); + expect(rows).toHaveLength(1); + expect(rows[0].keyId).toBe(apiKey.id); + expect(rows[0].requests).toBe(1); + // The reading is taken on the dialect the upstream actually spoke, which for this family is + // always a translated one — and the two regressions this fences are a row written with no + // metrics at all and a row written with every quantity at zero. + expect(tokenCountsFromUsage(rows[0])).toEqual({ input: 21, output: 3 }); +}); + +test('a turn dialled over the Anthropic Messages wire still answers the client in Gemini generateContent', async () => { + const { apiKey, repo } = await setupAppTest(); + + // The candidate serves `/messages` and nothing else, so the picker's first preference is + // unavailable and the turn leaves through the other pair. A chain that dialled anything else + // would call a Copilot method with no upstream mocked here. + const response = await withCopilot( + [{ id: 'anthropic-route', supported_endpoints: ['/messages'] }], + (path, body) => { + expect(path).toBe('/v1/messages'); + expect(body.messages).toBeDefined(); + expect(body.stream).toBe(true); + return sseAnthropicMessagesResponse({ + id: 'msg_route', + type: 'message', + role: 'assistant', + model: 'anthropic-route', + content: [{ type: 'text', text: ANSWER }], + stop_reason: 'end_turn', + usage: { input_tokens: 8, output_tokens: 2 }, + }); + }, + async () => await post(apiKey.key, 'anthropic-route:generateContent'), + ); + + expect(response.status).toBe(200); + const body = await response.json() as GeminiGenerateContentEvent; + expect(body.candidates).toHaveLength(1); + expect(body.candidates?.[0].finishReason).toBe('STOP'); + expect(geminiGenerateContentText(body)).toBe(ANSWER); + + await flushBackground(); + const rows = await repo.usage.listAll(); + expect(rows).toHaveLength(1); + expect(tokenCountsFromUsage(rows[0])).toEqual({ input: 8, output: 2 }); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/routes-openai-chat-completions_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-openai-chat-completions_test.ts new file mode 100644 index 0000000000..23885a54a5 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/routes-openai-chat-completions_test.ts @@ -0,0 +1,222 @@ +// `/v1/chat/completions`, from the client's socket to Copilot's and back. +// +// Everything else in this directory calls a chain, a stage or a helper directly. What no such +// test can state is that the route *works*: the parts it skips — the provider's own boundary +// boundary rules, the fetcher, the epilogue that writes the client's bytes and settles what the +// stream billed — are exactly where a turn has been lost before. So each row here drives the +// real Hono route against a Copilot upstream, reads the bytes the client would read, and says +// something about them that a broken chain could not produce. + +import { expect, test } from 'vitest'; + +import { tokenCountsFromUsage } from '../../../src/repo/usage-metrics.ts'; +import { copilotModels, parseSSEText, requestAppWithWarmModels as requestApp, setupAppTest, sseOpenAIChatCompletionsResponse, sseAnthropicMessagesResponse } from '../../test-utils/app.ts'; +import { flushBackground } from '../../test-utils/background-tracker.ts'; +import { jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; + +const ANSWER = 'hello from copilot'; + +/** A Copilot seat, as far as a turn can see it: the editor version probe, the token exchange, + * the catalog, and one data-plane endpoint whose answer the row decides. */ +const withCopilot = async ( + models: Parameters[0], + serve: (path: string, body: Record) => Response, + run: () => Promise, +): Promise => + await withMockedFetch( + async request => { + const url = new URL(request.url); + if (url.hostname === 'update.code.visualstudio.com') return jsonResponse(['1.110.1']); + if (url.pathname === '/copilot_internal/v2/token') { + return jsonResponse({ token: 'copilot-access-token', expires_at: 4102444800, refresh_in: 3600, endpoints: { api: 'https://api.individual.githubcopilot.com' } }); + } + if (url.pathname === '/models') return jsonResponse(copilotModels(models)); + return serve(url.pathname, await request.json() as Record); + }, + run, + ); + +/** The answer an OpenAI Chat Completions upstream gives, as the SSE every chat endpoint really speaks. */ +const upstreamTurn = (model: string): Response => + sseOpenAIChatCompletionsResponse({ + id: 'chatcmpl_route', + object: 'chat.completion', + created: 1_772_000_000, + model, + choices: [{ index: 0, message: { role: 'assistant', content: ANSWER }, finish_reason: 'stop' }], + usage: { prompt_tokens: 11, completion_tokens: 7, total_tokens: 18 }, + }); + +const post = async (apiKey: string, body: Record): Promise => + await requestApp('/v1/chat/completions', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey }, + body: JSON.stringify(body), + }); + +interface ChatChunk { + readonly object: string; + readonly choices: readonly { readonly delta?: { readonly content?: string }; readonly finish_reason: string | null }[]; +} + +test('a streaming turn reaches the client as OpenAI Chat Completions SSE, terminator included', async () => { + const { apiKey } = await setupAppTest(); + + const response = await withCopilot( + [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], + (path, body) => { + expect(path).toBe('/chat/completions'); + // The provider forces streaming on every chat endpoint, so what the client asked for is + // not what the wire carries — and the marker below is the Copilot boundary chain having + // run on a body it was free to write into. + expect(body.stream).toBe(true); + return upstreamTurn('gpt-route'); + }, + async () => await post(apiKey.key, { model: 'gpt-route', messages: [{ role: 'user', content: 'hi' }], stream: true }), + ); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')?.split(';')[0]).toBe('text/event-stream'); + + const frames = parseSSEText(await response.text()); + expect(frames.length > 1).toBe(true); + // The sentinel is what an OpenAI client reads as the end of the turn; a stream that stopped + // without it is a truncated answer presented as a whole one. + expect(frames.at(-1)?.data).toBe('[DONE]'); + + const chunks = frames.slice(0, -1).map(frame => JSON.parse(frame.data) as ChatChunk); + for (const chunk of chunks) expect(chunk.object).toBe('chat.completion.chunk'); + expect(chunks.map(chunk => chunk.choices[0]?.delta?.content ?? '').join('')).toBe(ANSWER); + expect(chunks.some(chunk => chunk.choices[0]?.finish_reason === 'stop')).toBe(true); +}); + +test('a non-streaming turn reaches the client as one assembled OpenAI Chat Completions body', async () => { + const { apiKey } = await setupAppTest(); + + const response = await withCopilot( + [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], + () => upstreamTurn('gpt-route'), + async () => await post(apiKey.key, { model: 'gpt-route', messages: [{ role: 'user', content: 'hi' }] }), + ); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')?.split(';')[0]).toBe('application/json'); + + const body = await response.json() as { + object: string; + choices: { index: number; message: { role: string; content: string }; finish_reason: string }[]; + usage: { prompt_tokens: number; completion_tokens: number }; + }; + expect(body.object).toBe('chat.completion'); + expect(body.choices).toHaveLength(1); + expect(body.choices[0].message.role).toBe('assistant'); + expect(body.choices[0].message.content).toBe(ANSWER); + expect(body.choices[0].finish_reason).toBe('stop'); + // What the upstream metered is stated to the client too, and it is the same reading the + // usage row below is written from. + expect(body.usage.prompt_tokens).toBe(11); + expect(body.usage.completion_tokens).toBe(7); +}); + +test('an upstream refusal reaches the client in the upstream-s own words and status', async () => { + const { apiKey } = await setupAppTest(); + const refusal = { error: { message: 'copilot said no', type: 'insufficient_quota', code: 'quota_exceeded' } }; + + const response = await withCopilot( + [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], + () => jsonResponse(refusal, 402), + async () => await post(apiKey.key, { model: 'gpt-route', messages: [{ role: 'user', content: 'hi' }] }), + ); + + expect(response.status).toBe(402); + // Not `{ error: { message, type: 'api_error' } }` — the answer to "why was I refused" is the + // refusing party's to give, so the gateway forwards it rather than quoting it. + expect(await response.json()).toEqual(refusal); +}); + +test('a refusal on a turn that asked to stream is a body, not an opened stream', async () => { + const { apiKey } = await setupAppTest(); + const refusal = { error: { message: 'copilot said no', type: 'insufficient_quota' } }; + + // The riskier half of the same statement: the client asked for SSE, so the seam has a + // stream to open and must not — nothing was ever generated, and an error written into a + // 200 stream is one an OpenAI client reads as a successful empty turn. + const response = await withCopilot( + [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], + () => jsonResponse(refusal, 402), + async () => await post(apiKey.key, { model: 'gpt-route', messages: [{ role: 'user', content: 'hi' }], stream: true }), + ); + + expect(response.status).toBe(402); + expect(response.headers.get('content-type')?.split(';')[0]).toBe('application/json'); + expect(await response.json()).toEqual(refusal); +}); + +test('a served turn writes one usage row carrying the tokens the upstream reported', async () => { + const { apiKey, repo } = await setupAppTest(); + + const response = await withCopilot( + [{ id: 'gpt-route', supported_endpoints: ['/chat/completions'] }], + () => upstreamTurn('gpt-route'), + async () => { + const answer = await post(apiKey.key, { model: 'gpt-route', messages: [{ role: 'user', content: 'hi' }], stream: true }); + // A stream's numbers arrive with its last chunk, so the row is written after the client + // has read it — a test that never read the body would settle nothing. + await answer.text(); + return answer; + }, + ); + + expect(response.status).toBe(200); + await flushBackground(); + + const rows = await repo.usage.listAll(); + expect(rows).toHaveLength(1); + expect(rows[0].keyId).toBe(apiKey.id); + expect(rows[0].requests).toBe(1); + // The two regressions this fences: a row written with no metrics at all, and a row written + // with every quantity at zero. + expect(tokenCountsFromUsage(rows[0])).toEqual({ input: 11, output: 7 }); +}); + +test('a turn dialled over the Anthropic Messages wire still answers the client in OpenAI Chat Completions', async () => { + const { apiKey, repo } = await setupAppTest(); + + // The candidate serves `/messages` and nothing else, so the picker's first preference — this + // protocol's own wire — is unavailable and the turn leaves through the handoff. A chain that + // dialled anything else would call a Copilot method that has no upstream mocked here. + const response = await withCopilot( + [{ id: 'anthropic-route', supported_endpoints: ['/messages'] }], + (path, body) => { + expect(path).toBe('/v1/messages'); + expect(body.stream).toBe(true); + return sseAnthropicMessagesResponse({ + id: 'msg_route', + type: 'message', + role: 'assistant', + model: 'anthropic-route', + content: [{ type: 'text', text: ANSWER }], + stop_reason: 'end_turn', + usage: { input_tokens: 13, output_tokens: 5 }, + }); + }, + async () => await post(apiKey.key, { model: 'anthropic-route', messages: [{ role: 'user', content: 'hi' }] }), + ); + + expect(response.status).toBe(200); + const body = await response.json() as { + object: string; + choices: { message: { role: string; content: string }; finish_reason: string }[]; + }; + expect(body.object).toBe('chat.completion'); + expect(body.choices[0].message.role).toBe('assistant'); + expect(body.choices[0].message.content).toBe(ANSWER); + expect(body.choices[0].finish_reason).toBe('stop'); + + // The reading is taken on the dialect the upstream actually spoke, so a translated turn is + // metered like any other rather than billing nothing. + await flushBackground(); + const rows = await repo.usage.listAll(); + expect(rows).toHaveLength(1); + expect(tokenCountsFromUsage(rows[0])).toEqual({ input: 13, output: 5 }); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/routes-openai-responses_test.ts b/packages/gateway/__tests__/data-plane/chat/routes-openai-responses_test.ts new file mode 100644 index 0000000000..593dd17a48 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/routes-openai-responses_test.ts @@ -0,0 +1,112 @@ +// `/v1/responses`, from the client's socket to Copilot's and back. +// +// Everything else in this directory calls a chain, a stage or a helper directly, against a +// provider that is a stub. What no such test can state is that the *dial* works: a provider's +// own boundary shapes the body it is handed — Copilot rewrites tool definitions, strips the +// fields its endpoint rejects, compresses inline images and re-stamps item ids — and the +// record that body is built from is deep-frozen. A body handed over as a shallow copy +// satisfies every type in the system and throws at the first write into a nested node, +// answering the client with a 502 raised where nothing can explain it. So this drives the +// real route against a Copilot upstream and reads what the client would read. + +import { expect, test } from 'vitest'; + +import { copilotModels, requestAppWithWarmModels as requestApp, setupAppTest, sseOpenAIResponsesResponse } from '../../test-utils/app.ts'; +import { jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; + +const ANSWER = 'hello from copilot'; + +/** A Copilot seat, as far as a turn can see it: the editor version probe, the token exchange, + * the catalog, and one data-plane endpoint whose answer the row decides. */ +const withCopilot = async ( + models: Parameters[0], + serve: (path: string, body: Record) => Response, + run: () => Promise, +): Promise => + await withMockedFetch( + async request => { + const url = new URL(request.url); + if (url.hostname === 'update.code.visualstudio.com') return jsonResponse(['1.110.1']); + if (url.pathname === '/copilot_internal/v2/token') { + return jsonResponse({ token: 'copilot-access-token', expires_at: 4102444800, refresh_in: 3600, endpoints: { api: 'https://api.individual.githubcopilot.com' } }); + } + if (url.pathname === '/models') return jsonResponse(copilotModels(models)); + return serve(url.pathname, await request.json() as Record); + }, + run, + ); + +const post = async (apiKey: string, body: Record): Promise => + await requestApp('/v1/responses', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey }, + body: JSON.stringify(body), + }); + +test('a turn reaches Copilot over its own OpenAI Responses endpoint and comes back as one response', async () => { + const { apiKey } = await setupAppTest(); + + let sent: Record | undefined; + const response = await withCopilot( + [{ id: 'gpt-responses', supported_endpoints: ['/responses'] }], + (path, body) => { + expect(path).toBe('/responses'); + sent = body; + return sseOpenAIResponsesResponse({ + id: 'resp_route', + object: 'response', + model: 'gpt-responses', + status: 'completed', + output: [{ + type: 'message', id: 'msg_route', role: 'assistant', status: 'completed', + content: [{ type: 'output_text', text: ANSWER, annotations: [] }], + }], + error: null, + incomplete_details: null, + usage: { input_tokens: 11, output_tokens: 7, total_tokens: 18 }, + }); + }, + async () => await post(apiKey.key, { + model: 'gpt-responses', + // Everything the Copilot boundary has an opinion about, so the body it was handed is one + // it rewrote rather than one it merely forwarded: a namespace tool whose empty + // description it fills, an image-generation tool its endpoint rejects, a service tier it + // does not expose, and the snapshot hint it refuses. + input: [{ type: 'message', role: 'user', content: [{ type: 'input_text', text: 'hi' }] }], + tools: [ + { type: 'namespace', name: 'repo', description: '', tools: [] }, + { type: 'image_generation' }, + ], + service_tier: 'flex', + store: true, + }), + ); + + // The regression this fences is a 502 whose body reads `Cannot add property …, object is not + // extensible`: the provider writing into a body whose nested nodes were still the record's. + expect(response.status).toBe(200); + + const body = await response.json() as { + object: string; + status: string; + output: { type: string; content?: { type: string; text: string }[] }[]; + usage: { input_tokens: number; output_tokens: number }; + }; + expect(body.object).toBe('response'); + expect(body.status).toBe('completed'); + expect(body.output.find(item => item.type === 'message')?.content?.[0]?.text).toBe(ANSWER); + expect(body.usage).toMatchObject({ input_tokens: 11, output_tokens: 7 }); + + // What the turn looked like on the wire. The hosted image-generation tool never reached the + // Copilot boundary that would have dropped it: Copilot carries the + // `openai-responses-image-generation-shim` flag, so the hosted-tool stage got there first and sent a + // function tool the model can call and this gateway executes. What the boundary did make of + // the rest is here — the description its namespace tool needs is filled, and neither field it + // refuses travels. + const tools = sent?.tools as { type: string; name?: string; description?: string }[]; + expect(tools.map(tool => tool.type)).toEqual(['namespace', 'function']); + expect(tools[0]!.description).toBe('Tools in the repo namespace.'); + expect(tools[1]!.name).toBe('image_generation'); + expect(sent).not.toHaveProperty('service_tier'); + expect(sent).toMatchObject({ store: false }); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts b/packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts new file mode 100644 index 0000000000..ca2d4b0fe6 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/run-dump-frames_test.ts @@ -0,0 +1,219 @@ +// The frames a chat turn was served are in its run record. +// +// A run record that holds only stage boundaries describes the shape of the turn and none of +// its content, and the dashboard's collected view is assembled by replaying exactly these +// events. The tee sits at the family's edge, so what lands in the record is the source +// protocol's frames — what the client read — rather than whatever the upstream spoke. +// +// Every edge that hands a client frames is covered here, in both shapes the client can be +// handed them: the stream itself, and the one body the same frames were folded into. +// `/v1/responses/compact` is one of those edges rather than a wire under the generate chain — +// it reads its frames itself, so recording them is its own to do — and it only ever folds, +// because a compaction is one resource. + +import { test, vi } from 'vitest'; + +import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts'; +import { eventsOf, installDumpStubs, runRecordOf } from '../../dump/test-fixtures.ts'; +import { copilotModels, flushAsyncWork, requestAppWithWarmModels as requestApp, setupAppTest, sseOpenAIChatCompletionsResponse, sseOpenAIResponsesResponse } from '../../test-utils/app.ts'; +import { createRunReader, type DumpEvent } from '@floway-dev/pipeline'; +import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; + +/** A Copilot seat, as far as a turn can see it: the editor version probe, the token exchange, + * the catalog, and one data-plane endpoint whose answer each turn decides for itself. */ +const withCopilot = async ( + models: Parameters[0], + serve: (request: Request) => Promise | Response, + run: () => Promise, +): Promise => + await withMockedFetch( + async request => { + const url = new URL(request.url); + if (url.hostname === 'update.code.visualstudio.com') return jsonResponse(['1.110.1']); + if (url.pathname === '/copilot_internal/v2/token') { + return jsonResponse({ token: 'copilot-access-token', expires_at: 4102444800, refresh_in: 3600, endpoints: { api: 'https://api.individual.githubcopilot.com' } }); + } + if (url.pathname === '/models') return jsonResponse(copilotModels(models)); + return await serve(request); + }, + run, + ); + +// The run format interns every node it has seen, so a frame arrives as a reference into the +// table the `object` events build up. Resolving it here is what makes the assertion about the +// frame's content rather than about an event existing. +const resolveFrames = (events: readonly Record[]): unknown[] => { + const table = new Map(); + for (const event of events) { + if (event.type !== 'object') continue; + const from = event.fromObjectId as number; + (event.nodes as unknown[]).forEach((node, index) => { table.set(from + index, node); }); + } + const resolve = (value: unknown): unknown => { + if (Array.isArray(value)) return value.map(resolve); + if (typeof value !== 'object' || value === null) return value; + const record = value as Record; + if ('$' in record) return resolve(table.get(record.$ as number)); + return Object.fromEntries(Object.entries(record).map(([key, child]) => [key, resolve(child)])); + }; + return events + .filter(event => event.type === 'stream.frame') + .flatMap(event => (event.frames as unknown[]).map(resolve)); +}; + +/** Runs one turn with a record open, and hands back the frames that record holds. + * + * Reading the client's body is part of the turn rather than an assertion about it: reading is + * what records, so a caller that never read would prove the opposite of what these tests + * claim. */ +const framesRecordedBy = async (turn: (apiKey: string) => Promise): Promise => { + const { apiKey, repo } = await setupAppTest(); + await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); + + const response = await turn(apiKey.key); + assertEquals(response.status, 200); + await response.text(); + + await flushAsyncWork(); + await vi.waitFor(() => assertEquals(dumps.stored.length, 1)); + const stored = dumps.stored[0]; + assertExists(stored); + const events = eventsOf(runRecordOf(stored.record)); + const read = createRunReader(); + let clientStream: number | undefined; + for (const event of events) { + const decoded = read(event as unknown as DumpEvent); + const frames = decoded?.facts?.['response.chat.clientFrames']; + if (typeof frames === 'object' && frames !== null && 'stream' in frames) clientStream = frames.stream as number; + } + assertExists(clientStream); + assertEquals(events.some(event => event.type === 'stream.end' && event.streamId === clientStream), true); + return resolveFrames(events); +}; + +/** The content is the point, not the count: a record holding frames with nothing in them is + * the same empty record with more events. */ +const assertFramesCarry = (frames: readonly unknown[], text: string): void => { + assertEquals(frames.length > 0, true); + assertEquals(JSON.stringify(frames).includes(text), true); +}; + +/** The output items the recorded frames closed, which is what a resource assembled from them + * is made of. */ +const closedItemTypes = (frames: readonly unknown[]): readonly string[] => + frames.flatMap(frame => { + const { event } = frame as { event?: { type?: string; item?: { type?: string } } }; + if (event?.type !== 'response.output_item.done') return []; + return event.item?.type === undefined ? [] : [event.item.type]; + }); + +const ANSWER = 'hello there'; + +const openaiChatCompletionsTurn = (stream: boolean) => async (apiKey: string): Promise => + await withCopilot( + [{ id: 'gpt-frames', supported_endpoints: ['/chat/completions'] }], + () => sseOpenAIChatCompletionsResponse({ + id: 'chatcmpl_frames', + object: 'chat.completion', + model: 'gpt-frames', + choices: [{ index: 0, message: { role: 'assistant', content: ANSWER }, finish_reason: 'stop' }], + usage: { prompt_tokens: 3, completion_tokens: 5, total_tokens: 8 }, + }), + async () => await requestApp('/v1/chat/completions', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey }, + body: JSON.stringify({ model: 'gpt-frames', messages: [{ role: 'user', content: 'hi' }], stream }), + }), + ); + +const openaiResponsesTurn = (stream: boolean) => async (apiKey: string): Promise => + await withCopilot( + [{ id: 'gpt-responses-frames', supported_endpoints: ['/responses'] }], + () => sseOpenAIResponsesResponse({ + id: 'resp_frames', + object: 'response', + model: 'gpt-responses-frames', + status: 'completed', + output: [{ + type: 'message', id: 'msg_frames', role: 'assistant', status: 'completed', + content: [{ type: 'output_text', text: ANSWER, annotations: [] }], + }], + error: null, + incomplete_details: null, + usage: { input_tokens: 11, output_tokens: 7, total_tokens: 18 }, + }), + async () => await requestApp('/v1/responses', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey }, + body: JSON.stringify({ + model: 'gpt-responses-frames', + input: [{ type: 'message', role: 'user', content: [{ type: 'input_text', text: 'hi' }] }], + stream, + }), + }), + ); + +/** The history a compaction is asked to stand in for. Copilot's compaction retains the + * messages it did not absorb, so this is what the resource carries beside the blob. */ +const HISTORY = 'everything that has been said so far'; + +const compactionTurn = async (apiKey: string): Promise => + await withCopilot( + [{ id: 'gpt-compact-frames', supported_endpoints: ['/responses'] }], + // Copilot has no compaction endpoint of its own: it replays the protocol over `/responses` + // with the trigger appended and `stream: false`, so the answer here is one JSON body. + async request => { + const body = await request.json() as { input: unknown[] }; + return Array.isArray(body.input) && (body.input[0] as { role?: string })?.role === 'system' ? sseOpenAIResponsesResponse({ + id: 'replay', object: 'response', model: 'gpt-compact-frames', status: 'completed', + output: [{ type: 'message', id: 'replay_msg', role: 'assistant', status: 'completed', content: [{ type: 'output_text', text: HISTORY, annotations: [] }] }], + error: null, incomplete_details: null, usage: { input_tokens: 2, output_tokens: 1, total_tokens: 3 }, + }) : jsonResponse({ + id: 'resp_compaction', + object: 'response', + model: 'gpt-compact-frames', + status: 'completed', + output: [{ type: 'compaction', id: 'cmp_frames', encrypted_content: 'OPAQUE_COMPACTION_BLOB' }], + error: null, + incomplete_details: null, + usage: { input_tokens: 21, output_tokens: 4, total_tokens: 25 }, + }); + }, + async () => await requestApp('/v1/responses/compact', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey }, + body: JSON.stringify({ + model: 'gpt-compact-frames', + input: [{ type: 'message', role: 'user', content: [{ type: 'input_text', text: HISTORY }] }], + }), + }), + ); + +test('a streamed chat turn records the frames it served', async () => { + assertFramesCarry(await framesRecordedBy(openaiChatCompletionsTurn(true)), ANSWER); +}); + +test('a collected chat turn records the frames it was assembled from', async () => { + // 「流式请求前端看到流+前端组装结果，非流式请求前端看到流+原始组装结果」 — the frames are + // recorded either way, which is what lets the operator see the stream behind a single body. + assertFramesCarry(await framesRecordedBy(openaiChatCompletionsTurn(false)), ANSWER); +}); + +test('a streamed responses turn records the frames it served', async () => { + assertFramesCarry(await framesRecordedBy(openaiResponsesTurn(true)), ANSWER); +}); + +test('a collected responses turn records the frames it was assembled from', async () => { + assertFramesCarry(await framesRecordedBy(openaiResponsesTurn(false)), ANSWER); +}); + +test('a compaction records the frames its resource was assembled from', async () => { + const frames = await framesRecordedBy(compactionTurn); + // These frames never reached a client — the client got the one resource they add up to — so + // the record is the only place the turn's own events survive at all. Both halves of what + // that resource was assembled from are in them: the history the upstream retained, and the + // compaction item the next turn inherits. + assertFramesCarry(frames, HISTORY); + assertEquals(closedItemTypes(frames).includes('compaction'), true); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/shared/alias-rules_test.ts b/packages/gateway/__tests__/data-plane/chat/shared/alias-rules_test.ts index d2540b709f..635a529fd3 100644 --- a/packages/gateway/__tests__/data-plane/chat/shared/alias-rules_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/shared/alias-rules_test.ts @@ -3,6 +3,9 @@ // are authoritative — an existing IR field is OVERWRITTEN by a matching // rule — and rules the target IR cannot express are silently dropped // (there's no wire slot to put them on). +// +// Every helper produces a payload rather than writing into the one it was handed, which the +// three frozen-input cases pin: what an overlay is handed at the dial is a record's value. import { test } from 'vitest'; @@ -35,202 +38,244 @@ const msgPayload = (overrides: Partial = {}): Anthropi test('openai-chat-completions: empty rules leave the payload unchanged', () => { const body = ccPayload({ reasoning_effort: 'high', verbosity: 'low', service_tier: 'priority' }); - applyRulesToUpstreamOpenAIChatCompletions(body, {}); - assertEquals(body.reasoning_effort, 'high'); - assertEquals(body.verbosity, 'low'); - assertEquals(body.service_tier, 'priority'); + const applied = applyRulesToUpstreamOpenAIChatCompletions(body, {}); + assertEquals(applied.reasoning_effort, 'high'); + assertEquals(applied.verbosity, 'low'); + assertEquals(applied.service_tier, 'priority'); }); test('openai-chat-completions: rules stamp every supported native field onto the IR', () => { const body = ccPayload(); - applyRulesToUpstreamOpenAIChatCompletions(body, { + const applied = applyRulesToUpstreamOpenAIChatCompletions(body, { reasoning: { effort: 'high' }, verbosity: 'low', serviceTier: 'priority', }); - assertEquals(body.reasoning_effort, 'high'); - assertEquals(body.verbosity, 'low'); - assertEquals(body.service_tier, 'priority'); + assertEquals(applied.reasoning_effort, 'high'); + assertEquals(applied.verbosity, 'low'); + assertEquals(applied.service_tier, 'priority'); }); test('openai-chat-completions: budget_tokens / adaptive / summary have no native slot — silently dropped', () => { const body = ccPayload(); - applyRulesToUpstreamOpenAIChatCompletions(body, { + const applied = applyRulesToUpstreamOpenAIChatCompletions(body, { reasoning: { budget_tokens: 1024, adaptive: true, summary: 'detailed' }, }); - assertEquals(body.reasoning_effort, undefined); + assertEquals(applied.reasoning_effort, undefined); // Nothing surfaces on the CC IR because none of those rules map to // OpenAI Chat Completions' native fields. - assertEquals('thinking_budget' in body, false); - assertEquals('adaptive_thinking' in body, false); - assertEquals('reasoning_summary' in body, false); + assertEquals('thinking_budget' in applied, false); + assertEquals('adaptive_thinking' in applied, false); + assertEquals('reasoning_summary' in applied, false); }); test('openai-chat-completions: alias rules overwrite existing IR fields', () => { const body = ccPayload({ reasoning_effort: 'low', verbosity: 'high', service_tier: 'default' }); - applyRulesToUpstreamOpenAIChatCompletions(body, { + const applied = applyRulesToUpstreamOpenAIChatCompletions(body, { reasoning: { effort: 'xhigh' }, verbosity: 'low', serviceTier: 'priority', }); - assertEquals(body.reasoning_effort, 'xhigh'); - assertEquals(body.verbosity, 'low'); - assertEquals(body.service_tier, 'priority'); + assertEquals(applied.reasoning_effort, 'xhigh'); + assertEquals(applied.verbosity, 'low'); + assertEquals(applied.service_tier, 'priority'); }); // ── OpenAI Responses target ── -test('responses: empty rules leave the payload unchanged', () => { +test('openai-responses: empty rules leave the payload unchanged', () => { const body = resPayload({ reasoning: { effort: 'high' }, text: { verbosity: 'low' }, service_tier: 'priority' }); - applyRulesToUpstreamOpenAIResponses(body, {}); - assertEquals(body.reasoning?.effort, 'high'); - assertEquals(body.text?.verbosity, 'low'); - assertEquals(body.service_tier, 'priority'); + const applied = applyRulesToUpstreamOpenAIResponses(body, {}); + assertEquals(applied.reasoning?.effort, 'high'); + assertEquals(applied.text?.verbosity, 'low'); + assertEquals(applied.service_tier, 'priority'); }); -test('responses: rules stamp every supported native field onto the IR', () => { +test('openai-responses: rules stamp every supported native field onto the IR', () => { const body = resPayload(); - applyRulesToUpstreamOpenAIResponses(body, { + const applied = applyRulesToUpstreamOpenAIResponses(body, { reasoning: { effort: 'high', summary: 'concise' }, verbosity: 'medium', serviceTier: 'flex', }); - assertEquals(body.reasoning?.effort, 'high'); - assertEquals(body.reasoning?.summary, 'concise'); - assertEquals(body.text?.verbosity, 'medium'); - assertEquals(body.service_tier, 'flex'); + assertEquals(applied.reasoning?.effort, 'high'); + assertEquals(applied.reasoning?.summary, 'concise'); + assertEquals(applied.text?.verbosity, 'medium'); + assertEquals(applied.service_tier, 'flex'); }); -test('responses: budget_tokens / adaptive have no native slot — silently dropped', () => { +test('openai-responses: budget_tokens / adaptive have no native slot — silently dropped', () => { const body = resPayload(); - applyRulesToUpstreamOpenAIResponses(body, { + const applied = applyRulesToUpstreamOpenAIResponses(body, { reasoning: { budget_tokens: 1024, adaptive: true }, }); - assertEquals(body.reasoning, undefined); - assertEquals('thinking_budget' in body, false); - assertEquals('adaptive_thinking' in body, false); + assertEquals(applied.reasoning, undefined); + assertEquals('thinking_budget' in applied, false); + assertEquals('adaptive_thinking' in applied, false); }); -test('responses: alias rules overwrite owned fields while preserving reasoning.context', () => { +test('openai-responses: alias rules overwrite owned fields while preserving reasoning.context', () => { const body = resPayload({ reasoning: { effort: 'low', summary: 'auto', context: 'future_mode' }, service_tier: 'default', text: { verbosity: 'high' }, }); - applyRulesToUpstreamOpenAIResponses(body, { + const applied = applyRulesToUpstreamOpenAIResponses(body, { reasoning: { effort: 'xhigh', summary: 'detailed' }, verbosity: 'low', serviceTier: 'priority', }); - assertEquals(body.reasoning, { effort: 'xhigh', summary: 'detailed', context: 'future_mode' }); - assertEquals(body.text?.verbosity, 'low'); - assertEquals(body.service_tier, 'priority'); + assertEquals(applied.reasoning, { effort: 'xhigh', summary: 'detailed', context: 'future_mode' }); + assertEquals(applied.text?.verbosity, 'low'); + assertEquals(applied.service_tier, 'priority'); }); // ── Anthropic Messages target ── -test('messages: empty rules leave the payload unchanged', () => { +test('anthropic-messages: empty rules leave the payload unchanged', () => { const body = msgPayload({ output_config: { effort: 'high' }, thinking: { type: 'enabled', budget_tokens: 512 }, speed: 'fast' }); - applyRulesToUpstreamAnthropicMessages(body, {}); - assertEquals(body.output_config?.effort, 'high'); - assertEquals(body.thinking?.budget_tokens, 512); - assertEquals(body.speed, 'fast'); + const applied = applyRulesToUpstreamAnthropicMessages(body, {}); + assertEquals(applied.output_config?.effort, 'high'); + assertEquals(applied.thinking?.budget_tokens, 512); + assertEquals(applied.speed, 'fast'); }); -test('messages: effort lands on output_config, budget+adaptive land on thinking', () => { +test('anthropic-messages: effort lands on output_config, budget+adaptive land on thinking', () => { const body = msgPayload(); - applyRulesToUpstreamAnthropicMessages(body, { + const applied = applyRulesToUpstreamAnthropicMessages(body, { reasoning: { effort: 'high', budget_tokens: 2048 }, }); - assertEquals(body.output_config?.effort, 'high'); - assertEquals(body.thinking?.type, 'enabled'); - assertEquals(body.thinking?.budget_tokens, 2048); + assertEquals(applied.output_config?.effort, 'high'); + assertEquals(applied.thinking?.type, 'enabled'); + assertEquals(applied.thinking?.budget_tokens, 2048); }); -test('messages: verbosity has no Anthropic-shaped slot — silently dropped', () => { +test('anthropic-messages: verbosity has no Anthropic-shaped slot — silently dropped', () => { const body = msgPayload(); - applyRulesToUpstreamAnthropicMessages(body, { verbosity: 'low' }); - assertEquals('verbosity' in body, false); + const applied = applyRulesToUpstreamAnthropicMessages(body, { verbosity: 'low' }); + assertEquals('verbosity' in applied, false); }); -test('messages: summary=concise|detailed collapses onto thinking.display=summarized (enables thinking)', () => { +test('anthropic-messages: summary=concise|detailed collapses onto thinking.display=summarized (enables thinking)', () => { const body = msgPayload(); - applyRulesToUpstreamAnthropicMessages(body, { reasoning: { summary: 'concise' } }); - assertEquals(body.thinking?.type, 'enabled'); - assertEquals(body.thinking?.display, 'summarized'); + const applied = applyRulesToUpstreamAnthropicMessages(body, { reasoning: { summary: 'concise' } }); + assertEquals(applied.thinking?.type, 'enabled'); + assertEquals(applied.thinking?.display, 'summarized'); }); -test('messages: summary=omitted collapses onto thinking.display=omitted', () => { +test('anthropic-messages: summary=omitted collapses onto thinking.display=omitted', () => { const body = msgPayload(); - applyRulesToUpstreamAnthropicMessages(body, { reasoning: { summary: 'omitted' } }); - assertEquals(body.thinking?.display, 'omitted'); + const applied = applyRulesToUpstreamAnthropicMessages(body, { reasoning: { summary: 'omitted' } }); + assertEquals(applied.thinking?.display, 'omitted'); }); -test('messages: summary=auto is a no-op (Anthropic default takes over)', () => { +test('anthropic-messages: summary=auto is a no-op (Anthropic default takes over)', () => { const body = msgPayload(); - applyRulesToUpstreamAnthropicMessages(body, { reasoning: { summary: 'auto' } }); - assertEquals(body.thinking, undefined); + const applied = applyRulesToUpstreamAnthropicMessages(body, { reasoning: { summary: 'auto' } }); + assertEquals(applied.thinking, undefined); }); -test('messages: adaptive=true sets thinking.type=adaptive and ignores budget_tokens', () => { +test('anthropic-messages: adaptive=true sets thinking.type=adaptive and ignores budget_tokens', () => { const body = msgPayload(); - applyRulesToUpstreamAnthropicMessages(body, { reasoning: { adaptive: true, budget_tokens: 4096 } }); - assertEquals(body.thinking?.type, 'adaptive'); + const applied = applyRulesToUpstreamAnthropicMessages(body, { reasoning: { adaptive: true, budget_tokens: 4096 } }); + assertEquals(applied.thinking?.type, 'adaptive'); }); -test('messages: adaptive=true strips a client-set budget_tokens from body.thinking', () => { +test('anthropic-messages: adaptive=true strips a client-set budget_tokens from body.thinking', () => { const body = msgPayload(); body.thinking = { type: 'enabled', budget_tokens: 5000 }; - applyRulesToUpstreamAnthropicMessages(body, { reasoning: { adaptive: true } }); - assertEquals(body.thinking?.type, 'adaptive'); + const applied = applyRulesToUpstreamAnthropicMessages(body, { reasoning: { adaptive: true } }); + assertEquals(applied.thinking?.type, 'adaptive'); // The prior client budget must not leak into the adaptive block — adaptive // auto-determines the budget and a sibling budget_tokens violates the // rules-are-authoritative contract. - assertEquals((body.thinking as { budget_tokens?: number }).budget_tokens, undefined); + assertEquals((applied.thinking as { budget_tokens?: number }).budget_tokens, undefined); }); -test('messages: serviceTier=fast maps to speed=fast (cross-protocol bridge)', () => { +test('anthropic-messages: serviceTier=fast maps to speed=fast (cross-protocol bridge)', () => { const body = msgPayload(); - applyRulesToUpstreamAnthropicMessages(body, { serviceTier: 'fast' }); - assertEquals(body.speed, 'fast'); - assertEquals(body.service_tier, undefined); + const applied = applyRulesToUpstreamAnthropicMessages(body, { serviceTier: 'fast' }); + assertEquals(applied.speed, 'fast'); + assertEquals(applied.service_tier, undefined); }); -test('messages: serviceTier=priority maps to speed=fast (OpenAI renamed the lane, both spellings reach it)', () => { +test('anthropic-messages: non-accelerated serviceTier lands on service_tier directly', () => { const body = msgPayload(); - applyRulesToUpstreamAnthropicMessages(body, { serviceTier: 'priority' }); - assertEquals(body.speed, 'fast'); - assertEquals(body.service_tier, undefined); + const applied = applyRulesToUpstreamAnthropicMessages(body, { serviceTier: 'flex' }); + assertEquals(applied.service_tier, 'flex'); + assertEquals(applied.speed, undefined); }); -test('messages: non-accelerated serviceTier lands on service_tier directly', () => { - const body = msgPayload(); - applyRulesToUpstreamAnthropicMessages(body, { serviceTier: 'flex' }); - assertEquals(body.service_tier, 'flex'); - assertEquals(body.speed, undefined); -}); - -test('messages: serviceTier=fast clears a pre-existing body.service_tier on the same payload', () => { +test('anthropic-messages: serviceTier=fast clears a pre-existing body.service_tier on the same payload', () => { // Upstream must never see both `speed` and `service_tier` set on the // same request — Anthropic treats them as alternates and the wire // semantics for a conflict are undefined. The overlay clears the // sibling field whichever branch it takes. - const body = msgPayload({ service_tier: 'flex' }); - applyRulesToUpstreamAnthropicMessages(body, { serviceTier: 'fast' }); - assertEquals(body.speed, 'fast'); - assertEquals(body.service_tier, undefined); + const body = msgPayload({ service_tier: 'priority' }); + const applied = applyRulesToUpstreamAnthropicMessages(body, { serviceTier: 'fast' }); + assertEquals(applied.speed, 'fast'); + assertEquals(applied.service_tier, undefined); }); -test('messages: non-accelerated serviceTier clears a pre-existing body.speed on the same payload', () => { +test('anthropic-messages: non-accelerated serviceTier clears a pre-existing body.speed on the same payload', () => { const body = msgPayload({ speed: 'fast' }); - applyRulesToUpstreamAnthropicMessages(body, { serviceTier: 'flex' }); - assertEquals(body.service_tier, 'flex'); - assertEquals(body.speed, undefined); + const applied = applyRulesToUpstreamAnthropicMessages(body, { serviceTier: 'flex' }); + assertEquals(applied.service_tier, 'flex'); + assertEquals(applied.speed, undefined); }); -test('messages: alias rules overwrite existing thinking + output_config fields', () => { +test('anthropic-messages: alias rules overwrite existing thinking + output_config fields', () => { const body = msgPayload({ output_config: { effort: 'low' }, thinking: { type: 'enabled', budget_tokens: 100 } }); - applyRulesToUpstreamAnthropicMessages(body, { reasoning: { effort: 'xhigh', budget_tokens: 9999 } }); - assertEquals(body.output_config?.effort, 'xhigh'); - assertEquals(body.thinking?.budget_tokens, 9999); + const applied = applyRulesToUpstreamAnthropicMessages(body, { reasoning: { effort: 'xhigh', budget_tokens: 9999 } }); + assertEquals(applied.output_config?.effort, 'xhigh'); + assertEquals(applied.thinking?.budget_tokens, 9999); +}); + +// ── The frozen payload every overlay is actually handed ── +// +// At the dial the payload descends from a record, so the object an overlay is handed is frozen +// and every rule it could want to set is a write that throws. Each of the three is exercised +// with the rules that touch the most of its payload, which is where an in-place write would be. + +test('openai-chat-completions: produces a payload without writing into the frozen one', () => { + const body = Object.freeze(ccPayload({ reasoning_effort: 'low' })); + const applied = applyRulesToUpstreamOpenAIChatCompletions(body, { + reasoning: { effort: 'xhigh' }, + verbosity: 'low', + serviceTier: 'priority', + }); + assertEquals(applied.reasoning_effort, 'xhigh'); + assertEquals(body.reasoning_effort, 'low'); +}); + +test('openai-responses: produces a payload without writing into the frozen one', () => { + const body = Object.freeze(resPayload({ reasoning: { effort: 'low' }, text: { verbosity: 'high' } })); + const applied = applyRulesToUpstreamOpenAIResponses(body, { + reasoning: { effort: 'xhigh', summary: 'detailed' }, + verbosity: 'low', + serviceTier: 'priority', + }); + assertEquals(applied.reasoning?.effort, 'xhigh'); + assertEquals(body.reasoning?.effort, 'low'); +}); + +test('anthropic-messages: produces a payload without writing into the frozen one', () => { + const body = Object.freeze(msgPayload({ thinking: { type: 'enabled', budget_tokens: 100 }, service_tier: 'priority' })); + const applied = applyRulesToUpstreamAnthropicMessages(body, { + reasoning: { effort: 'xhigh', adaptive: true, summary: 'concise' }, + serviceTier: 'fast', + }); + assertEquals(applied.thinking?.type, 'adaptive'); + assertEquals(applied.speed, 'fast'); + assertEquals(applied.service_tier, undefined); + assertEquals(body.thinking?.type, 'enabled'); + assertEquals(body.service_tier, 'priority'); +}); + +test('anthropic-messages: priority selects native fast speed without mutating the input', () => { + const body = Object.freeze(msgPayload({ service_tier: 'flex' })); + const applied = applyRulesToUpstreamAnthropicMessages(body, { serviceTier: 'priority' }); + assertEquals(applied.speed, 'fast'); + assertEquals(applied.service_tier, undefined); + assertEquals(body.service_tier, 'flex'); }); diff --git a/packages/gateway/__tests__/data-plane/chat/shared/errors_test.ts b/packages/gateway/__tests__/data-plane/chat/shared/errors_test.ts index 9e77a7c03a..42945dadb8 100644 --- a/packages/gateway/__tests__/data-plane/chat/shared/errors_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/shared/errors_test.ts @@ -1,25 +1,17 @@ +// The one refusal that travels as a throw, and the guard that keeps it from catching anything +// else. What it carries has to survive the trip intact: the message names which carried target +// went missing, and only the selection several frames down knows that. + import { test } from 'vitest'; import { type ChatServeFailure, throwChatServeFailure, tryCatchChatServeFailure } from '../../../../src/data-plane/chat/shared/errors.ts'; import { assertEquals, assertThrows } from '@floway-dev/test-utils'; -const cases: readonly ChatServeFailure[] = [ - { kind: 'model-missing', model: 'gpt-9', failedUpstreams: [] }, - { kind: 'model-missing', model: 'gpt-9', failedUpstreams: ['Azure prod'] }, - { kind: 'model-unsupported', model: 'gpt-9', failedUpstreams: [] }, - { kind: 'model-unsupported', model: 'gpt-9', failedUpstreams: ['Azure prod', 'Custom'] }, - { kind: 'routing-unavailable', message: 'no upstream can serve this' }, -]; - -for (const failure of cases) { - const label = 'failedUpstreams' in failure && failure.failedUpstreams.length - ? `${failure.kind} (with ${failure.failedUpstreams.length} failed upstream(s))` - : failure.kind; - test(`round-trips ${label} through throw/catch`, () => { - const error = assertThrows(() => throwChatServeFailure(failure)); - assertEquals(tryCatchChatServeFailure(error), failure); - }); -} +test('round-trips routing-unavailable through throw/catch', () => { + const failure: ChatServeFailure = { kind: 'routing-unavailable', message: 'no upstream can serve this' }; + const error = assertThrows(() => throwChatServeFailure(failure)); + assertEquals(tryCatchChatServeFailure(error), failure); +}); test('returns null for an error not raised by throwChatServeFailure', () => { assertEquals(tryCatchChatServeFailure(new Error('something else')), null); diff --git a/packages/gateway/__tests__/data-plane/chat/shared/first-output-token_test.ts b/packages/gateway/__tests__/data-plane/chat/shared/first-output-token_test.ts index 4ecbfd27a0..99c888b92e 100644 --- a/packages/gateway/__tests__/data-plane/chat/shared/first-output-token_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/shared/first-output-token_test.ts @@ -5,7 +5,7 @@ import type { ProtocolFrame } from '@floway-dev/protocols/common'; const eventFrame = (event: T): ProtocolFrame => ({ type: 'event', event }); -describe('isFirstOutputTokenFrame — messages', () => { +describe('isFirstOutputTokenFrame — anthropicMessages', () => { it('accepts text_delta', () => { expect(isFirstOutputTokenFrame(eventFrame({ type: 'content_block_delta', delta: { type: 'text_delta', text: 'hi' } }), 'anthropicMessages')).toBe(true); }); @@ -35,7 +35,7 @@ describe('isFirstOutputTokenFrame — messages', () => { }); }); -describe('isFirstOutputTokenFrame — responses', () => { +describe('isFirstOutputTokenFrame — openaiResponses', () => { it('accepts response.output_text.delta', () => { expect(isFirstOutputTokenFrame(eventFrame({ type: 'response.output_text.delta', delta: 'hi' }), 'openaiResponses')).toBe(true); }); @@ -68,7 +68,7 @@ describe('isFirstOutputTokenFrame — responses', () => { }); }); -describe('isFirstOutputTokenFrame — openai-chat-completions', () => { +describe('isFirstOutputTokenFrame — openaiChatCompletions', () => { it('accepts chunk with delta.content', () => { expect(isFirstOutputTokenFrame(eventFrame({ choices: [{ delta: { content: 'hi' } }] }), 'openaiChatCompletions')).toBe(true); }); diff --git a/packages/gateway/__tests__/data-plane/chat/shared/provider-stream-result_test.ts b/packages/gateway/__tests__/data-plane/chat/shared/provider-stream-result_test.ts deleted file mode 100644 index 4041576a12..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/shared/provider-stream-result_test.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { describe, expect, test } from 'vitest'; - -import { providerStreamResultToExecuteResult } from '../../../../src/data-plane/chat/shared/provider-stream-result.ts'; -import { mockGatewayCtx } from '../../../test-utils/gateway-ctx.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ProviderStreamResult } from '@floway-dev/provider'; -import { stubModelCandidate } from '@floway-dev/test-utils'; - -const iter = (items: readonly T[]): AsyncIterable => ({ - async *[Symbol.asyncIterator]() { for (const item of items) yield item; }, -}); - -const okStreamResult = (events: AsyncIterable>): ProviderStreamResult => ({ - ok: true, - events, - modelKey: 'test-model-key', -}); - -const drainEvents = async (result: Awaited>>): Promise[]> => { - if (result.type !== 'events') throw new Error(`expected events result, got ${result.type}`); - const collected: ProtocolFrame[] = []; - for await (const frame of result.events) collected.push(frame); - return collected; -}; - -describe('providerStreamResultToExecuteResult (first-output-token stamping)', () => { - test('stamps firstOutputTokenAt on the first generated-token frame (messages thinking_delta)', async () => { - const ctx = mockGatewayCtx(); - const frames: ProtocolFrame[] = [ - { type: 'event', event: { type: 'message_start' } }, - { type: 'event', event: { type: 'content_block_delta', delta: { type: 'thinking_delta', thinking: '...' } } }, - { type: 'event', event: { type: 'content_block_delta', delta: { type: 'text_delta', text: 'hi' } } }, - { type: 'event', event: { type: 'content_block_delta', delta: { type: 'text_delta', text: ' there' } } }, - ]; - const result = await providerStreamResultToExecuteResult(okStreamResult(iter(frames)), stubModelCandidate(), 'anthropicMessages', ctx, () => null); - const collected = await drainEvents(result); - expect(collected).toEqual(frames); - expect(ctx.attempt.timing.firstOutputTokenAt).not.toBe(null); - }); - - test('leaves firstOutputTokenAt null when only envelope frames appear', async () => { - const ctx = mockGatewayCtx(); - const frames: ProtocolFrame[] = [ - { type: 'event', event: { type: 'response.created' } }, - { type: 'event', event: { type: 'response.output_item.added' } }, - ]; - const result = await providerStreamResultToExecuteResult(okStreamResult(iter(frames)), stubModelCandidate(), 'openaiResponses', ctx, () => null); - await drainEvents(result); - expect(ctx.attempt.timing.firstOutputTokenAt).toBe(null); - }); - - test('stamps at most once even for many output-content frames', async () => { - const ctx = mockGatewayCtx(); - const frames: ProtocolFrame[] = [ - { type: 'event', event: { choices: [{ delta: { content: 'a' } }] } }, - { type: 'event', event: { choices: [{ delta: { content: 'b' } }] } }, - { type: 'event', event: { choices: [{ delta: { content: 'c' } }] } }, - ]; - const result = await providerStreamResultToExecuteResult(okStreamResult(iter(frames)), stubModelCandidate(), 'openaiChatCompletions', ctx, () => null); - if (result.type !== 'events') throw new Error(`expected events result, got ${result.type}`); - const stampsAfterEachFrame: (number | null)[] = []; - for await (const _ of result.events) stampsAfterEachFrame.push(ctx.attempt.timing.firstOutputTokenAt); - expect(stampsAfterEachFrame[0]).not.toBe(null); - // The subsequent frames must observe the exact same stamp — the stamping - // hook never overwrites once firstOutputTokenAt has been set. - expect(stampsAfterEachFrame[1]).toBe(stampsAfterEachFrame[0]); - expect(stampsAfterEachFrame[2]).toBe(stampsAfterEachFrame[0]); - }); -}); - -test('an abandoned stream still settles its metadata instead of hanging the caller', async () => { - // Every streaming response resolves its cost through finalMetadata, and the - // respond layer awaits it in a finally. A transport that walks away without - // closing the generator would hang that await forever. - const abort = new AbortController(); - const ctx = { ...mockGatewayCtx(), abortSignal: abort.signal }; - const frames: ProtocolFrame[] = [{ type: 'event', event: { type: 'response.created' } }]; - - const result = await providerStreamResultToExecuteResult(okStreamResult(iter(frames)), stubModelCandidate(), 'openaiResponses', ctx, () => null); - expect(result.type).toBe('events'); - if (result.type !== 'events') return; - - // Never iterate the events; just abandon them. - abort.abort(); - - expect((await result.finalMetadata!).modelIdentity).toBeDefined(); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/shared/respond_test.ts b/packages/gateway/__tests__/data-plane/chat/shared/source-stream-state_test.ts similarity index 96% rename from packages/gateway/__tests__/data-plane/chat/shared/respond_test.ts rename to packages/gateway/__tests__/data-plane/chat/shared/source-stream-state_test.ts index ec3a3dc54a..ec19efd66b 100644 --- a/packages/gateway/__tests__/data-plane/chat/shared/respond_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/shared/source-stream-state_test.ts @@ -1,6 +1,6 @@ import { test } from 'vitest'; -import { SourceStreamState } from '../../../../src/data-plane/chat/shared/respond.ts'; +import { SourceStreamState } from '../../../../src/data-plane/chat/shared/source-stream-state.ts'; import { assertEquals } from '@floway-dev/test-utils'; // ── SourceStreamState classification ── diff --git a/packages/gateway/__tests__/data-plane/chat/shared/translate-traverse_test.ts b/packages/gateway/__tests__/data-plane/chat/shared/translate-traverse_test.ts deleted file mode 100644 index e467fc77d1..0000000000 --- a/packages/gateway/__tests__/data-plane/chat/shared/translate-traverse_test.ts +++ /dev/null @@ -1,312 +0,0 @@ -import { describe, expect, test } from 'vitest'; - -import { traverseTranslation } from '../../../../src/data-plane/chat/shared/translate-traverse.ts'; -import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; - -// Fake event types — no real protocol types needed. -type SrcEvent = { kind: 'src'; value: string }; -type TgtEvent = { kind: 'tgt'; value: string }; - -const translate = async (payload: string): Promise<{ - target: string; - events: (e: AsyncIterable>) => AsyncIterable>; -}> => ({ - target: `translated(${payload})`, - async *events(frames) { - for await (const frame of frames) { - if (frame.type === 'done') { - yield doneFrame(); - } else { - yield eventFrame({ kind: 'src', value: frame.event.value }); - } - } - }, -}); - -const makeTgtFrames = async function* (values: string[]): AsyncGenerator> { - for (const v of values) yield eventFrame({ kind: 'tgt', value: v }); - yield doneFrame(); -}; - -const collectFrames = async (iterable: AsyncIterable>): Promise[]> => { - const out: ProtocolFrame[] = []; - for await (const frame of iterable) out.push(frame); - return out; -}; - -const fakeModelIdentity = { modelId: 'fake' } as never; - -describe('traverseTranslation', () => { - test('events inner — outer wraps with trip.events', async () => { - const innerAttempt = async (_translated: string): Promise>> => ({ - type: 'events', - events: makeTgtFrames(['a', 'b']), - modelIdentity: fakeModelIdentity, - }); - - const result = await traverseTranslation('payload', translate, innerAttempt); - - expect(result.type).toBe('events'); - if (result.type !== 'events') throw new Error('unreachable'); - - const frames = await collectFrames(result.events); - expect(frames).toEqual([ - { type: 'event', event: { kind: 'src', value: 'a' } }, - { type: 'event', event: { kind: 'src', value: 'b' } }, - { type: 'done' }, - ]); - expect(result.modelIdentity).toBe(fakeModelIdentity); - }); - - test('result inner — outer returns the result unchanged', async () => { - const inner: ExecuteResult> = { - type: 'api-error', - source: 'upstream', - status: 404, - headers: new Headers(), - body: new Uint8Array(), - }; - const innerAttempt = async (_translated: string): Promise>> => inner; - - const result = await traverseTranslation('payload', translate, innerAttempt); - - expect(result).toBe(inner); - }); - - test('failure inner — outer returns the failure unchanged', async () => { - const inner: ExecuteResult> = { - type: 'internal-error', - status: 500, - error: new Error('boom') as never, - }; - const innerAttempt = async (_translated: string): Promise>> => inner; - - const result = await traverseTranslation('payload', translate, innerAttempt); - - expect(result).toBe(inner); - }); - - test('upstream api-error with trip.apiError rewrite — outer returns the rewritten envelope', async () => { - const inner: ExecuteResult> = { - type: 'api-error', - source: 'upstream', - status: 400, - headers: new Headers({ 'x-upstream-request-id': 'abc' }), - body: new TextEncoder().encode('raw upstream'), - }; - const translateWithRewrite = async (payload: string): Promise<{ - target: string; - events: (e: AsyncIterable>) => AsyncIterable>; - apiError: (upstream: { status: number; headers: Headers; body: Uint8Array }) => { status: number; headers: Headers; body: Uint8Array }; - }> => ({ - ...(await translate(payload)), - apiError: () => ({ status: 400, headers: new Headers({ 'content-type': 'application/json' }), body: new TextEncoder().encode('rewritten') }), - }); - - const result = await traverseTranslation('payload', translateWithRewrite, async () => inner); - expect(result.type).toBe('api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - expect(new TextDecoder().decode(result.body)).toBe('rewritten'); - expect(result.headers.get('content-type')).toBe('application/json'); - expect(result.headers.get('x-upstream-request-id')).toBeNull(); - // Fields not touched by the rewriter (source, performance, upstream) survive. - expect(result.source).toBe('upstream'); - }); - - test('upstream api-error with trip.apiError returning undefined — outer passes the envelope through', async () => { - const inner: ExecuteResult> = { - type: 'api-error', - source: 'upstream', - status: 502, - headers: new Headers(), - body: new TextEncoder().encode('bad gateway'), - }; - const translateWithRewrite = async (payload: string): Promise<{ - target: string; - events: (e: AsyncIterable>) => AsyncIterable>; - apiError: () => undefined; - }> => ({ ...(await translate(payload)), apiError: () => undefined }); - - const result = await traverseTranslation('payload', translateWithRewrite, async () => inner); - expect(result).toBe(inner); - }); - - test('gateway-sourced api-error — trip.apiError is NOT invoked', async () => { - const inner: ExecuteResult> = { - type: 'api-error', - source: 'gateway', - status: 400, - headers: new Headers(), - body: new TextEncoder().encode('synthesized'), - }; - let called = false; - const translateWithRewrite = async (payload: string): Promise<{ - target: string; - events: (e: AsyncIterable>) => AsyncIterable>; - apiError: () => { status: number; headers: Headers; body: Uint8Array }; - }> => ({ - ...(await translate(payload)), - apiError: () => { - called = true; - return { status: 999, headers: new Headers(), body: new Uint8Array() }; - }, - }); - - const result = await traverseTranslation('payload', translateWithRewrite, async () => inner); - expect(called).toBe(false); - expect(result).toBe(inner); - }); -}); - -// The pre-translation capture hook sees the ORIGINAL target-protocol frames -// (events path) and the verbatim upstream api-error envelope (api-error path), -// before `trip.events` / `trip.apiError` rewrite them. `capture === undefined` -// (api key opts out of dumps) skips all capture work. -describe('traverseTranslation upstream capture', () => { - test('events path — capture receives each original target frame, in order, before translation', async () => { - const innerAttempt = async (_translated: string): Promise>> => ({ - type: 'events', - events: makeTgtFrames(['a', 'b']), - modelIdentity: fakeModelIdentity, - }); - const captured: ProtocolFrame[] = []; - const capture = { - upstreamFrame: (frame: ProtocolFrame) => captured.push(frame), - upstreamApiError: () => { throw new Error('should not be called on events path'); }, - }; - - const result = await traverseTranslation('payload', translate, innerAttempt, capture); - expect(result.type).toBe('events'); - if (result.type !== 'events') throw new Error('unreachable'); - // Drain the translated stream so the tee pulls the inner frames. - await collectFrames(result.events); - - // Capture saw the original target frames (incl. done) in order. - expect(captured).toEqual([ - { type: 'event', event: { kind: 'tgt', value: 'a' } }, - { type: 'event', event: { kind: 'tgt', value: 'b' } }, - { type: 'done' }, - ]); - }); - - test('events path — downstream still receives the translated frames when capture is set', async () => { - const innerAttempt = async (_translated: string): Promise>> => ({ - type: 'events', - events: makeTgtFrames(['a']), - modelIdentity: fakeModelIdentity, - }); - const capture = { upstreamFrame: () => {}, upstreamApiError: () => {} }; - - const result = await traverseTranslation('payload', translate, innerAttempt, capture); - if (result.type !== 'events') throw new Error('unreachable'); - const frames = await collectFrames(result.events); - expect(frames).toEqual([ - { type: 'event', event: { kind: 'src', value: 'a' } }, - { type: 'done' }, - ]); - }); - - test('api-error path — capture receives the verbatim upstream envelope before any rewrite', async () => { - const inner: ExecuteResult> = { - type: 'api-error', - source: 'upstream', - status: 413, - headers: new Headers({ 'x-upstream': 'yes' }), - body: new TextEncoder().encode('raw upstream body'), - }; - const captured: { status: number; headers: Headers; body: Uint8Array }[] = []; - const capture = { - upstreamFrame: () => { throw new Error('should not be called on api-error path'); }, - upstreamApiError: (error: { status: number; headers: Headers; body: Uint8Array }) => captured.push(error), - }; - const translateWithRewrite = async (payload: string): Promise<{ - target: string; - events: (e: AsyncIterable>) => AsyncIterable>; - apiError: () => { status: number; headers: Headers; body: Uint8Array }; - }> => ({ - ...(await translate(payload)), - apiError: () => ({ status: 400, headers: new Headers(), body: new TextEncoder().encode('rewritten') }), - }); - - const result = await traverseTranslation('payload', translateWithRewrite, async () => inner, capture); - expect(result.type).toBe('api-error'); - if (result.type !== 'api-error') throw new Error('unreachable'); - // Capture saw the ORIGINAL envelope, not the rewrite. - expect(captured).toHaveLength(1); - expect(captured[0].status).toBe(413); - expect(captured[0].headers.get('x-upstream')).toBe('yes'); - expect(new TextDecoder().decode(captured[0].body)).toBe('raw upstream body'); - // Downstream got the rewrite. - expect(new TextDecoder().decode(result.body)).toBe('rewritten'); - }); - - test('api-error path — capture fires even when trip.apiError is undefined (no rewrite)', async () => { - const inner: ExecuteResult> = { - type: 'api-error', - source: 'upstream', - status: 502, - headers: new Headers(), - body: new TextEncoder().encode('bad gateway'), - }; - const captured: { status: number }[] = []; - const capture = { - upstreamFrame: () => {}, - upstreamApiError: (error: { status: number; headers: Headers; body: Uint8Array }) => captured.push({ status: error.status }), - }; - - await traverseTranslation('payload', translate, async () => inner, capture); - expect(captured).toEqual([{ status: 502 }]); - }); - - test('internal-error path — capture is NOT invoked', async () => { - const inner: ExecuteResult> = { - type: 'internal-error', - status: 500, - error: new Error('boom') as never, - }; - let called = false; - const capture = { - upstreamFrame: () => { called = true; }, - upstreamApiError: () => { called = true; }, - }; - - const result = await traverseTranslation('payload', translate, async () => inner, capture); - expect(called).toBe(false); - expect(result).toBe(inner); - }); - - test('gateway-sourced api-error — capture is NOT invoked', async () => { - const inner: ExecuteResult> = { - type: 'api-error', - source: 'gateway', - status: 400, - headers: new Headers(), - body: new TextEncoder().encode('synthesized'), - }; - let called = false; - const capture = { - upstreamFrame: () => { called = true; }, - upstreamApiError: () => { called = true; }, - }; - - await traverseTranslation('payload', translate, async () => inner, capture); - expect(called).toBe(false); - }); - - test('capture undefined — no tee, downstream still works', async () => { - const innerAttempt = async (_translated: string): Promise>> => ({ - type: 'events', - events: makeTgtFrames(['a']), - modelIdentity: fakeModelIdentity, - }); - - const result = await traverseTranslation('payload', translate, innerAttempt, undefined); - if (result.type !== 'events') throw new Error('unreachable'); - const frames = await collectFrames(result.events); - expect(frames).toEqual([ - { type: 'event', event: { kind: 'src', value: 'a' } }, - { type: 'done' }, - ]); - }); -}); diff --git a/packages/gateway/__tests__/data-plane/chat/shared/usage_test.ts b/packages/gateway/__tests__/data-plane/chat/shared/usage_test.ts new file mode 100644 index 0000000000..1a171e30cf --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/shared/usage_test.ts @@ -0,0 +1,37 @@ +import { expect, test } from 'vitest'; + +import { createAnthropicMessagesBillableUsageReader } from '../../../../src/data-plane/chat/anthropic-messages/usage.ts'; +import { billableUsageFromOpenAIChatCompletionsEvent } from '../../../../src/data-plane/chat/openai-chat-completions/usage.ts'; +import { billableUsageFromOpenAIResponsesResult } from '../../../../src/data-plane/chat/openai-responses/usage.ts'; +import { chatUsageMeasurement } from '../../../../src/data-plane/chat/shared/usage.ts'; +import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; + +for (const protocol of ['openaiChatCompletions', 'anthropicMessages', 'openaiResponses'] as const) test(`${protocol} preserves reported zero input and output as measured quantities`, () => { + const usage = protocol === 'openaiChatCompletions' + ? billableUsageFromOpenAIChatCompletionsEvent({ id: 'c', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [], usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 } }) + : protocol === 'openaiResponses' + ? billableUsageFromOpenAIResponsesResult({ usage: { input_tokens: 0, output_tokens: 0, total_tokens: 0 } }) + : createAnthropicMessagesBillableUsageReader()({ type: 'message_start', message: { id: 'm', type: 'message', role: 'assistant', model: 'm', content: [], stop_reason: null, stop_sequence: null, usage: { input_tokens: 0, output_tokens: 0 } } }); + if (usage === null) throw new Error('A reported zero usage block must remain a measurement'); + expect(chatUsageMeasurement(usage)).toEqual({ quantities: { input_tokens: '0', output_tokens: '0' }, pricingFacts: { inputTokens: 0 }, dumpTokenUsage: { input: 0, output: 0 } }); +}); + +test('absent native usage remains absent rather than reporting zero', () => { + const chunk: OpenAIChatCompletionsStreamEvent = { id: 'c', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [] }; + expect(billableUsageFromOpenAIChatCompletionsEvent(chunk)).toBeNull(); + expect(billableUsageFromOpenAIResponsesResult({})).toBeNull(); + expect(createAnthropicMessagesBillableUsageReader()({ type: 'message_stop' })).toBeNull(); +}); + +test('Messages partial input accounting preserves zero without manufacturing unreported output', () => { + const usage = createAnthropicMessagesBillableUsageReader()({ type: 'message_start', message: { usage: { input_tokens: 0 } } } as AnthropicMessagesStreamEvent); + if (usage === null) throw new Error('Reported input must remain measurable'); + expect(chatUsageMeasurement(usage)).toEqual({ quantities: { input_tokens: '0' }, pricingFacts: { inputTokens: 0 }, dumpTokenUsage: { input: 0 } }); +}); + +test('Messages partial output accounting preserves zero without manufacturing input or its pricing selector', () => { + const usage = createAnthropicMessagesBillableUsageReader()({ type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 0 } }); + if (usage === null) throw new Error('Reported output must remain measurable'); + expect(chatUsageMeasurement(usage)).toEqual({ quantities: { output_tokens: '0' }, pricingFacts: {}, dumpTokenUsage: { output: 0 } }); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts new file mode 100644 index 0000000000..dd29378bda --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/translated-wires_test.ts @@ -0,0 +1,556 @@ +// The translated wires, run end to end — one row per (source, target) pair. +// +// The property each row states is the same, and it is the one nothing else can state: a +// candidate reachable *only* over another protocol's endpoint is dialled there rather than +// refused or dialled on the wrong wire, and what comes back reaches the client in the +// client's own protocol. The candidates here carry exactly one endpoint each, so a chain that +// picked any other wire would call a provider method that throws. +// +// Beside the matrix, three things that are about the fork rather than about a pair: that +// failover moves from a native candidate onto a translated one, that a pair's own refusal +// rewrite survives the trip, and that a rule which speaks about one protocol's wire does not +// run on a turn that leaves for another — which a target-API guard inside each rule could also +// say, and which position says here without a rule having to know where it sits. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { anthropicMessagesServePipeline } from '../../../src/data-plane/chat/anthropic-messages/pipeline.ts'; +import { anthropicMessagesWire } from '../../../src/data-plane/chat/anthropic-messages/wire.ts'; +import { geminiGenerateContentServePipeline } from '../../../src/data-plane/chat/gemini-generate-content/pipeline.ts'; +import { handOff } from '../../../src/data-plane/chat/handoff.ts'; +import { openaiChatCompletionsServePipeline } from '../../../src/data-plane/chat/openai-chat-completions/pipeline.ts'; +import { openaiResponsesServePipeline } from '../../../src/data-plane/chat/openai-responses/pipeline.ts'; +import { createCandidateRegistry } from '../../../src/data-plane/pipeline/candidates.ts'; +import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; +import { initRepo } from '../../../src/repo/index.ts'; +import { chatFixtureHttpServices, stubChatProviderPipelines } from '../../test-utils/chat-provider-pipelines.ts'; +import { mockChatGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { compose, move, run, type Pipeline } from '@floway-dev/pipeline'; +import { PROMPT_TOO_LONG_MESSAGE, type AnthropicMessagesPayload, type AnthropicMessagesResult, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import { doneFrame, eventFrame, type ModelEndpoints } from '@floway-dev/protocols/common'; +import type { GeminiGenerateContentPayload, GeminiGenerateContentResult } from '@floway-dev/protocols/gemini-generate-content'; +import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsResult, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import { directFetcher, type FlagId, type ModelCandidate } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel, type ProviderOpenAIResponsesResult, type ProviderStreamResult } from '@floway-dev/test-utils'; + +vi.mock('../../../src/data-plane/providers/resolution.ts', async importOriginal => ({ + ...(await importOriginal()), + enumerateModelCandidates: vi.fn(), +})); + +const MODEL = 'model-x'; + +let live: readonly ModelCandidate[] = []; + +const resolves = (candidates: readonly ModelCandidate[]): void => { + live = candidates; + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] } as never); +}; + +interface Calls { + readonly callOpenAIChatCompletions?: (model: unknown, body: unknown) => Promise>; + readonly callAnthropicMessages?: (model: unknown, body: unknown) => Promise>; + readonly callOpenAIResponses?: (model: unknown, body: unknown) => Promise; +} + +/** A candidate reachable over exactly the endpoints it is given. Handing it one endpoint is + * what pins the wire: every provider method the stub was not given throws, so a chain that + * picked another wire fails loudly rather than quietly answering. */ +const candidate = ( + upstreamId: string, + endpoints: ModelEndpoints, + calls: Calls, + flags: readonly FlagId[] = [], +): ModelCandidate => ({ + provider: { + upstreamId, kind: 'custom', name: upstreamId, inboundHeaderAllowlist: [], + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + pipelines: stubChatProviderPipelines(calls as never), + instance: stubProvider(), + }, + model: stubInternalModel( + { + id: MODEL, + endpoints, + providerModels: { [upstreamId]: stubProviderModel({ id: MODEL, endpoints, enabledFlags: new Set(flags) }) }, + }, + upstreamId, + ), + fetcher: directFetcher, +} as unknown as ModelCandidate); + +// ── What each wire answers with ─────────────────────────────────────────────────────────── + +const openaiChatCompletionsTurn = (text: string): ProviderStreamResult => ({ + ok: true, + modelKey: 'k', + headers: new Headers(), + events: (async function* () { + yield eventFrame({ + id: 'c1', object: 'chat.completion.chunk', created: 1, model: MODEL, + choices: [{ index: 0, delta: { role: 'assistant', content: text }, finish_reason: null }], + } as OpenAIChatCompletionsStreamEvent); + yield eventFrame({ + id: 'c1', object: 'chat.completion.chunk', created: 1, model: MODEL, + choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], + } as OpenAIChatCompletionsStreamEvent); + yield doneFrame(); + })(), +}); + +const anthropicMessagesTurn = (text: string): ProviderStreamResult => ({ + ok: true, + modelKey: 'k', + headers: new Headers(), + events: (async function* () { + yield eventFrame({ + type: 'message_start', + message: { + id: 'msg_1', type: 'message', role: 'assistant', content: [], model: MODEL, + stop_reason: null, stop_sequence: null, usage: { input_tokens: 3, output_tokens: 0 }, + }, + } as AnthropicMessagesStreamEvent); + yield eventFrame({ type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } } as AnthropicMessagesStreamEvent); + yield eventFrame({ type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text } } as AnthropicMessagesStreamEvent); + yield eventFrame({ type: 'content_block_stop', index: 0 } as AnthropicMessagesStreamEvent); + yield eventFrame({ type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 2 } } as AnthropicMessagesStreamEvent); + yield eventFrame({ type: 'message_stop' } as AnthropicMessagesStreamEvent); + yield doneFrame(); + })(), +}); + +const openaiResponsesResult = (text: string): OpenAIResponsesResult => ({ + id: 'resp_1', + object: 'response', + model: MODEL, + status: 'completed', + output: [{ + type: 'message', id: 'msg_1', role: 'assistant', status: 'completed', + content: [{ type: 'output_text', text, annotations: [] }], + }], + error: null, + incomplete_details: null, +}); + +const openaiResponsesTurn = (text: string): ProviderOpenAIResponsesResult => ({ + action: 'generate', + ok: true, + modelKey: 'k', + headers: new Headers(), + events: (async function* () { + yield eventFrame({ + type: 'response.output_item.added', + sequence_number: 1, + output_index: 0, + item: { type: 'message', id: 'msg_1', role: 'assistant', status: 'in_progress', content: [] }, + } as OpenAIResponsesStreamEvent); + yield eventFrame({ + type: 'response.output_text.delta', + sequence_number: 2, item_id: 'msg_1', output_index: 0, content_index: 0, delta: text, + } as OpenAIResponsesStreamEvent); + yield eventFrame({ type: 'response.completed', sequence_number: 3, response: openaiResponsesResult(text) } as OpenAIResponsesStreamEvent); + yield doneFrame(); + })(), +}); + +/** A refusal, in the words an OpenAI-shaped upstream uses for a turn that will not fit. */ +const contextExceeded = async (): Promise> => ({ + ok: false, + modelKey: 'k', + response: Response.json( + { error: { code: 'context_length_exceeded', type: 'invalid_request_error', message: "This model's maximum context length is 8192 tokens" } }, + { status: 400 }, + ), +}); + +// ── Running one source protocol's chain ─────────────────────────────────────────────────── + +const serve = async ( + pipeline: Pipeline, + facts: Record, +) => { + const gateway = mockChatGatewayCtx({ wantsStream: false }); + return await run(pipeline, move(facts) as never, { + gateway, + background: () => {}, + ...chatFixtureHttpServices(gateway), + ...createCandidateRegistry(), + selectAffinity: (selected: ModelCandidate) => { gateway.affinity.select(selected); }, + resolveAttempt: (selector: { readonly upstreamId: string }) => { + const found = live.find(c => c.provider.upstreamId === selector.upstreamId); + if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); + return found; + }, + } as never); +}; + +const openaiChatCompletionsPayload = { model: MODEL, messages: [{ role: 'user', content: 'hi' }] } as unknown as OpenAIChatCompletionsPayload; +const anthropicMessagesPayload = { model: MODEL, max_tokens: 64, messages: [{ role: 'user', content: 'hi' }] } as unknown as AnthropicMessagesPayload; +const openaiResponsesPayload = { model: MODEL, input: [{ type: 'message', role: 'user', content: 'hi' }] } as unknown as CanonicalOpenAIResponsesPayload; +const geminiGenerateContentPayload = { contents: [{ role: 'user' as const, parts: [{ text: 'hi' }] }] } satisfies GeminiGenerateContentPayload; + +const serveOpenAIChatCompletions = async (payload: OpenAIChatCompletionsPayload = openaiChatCompletionsPayload) => + await serve(openaiChatCompletionsServePipeline(), { + 'ingress.http.headers': [], + 'ingress.chat.sourceProtocol': 'openaiChatCompletions', + 'ingress.chat.openaiChatCompletions.wantsStream': false, + 'ingress.chat.openaiChatCompletions.wantsUsageChunk': false, + 'request.chat.openaiChatCompletions': payload, + 'serve.model': MODEL, + }); + +const serveAnthropicMessages = async (payload: AnthropicMessagesPayload = anthropicMessagesPayload) => + await serve(anthropicMessagesServePipeline(), { + 'ingress.http.headers': [], + 'ingress.chat.sourceProtocol': 'anthropicMessages', + 'ingress.chat.anthropicMessages.wantsStream': false, + 'request.chat.anthropicMessages': payload, + 'serve.model': MODEL, + }); + +const serveOpenAIResponses = async (payload: CanonicalOpenAIResponsesPayload = openaiResponsesPayload) => + await serve(openaiResponsesServePipeline(), { + 'ingress.http.headers': [], + 'ingress.chat.sourceProtocol': 'openaiResponses', + 'ingress.chat.openaiResponses.wantsStream': false, + 'request.chat.openaiResponses': payload, + 'serve.model': MODEL, + }); + +const serveGeminiGenerateContent = async (payload: GeminiGenerateContentPayload = geminiGenerateContentPayload) => + await serve(geminiGenerateContentServePipeline(), { + 'ingress.http.headers': [], + 'ingress.chat.sourceProtocol': 'geminiGenerateContent', + 'ingress.chat.geminiGenerateContent.wantsStream': false, + 'request.chat.geminiGenerateContent': payload, + 'serve.model': MODEL, + }); + +// ── What the client was answered with, per protocol ─────────────────────────────────────── + +const openaiChatCompletionsText = (rendered: unknown): string | null | undefined => + (rendered as OpenAIChatCompletionsResult).choices[0]?.message.content as string | null | undefined; + +/** The answer's own text, past the carrier the edge writes back: on this protocol the turn's + * state rides as a redacted-thinking block at the head of the content. */ +const anthropicMessagesText = (rendered: unknown): string | undefined => { + const block = (rendered as AnthropicMessagesResult).content.find(part => part.type === 'text'); + return block?.type === 'text' ? block.text : undefined; +}; + +const openaiResponsesText = (rendered: unknown): string | undefined => { + const item = (rendered as OpenAIResponsesResult).output.find(output => output.type === 'message'); + const part = item?.type === 'message' ? item.content[0] : undefined; + return part?.type === 'output_text' ? part.text : undefined; +}; + +const geminiGenerateContentText = (rendered: unknown): string | undefined => + (rendered as GeminiGenerateContentResult).candidates?.[0]?.content.parts.map(part => part.text ?? '').join(''); + +beforeEach(() => { + vi.mocked(enumerateModelCandidates).mockReset(); + initRepo({ + apiKeys: { update: async () => {} }, + usage: { record: async () => {} }, + performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, + } as never); +}); + +describe('a chat family reaching a candidate over another protocol', () => { + it('serves /v1/chat/completions on an Anthropic Messages-only candidate', async () => { + const callAnthropicMessages = vi.fn(async () => anthropicMessagesTurn('hello')); + resolves([candidate('up_a', { anthropicMessages: {} }, { callAnthropicMessages })]); + + const { facts, drain } = await serveOpenAIChatCompletions(); + + expect(callAnthropicMessages).toHaveBeenCalledTimes(1); + expect(facts['response.http.status']).toBe(200); + expect(openaiChatCompletionsText(facts['response.chat.openaiChatCompletions.rendered'])).toBe('hello'); + await drain(); + }); + + it('serves /v1/chat/completions on an OpenAI Responses-only candidate', async () => { + const callOpenAIResponses = vi.fn(async () => openaiResponsesTurn('hello')); + resolves([candidate('up_a', { openaiResponses: {} }, { callOpenAIResponses })]); + + const { facts, drain } = await serveOpenAIChatCompletions(); + + expect(callOpenAIResponses).toHaveBeenCalledTimes(1); + expect(openaiChatCompletionsText(facts['response.chat.openaiChatCompletions.rendered'])).toBe('hello'); + await drain(); + }); + + it('serves /v1/messages on an OpenAI Responses-only candidate', async () => { + const callOpenAIResponses = vi.fn(async () => openaiResponsesTurn('hello')); + resolves([candidate('up_a', { openaiResponses: {} }, { callOpenAIResponses })]); + + const { facts, drain } = await serveAnthropicMessages(); + + expect(callOpenAIResponses).toHaveBeenCalledTimes(1); + expect(anthropicMessagesText(facts['response.chat.anthropicMessages.rendered'])).toBe('hello'); + await drain(); + }); + + it('serves /v1/messages on an OpenAI Chat Completions-only candidate', async () => { + const callOpenAIChatCompletions = vi.fn(async () => openaiChatCompletionsTurn('hello')); + resolves([candidate('up_a', { openaiChatCompletions: {} }, { callOpenAIChatCompletions })]); + + const { facts, drain } = await serveAnthropicMessages(); + + expect(callOpenAIChatCompletions).toHaveBeenCalledTimes(1); + expect(anthropicMessagesText(facts['response.chat.anthropicMessages.rendered'])).toBe('hello'); + await drain(); + }); + + it('serves /v1/responses on an Anthropic Messages-only candidate', async () => { + const callAnthropicMessages = vi.fn(async () => anthropicMessagesTurn('hello')); + resolves([candidate('up_a', { anthropicMessages: {} }, { callAnthropicMessages })]); + + const { facts, drain } = await serveOpenAIResponses(); + + expect(callAnthropicMessages).toHaveBeenCalledTimes(1); + expect(openaiResponsesText(facts['response.chat.openaiResponses.rendered'])).toBe('hello'); + await drain(); + }); + + it('serves /v1/responses on an OpenAI Chat Completions-only candidate', async () => { + const callOpenAIChatCompletions = vi.fn(async () => openaiChatCompletionsTurn('hello')); + resolves([candidate('up_a', { openaiChatCompletions: {} }, { callOpenAIChatCompletions })]); + + const { facts, drain } = await serveOpenAIResponses(); + + expect(callOpenAIChatCompletions).toHaveBeenCalledTimes(1); + expect(openaiResponsesText(facts['response.chat.openaiResponses.rendered'])).toBe('hello'); + await drain(); + }); + + // Gemini generateContent has no wire of its own, so all three of its rows are translated ones. + it('serves :generateContent on an OpenAI Chat Completions-only candidate', async () => { + const callOpenAIChatCompletions = vi.fn(async () => openaiChatCompletionsTurn('hello')); + resolves([candidate('up_a', { openaiChatCompletions: {} }, { callOpenAIChatCompletions })]); + + const { facts, drain } = await serveGeminiGenerateContent(); + + expect(callOpenAIChatCompletions).toHaveBeenCalledTimes(1); + expect(geminiGenerateContentText(facts['response.chat.geminiGenerateContent.rendered'])).toBe('hello'); + await drain(); + }); + + it('serves :generateContent on an Anthropic Messages-only candidate', async () => { + const callAnthropicMessages = vi.fn(async () => anthropicMessagesTurn('hello')); + resolves([candidate('up_a', { anthropicMessages: {} }, { callAnthropicMessages })]); + + const { facts, drain } = await serveGeminiGenerateContent(); + + expect(callAnthropicMessages).toHaveBeenCalledTimes(1); + expect(geminiGenerateContentText(facts['response.chat.geminiGenerateContent.rendered'])).toBe('hello'); + await drain(); + }); + + it('serves :generateContent on an OpenAI Responses-only candidate', async () => { + const callOpenAIResponses = vi.fn(async () => openaiResponsesTurn('hello')); + resolves([candidate('up_a', { openaiResponses: {} }, { callOpenAIResponses })]); + + const { facts, drain } = await serveGeminiGenerateContent(); + + expect(callOpenAIResponses).toHaveBeenCalledTimes(1); + expect(geminiGenerateContentText(facts['response.chat.geminiGenerateContent.rendered'])).toBe('hello'); + await drain(); + }); +}); + +describe('the fork over wires', () => { + // Failover is an ordinary middle stage: what it re-runs is the whole suffix including the + // stage that picks the wire, so the next candidate re-picks. Nothing here is a mechanism of + // failover's own — the first candidate goes native and the second goes translated because + // that is what each one's own endpoints say. + it('moves from a refused native wire onto a translated one', async () => { + const callOpenAIChatCompletions = vi.fn(async () => ({ + ok: false as const, modelKey: 'k', response: Response.json({ error: { message: 'slow down' } }, { status: 429 }), + })); + const callAnthropicMessages = vi.fn(async () => anthropicMessagesTurn('second')); + resolves([ + candidate('up_a', { openaiChatCompletions: {} }, { callOpenAIChatCompletions }), + candidate('up_b', { anthropicMessages: {} }, { callAnthropicMessages }), + ]); + + const { facts, drain } = await serveOpenAIChatCompletions(); + + expect(callOpenAIChatCompletions).toHaveBeenCalledTimes(1); + expect(callAnthropicMessages).toHaveBeenCalledTimes(1); + expect(facts['response.http.status']).toBe(200); + expect(openaiChatCompletionsText(facts['response.chat.openaiChatCompletions.rendered'])).toBe('second'); + await drain(); + }); + + // The canonical rewrite: Claude Code reads Anthropic's `prompt is too long:` shape to know + // it must auto-compact, and an OpenAI-shaped upstream states the same condition in its own + // words. The pair owns the translation of that refusal, and the handoff is what asks it. + it('rewrites a context-window refusal into the pair-s own envelope', async () => { + resolves([candidate('up_a', { openaiChatCompletions: {} }, { callOpenAIChatCompletions: contextExceeded })]); + + const { facts, drain } = await serveAnthropicMessages(); + + expect(facts['response.http.status']).toBe(400); + expect(facts['response.chat.anthropicMessages.rendered']).toEqual({ + type: 'error', + error: { type: 'invalid_request_error', message: PROMPT_TOO_LONG_MESSAGE }, + }); + await drain(); + }); + + // A refusal a pair says nothing about keeps the status the upstream sent and loses the + // envelope it came in. The object was the *target* protocol's — an OpenAI `{error:{message}}` + // reaching a client that reads Anthropic — and answering one protocol in another's words is + // what the pair exists to prevent. What crosses is the status and the sentence; the client's + // own edge writes the shape. + it('carries a refusal the pair does not speak about without its foreign envelope', async () => { + const callOpenAIChatCompletions = vi.fn(async () => ({ + ok: false as const, modelKey: 'k', response: Response.json({ error: { message: 'no' } }, { status: 403 }), + })); + resolves([candidate('up_a', { openaiChatCompletions: {} }, { callOpenAIChatCompletions })]); + + const { facts, drain } = await serveAnthropicMessages(); + + expect(facts['response.http.status']).toBe(403); + const rendered = facts['response.chat.anthropicMessages.rendered'] as { type: string; error: { type: string; message: string } }; + expect(rendered.type).toBe('error'); + expect(rendered.error.type).toBe('permission_error'); + // The upstream's own words survive as the sentence, which is all that can be kept of a + // body written in a protocol this client does not read. + expect(rendered.error.message).toContain('no'); + await drain(); + }); + + // The other half of the same rule: a rule that *can* speak carries the object across, because + // by then it is in the client's own protocol. + it('carries the envelope a pair did rewrite', async () => { + resolves([candidate('up_a', { openaiChatCompletions: {} }, { callOpenAIChatCompletions: contextExceeded })]); + + const { facts, drain } = await serveAnthropicMessages(); + + expect(facts['response.chat.anthropicMessages.rendered']).toEqual({ + type: 'error', + error: { type: 'invalid_request_error', message: PROMPT_TOO_LONG_MESSAGE }, + }); + await drain(); + }); +}); + +describe('a rule that speaks about one protocol-s wire', () => { + // The Anthropic Messages fold rewrites *every* inline system message, because Anthropic's top-level + // `system` is the only first-position slot there. The OpenAI Chat Completions fold rewrites only a + // system message past the leading run. So a leading system message tells the two apart: a + // turn that left for OpenAI Chat Completions must be folded by the OpenAI Chat Completions rule, which + // leaves it alone, and not by the Anthropic Messages one, which would not. + it('does not run on a turn that leaves for another protocol', async () => { + let sent: { messages: { role: string }[] } | undefined; + const payload = { + model: MODEL, + max_tokens: 64, + messages: [{ role: 'system', content: 'lead' }, { role: 'user', content: 'hi' }], + } as unknown as AnthropicMessagesPayload; + resolves([candidate( + 'up_a', + { openaiChatCompletions: {} }, + { + callOpenAIChatCompletions: async (_model, body) => { + sent = body as { messages: { role: string }[] }; + return openaiChatCompletionsTurn('hello'); + }, + }, + ['rewrite-mid-conv-system-to-user'], + )]); + + const { drain } = await serveAnthropicMessages(payload); + + expect(sent?.messages.map(message => message.role)).toEqual(['system', 'user']); + await drain(); + }); + + // The other direction, and the one that pays for the whole arrangement: the usage chunk is + // asked for by the OpenAI Chat Completions wire, so an Anthropic Messages turn that reaches an upstream over + // that endpoint is metered like any other turn on it. A rule left in the OpenAI Chat Completions + // *source* chain would ask for nothing here, and this turn would bill zero. + it('asks for the usage chunk on a turn that arrived from another protocol', async () => { + let sent: { stream_options?: unknown } | undefined; + resolves([candidate('up_a', { openaiChatCompletions: {} }, { + callOpenAIChatCompletions: async (_model, body) => { + sent = body as { stream_options?: unknown }; + return openaiChatCompletionsTurn('hello'); + }, + })]); + + const { drain } = await serveAnthropicMessages(); + + expect(sent?.stream_options).toEqual({ include_usage: true, continuous_usage_stats: true }); + await drain(); + }); + + // The same arrangement for a rule no guard ever covered. The old surface ran the *whole* + // target array on a translated body, so a turn arriving over a translation was shaped for + // the wire it landed on. A rule left in the Anthropic Messages source chain would scrub nothing here. + it('applies the target wire-s unguarded rules to a turn that arrived translated', async () => { + let sent: { system?: unknown } | undefined; + resolves([candidate('up_a', { anthropicMessages: {} }, { + callAnthropicMessages: async (_model, body) => { + sent = body as { system?: unknown }; + return anthropicMessagesTurn('hello'); + }, + }, ['strip-billing-attribution'])]); + + const { drain } = await serveOpenAIChatCompletions({ + ...openaiChatCompletionsPayload, + messages: [ + { role: 'system', content: 'be brief\nx-anthropic-billing-header: acct-42' }, + { role: 'user', content: 'hi' }, + ], + } as OpenAIChatCompletionsPayload); + + expect(String(JSON.stringify(sent?.system))).not.toContain('x-anthropic-billing-header'); + await drain(); + }); + + // And a rule the *other* wire owns stays off this turn. `stream_options` is an OpenAI Chat + // Completions field, so an OpenAI Chat Completions turn dialled over Anthropic Messages must + // not carry it — which a target-API guard inside the rule could also say, and which position + // says here. + it('does not carry another wire-s request rules onto the wire it dialled', async () => { + let sent: Record | undefined; + resolves([candidate('up_a', { anthropicMessages: {} }, { + callAnthropicMessages: async (_model, body) => { + sent = body as Record; + return anthropicMessagesTurn('hello'); + }, + })]); + + const { drain } = await serveOpenAIChatCompletions(); + + expect(sent).toBeDefined(); + expect('stream_options' in sent!).toBe(false); + await drain(); + }); + + // The other half, and this one assembly says rather than a run: the wire's own rules read + // the wire's own request key, so `compose` refuses to place them *below* a handoff that has + // taken it. It is one array's arrangement and not a proof — `compose` walks one array, and + // the same rule left above the fork is invisible to it. What keeps a wire's rule off a + // translated turn is that the rule lives in the wire's own chain. + it('cannot be placed in a chain whose payload a handoff consumed', () => { + const leaving = handOff({ + from: { request: 'request.chat.anthropicMessages', response: 'response.chat.anthropicMessages' }, + to: { request: 'request.chat.openaiChatCompletions', response: 'response.chat.openaiChatCompletions' }, + trip: () => { throw new Error('not dialled'); }, + }); + const arriving = handOff({ + from: { request: 'request.chat.openaiChatCompletions', response: 'response.chat.openaiChatCompletions' }, + to: { request: 'request.chat.anthropicMessages', response: 'response.chat.anthropicMessages' }, + trip: () => { throw new Error('not dialled'); }, + }); + + expect(() => compose('leaving', [leaving, ...anthropicMessagesWire('r')])) + .toThrow(/needs request\.chat\.anthropicMessages, which handOff:.* consumed above it/); + expect(() => compose('arriving', [arriving, ...anthropicMessagesWire('r')])).not.toThrow(); + }); +}); diff --git a/packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts b/packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts new file mode 100644 index 0000000000..e9d6c537b1 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/chat/translation-refusal_test.ts @@ -0,0 +1,87 @@ +// A body the target protocol cannot represent is the client's fault. +// +// A translated wire runs the client's body through a translator, and a translator refuses input +// it would otherwise have to coerce or silently drop. That refusal is rendered as a 400 in the +// client's own protocol; a run that let it escape as a throw would answer 500 with a stack +// trace, telling the caller that the gateway broke on a request that was simply not serveable +// over that wire. +// +// It is answered rather than thrown for a second reason: the verdict belongs to one candidate. +// The next candidate may be reachable on a wire whose protocol *can* carry the body, and +// failover re-runs the suffix to find out. +// +// Both of Gemini generateContent's actions cross a protocol boundary and both are covered: generation reaches +// its target through a handoff, and `:countTokens` through the pair that asks the question in +// Anthropic Messages. A refusal is answered the same way on either, because what the caller +// reads is the same protocol whichever action it asked for. + +import { test } from 'vitest'; + +import { copilotModels, requestAppWithWarmModels as requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { assertEquals, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; + +const upstream = (models: Parameters[0]) => async (request: Request): Promise => { + const url = new URL(request.url); + if (url.hostname === 'update.code.visualstudio.com') return jsonResponse(['1.110.1']); + if (url.pathname === '/copilot_internal/v2/token') { + return jsonResponse({ token: 'copilot-access-token', expires_at: 4102444800, refresh_in: 3600, endpoints: { api: 'https://api.individual.githubcopilot.com' } }); + } + if (url.pathname === '/models') return jsonResponse(copilotModels(models)); + throw new Error(`Unhandled fetch ${request.url}`); +}; + +// A `functionResponse` part in model content: a shape Gemini generateContent defines and neither target protocol +// has anywhere to put, so the translator refuses it rather than dropping it. +const untranslatableGeminiGenerateContentBody = { + contents: [ + { role: 'model', parts: [{ functionResponse: { name: 'f', response: { ok: true } } }] }, + { role: 'user', parts: [{ text: 'hi' }] }, + ], +}; + +/** What a Gemini generateContent client reads a refusal out of. `error.status` is the field it keys on, and it + * is not a field the gateway's internal-error envelope has — nor is `stack` a field a + * caller-input refusal has any business carrying. */ +const assertGoogleRpcRefusal = async (response: Response): Promise => { + assertEquals(response.status, 400); + const body = await response.json() as { error: { code: number; status: string; message: string } }; + assertEquals(body.error.code, 400); + assertEquals(body.error.status, 'INVALID_ARGUMENT'); + assertEquals(body.error.message.includes('not supported in model content'), true); + assertEquals('stack' in body.error, false); +}; + +test('a body the target protocol cannot carry is refused in the client own protocol', async () => { + const { apiKey } = await setupAppTest(); + + await withMockedFetch( + upstream([{ id: 'gpt-translate', supported_endpoints: ['/chat/completions'] }]), + async () => { + await assertGoogleRpcRefusal(await requestApp('/v1beta/models/gpt-translate:generateContent', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + body: JSON.stringify(untranslatableGeminiGenerateContentBody), + })); + }, + ); +}); + +// Counting crosses the same boundary. It has no handoff — what crosses is one measurement, not +// frames — so the refusal it can hit is its own to answer, and answering it anywhere else means +// a caller asking what a turn would cost is told the gateway broke. +test('a body the counting wire cannot carry is refused in the client own protocol', async () => { + const { apiKey } = await setupAppTest(); + + await withMockedFetch( + // Counting is reachable only over an upstream's own Anthropic Messages endpoint, so this is the one + // candidate shape that gets as far as the translation. + upstream([{ id: 'claude-count', supported_endpoints: ['/messages'] }]), + async () => { + await assertGoogleRpcRefusal(await requestApp('/v1beta/models/claude-count:countTokens', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + body: JSON.stringify(untranslatableGeminiGenerateContentBody), + })); + }, + ); +}); diff --git a/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts index 124e6bc00f..abf1cdd4ab 100644 --- a/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts @@ -14,8 +14,8 @@ import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts'; import { stubProviderPipeline } from '../test-utils/provider-pipeline.ts'; import { isOwned, move, run } from '@floway-dev/pipeline'; import type { SseFrame } from '@floway-dev/protocols/common'; -import { directFetcher, type ModelCandidate, type ProviderCallResult } from '@floway-dev/provider'; -import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; +import { directFetcher, type ModelCandidate } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel, type ProviderCallResult } from '@floway-dev/test-utils'; vi.mock('../../src/data-plane/providers/resolution.ts', async importOriginal => ({ ...(await importOriginal()), diff --git a/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts index 8c694df07c..b6905b4bae 100644 --- a/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts @@ -14,8 +14,8 @@ import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts'; import { stubProviderPipeline } from '../test-utils/provider-pipeline.ts'; import { compose, defineStage, isDeferred, move, run } from '@floway-dev/pipeline'; import type { SseFrame } from '@floway-dev/protocols/common'; -import { directFetcher, type ModelCandidate, type ProviderCallResult, type UpstreamCallOptions, type ProviderRequest, type ProviderResponse, type ProviderOperationPayloads } from '@floway-dev/provider'; -import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; +import { directFetcher, type ModelCandidate, type UpstreamCallOptions, type ProviderRequest, type ProviderResponse, type ProviderOperationPayloads } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel, type ProviderCallResult } from '@floway-dev/test-utils'; vi.mock('../../src/data-plane/providers/resolution.ts', async importOriginal => ({ ...(await importOriginal()), diff --git a/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts b/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts index 738be26a3f..a2efbf692b 100644 --- a/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts @@ -311,7 +311,7 @@ test('/v1/completions handler also serves the unversioned /completions path', as // Dual-path coverage: usage + performance + dump telemetry must land // equivalently on both the non-streaming JSON and the streaming SSE // branches. The handler does not force `stream: true` upstream the way -// openai-chat-completions does (no interceptor framework to feed), so the two +// openai-chat-completions does (no chat chain to feed), so the two // paths really do exercise different scaffold branches and the assertions // here keep them honest. test('/v1/completions non-streaming records usage row, performance neutral row (text_completion operation, no TTFT/TPOT), and a bytes-body dump record', async () => { @@ -360,10 +360,9 @@ test('/v1/completions non-streaming records usage row, performance neutral row ( assertEquals(dump.meta.model, 'davinci-002'); assertEquals(dump.meta.inputTokens, 7); assertEquals(dump.meta.outputTokens, 2); - // The shape follows the endpoint: a pipelined turn is recorded as its whole run, so what - // is stored is the event stream rather than the two edges. The metadata is common to both, - // which is what lets the dashboard list them together. - assertEquals(runRecordOf(dumpStubs.stored[0]!.record).shape, 'run'); + // A turn is recorded as its whole run, so what is stored is the event stream and the metadata + // beside it — which is what lets the dashboard list a turn without opening it. + assertEquals(eventsOf(runRecordOf(dumpStubs.stored[0]!.record)).length > 0, true); }); // A stream that stops before its terminator did not produce what it said it would, and the diff --git a/packages/gateway/__tests__/data-plane/openai-completions/usage_test.ts b/packages/gateway/__tests__/data-plane/openai-completions/usage_test.ts index 5d2eac8481..e94b7bf1dd 100644 --- a/packages/gateway/__tests__/data-plane/openai-completions/usage_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-completions/usage_test.ts @@ -87,7 +87,7 @@ test('tokenUsageFromOpenAICompletionsUsage runs serviceTier through billableServ test('tokenUsageFromOpenAICompletionsUsage folds the cache counts back in on the totals alone', () => { // The same accounting the chat targets normalize on the usage chunk: this - // endpoint has no interceptor chain, so the fold happens on the billing read. + // endpoint runs no chat rules, so the fold happens on the billing read. assertEquals( tokenUsageFromOpenAICompletionsUsage( { prompt_tokens: 479, completion_tokens: 373, total_tokens: 14164, prompt_tokens_details: { cached_tokens: 13312 } }, diff --git a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts index 7d1fce498b..34a08407fd 100644 --- a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts +++ b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts @@ -7,6 +7,13 @@ import { describe, expect, it } from 'vitest'; import { searchServePipeline } from '../../../src/data-plane/alpha-search/pipeline.ts'; +import { anthropicMessagesCountTokensPipeline } from '../../../src/data-plane/chat/anthropic-messages/count-tokens.ts'; +import { anthropicMessagesServePipeline } from '../../../src/data-plane/chat/anthropic-messages/pipeline.ts'; +import { geminiGenerateContentCountTokensPipeline } from '../../../src/data-plane/chat/gemini-generate-content/count-tokens.ts'; +import { geminiGenerateContentServePipeline } from '../../../src/data-plane/chat/gemini-generate-content/pipeline.ts'; +import { openaiChatCompletionsServePipeline } from '../../../src/data-plane/chat/openai-chat-completions/pipeline.ts'; +import { openaiResponsesCompactPipeline } from '../../../src/data-plane/chat/openai-responses/compact.ts'; +import { openaiResponsesServePipeline, openaiResponsesWebSocketPipeline } from '../../../src/data-plane/chat/openai-responses/pipeline.ts'; import { openaiAudioTranscriptionServePipeline } from '../../../src/data-plane/openai-audio/pipeline.ts'; import { openaiCompletionsServePipeline } from '../../../src/data-plane/openai-completions/pipeline.ts'; import { openaiEmbeddingsServePipeline } from '../../../src/data-plane/openai-embeddings/pipeline.ts'; @@ -16,11 +23,19 @@ import { rerankServePipeline } from '../../../src/data-plane/rerank/pipeline.ts' const FAMILIES: readonly (readonly [string, () => { readonly name: string }])[] = [ ['OpenAI Embeddings', () => openaiEmbeddingsServePipeline], ['rerank', () => rerankServePipeline({ sourceProtocol: 'cohere-v2', raw: {}, query: 'q', documents: ['a'] } as never)], - ['OpenAI Images Generations', () => openaiImagesServePipeline({ operation: 'generations', parameters: {} } as never)], - ['OpenAI Images Edits', () => openaiImagesServePipeline({ operation: 'edits', images: [], parameters: {} } as never)], + ['OpenAI Images generations', () => openaiImagesServePipeline({ operation: 'generations', parameters: {} } as never)], + ['OpenAI Images edits', () => openaiImagesServePipeline({ operation: 'edits', images: [], parameters: {} } as never)], ['OpenAI Completions', () => openaiCompletionsServePipeline], ['OpenAI Audio Transcriptions', () => openaiAudioTranscriptionServePipeline], ['alpha search', () => searchServePipeline({ kind: 'search' } as never)], + ['OpenAI Chat Completions', () => openaiChatCompletionsServePipeline()], + ['Anthropic Messages', () => anthropicMessagesServePipeline()], + ['Gemini generateContent', () => geminiGenerateContentServePipeline()], + ['OpenAI Responses', () => openaiResponsesServePipeline()], + ['OpenAI Responses WebSocket', () => openaiResponsesWebSocketPipeline()], + ['Anthropic Messages count_tokens', () => anthropicMessagesCountTokensPipeline()], + ['Gemini generateContent countTokens', () => geminiGenerateContentCountTokensPipeline()], + ['OpenAI Responses compact', () => openaiResponsesCompactPipeline()], ]; describe('every family assembles', () => { @@ -30,3 +45,10 @@ describe('every family assembles', () => { }); } }); + +it('Responses transport entries require the facts of their actual edge', () => { + const http = openaiResponsesServePipeline(); + const websocket = openaiResponsesWebSocketPipeline(); + expect(http.entryNeeds).not.toContain('ingress.chat.openaiResponses.eventId'); + expect(websocket.entryNeeds).toContain('ingress.chat.openaiResponses.eventId'); +}); diff --git a/packages/gateway/__tests__/data-plane/pipeline/attempt-body_test.ts b/packages/gateway/__tests__/data-plane/pipeline/attempt-body_test.ts new file mode 100644 index 0000000000..b871cbb332 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/pipeline/attempt-body_test.ts @@ -0,0 +1,72 @@ +// A provider is handed the record's own values. +// +// Nothing is deep-copied: the object built here exists to carry the addressed model and then +// drop it, and everything under it is the record's, frozen. A provider that tried to shape its +// payload by writing into what it was given would be writing into a fact — so what is pinned +// here is that the refusal reaches it, at the depth where the rule that used to do it worked. + +import { describe, it } from 'vitest'; + +import { bodyForAttempt } from '../../../src/data-plane/pipeline/attempt-body.ts'; +import { move } from '@floway-dev/pipeline'; +import type { ModelCandidate } from '@floway-dev/provider'; +import { assertEquals } from '@floway-dev/test-utils'; + +const candidateFor = (id: string, rules?: ModelCandidate['rules']): ModelCandidate => + ({ model: { id }, ...(rules === undefined ? {} : { rules }) }) as ModelCandidate; + +const recordedRequest = () => { + const { request } = move({ + request: { + model: 'alias-name', + messages: [{ role: 'system', content: 'be brief' }, { role: 'user', content: 'hi' }], + }, + }); + return request; +}; + +describe('bodyForAttempt', () => { + it('hands over children no one can write into', () => { + const recorded = recordedRequest(); + const body = bodyForAttempt(recorded, candidateFor('upstream-model'), body => body); + + // What the Copilot cache-control rule used to do, at the node it did it to. It rebuilds + // now, and this is why: the message it was reaching into is the record's own. + let refused: unknown; + try { + (body.messages[0] as Record).copilot_cache_control = { type: 'ephemeral' }; + } catch (error) { refused = error; } + + assertEquals(refused instanceof TypeError, true); + assertEquals('copilot_cache_control' in recorded.messages[0], false); + assertEquals(recorded.model, 'alias-name'); + }); + + it('addresses the model the candidate resolved and drops the one the client named', () => { + const body = bodyForAttempt(recordedRequest(), candidateFor('upstream-model'), body => body); + assertEquals('model' in body, false); + }); + + it('applies the alias rules over the body it addressed', () => { + const rules = { reasoning: { effort: 'high' } } as NonNullable; + let addressed: string | undefined; + const body = bodyForAttempt( + recordedRequest(), + candidateFor('upstream-model', rules), + (payload, applied) => { + addressed = payload.model; + return { ...payload, reasoning_effort: applied.reasoning?.effort }; + }, + ); + assertEquals((body as Record).reasoning_effort, 'high'); + // The overlay ran over the model this attempt is for rather than the alias the client + // addressed, which is what lets a rule speak about the upstream it is actually dialling. + assertEquals(addressed, 'upstream-model'); + }); + + it('applies nothing where the alias carries no rules', () => { + let ran = false; + bodyForAttempt(recordedRequest(), candidateFor('upstream-model'), body => { ran = true; return body; }); + assertEquals(ran, false); + }); +}); diff --git a/packages/gateway/__tests__/data-plane/pipeline/failure-render_test.ts b/packages/gateway/__tests__/data-plane/pipeline/failure-render_test.ts new file mode 100644 index 0000000000..5cbfffcfb6 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/pipeline/failure-render_test.ts @@ -0,0 +1,95 @@ +// What a client is sent when a turn failed, and the status it is sent with. +// +// The three tiers are easy to collapse by accident — an edge that reaches for the protocol's +// default first will bury both an upstream's own words and a refusal's own envelope, and each +// family renders its own errors, so the rule is worth holding in one place and pinning here. +// +// The status travels with the body because only the third tier may change it, and one protocol +// must: a Google-RPC envelope states its code inside the body, so an edge free to pick a status +// separately can state two different things about one failure. + +import { describe, it } from 'vitest'; + +import { mintGeminiGenerateContentFailure } from '../../../src/data-plane/chat/gemini-generate-content/errors.ts'; +import { mintedAs, renderFailure } from '../../../src/data-plane/pipeline/facts.ts'; +import { assertEquals } from '@floway-dev/test-utils'; + +const protocolDefault = mintedAs(() => ({ error: { message: 'rendered from the status', type: 'api_error' } })); + +describe('renderFailure', () => { + it('forwards the body the upstream sent', () => { + const upstream = { error: { message: 'model is overloaded', type: 'overloaded_error' } }; + assertEquals( + renderFailure({ status: 529, message: 'upstream refused', body: upstream }, protocolDefault), + { body: upstream, status: 529 }, + ); + }); + + it('writes the refusal own envelope where the gateway refused', () => { + const envelope = { error: { message: 'no route', type: 'invalid_request_error', code: 'responses_item_routing_unavailable' } }; + assertEquals( + renderFailure({ status: 400, message: 'no route', envelope }, protocolDefault), + { body: envelope, status: 400 }, + ); + }); + + it('prefers what the upstream said over what the gateway would have written', () => { + const upstream = { error: { message: 'context length exceeded' } }; + const envelope = { error: { message: 'no route' } }; + assertEquals( + renderFailure({ status: 400, message: 'no route', body: upstream, envelope }, protocolDefault), + { body: upstream, status: 400 }, + ); + }); + + it('falls back to the protocol own rendering when neither is present', () => { + assertEquals( + renderFailure({ status: 502, message: 'dial failed' }, protocolDefault), + { body: { error: { message: 'rendered from the status', type: 'api_error' } }, status: 502 }, + ); + }); + + it('renders the protocol default where the upstream body was not an object', () => { + // A gateway that forwarded `"Bad Gateway"` verbatim would answer a JSON protocol with a + // bare string, which no client of it can read. + assertEquals( + renderFailure({ status: 502, message: 'dial failed', body: 'Bad Gateway' }, protocolDefault), + { body: { error: { message: 'rendered from the status', type: 'api_error' } }, status: 502 }, + ); + }); +}); + +// Gemini generateContent is the reason the status is the third tier's to decide, so the rule +// is pinned on the protocol that exercises it rather than only on the helper. +describe('the Gemini generateContent third tier', () => { + it('sends a code it has no name for as the one INTERNAL belongs to', () => { + // 418 maps to `INTERNAL`, and `INTERNAL` is 500's name. Sending it as a 418 would answer + // with `code: 418` beside `status: "INTERNAL"` — two statements about one failure. + assertEquals( + renderFailure({ status: 418, message: 'upstream was a teapot' }, mintGeminiGenerateContentFailure), + { body: { error: { code: 500, message: 'upstream was a teapot', status: 'INTERNAL' } }, status: 500 }, + ); + }); + + it('leaves a code it can name alone', () => { + assertEquals( + renderFailure({ status: 429, message: 'slow down' }, mintGeminiGenerateContentFailure), + { body: { error: { code: 429, message: 'slow down', status: 'RESOURCE_EXHAUSTED' } }, status: 429 }, + ); + }); + + it('does not reword an upstream body or a refusal envelope', () => { + // Neither tier is this gateway's to restate, so a 418 survives both ways when someone + // else already answered with it. + const upstream = { error: { code: 418, message: 'upstream said so', status: 'WHATEVER' } }; + assertEquals( + renderFailure({ status: 418, message: 'ignored', body: upstream }, mintGeminiGenerateContentFailure), + { body: upstream, status: 418 }, + ); + const envelope = { error: { code: 418, message: 'we said so', status: 'INVALID_ARGUMENT' } }; + assertEquals( + renderFailure({ status: 418, message: 'ignored', envelope }, mintGeminiGenerateContentFailure), + { body: envelope, status: 418 }, + ); + }); +}); diff --git a/packages/gateway/__tests__/data-plane/pipeline/settlement_test.ts b/packages/gateway/__tests__/data-plane/pipeline/settlement_test.ts index dc25b65eed..a0c3f497a4 100644 --- a/packages/gateway/__tests__/data-plane/pipeline/settlement_test.ts +++ b/packages/gateway/__tests__/data-plane/pipeline/settlement_test.ts @@ -29,13 +29,10 @@ describe('pipeline settlement', () => { const dump = openRunDump(key, { method: 'POST', path: '/v1/responses', body: { bytes: new Uint8Array(), streamError: null }, }, background, true, timing); - const gateway = { - ...mockGatewayCtx({ - backgroundScheduler: background, - attempt: { timing, telemetry: mockPerfTelemetryContext({ keyId: key.id }) }, - }), - dump, - }; + const gateway = mockGatewayCtx({ + dump, backgroundScheduler: background, + attempt: { timing, telemetry: mockPerfTelemetryContext({ keyId: key.id }) }, + }); const identity = { model: 'm', upstream: 'u', modelKey: 'm', pricing: null }; if (failed) dump!.failed(new Error('Original source failure', { cause: new Error('socket reset') })); const active = pending.length; diff --git a/packages/gateway/__tests__/data-plane/rerank-status_test.ts b/packages/gateway/__tests__/data-plane/rerank-status_test.ts index 641b2e0dc4..0870b2d4f5 100644 --- a/packages/gateway/__tests__/data-plane/rerank-status_test.ts +++ b/packages/gateway/__tests__/data-plane/rerank-status_test.ts @@ -14,8 +14,8 @@ import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts'; import { stubProviderPipeline } from '../test-utils/provider-pipeline.ts'; import { move, run } from '@floway-dev/pipeline'; import type { CanonicalRerankRequest } from '@floway-dev/protocols/rerank'; -import { directFetcher, type ModelCandidate, type ProviderRerankCallResult } from '@floway-dev/provider'; -import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; +import { directFetcher, type ModelCandidate } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel, type ProviderRerankCallResult } from '@floway-dev/test-utils'; vi.mock('../../src/data-plane/providers/resolution.ts', () => ({ enumerateModelCandidates: vi.fn(), diff --git a/packages/gateway/__tests__/data-plane/shared/gateway-ctx_test.ts b/packages/gateway/__tests__/data-plane/shared/gateway-ctx_test.ts index 7a5c391b13..ec870debe6 100644 --- a/packages/gateway/__tests__/data-plane/shared/gateway-ctx_test.ts +++ b/packages/gateway/__tests__/data-plane/shared/gateway-ctx_test.ts @@ -56,7 +56,7 @@ describe('createGatewayCtxFromHono', () => { let ctx: ReturnType | undefined; app.get('/test', c => { c.set('apiKey', buildApiKey({ id: 'key-1', upstreamIds: ['up-1', 'up-2'] })); - ctx = createGatewayCtxFromHono(c, { wantsStream: true, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER }); + ctx = createGatewayCtxFromHono(c, { wantsStream: true, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER, dump: null }); return c.text('ok'); }); await app.request('/test'); @@ -69,7 +69,7 @@ describe('createGatewayCtxFromHono', () => { const app = makeApp(); let ctx: ReturnType | undefined; app.get('/test', c => { - ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER }); + ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER, dump: null }); return c.text('ok'); }); await app.request('/test'); @@ -82,7 +82,7 @@ describe('createGatewayCtxFromHono', () => { const app = makeApp(); let ctx: ReturnType | undefined; app.get('/test', c => { - ctx = createGatewayCtxFromHono(c, { wantsStream: true, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER }); + ctx = createGatewayCtxFromHono(c, { wantsStream: true, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER, dump: null }); return c.text('ok'); }); await app.request('/test'); @@ -94,7 +94,7 @@ describe('createGatewayCtxFromHono', () => { const app = makeApp(); let ctx: ReturnType | undefined; app.get('/test', c => { - ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER }); + ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER, dump: null }); return c.text('ok'); }); await app.request('/test'); @@ -106,7 +106,7 @@ describe('createGatewayCtxFromHono', () => { const app = makeApp(); let ctx: ReturnType | undefined; app.get('/test', c => { - ctx = createGatewayCtxFromHono(c, { wantsStream: true, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER }); + ctx = createGatewayCtxFromHono(c, { wantsStream: true, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER, dump: null }); return c.text('ok'); }); await app.request('/test'); @@ -119,7 +119,7 @@ describe('createGatewayCtxFromHono', () => { const app = makeApp(); let ctx: ReturnType | undefined; app.get('/test', c => { - ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER }); + ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER, dump: null }); return c.text('ok'); }); await app.request('/test'); @@ -134,7 +134,7 @@ describe('createGatewayCtxFromHono', () => { let controller: AbortController | undefined; app.get('/test', c => { controller = new AbortController(); - ctx = createGatewayCtxFromHono(c, { wantsStream: true, downstreamAbortController: controller, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER }); + ctx = createGatewayCtxFromHono(c, { wantsStream: true, downstreamAbortController: controller, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER, dump: null }); return c.text('ok'); }); await app.request('/test'); @@ -150,7 +150,7 @@ describe('createGatewayCtxFromHono', () => { let ctx: ReturnType | undefined; const scheduler: BackgroundScheduler = () => {}; app.get('/test', c => { - ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: scheduler }); + ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: scheduler, dump: null }); return c.text('ok'); }); await app.request('/test'); @@ -166,21 +166,21 @@ describe('createGatewayCtxFromHono', () => { app.get('/cap-only', c => { // Unrestricted key (apiKey.upstreamIds null) under a capped user. c.set('user', buildUser({ upstreamIds: ['up-a'] })); - collected.capOnly = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER }).upstreamIds; + collected.capOnly = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER, dump: null }).upstreamIds; return c.text('ok'); }); app.get('/both', c => { // Per-key whitelist further narrows the user cap and preserves per-key order. c.set('user', buildUser({ upstreamIds: ['up-a', 'up-b'] })); c.set('apiKey', buildApiKey({ upstreamIds: ['up-b', 'up-c'] })); - collected.both = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER }).upstreamIds; + collected.both = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER, dump: null }).upstreamIds; return c.text('ok'); }); app.get('/key-only', c => { // Uncapped user with a per-key whitelist falls through to the per-key // list verbatim. c.set('apiKey', buildApiKey({ upstreamIds: ['up-x'] })); - collected.keyOnly = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER }).upstreamIds; + collected.keyOnly = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: EMPTY_REQUEST_BODY, backgroundScheduler: NOOP_SCHEDULER, dump: null }).upstreamIds; return c.text('ok'); }); await app.request('/cap-only'); diff --git a/packages/gateway/__tests__/data-plane/shared/iterate-candidates_test.ts b/packages/gateway/__tests__/data-plane/shared/iterate-candidates_test.ts deleted file mode 100644 index 0d05d21b96..0000000000 --- a/packages/gateway/__tests__/data-plane/shared/iterate-candidates_test.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import type { GatewayCtx } from '../../../src/data-plane/shared/gateway-ctx.ts'; -import { iterateCandidates } from '../../../src/data-plane/shared/iterate-candidates.ts'; -import { mockGatewayCtx } from '../../test-utils/gateway-ctx.ts'; -import type { ModelCandidate, PerformanceTelemetryContext } from '@floway-dev/provider'; -import { mockPerfTelemetryContext, stubModelCandidate, stubProvider } from '@floway-dev/test-utils'; - -const stubCandidate = (id: string, upstream = 'up'): ModelCandidate => - stubModelCandidate({ - model: { id }, - provider: { - upstreamId: upstream, - kind: 'custom', - name: upstream, - inboundHeaderAllowlist: [], - disabledPublicModelIds: [], - modelPrefix: null, - modelsCache: null, pipelines: {}, - instance: stubProvider(), - }, - }); - -// iterateCandidates reads `attempt`, `apiKeyId`, and `runtimeLocation` from -// ctx — the last two feed `upstreamPerformanceContext` when stamping -// `attempt.telemetry`. Building through mockGatewayCtx keeps the stub aligned -// with the real GatewayCtx shape. -const stubCtx = (attempt: GatewayCtx['attempt']): GatewayCtx => mockGatewayCtx({ attempt }); - -describe('iterateCandidates', () => { - it('clears the timing slots and stamps telemetry from the current candidate on entry', async () => { - const attempt = { timing: { upstreamCallStartedAt: 999, firstOutputTokenAt: 999 }, telemetry: mockPerfTelemetryContext({ upstream: 'carryover' }) as PerformanceTelemetryContext | undefined }; - const ctx = stubCtx(attempt); - const observed: Array<{ upstreamCallStartedAt: number | null; firstOutputTokenAt: number | null; upstream: string | undefined }> = []; - - await iterateCandidates( - [stubCandidate('a', 'up_a'), stubCandidate('b', 'up_b'), stubCandidate('c', 'up_c')], - 'test', - ctx, - 'chat', - async candidate => { - observed.push({ - upstreamCallStartedAt: attempt.timing.upstreamCallStartedAt, - firstOutputTokenAt: attempt.timing.firstOutputTokenAt, - upstream: attempt.telemetry?.upstream, - }); - // Simulate an attempt that stamps timing then fails, so the loop - // advances to the next candidate. - attempt.timing.upstreamCallStartedAt = 100; - attempt.timing.firstOutputTokenAt = 200; - return candidate.model.id === 'c' - ? { type: 'events' as const } - : { type: 'api-error' as const }; - }, - ); - - // Every attempt must observe cleared timing slots on entry — a prior - // candidate's stamps (or the caller-supplied carryover) must not - // survive into the next attempt. Telemetry must reflect the CURRENT - // candidate: regressing this reintroduces the mid-attempt-throw - // misattribution the stamp was hoisted to prevent. - expect(observed).toEqual([ - { upstreamCallStartedAt: null, firstOutputTokenAt: null, upstream: 'up_a' }, - { upstreamCallStartedAt: null, firstOutputTokenAt: null, upstream: 'up_b' }, - { upstreamCallStartedAt: null, firstOutputTokenAt: null, upstream: 'up_c' }, - ]); - }); - - it('returns the first success and stops iterating', async () => { - const ctx = stubCtx({ timing: { upstreamCallStartedAt: null, firstOutputTokenAt: null }, telemetry: undefined }); - let calls = 0; - const result = await iterateCandidates( - [stubCandidate('a'), stubCandidate('b'), stubCandidate('c')], - 'test', - ctx, - 'chat', - async () => { - calls++; - return { type: 'events' as const }; - }, - ); - - expect(result).toEqual({ type: 'events' }); - expect(calls).toBe(1); - }); - - it('returns the last failure once every candidate errors', async () => { - const ctx = stubCtx({ timing: { upstreamCallStartedAt: null, firstOutputTokenAt: null }, telemetry: undefined }); - let index = 0; - const failures = [ - { type: 'api-error' as const, marker: 'first' }, - { type: 'internal-error' as const, marker: 'last' }, - ]; - const result = await iterateCandidates( - [stubCandidate('a'), stubCandidate('b')], - 'test', - ctx, - 'chat', - async () => failures[index++]!, - ); - - expect(result).toEqual(failures[1]); - }); - - it('treats non-2xx plain results as failure so the next candidate runs', async () => { - const ctx = stubCtx({ timing: { upstreamCallStartedAt: null, firstOutputTokenAt: null }, telemetry: undefined }); - const attempts: number[] = []; - const result = await iterateCandidates( - [stubCandidate('a'), stubCandidate('b')], - 'test', - ctx, - 'chat', - async candidate => { - attempts.push(attempts.length); - return candidate.model.id === 'a' - ? { type: 'plain' as const, status: 500 } - : { type: 'plain' as const, status: 200 }; - }, - ); - - expect(attempts).toEqual([0, 1]); - expect(result).toEqual({ type: 'plain', status: 200 }); - }); -}); diff --git a/packages/gateway/__tests__/data-plane/tools/web-search/alpha-search/upstream_test.ts b/packages/gateway/__tests__/data-plane/tools/web-search/alpha-search/upstream_test.ts index 8b5f6a182f..feb3b112e5 100644 --- a/packages/gateway/__tests__/data-plane/tools/web-search/alpha-search/upstream_test.ts +++ b/packages/gateway/__tests__/data-plane/tools/web-search/alpha-search/upstream_test.ts @@ -1,5 +1,9 @@ import { test, vi } from 'vitest'; +import { providerEntry } from '../../../../../src/data-plane/pipeline/provider-entry.ts'; +import { mockGatewayCtx } from '../../../../test-utils/gateway-ctx.ts'; +import { stubProviderPipeline } from '../../../../test-utils/provider-pipeline.ts'; +import { run } from '@floway-dev/pipeline'; import type { InboundHeaderMatcher, ModelCandidate } from '@floway-dev/provider'; import { assertEquals, assertExists, stubModelCandidate, stubProvider } from '@floway-dev/test-utils'; @@ -16,7 +20,7 @@ vi.mock('../../../../../src/data-plane/providers/resolution.ts', async importOri }; }); -const { resolveAlphaSearchDispatcher } = await import('../../../../../src/data-plane/tools/web-search/alpha-search/upstream.ts'); +const { resolveAlphaSearchCandidate } = await import('../../../../../src/data-plane/tools/web-search/alpha-search/upstream.ts'); const dispatcherFor = async (kind: 'codex' | 'custom', inboundHeaderAllowlist: readonly InboundHeaderMatcher[] = []) => { let observedHeaders: Headers | undefined; @@ -26,21 +30,30 @@ const dispatcherFor = async (kind: 'codex' | 'custom', inboundHeaderAllowlist: r upstreamId: 'search-upstream', kind, inboundHeaderAllowlist, - instance: stubProvider({ - callAlphaSearch: async (_model, _body, _signal, opts) => { + pipelines: { + alphaSearch: stubProviderPipeline('alphaSearch', async (_model, _body, _signal, opts) => { observedHeaders = opts.headers; return { response: new Response('{}'), modelKey: 'search-model' }; - }, - }), + }), + }, + instance: stubProvider(), }; resolvedCandidate = stubModelCandidate({ provider }); - const dispatcher = await resolveAlphaSearchDispatcher({ + const dispatcher = await resolveAlphaSearchCandidate({ config: { upstreamId: provider.upstreamId, model: 'search-model' }, upstreamIds: null, scheduler: promise => { void promise; }, runtimeLocation: 'TEST', }); - return { dispatcher, observedHeaders: () => observedHeaders }; + return { + dispatcher: async (body: Record, _signal: AbortSignal | undefined, headers: Headers) => { + const pipeline = dispatcher.provider.pipelines.alphaSearch; + if (pipeline === undefined) throw new Error('Missing selected Alpha Search pipeline'); + const entry = providerEntry({ 'route.attempt': { candidateId: 0, upstreamId: 'search-upstream', modelId: 'search-model', flags: [] }, 'ingress.http.headers': [...headers] }, dispatcher, body); + const executed = await run(pipeline, entry, { gateway: mockGatewayCtx() }); + await executed.drain(); + }, observedHeaders: () => observedHeaders, + }; }; test('Codex Alpha Search receives only its declared turn metadata', async () => { diff --git a/packages/gateway/__tests__/dump/accumulator_test.ts b/packages/gateway/__tests__/dump/accumulator_test.ts deleted file mode 100644 index a0cfac0cb8..0000000000 --- a/packages/gateway/__tests__/dump/accumulator_test.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { expect, test } from 'vitest'; - -import { installDumpStubs } from './test-fixtures.ts'; -import { DumpAccumulator } from '../../src/dump/accumulator.ts'; -import { initDumpBroker, initDumpStore } from '../../src/dump/registry.ts'; -import type { ApiKey } from '../../src/repo/types.ts'; - -const buildApiKey = (overrides: Partial = {}): ApiKey => ({ - id: 'key_dump_acc_test', - userId: 1, - name: 'test', - key: 'sk-test', - serverSecret: '00'.repeat(32), - createdAt: '2026-01-01T00:00:00.000Z', - upstreamIds: null, - deletedAt: null, - dumpRetentionSeconds: 3600, - openaiResponsesRetentionSeconds: 0, - ...overrides, -}); - -const makeSnapshot = () => ({ - method: 'POST', - path: '/v1/chat/completions', - headers: [['content-type', 'application/json']] as Array<[string, string]>, - bodyByteLength: 2, - streamError: null, -}); - -test('DumpAccumulator stamps TTFT on a streamed record when first token arrived', async () => { - const stubs = installDumpStubs(initDumpStore, initDumpBroker); - const tasks: Promise[] = []; - const scheduler = (p: Promise) => { tasks.push(p); }; - - const attempt = { upstreamCallStartedAt: 100, firstOutputTokenAt: 250 }; - const dump = new DumpAccumulator( - buildApiKey(), - makeSnapshot(), - new Uint8Array([123, 125]), - Date.now(), - scheduler, - true, // wantsStream - attempt, - ); - - dump.finalize(200, []); - await Promise.all(tasks); - - expect(stubs.stored).toHaveLength(1); - expect(stubs.stored[0]!.record.meta.ttftMs).toBe(150); -}); - -test('DumpAccumulator leaves TTFT absent on a non-streaming record', async () => { - const stubs = installDumpStubs(initDumpStore, initDumpBroker); - const tasks: Promise[] = []; - const scheduler = (p: Promise) => { tasks.push(p); }; - - const attempt = { upstreamCallStartedAt: 100, firstOutputTokenAt: 250 }; - const dump = new DumpAccumulator( - buildApiKey(), - makeSnapshot(), - new Uint8Array([123, 125]), - Date.now(), - scheduler, - false, // wantsStream: false - attempt, - ); - - dump.finalize(200, []); - await Promise.all(tasks); - - expect(stubs.stored).toHaveLength(1); - expect(stubs.stored[0]!.record.meta.ttftMs).toBeNull(); -}); - -test('DumpAccumulator leaves TTFT absent on a failure before any output', async () => { - const stubs = installDumpStubs(initDumpStore, initDumpBroker); - const tasks: Promise[] = []; - const scheduler = (p: Promise) => { tasks.push(p); }; - - const attempt = { upstreamCallStartedAt: 100, firstOutputTokenAt: null }; - const dump = new DumpAccumulator( - buildApiKey(), - makeSnapshot(), - new Uint8Array([123, 125]), - Date.now(), - scheduler, - true, // wantsStream: true - attempt, - ); - - dump.failed('upstream 502 error before output'); - dump.finalize(502, []); - await Promise.all(tasks); - - expect(stubs.stored).toHaveLength(1); - expect(stubs.stored[0]!.record.meta.ttftMs).toBeNull(); -}); - -test('DumpAccumulator leaves TTFT absent on a WebSocket prewarm response with no upstream call', async () => { - const stubs = installDumpStubs(initDumpStore, initDumpBroker); - const tasks: Promise[] = []; - const scheduler = (p: Promise) => { tasks.push(p); }; - - const attempt = { upstreamCallStartedAt: null, firstOutputTokenAt: null }; - const dump = new DumpAccumulator( - buildApiKey(), - makeSnapshot(), - new Uint8Array([123, 125]), - Date.now(), - scheduler, - true, // wantsStream: true - attempt, - ); - - dump.finalize(200, []); - await Promise.all(tasks); - - expect(stubs.stored).toHaveLength(1); - expect(stubs.stored[0]!.record.meta.ttftMs).toBeNull(); -}); diff --git a/packages/gateway/__tests__/dump/http-capture_test.ts b/packages/gateway/__tests__/dump/http-capture_test.ts deleted file mode 100644 index 8b65ca431c..0000000000 --- a/packages/gateway/__tests__/dump/http-capture_test.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { expect, test } from 'vitest'; - -import { HttpCapture } from '../../src/dump/http-capture.ts'; -import { parseSSEStream } from '@floway-dev/protocols/common'; -import { jsonRequestBody } from '@floway-dev/provider'; - -const utf8 = (text: string) => new TextEncoder().encode(text); - -test('captures the serialized provider request and raw malformed SSE before parsing', async () => { - const capture = new HttpCapture(); - const raw = ': keepalive\r\ndata: {broken json}\r\n\r\ndata: tail\n\n'; - const requestBody = { model: 'resolved-model', messages: [{ role: 'user', content: '上游' }] }; - const fetcher = capture.wrapFetcher(async (url, init) => { - expect(url).toBe('https://upstream.test/chat'); - expect(new TextDecoder().decode(init.body as Uint8Array)).toBe(JSON.stringify(requestBody)); - return new Response(raw, { status: 200, headers: { 'content-type': 'text/event-stream', 'x-request-id': 'trace' } }); - }, 'upstream-1'); - const response = await fetcher('https://upstream.test/chat', { method: 'POST', headers: { 'content-type': 'application/json' }, body: jsonRequestBody(requestBody) }); - await expect((async () => { for await (const frame of parseSSEStream(response.body!)) JSON.parse(frame.data); })()).rejects.toThrow(); - expect(capture.exchanges[0]?.request.body).toEqual({ encoding: 'utf8', data: JSON.stringify(requestBody) }); - expect(capture.exchanges[0]?.response?.body).toEqual({ encoding: 'utf8', data: raw }); - expect(capture.exchanges[0]?.response?.headers).toContainEqual(['x-request-id', 'trace']); - expect(capture.exchanges[0]?.response?.complete).toBe(false); -}); - -test('preserves bytes received before a network failure and propagates the original error', async () => { - const capture = new HttpCapture(); - const failure = new Error('socket reset'); - let pulled = false; - const upstream = new ReadableStream({ - pull(controller) { - if (pulled) controller.error(failure); - else { pulled = true; controller.enqueue(utf8('data: partial\n')); } - }, - }, { highWaterMark: 0 }); - const fetcher = capture.wrapFetcher(async () => new Response(upstream, { headers: { 'content-type': 'text/event-stream' } }), 'u'); - const response = await fetcher('https://upstream.test', {}); - await expect(response.text()).rejects.toBe(failure); - expect(capture.exchanges[0]?.response).toMatchObject({ body: { encoding: 'utf8', data: 'data: partial\n' }, complete: false }); - expect(capture.exchanges[0]?.response?.error).toContain('socket reset'); -}); - -test('records every retry independently including empty response and dial failure', async () => { - const capture = new HttpCapture(); - const failure = new Error('dial failed'); - await expect(capture.wrapFetcher(async () => { throw failure; }, 'a')('https://a.test', {})).rejects.toBe(failure); - const response = await capture.wrapFetcher(async () => new Response('ok', { headers: { 'content-type': 'text/plain' } }), 'b')('https://b.test', {}); - expect(await response.text()).toBe('ok'); - await capture.wrapFetcher(async () => new Response(null, { status: 204 }), 'c')('https://c.test', {}); - expect(capture.exchanges).toHaveLength(3); - expect(capture.exchanges[0]?.error).toContain('dial failed'); - expect(capture.exchanges[1]?.response).toMatchObject({ complete: true, body: { data: 'ok' } }); - expect(capture.exchanges[2]?.response).toMatchObject({ complete: true, status: 204, body: { data: '' } }); -}); - -test('captures binary bytes without lossy UTF-8 decoding', async () => { - const capture = new HttpCapture(); - const response = await capture.wrapFetcher(async () => new Response(new Uint8Array([255, 128]), { headers: { 'content-type': 'text/plain' } }), 'u')('https://u.test', {}); - expect([...new Uint8Array(await response.arrayBuffer())]).toEqual([255, 128]); - expect(capture.exchanges[0]?.response?.body).toEqual({ encoding: 'base64', data: '/4A=' }); -}); - -test('cancelling a pending read remains incomplete and never closes an already cancelled controller', async () => { - const capture = new HttpCapture(); - let cancelReason: unknown; - const response = await capture.wrapFetcher(async () => new Response(new ReadableStream({ cancel(reason) { cancelReason = reason; } })), 'u')('https://u.test', {}); - const reader = response.body!.getReader(); - const pending = reader.read(); - await reader.cancel('client cancelled'); - await pending; - expect(cancelReason).toBe('client cancelled'); - expect(capture.exchanges[0]?.response).toMatchObject({ complete: false, error: 'client cancelled' }); -}); - -test('raw text preserves the UTF-8 BOM', async () => { - const capture = new HttpCapture(); - const bytes = new Uint8Array([239, 187, 191, 120]); - const response = await capture.wrapFetcher(async () => new Response(bytes, { headers: { 'content-type': 'text/plain' } }), 'u')('https://u.test', {}); - await response.arrayBuffer(); - expect(capture.exchanges[0]?.response?.body).toEqual({ encoding: 'utf8', data: '\uFEFFx' }); -}); diff --git a/packages/gateway/__tests__/dump/run-sink_test.ts b/packages/gateway/__tests__/dump/run-sink_test.ts index 71e8dfdc97..9358b37fc8 100644 --- a/packages/gateway/__tests__/dump/run-sink_test.ts +++ b/packages/gateway/__tests__/dump/run-sink_test.ts @@ -3,9 +3,9 @@ import { expect, test } from 'vitest'; import { installDumpStubs } from './test-fixtures.ts'; import { initDumpBroker, initDumpStore } from '../../src/dump/registry.ts'; import { openRunDump } from '../../src/dump/run-sink.ts'; -import type { StoredDumpRecord, StoredDumpRunRecord } from '../../src/dump/types.ts'; import type { ApiKey } from '../../src/repo/types.ts'; import { flushBackground, trackBackground } from '../test-utils/background-tracker.ts'; +import type { StoredDumpRecord } from '@floway-dev/dump/types'; import { compose, defineStage, defer, move, run, type DumpEvent, type Event } from '@floway-dev/pipeline'; import { assertEquals } from '@floway-dev/test-utils'; @@ -53,15 +53,14 @@ const shout = defineStage, Pick, Pick, Pick>('shout-it', [shout, answer]); -const runRecordOf = (stored: { record: StoredDumpRecord } | undefined): StoredDumpRunRecord => { +const runRecordOf = (stored: { record: StoredDumpRecord } | undefined): StoredDumpRecord => { if (!stored) throw new Error('expected a stored dump record'); - if (stored.record.shape !== 'run') throw new Error(`expected the run shape, got ${stored.record.shape}`); return stored.record; }; -const ndjson = (record: StoredDumpRunRecord): string => new TextDecoder().decode(record.events); +const ndjson = (record: StoredDumpRecord): string => new TextDecoder().decode(record.events); -const lines = (record: StoredDumpRunRecord): DumpEvent[] => +const lines = (record: StoredDumpRecord): DumpEvent[] => ndjson(record).split('\n').filter(Boolean).map(line => JSON.parse(line) as DumpEvent); test('a run under a key with retention stores its whole event stream as NDJSON', async () => { diff --git a/packages/gateway/__tests__/dump/storage-codec_test.ts b/packages/gateway/__tests__/dump/storage-codec_test.ts index cd691a1d38..72c0f5c6e5 100644 --- a/packages/gateway/__tests__/dump/storage-codec_test.ts +++ b/packages/gateway/__tests__/dump/storage-codec_test.ts @@ -2,46 +2,13 @@ import { expect, test } from 'vitest'; import { fakeMeta } from './test-fixtures.ts'; import { - decodeDumpHeaders, decodePersistedDumpMetadata, - encodeDumpHeaders, encodePersistedDumpMetadata, } from '../../src/dump/storage-codec.ts'; -import { dumpCodec } from '@floway-dev/dump/codec'; - -test('dump storage headers preserve duplicate pairs and their order', () => { - const headers: Array<[string, string]> = [ - ['set-cookie', 'a=1'], - ['x-empty', ''], - ['set-cookie', 'b=2'], - ]; - - expect(decodeDumpHeaders(encodeDumpHeaders(headers, 'test headers'), 'test headers')).toEqual(headers); -}); - -test('dump broker frames round-trip metadata through the shared schema', () => { - const metadata = fakeMeta({ - upstream: { id: 'upstream-a', name: 'A', kind: 'custom', hue: 42 }, - error: { kind: 'failed', reason: 'connection closed' }, - }); - - expect(dumpCodec.decode(dumpCodec.encode(metadata))).toEqual(metadata); -}); - -test('dump broker frames reject valid JSON with malformed metadata', () => { - const frame = { - event: 'appended', - data: { ...fakeMeta(), status: '200' }, - }; - - expect(() => dumpCodec.decode(JSON.stringify(frame))) - .toThrow(/Invalid dump broker frame.*status/su); -}); // `targetApi` is the target protocol of a translated turn. It round-trips // through persisted metadata, and old `meta_json` written before the field -// existed (or native turns with no target) parses to null — the dashboard -// renders no upstream tab for those records. +// existed (or native turns with no target) has no target wire to display. test('persisted metadata round-trips targetApi', () => { const meta = fakeMeta({ targetApi: 'openaiResponses' }); const decoded = decodePersistedDumpMetadata( diff --git a/packages/gateway/__tests__/dump/test-fixtures.ts b/packages/gateway/__tests__/dump/test-fixtures.ts index 4ed29e0d12..d365b4e1c0 100644 --- a/packages/gateway/__tests__/dump/test-fixtures.ts +++ b/packages/gateway/__tests__/dump/test-fixtures.ts @@ -1,6 +1,4 @@ -import type { DumpStore } from '../../src/dump/store-contract.ts'; -import type { StoredDumpEdgeRecord, StoredDumpRecord, StoredDumpRunRecord } from '../../src/dump/types.ts'; -import type { DumpBroker, DumpMetadata } from '@floway-dev/dump'; +import type { DumpBroker, DumpStore, DumpMetadata, StoredDumpRecord } from '@floway-dev/dump/types'; import { encodeRun, toNdjson, type Event } from '@floway-dev/pipeline'; export const fakeMeta = (overrides: Partial = {}): DumpMetadata => ({ @@ -21,38 +19,21 @@ export const fakeMeta = (overrides: Partial = {}): DumpMetadata => ...overrides, }); -export const fakeRecord = (overrides: Partial = {}): StoredDumpEdgeRecord => ({ - shape: 'edge', - meta: fakeMeta(overrides), - request: { method: 'POST', path: '/v1/x', headers: [], body: new Uint8Array() }, - response: { status: 200, headers: [], body: { type: 'none' } }, -}); - // A run record holds one NDJSON stream and no edge halves, so a fixture takes // the events a run emitted and encodes them the way the sink does — object // space, folding and all. -export const fakeRunRecord = (events: readonly Event[], overrides: Partial = {}): StoredDumpRunRecord => ({ - shape: 'run', +export const fakeRunRecord = (events: readonly Event[], overrides: Partial = {}): StoredDumpRecord => ({ meta: fakeMeta(overrides), events: new TextEncoder().encode(toNdjson(encodeRun(events))), }); -// A reader hands back whichever shape was written, so a test asserting on the -// request or response halves says which one it wrote. -export const edgeRecordOf = (record: StoredDumpRecord | null | undefined): StoredDumpEdgeRecord => { +export const runRecordOf = (record: StoredDumpRecord | null | undefined): StoredDumpRecord => { if (!record) throw new Error('expected a stored dump record'); - if (record.shape !== 'edge') throw new Error(`expected the edge shape, got ${record.shape}`); - return record; -}; - -export const runRecordOf = (record: StoredDumpRecord | null | undefined): StoredDumpRunRecord => { - if (!record) throw new Error('expected a stored dump record'); - if (record.shape !== 'run') throw new Error(`expected the run shape, got ${record.shape}`); return record; }; /** The events a run recorded, decoded from the NDJSON one line at a time. */ -export const eventsOf = (record: StoredDumpRunRecord): readonly Record[] => +export const eventsOf = (record: StoredDumpRecord): readonly Record[] => new TextDecoder().decode(record.events).split('\n').filter(line => line.length > 0) .map(line => JSON.parse(line) as Record); @@ -85,19 +66,9 @@ export const installDumpStubs = ( const throws: Partial> = {}; const store: DumpStore = { - async prepareRequestBody(body) { - return { encoding: 'identity', bytes: body, decodedByteLength: body.byteLength }; - }, async put(keyId, record) { if (throws.put) throw throws.put; - // The run shape is already stored-shaped; only the edge shape carries a - // separately prepared request body to rehydrate. - if (record.shape === 'edge' && record.request.body.encoding !== 'identity') { - throw new Error('dump test stub expected identity request body'); - } - const storedRecord: StoredDumpRecord = record.shape === 'run' - ? record - : { ...record, request: { ...record.request, body: record.request.body.bytes } }; + const storedRecord: StoredDumpRecord = record; stored.push({ keyId, record: storedRecord }); const list = records.get(keyId) ?? []; list.unshift(storedRecord); diff --git a/packages/gateway/__tests__/dump/wire_test.ts b/packages/gateway/__tests__/dump/wire_test.ts deleted file mode 100644 index 5b28345ef6..0000000000 --- a/packages/gateway/__tests__/dump/wire_test.ts +++ /dev/null @@ -1,195 +0,0 @@ -import { expect, test } from 'vitest'; - -import type { DumpEdgeRecord, StoredDumpEdgeRecord } from '../../src/dump/types.ts'; -import { dumpRecordToWire } from '../../src/dump/wire.ts'; -import { assertEquals } from '@floway-dev/test-utils'; - -const baseStored = (overrides: Partial = {}): StoredDumpEdgeRecord => ({ - shape: 'edge', - meta: { - id: 'rec', - startedAt: 0, - completedAt: 1, - method: 'POST', - path: '/v1/x', - status: 200, - upstream: null, - model: null, - inputTokens: null, - outputTokens: null, - requestBytes: 0, - responseBytes: 0, - durationMs: 1, - error: null, - }, - request: { method: 'POST', path: '/v1/x', headers: [], body: new Uint8Array() }, - response: { status: 200, headers: [], body: { type: 'none' } }, - ...overrides, -}); - -const edgeWire = (record: StoredDumpEdgeRecord): DumpEdgeRecord => { - const wire = dumpRecordToWire(record); - if (wire.shape !== 'edge') throw new Error('expected the edge shape'); - return wire; -}; - -// A textual request content-type with valid UTF-8 bytes serializes as utf8 on -// the wire — the dashboard reads `data` directly as a string. -test('dumpRecordToWire encodes a textual request body as utf8', () => { - const wire = edgeWire(baseStored({ - request: { - method: 'POST', - path: '/v1/messages', - headers: [['content-type', 'application/json']], - body: new TextEncoder().encode('{"k":"v"}'), - }, - })); - assertEquals(wire.request.body.encoding, 'utf8'); - assertEquals(wire.request.body.data, '{"k":"v"}'); -}); - -test('dumpRecordToWire recognizes structured textual suffixes without accepting near matches', () => { - const structured = edgeWire(baseStored({ - request: { - method: 'POST', - path: '/v1/x', - headers: [['content-type', 'Application/Problem+JSON; charset=utf-8']], - body: new TextEncoder().encode('{"error":"bad"}'), - }, - })); - assertEquals(structured.request.body.encoding, 'utf8'); - - const nearMatch = edgeWire(baseStored({ - request: { - method: 'POST', - path: '/v1/x', - headers: [['content-type', 'application/jsonish']], - body: new TextEncoder().encode('{"not":"json"}'), - }, - })); - assertEquals(nearMatch.request.body.encoding, 'base64'); -}); - -// A binary content-type round-trips through base64 so JSON serialization -// preserves every byte; the dashboard decodes base64 client-side. -test('dumpRecordToWire encodes a binary response body as base64', () => { - const png = new Uint8Array([0x89, 0x50, 0x4E, 0x47]); // PNG magic - const wire = edgeWire(baseStored({ - response: { - status: 200, - headers: [['content-type', 'image/png']], - body: { type: 'bytes', body: png }, - }, - })); - if (wire.response.body.type !== 'bytes') throw new Error('expected bytes'); - assertEquals(wire.response.body.body.encoding, 'base64'); - // Decoding the base64 back to bytes must reproduce the original sequence. - const binary = atob(wire.response.body.body.data); - assertEquals(Array.from(binary, c => c.charCodeAt(0)), [0x89, 0x50, 0x4E, 0x47]); -}); - -// A content-type that claims to be text but carries bytes that do not decode -// as UTF-8 falls through to base64 so the wire never silently corrupts. -test('dumpRecordToWire falls back to base64 when textual content-type carries non-UTF-8 bytes', () => { - const bytes = new Uint8Array([0xFF, 0xFE, 0xFD]); - const wire = edgeWire(baseStored({ - request: { - method: 'POST', - path: '/v1/x', - headers: [['content-type', 'text/plain']], - body: bytes, - }, - })); - assertEquals(wire.request.body.encoding, 'base64'); - const binary = atob(wire.request.body.data); - assertEquals(Array.from(binary, c => c.charCodeAt(0)), [0xFF, 0xFE, 0xFD]); -}); - -// `stream` and `none` response bodies pass through wire serialization -// unchanged because they carry no raw bytes. -test('dumpRecordToWire passes stream + none response bodies through', () => { - const streamWire = edgeWire(baseStored({ - response: { status: 200, headers: [], body: { type: 'stream', events: [] } }, - })); - assertEquals(streamWire.response.body.type, 'stream'); - - const noneWire = edgeWire(baseStored({ - response: { status: null, headers: [], body: { type: 'none' } }, - })); - assertEquals(noneWire.response.body.type, 'none'); -}); - -// A run record crosses to the wire as the stored bytes decoded, because a line -// of NDJSON is one event and one SSE `data:` payload — what the dashboard reads -// here is what a live observer will read frame by frame. -test('dumpRecordToWire hands a run record its NDJSON verbatim', () => { - const ndjson = '{"type":"stage.entered","stageId":1,"name":"serve","parentStageId":null}\n' - + '{"type":"stage.leaved","stageId":1,"facts":{"response.http.status":200}}\n'; - const wire = dumpRecordToWire({ - shape: 'run', - meta: baseStored().meta, - events: new TextEncoder().encode(ndjson), - }); - if (wire.shape !== 'run') throw new Error('expected the run shape'); - assertEquals(wire.events, ndjson); - assertEquals(wire.meta.id, 'rec'); -}); - -// The gateway wrote those bytes itself, so bytes that are not UTF-8 mean a -// corrupted record and the reader says so rather than serving mojibake. -test('dumpRecordToWire refuses a run stream that is not UTF-8', () => { - expect(() => dumpRecordToWire({ - shape: 'run', - meta: baseStored().meta, - events: new Uint8Array([0xFF, 0xFE, 0xFD]), - })).toThrow(); -}); - -// The optional parallel `upstream` body (pre-translation view) round-trips -// through the wire with the same bytes/stream semantics as the downstream -// body. Absent on native turns and old records — `upstream` stays undefined. -test('dumpRecordToWire maps the upstream stream body when present', () => { - const wire = edgeWire(baseStored({ - meta: { ...baseStored().meta, targetApi: 'openaiResponses' }, - response: { - status: 200, - headers: [], - body: { type: 'stream', events: [] }, - upstream: { - status: null, - headers: [], - body: { type: 'stream', events: [{ frame: { type: 'done' }, ts: 0 }] }, - }, - }, - })); - assertEquals(wire.response.upstream?.body.type, 'stream'); - assertEquals(wire.meta.targetApi, 'openaiResponses'); -}); - -test('dumpRecordToWire maps the upstream bytes body using its own content-type', () => { - const wire = edgeWire(baseStored({ - meta: { ...baseStored().meta, targetApi: 'openaiChatCompletions' }, - response: { - status: 200, - headers: [], - body: { type: 'none' }, - upstream: { - status: 413, - headers: [['content-type', 'application/json']], - body: { type: 'bytes', body: new TextEncoder().encode('{"error":"too big"}') }, - }, - }, - })); - if (wire.response.upstream === undefined) throw new Error('expected upstream'); - assertEquals(wire.response.upstream.status, 413); - if (wire.response.upstream.body.type !== 'bytes') throw new Error('expected bytes'); - assertEquals(wire.response.upstream.body.body.encoding, 'utf8'); - assertEquals(wire.response.upstream.body.body.data, '{"error":"too big"}'); -}); - -test('dumpRecordToWire omits upstream when absent (native turn / old record)', () => { - const wire = edgeWire(baseStored({ - response: { status: 200, headers: [], body: { type: 'none' } }, - })); - assertEquals(wire.response.upstream, undefined); -}); diff --git a/packages/gateway/__tests__/migrations/drop-edge-dump-records_test.ts b/packages/gateway/__tests__/migrations/drop-edge-dump-records_test.ts new file mode 100644 index 0000000000..8dbf0016c9 --- /dev/null +++ b/packages/gateway/__tests__/migrations/drop-edge-dump-records_test.ts @@ -0,0 +1,55 @@ +import { DatabaseSync } from 'node:sqlite'; + +import { expect, test } from 'vitest'; + +import { migrationSqlByFilename } from '../repo/test-sqlite.ts'; + +const MIGRATION = '0086_drop_edge_dump_records.sql'; + +test('deletes historical edge records and retires all their files while preserving run records', () => { + const db = new DatabaseSync(':memory:'); + try { + for (const [filename, sql] of migrationSqlByFilename) { + if (filename === MIGRATION) break; + db.exec(sql); + } + db.prepare(`INSERT INTO api_keys (id, user_id, name, key, created_at, server_secret, dump_retention_seconds) + VALUES (?, 1, ?, ?, ?, ?, 3600)`).run('key', 'Migration key', 'migration-key', '2026-01-01T00:00:00Z', '44'.repeat(32)); + const stage = db.prepare(`INSERT INTO spilled_files (file_key, owner_kind, owner_key, state, collect_after) + VALUES (?, ?, ?, 'staged', 10000)`); + const insert = db.prepare(`INSERT INTO dump_records + (key_id, id, created_at, meta_json, request_headers_json, request_body_descriptor, response_body_descriptor, response_upstream_body_descriptor) + VALUES ('key', ?, 1000, '{}', '[]', ?, ?, ?)`); + const file = (id: string, part: 'request' | 'response' | 'response-upstream', type: string) => { + const key = `dumps/v1/key/${id}-${part}.gz`; + stage.run(key, `dump-${part}`, JSON.stringify(['key', id])); + return JSON.stringify({ key, type }); + }; + insert.run('edge-bytes', file('edge-bytes', 'request', 'bytes'), file('edge-bytes', 'response', 'bytes'), file('edge-bytes', 'response-upstream', 'events')); + insert.run('edge-events', null, file('edge-events', 'response', 'events'), null); + insert.run('edge-empty', file('edge-empty', 'request', 'bytes'), null, null); + insert.run('retained-run', null, file('retained-run', 'response', 'run'), null); + + expect(db.prepare('SELECT id FROM dump_records ORDER BY id').all()).toEqual([ + { id: 'edge-bytes' }, { id: 'edge-empty' }, { id: 'edge-events' }, { id: 'retained-run' }, + ]); + expect(db.prepare('SELECT DISTINCT state, collect_after FROM spilled_files').all()).toEqual([{ state: 'owned', collect_after: null }]); + + const retainedRun = db.prepare("SELECT * FROM dump_records WHERE id = 'retained-run'").get(); + const migration = migrationSqlByFilename.find(([filename]) => filename === MIGRATION)!; + db.exec(migration[1]); + + expect(db.prepare('SELECT id FROM dump_records').all()).toEqual([{ id: 'retained-run' }]); + expect(db.prepare("SELECT * FROM dump_records WHERE id = 'retained-run'").get()).toEqual(retainedRun); + expect(db.prepare('SELECT file_key, state, collect_after FROM spilled_files WHERE owner_key != ? ORDER BY file_key') + .all(JSON.stringify(['key', 'retained-run']))).toEqual([ + { file_key: 'dumps/v1/key/edge-bytes-request.gz', state: 'retired', collect_after: 0 }, + { file_key: 'dumps/v1/key/edge-bytes-response-upstream.gz', state: 'retired', collect_after: 0 }, + { file_key: 'dumps/v1/key/edge-bytes-response.gz', state: 'retired', collect_after: 0 }, + { file_key: 'dumps/v1/key/edge-empty-request.gz', state: 'retired', collect_after: 0 }, + { file_key: 'dumps/v1/key/edge-events-response.gz', state: 'retired', collect_after: 0 }, + ]); + expect(db.prepare('SELECT state, collect_after FROM spilled_files WHERE owner_key = ?') + .get(JSON.stringify(['key', 'retained-run']))).toEqual({ state: 'owned', collect_after: null }); + } finally { db.close(); } +}); diff --git a/packages/gateway/__tests__/migrations/dump-upstream-body_test.ts b/packages/gateway/__tests__/migrations/dump-upstream-body_test.ts index b36265e9fb..b52278a61a 100644 --- a/packages/gateway/__tests__/migrations/dump-upstream-body_test.ts +++ b/packages/gateway/__tests__/migrations/dump-upstream-body_test.ts @@ -6,13 +6,9 @@ import { migrationSqlByFilename } from '../repo/test-sqlite.ts'; const MIGRATION = '0084_dump_upstream_body.sql'; -// The upstream body column is additive: nullable, and the three spilled-files -// triggers from 0066 are rebuilt to cover the new descriptor. The rebuilt -// triggers' behavior is exercised end-to-end by the dump-store round-trip -// tests (an upstream file is staged, adopted to `owned`, and rehydrated); this -// test asserts the migration itself: the column appears, it is nullable, and -// the rebuilt triggers are present by name. -test('0089 adds a nullable response_upstream_body_descriptor column', () => { +// Historical upstream-body storage remains in the schema; the run-only migration +// retires its records and files. The column and registry triggers still exist. +test('0084 adds a nullable response_upstream_body_descriptor column', () => { const before = new DatabaseSync(':memory:'); for (const [filename, sql] of migrationSqlByFilename) { if (filename === MIGRATION) break; @@ -29,7 +25,7 @@ test('0089 adds a nullable response_upstream_body_descriptor column', () => { const col = columnsAfter.find(c => c.name === 'response_upstream_body_descriptor'); expect(col).toBeDefined(); expect(col!.notnull).toBe(0); // nullable - // The rebuilt triggers are present (rebuilt, not dropped) after 0089. + // The rebuilt registry triggers survive the later run-only migration. for (const trigger of [ 'dump_records_validate_spilled_files', 'dump_records_adopt_spilled_files', diff --git a/packages/gateway/__tests__/repo/dump-store_test.ts b/packages/gateway/__tests__/repo/dump-store_test.ts index 5b10d2e4fd..bcf3a00317 100644 --- a/packages/gateway/__tests__/repo/dump-store_test.ts +++ b/packages/gateway/__tests__/repo/dump-store_test.ts @@ -6,11 +6,11 @@ import { expect, test } from 'vitest'; import { createSqliteTestDb, mapRunChangeCount } from './test-sqlite.ts'; import { decodeDumpBodyDescriptor } from '../../src/dump/storage-codec.ts'; -import type { DumpWriteEdgeRecord, StoredDumpEdgeRecord, StoredDumpRecord, StoredDumpRunRecord } from '../../src/dump/types.ts'; import { FileDumpStore } from '../../src/repo/dump-store.ts'; import { initRepo } from '../../src/repo/index.ts'; import { SqlRepo } from '../../src/repo/sql.ts'; import { collectSpilledFiles } from '../../src/scheduled/spilled-files.ts'; +import type { StoredDumpRecord } from '@floway-dev/dump/types'; import { encodeRun, toNdjson, type Facts } from '@floway-dev/pipeline'; import { initFileStore, MemoryFileStore } from '@floway-dev/platform'; import type { FileStore, SqlDatabase } from '@floway-dev/platform'; @@ -33,176 +33,17 @@ const openDb = async (): Promise => { return db; }; -const utf8 = (s: string): Uint8Array => new TextEncoder().encode(s); - -const requestBody = utf8('{"hello":"world"}'); - -const baseRecord = (id: string, completedAt: number): DumpWriteEdgeRecord => ({ - shape: 'edge', - meta: { - id, startedAt: completedAt - 1, completedAt, method: 'POST', path: '/v1/x', status: 200, - upstream: null, model: 'm', inputTokens: 1, outputTokens: 2, - requestBytes: 3, responseBytes: 4, durationMs: 1, error: null, - }, - request: { - method: 'POST', path: '/v1/x', - headers: [['content-type', 'application/json']], - body: { encoding: 'identity', bytes: requestBody, decodedByteLength: requestBody.byteLength }, - }, - response: { - status: 200, - headers: [['content-type', 'application/json']], - body: { type: 'bytes', body: utf8('{"id":"abc"}') }, - }, -}); - -// Every fixture above writes the edge shape, and a reader hands back what was -// written — so a test that asserts on the request or response halves says so. -const edgeOf = (record: StoredDumpRecord | null): StoredDumpEdgeRecord => { - assertExists(record); - if (record.shape !== 'edge') throw new Error('expected the edge shape'); - return record; -}; - -test('FileDumpStore prepares request gzip before terminal persistence', async () => { - const db = await openDb(); - const files = new MemoryFileStore(); - const store = new FileDumpStore(db, files); - const raw = utf8(`{"content":"${'repeatable '.repeat(4096)}"}`); - const prepared = await store.prepareRequestBody(raw); - const base = baseRecord('01HZZ0000000000000000000P1', Date.UTC(2026, 5, 1, 12, 0, 0)); - const record: DumpWriteEdgeRecord = { - ...base, - meta: { ...base.meta, requestBytes: raw.byteLength }, - request: { - method: 'POST', - path: '/v1/x', - headers: [['content-type', 'application/json']], - body: prepared, - }, - }; - - assertEquals(prepared.encoding, 'gzip'); - assertEquals(prepared.decodedByteLength, raw.byteLength); - assertEquals(prepared.bytes.byteLength < raw.byteLength, true); - await store.put('key_x', record); - const fetched = edgeOf(await store.get('key_x', record.meta.id)); - assertEquals(Array.from(fetched.request.body), Array.from(raw)); -}); - -test('FileDumpStore round-trips a JSON record through gzip', async () => { - const db = await openDb(); - const files = new MemoryFileStore(); - const store = new FileDumpStore(db, files); - const record = baseRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 12, 0, 0)); - - await store.put('key_x', record); - const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000A1')); - assertEquals(fetched.meta.id, record.meta.id); - assertEquals(new TextDecoder().decode(fetched.request.body), '{"hello":"world"}'); - if (fetched.response.body.type !== 'bytes') throw new Error('expected bytes'); - assertEquals(new TextDecoder().decode(fetched.response.body.body), '{"id":"abc"}'); -}); - -test('FileDumpStore round-trips ttftMs on persisted records', async () => { - const db = await openDb(); - const files = new MemoryFileStore(); - const store = new FileDumpStore(db, files); - const base = baseRecord('01HZZ0000000000000000000TT', Date.UTC(2026, 5, 1, 12, 0, 0)); - const record: DumpWriteEdgeRecord = { - ...base, - meta: { ...base.meta, ttftMs: 250 }, - }; - - await store.put('key_x', record); - const fetched = await store.get('key_x', '01HZZ0000000000000000000TT'); - assertExists(fetched); - assertEquals(fetched.meta.ttftMs, 250); - - const listed = await store.list('key_x', { limit: 10 }); - const item = listed.find(m => m.id === '01HZZ0000000000000000000TT'); - assertExists(item); - assertEquals(item.ttftMs, 250); -}); - -test('FileDumpStore preserves the original content-type header on binary bodies', async () => { - const db = await openDb(); - const files = new MemoryFileStore(); - const store = new FileDumpStore(db, files); - const pngMagic = new Uint8Array([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]); - const record: DumpWriteEdgeRecord = { - ...baseRecord('01HZZ0000000000000000000PNG', Date.UTC(2026, 5, 1, 12, 0, 0)), - response: { - status: 200, - headers: [['content-type', 'image/png']], - body: { type: 'bytes', body: pngMagic }, - }, - }; - - await store.put('key_x', record); - const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000PNG')); - // The header pair must survive verbatim — no `;base64` suffix tacked on. - assertEquals(fetched.response.headers.find(([k]) => k === 'content-type')?.[1], 'image/png'); - if (fetched.response.body.type !== 'bytes') throw new Error('expected bytes'); - assertEquals(Array.from(fetched.response.body.body), Array.from(pngMagic)); -}); - -test('FileDumpStore preserves the bytes discriminator on an empty-body response', async () => { - const db = await openDb(); - const files = new MemoryFileStore(); - const store = new FileDumpStore(db, files); - // 204-style: real upstream response with status + headers but a zero-length - // body. Persistence drops the body file (nothing to gzip), but headers are - // still written — the read path must surface this as `bytes`, not `none`. - const record: DumpWriteEdgeRecord = { - ...baseRecord('01HZZ0000000000000000000E1', Date.UTC(2026, 5, 1, 12, 0, 0)), - response: { - status: 204, - headers: [['content-type', 'application/json']], - body: { type: 'bytes', body: new Uint8Array() }, - }, - }; - - await store.put('key_x', record); - const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000E1')); - if (fetched.response.body.type !== 'bytes') throw new Error('expected bytes'); - assertEquals(fetched.response.body.body.byteLength, 0); - assertEquals(fetched.response.headers.find(([k]) => k === 'content-type')?.[1], 'application/json'); -}); - -test('FileDumpStore round-trips an SSE record as a stream events array', async () => { - const db = await openDb(); - const files = new MemoryFileStore(); - const store = new FileDumpStore(db, files); - const record: DumpWriteEdgeRecord = { - ...baseRecord('01HZZ0000000000000000000A2', Date.UTC(2026, 5, 1, 12, 0, 0)), - response: { - status: 200, - headers: [['content-type', 'text/event-stream']], - body: { - type: 'stream', - events: [ - { frame: { type: 'event', event: { type: 'message_start' } }, ts: 10 }, - { frame: { type: 'done' }, ts: 20 }, - ], - }, - }, - }; - await store.put('key_x', record); - const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000A2')); - if (fetched.response.body.type !== 'stream') throw new Error('expected stream'); - assertEquals(fetched.response.body.events.length, 2); - assertEquals(fetched.response.body.events[0]!.frame.type, 'event'); -}); - -// A pipelined turn stores the run rather than its edges: one NDJSON body file -// under the same contract, and a row that carries the same metadata. -const runRecord = (id: string, completedAt: number): StoredDumpRunRecord => { +// A turn is stored as its run: one NDJSON body file under the file-store contract, and a row +// carrying the metadata a list renders without opening it. +const runRecord = (id: string, completedAt: number): StoredDumpRecord => { const entered: Facts = { 'request.http.path': '/v1/embeddings' }; const left: Facts = { ...entered, 'response.http.status': 200 }; return { - shape: 'run', - meta: baseRecord(id, completedAt).meta, + meta: { + id, startedAt: completedAt - 1, completedAt, method: 'POST', path: '/v1/x', status: 200, + upstream: null, model: 'm', inputTokens: 1, outputTokens: 2, + requestBytes: 3, responseBytes: 4, durationMs: 1, error: null, + }, events: new TextEncoder().encode(toNdjson(encodeRun([ { type: 'stage.entered', stageId: 1, name: 'serve', parentStageId: null, facts: entered }, { type: 'stage.entered', stageId: 2, name: 'dial', parentStageId: 1, facts: entered }, @@ -221,11 +62,10 @@ test('FileDumpStore round-trips a run record as its NDJSON event stream', async await store.put('key_x', record); const fetched = await store.get('key_x', record.meta.id); assertExists(fetched); - if (fetched.shape !== 'run') throw new Error(`expected the run shape, got ${fetched.shape}`); assertEquals(new TextDecoder().decode(fetched.events), new TextDecoder().decode(record.events)); - // The stream is a body file like the other two: gzipped, named for what it - // holds, and pointed at by the row's descriptor. + // The stream is a body file: gzipped, named for what it holds, and pointed at by the row's + // response descriptor — which is also what says the row has a run to read. const row = await db.prepare('SELECT request_body_descriptor, response_body_descriptor FROM dump_records WHERE key_id = ? AND id = ?') .bind('key_x', record.meta.id) .first<{ request_body_descriptor: string | null; response_body_descriptor: string }>(); @@ -236,17 +76,16 @@ test('FileDumpStore round-trips a run record as its NDJSON event stream', async assertEquals(descriptor.key.endsWith('.run.gz'), true); }); -test('FileDumpStore lists a run record alongside an edge-shaped one', async () => { +test('FileDumpStore lists what a row renders without opening the stream', async () => { const db = await openDb(); const store = new FileDumpStore(db, new MemoryFileStore()); const base = Date.UTC(2026, 5, 1, 12, 0, 0); - await store.put('key_x', baseRecord('01HZZ00000000000000000EDG1', base)); + await store.put('key_x', runRecord('01HZZ00000000000000000RUN1', base)); await store.put('key_x', runRecord('01HZZ00000000000000000RUN2', base + 1)); const listed = await store.list('key_x', { limit: 10 }); - assertEquals(listed.map(meta => meta.id), ['01HZZ00000000000000000RUN2', '01HZZ00000000000000000EDG1']); - // One metadata shape for both: the list says nothing about how a turn was - // served, and every column a row renders is filled either way. + assertEquals(listed.map(meta => meta.id), ['01HZZ00000000000000000RUN2', '01HZZ00000000000000000RUN1']); + // Every column a row renders is filled from the metadata, so a list costs no file read. assertEquals(listed.map(meta => meta.model), ['m', 'm']); assertEquals(listed.map(meta => meta.status), [200, 200]); }); @@ -281,100 +120,10 @@ test('FileDumpStore retires an expired run record and collects its stream file', assertEquals(remaining.every(row => !row.file_key.includes('2026060109')), true); }); -// The optional pre-translation upstream body spills to its own file under the -// `resp.up` side and round-trips independently of the downstream body. The -// `meta.targetApi` names the target protocol so the dashboard can render the -// upstream frames with the right serializer. -test('FileDumpStore round-trips the upstream stream body alongside the downstream body', async () => { - const db = await openDb(); - const files = new MemoryFileStore(); - const store = new FileDumpStore(db, files); - const record: DumpWriteEdgeRecord = { - ...baseRecord('01HZZ0000000000000000000UP1', Date.UTC(2026, 5, 1, 12, 0, 0)), - meta: { - ...baseRecord('01HZZ0000000000000000000UP1', Date.UTC(2026, 5, 1, 12, 0, 0)).meta, - targetApi: 'openaiResponses', - }, - response: { - status: 200, - headers: [['content-type', 'text/event-stream']], - body: { - type: 'stream', - events: [{ frame: { type: 'done' }, ts: 20 }], - }, - upstream: { - status: null, - headers: [], - body: { - type: 'stream', - events: [ - { frame: { type: 'event', event: { type: 'response.created' } }, ts: 5 }, - { frame: { type: 'done' }, ts: 15 }, - ], - }, - }, - }, - }; - await store.put('key_x', record); - const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000UP1')); - assertExists(fetched); - assertEquals(fetched.meta.targetApi, 'openaiResponses'); - const upstream = fetched.response.upstream; - assertExists(upstream); - if (upstream.body.type !== 'stream') throw new Error('expected upstream stream'); - assertEquals(upstream.body.events.length, 2); - assertEquals(upstream.body.events[0]!.frame.type, 'event'); -}); - -test('FileDumpStore round-trips the upstream bytes body (api-error envelope)', async () => { - const db = await openDb(); - const files = new MemoryFileStore(); - const store = new FileDumpStore(db, files); - const record: DumpWriteEdgeRecord = { - ...baseRecord('01HZZ0000000000000000000UP2', Date.UTC(2026, 5, 1, 12, 0, 0)), - meta: { - ...baseRecord('01HZZ0000000000000000000UP2', Date.UTC(2026, 5, 1, 12, 0, 0)).meta, - targetApi: 'openaiChatCompletions', - }, - response: { - status: 200, - headers: [['content-type', 'text/event-stream']], - body: { type: 'stream', events: [] }, - upstream: { - status: 413, - headers: [['content-type', 'application/json']], - body: { type: 'bytes', body: utf8('{"error":"too large"}') }, - }, - }, - }; - await store.put('key_x', record); - const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000UP2')); - assertExists(fetched); - const upstream = fetched.response.upstream; - assertExists(upstream); - if (upstream.body.type !== 'bytes') throw new Error('expected upstream bytes'); - assertEquals(new TextDecoder().decode(upstream.body.body), '{"error":"too large"}'); -}); - -// A record written without an upstream body (native turn, or written before -// this field existed) round-trips with `upstream` absent — the dashboard -// renders no upstream tab. `meta.targetApi` is also absent on native turns. -test('FileDumpStore omits upstream and targetApi for a native (non-translated) record', async () => { - const db = await openDb(); - const files = new MemoryFileStore(); - const store = new FileDumpStore(db, files); - const record = baseRecord('01HZZ0000000000000000000NT1', Date.UTC(2026, 5, 1, 12, 0, 0)); - await store.put('key_x', record); - const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000NT1')); - assertExists(fetched); - assertEquals(fetched.response.upstream, undefined); - assertEquals(fetched.meta.targetApi, undefined); -}); - test('FileDumpStore rejects malformed metadata with its row identity', async () => { const db = await openDb(); const store = new FileDumpStore(db, new MemoryFileStore()); - const record = baseRecord('01HZZ000000000000000000BADM', Date.UTC(2026, 5, 1, 12)); + const record = runRecord('01HZZ000000000000000000BADM', Date.UTC(2026, 5, 1, 12)); await store.put('key_x', record); await db.prepare('UPDATE dump_records SET meta_json = ? WHERE key_id = ? AND id = ?') .bind(JSON.stringify({ ...record.meta, upstream: undefined, status: '200' }), 'key_x', record.meta.id) @@ -384,59 +133,33 @@ test('FileDumpStore rejects malformed metadata with its row identity', async () .rejects.toThrow(/Invalid dump record 01HZZ000000000000000000BADM metadata.*status/su); }); -test('FileDumpStore rejects malformed header pairs with their record and side', async () => { +test('FileDumpStore rejects a malformed body descriptor before file access', async () => { const db = await openDb(); const store = new FileDumpStore(db, new MemoryFileStore()); - const record = baseRecord('01HZZ000000000000000000BADH', Date.UTC(2026, 5, 1, 12)); + const record = runRecord('01HZZ000000000000000000BADD', Date.UTC(2026, 5, 1, 12)); await store.put('key_x', record); - await db.prepare('UPDATE dump_records SET request_headers_json = ? WHERE key_id = ? AND id = ?') - .bind(JSON.stringify([['x-bad', 1]]), 'key_x', record.meta.id) + await db.prepare('UPDATE dump_records SET response_body_descriptor = ? WHERE key_id = ? AND id = ?') + .bind(JSON.stringify({ key: 'dumps/v1/key_x/run.gz', type: 'chunks' }), 'key_x', record.meta.id) .run(); await expect(store.get('key_x', record.meta.id)) - .rejects.toThrow(/Invalid dump record 01HZZ000000000000000000BADH request headers/u); + .rejects.toThrow(/Invalid dump record 01HZZ000000000000000000BADD response body descriptor.*type/su); }); -test('FileDumpStore rejects malformed body descriptors before file access', async () => { +// A row whose descriptor is well-formed but names something other than a run has no stream to +// read, and the migration that deleted the shape which wrote those leaves none — so it is +// corruption, and the reader says so rather than handing back half a record. +test('FileDumpStore refuses a row that points at no run stream', async () => { const db = await openDb(); const store = new FileDumpStore(db, new MemoryFileStore()); - const record = baseRecord('01HZZ000000000000000000BADD', Date.UTC(2026, 5, 1, 12)); + const record = runRecord('01HZZ000000000000000000NORUN', Date.UTC(2026, 5, 1, 12)); await store.put('key_x', record); - await db.prepare('UPDATE dump_records SET request_body_descriptor = ? WHERE key_id = ? AND id = ?') - .bind(JSON.stringify({ key: 'dumps/v1/key_x/request.gz', type: 'chunks' }), 'key_x', record.meta.id) + await db.prepare('UPDATE dump_records SET response_body_descriptor = ? WHERE key_id = ? AND id = ?') + .bind(JSON.stringify({ key: 'dumps/v1/key_x/run.gz', type: 'bytes' }), 'key_x', record.meta.id) .run(); await expect(store.get('key_x', record.meta.id)) - .rejects.toThrow(/Invalid dump record 01HZZ000000000000000000BADD request body descriptor.*type/su); -}); - -test('FileDumpStore rejects malformed stream events with record and file context', async () => { - const db = await openDb(); - const files = new MemoryFileStore(); - const store = new FileDumpStore(db, files); - const record: DumpWriteEdgeRecord = { - ...baseRecord('01HZZ000000000000000000BADE', Date.UTC(2026, 5, 1, 12)), - response: { - status: 200, - headers: [['content-type', 'text/event-stream']], - body: { type: 'stream', events: [{ frame: { type: 'done' }, ts: 1 }] }, - }, - }; - await store.put('key_x', record); - const row = await db.prepare('SELECT response_body_descriptor FROM dump_records WHERE key_id = ? AND id = ?') - .bind('key_x', record.meta.id) - .first<{ response_body_descriptor: string }>(); - if (row === null) throw new Error('expected stored dump row'); - const descriptor = decodeDumpBodyDescriptor(row.response_body_descriptor, 'test response descriptor'); - const malformedEvents = utf8(JSON.stringify([{ frame: { type: 'done' }, ts: 'late' }])); - const compressed = new Uint8Array(await new Response( - new Blob([malformedEvents as BlobPart]).stream().pipeThrough(new CompressionStream('gzip')), - ).arrayBuffer()); - await files.put(descriptor.key, compressed); - - await expect(store.get('key_x', record.meta.id)).rejects.toThrow( - new RegExp(`Invalid dump record ${record.meta.id} response events at key=${descriptor.key}.*ts`, 'su'), - ); + .rejects.toThrow(/Invalid dump record 01HZZ000000000000000000NORUN response body descriptor.*type/su); }); test('FileDumpStore.list paginates newest-first with the (createdAt, id) cursor', async () => { @@ -445,7 +168,7 @@ test('FileDumpStore.list paginates newest-first with the (createdAt, id) cursor' const store = new FileDumpStore(db, files); const base = Date.UTC(2026, 5, 1, 12, 0, 0); for (let i = 0; i < 5; i++) { - await store.put('key_x', baseRecord(`01HZZ000000000000000000A0${i}`, base + i)); + await store.put('key_x', runRecord(`01HZZ000000000000000000A0${i}`, base + i)); } const first = await store.list('key_x', { limit: 2 }); assertEquals(first.map(m => m.id), ['01HZZ000000000000000000A04', '01HZZ000000000000000000A03']); @@ -462,8 +185,8 @@ test('FileDumpStore applies retention immediately and retires exact expired file const store = new FileDumpStore(db, files); const now = Date.UTC(2026, 5, 1, 12, 0, 0); // Old bucket 9:xx, current bucket 12:xx. - await store.put('key_x', baseRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 9, 0, 0))); - await store.put('key_x', baseRecord('01HZZ0000000000000000000A2', now)); + await store.put('key_x', runRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 9, 0, 0))); + await store.put('key_x', runRecord('01HZZ0000000000000000000A2', now)); await repo.apiKeys.update('key_x', { dumpRetentionSeconds: 2 * 3600 }); const originalNow = Date.now; Date.now = () => now + 1; @@ -492,7 +215,7 @@ test('growing dump retention can reveal a row not yet physically deleted', async const files = new MemoryFileStore(); const store = new FileDumpStore(db, files); const now = Date.UTC(2026, 5, 1, 12); - await store.put('key_x', baseRecord('01HZZ0000000000000000000A4', now - 3 * 3600_000)); + await store.put('key_x', runRecord('01HZZ0000000000000000000A4', now - 3 * 3600_000)); const originalNow = Date.now; Date.now = () => now; try { @@ -512,8 +235,8 @@ test('FileDumpStore retires every dump record when retention is disabled and col const files = new MemoryFileStore(); initFileStore(files); const store = new FileDumpStore(db, files); - await store.put('key_x', baseRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 9, 0, 0))); - await store.put('key_x', baseRecord('01HZZ0000000000000000000A2', Date.UTC(2026, 5, 1, 12, 0, 0))); + await store.put('key_x', runRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 9, 0, 0))); + await store.put('key_x', runRecord('01HZZ0000000000000000000A2', Date.UTC(2026, 5, 1, 12, 0, 0))); await repo.apiKeys.update('key_x', { dumpRetentionSeconds: null }); assertEquals((await store.list('key_x', { limit: 10 })).length, 0); assertEquals(await store.deleteExpiredBatch('key_x', Date.now(), 100), 2); @@ -527,8 +250,8 @@ test('FileDumpStore counts returned dump rows instead of trigger-amplified chang const repo = new SqlRepo(db); const files = new MemoryFileStore(); const store = new FileDumpStore(db, files); - await store.put('key_x', baseRecord('01HZZ0000000000000000000C1', Date.UTC(2026, 5, 1, 9))); - await store.put('key_x', baseRecord('01HZZ0000000000000000000C2', Date.UTC(2026, 5, 1, 10))); + await store.put('key_x', runRecord('01HZZ0000000000000000000C1', Date.UTC(2026, 5, 1, 9))); + await store.put('key_x', runRecord('01HZZ0000000000000000000C2', Date.UTC(2026, 5, 1, 10))); await repo.apiKeys.update('key_x', { dumpRetentionSeconds: null }); const d1LikeStore = new FileDumpStore(mapRunChangeCount(db, changes => changes * 3), files); @@ -542,8 +265,8 @@ test('FileDumpStore counts returned active dump rows instead of trigger-amplifie const files = new MemoryFileStore(); const store = new FileDumpStore(db, files); const now = Date.UTC(2026, 5, 1, 12); - await store.put('key_x', baseRecord('01HZZ0000000000000000000C3', Date.UTC(2026, 5, 1, 9))); - await store.put('key_x', baseRecord('01HZZ0000000000000000000C4', Date.UTC(2026, 5, 1, 10))); + await store.put('key_x', runRecord('01HZZ0000000000000000000C3', Date.UTC(2026, 5, 1, 9))); + await store.put('key_x', runRecord('01HZZ0000000000000000000C4', Date.UTC(2026, 5, 1, 10))); await repo.apiKeys.update('key_x', { dumpRetentionSeconds: 3600 }); const d1LikeStore = new FileDumpStore(mapRunChangeCount(db, changes => changes * 3), files); @@ -558,16 +281,18 @@ test('a record-ID race leaves only the losing write\'s uniquely keyed files coll const files = new MemoryFileStore(); initFileStore(files); const store = new FileDumpStore(db, files); - const record = baseRecord('01HZZ0000000000000000000A3', Date.now()); + const record = runRecord('01HZZ0000000000000000000A3', Date.now()); await store.put('key_x', record); + // One file per record now — the run's own stream — so the winner owns one and the loser + // staged one under a key of its own. await expect(store.put('key_x', record)).rejects.toThrow(); - expect((await db.prepare("SELECT COUNT(*) AS count FROM spilled_files WHERE state = 'owned'").first<{ count: number }>())?.count).toBe(2); - expect((await db.prepare("SELECT COUNT(*) AS count FROM spilled_files WHERE state = 'staged'").first<{ count: number }>())?.count).toBe(2); + expect((await db.prepare("SELECT COUNT(*) AS count FROM spilled_files WHERE state = 'owned'").first<{ count: number }>())?.count).toBe(1); + expect((await db.prepare("SELECT COUNT(*) AS count FROM spilled_files WHERE state = 'staged'").first<{ count: number }>())?.count).toBe(1); await collectSpilledFiles(Date.now() + 60 * 60 * 1000 + 1); expect((await db.prepare("SELECT COUNT(*) AS count FROM spilled_files WHERE state = 'staged'").first<{ count: number }>())?.count).toBe(0); - expect((await db.prepare("SELECT COUNT(*) AS count FROM spilled_files WHERE state = 'owned'").first<{ count: number }>())?.count).toBe(2); + expect((await db.prepare("SELECT COUNT(*) AS count FROM spilled_files WHERE state = 'owned'").first<{ count: number }>())?.count).toBe(1); assertExists(await store.get('key_x', record.meta.id)); }); @@ -614,44 +339,23 @@ test('FileDumpStore: put + get round-trips through real-filesystem IO', async () try { const db = await openDb(); const store = new FileDumpStore(db, new TmpDirFileStore(join(root, 'files'))); - const record = baseRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 12, 0, 0)); + const record = runRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 12, 0, 0)); await store.put('key_x', record); - const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000A1')); - assertEquals(new TextDecoder().decode(fetched.request.body), '{"hello":"world"}'); - if (fetched.response.body.type !== 'bytes') throw new Error('expected bytes'); - assertEquals(new TextDecoder().decode(fetched.response.body.body), '{"id":"abc"}'); + const fetched = await store.get('key_x', '01HZZ0000000000000000000A1'); + assertExists(fetched); + assertEquals(new TextDecoder().decode(fetched.events), new TextDecoder().decode(record.events)); } finally { await rm(root, { recursive: true, force: true }); } }); -test('FileDumpStore round-trips raw exchanges and parsed events in one owned spill', async () => { - const db = await openDb(); - const files = new MemoryFileStore(); - const store = new FileDumpStore(db, files); - const record = baseRecord('raw-exchange', Date.now()); - record.capture = { - exchanges: [{ upstreamId: 'u', request: { url: 'https://u.test', method: 'POST', headers: [], body: { encoding: 'utf8', data: '{"model":"upstream"}' } }, response: { status: 200, headers: [['content-type', 'text/event-stream']], body: { encoding: 'utf8', data: 'data: {broken\n' }, complete: false, error: 'socket reset' }, error: null }], - response: { body: { encoding: 'utf8', data: 'downstream' }, complete: true, error: null }, - }; - record.response = { ...record.response, upstream: { status: 200, headers: [['x-trace', 'upstream']], body: { type: 'stream', events: [{ ts: 0, frame: { type: 'done' } }] } } }; - await store.put('key_x', record); - const stored = await store.get('key_x', record.meta.id); - expect(edgeOf(stored).capture).toEqual(record.capture); - expect(edgeOf(stored!).response.upstream).toEqual(record.response.upstream); - const spill = await db.prepare("SELECT file_key, state FROM spilled_files WHERE owner_kind = 'dump-response-upstream'").first<{ file_key: string; state: string }>(); - expect(spill?.state).toBe('owned'); - await db.prepare('DELETE FROM dump_records WHERE id = ?').bind(record.meta.id).run(); - expect((await db.prepare('SELECT state FROM spilled_files WHERE file_key = ?').bind(spill!.file_key).first<{ state: string }>())?.state).toBe('retired'); -}); - test('FileDumpStore filters all retained history before applying the page limit', async () => { const db = await openDb(); const store = new FileDumpStore(db, new MemoryFileStore()); const now = Date.now(); for (let i = 0; i < 5; i++) { - const record = baseRecord(`filter-${i}`, now - i); + const record = runRecord(`filter-${i}`, now - i); record.meta.model = i === 4 ? 'needle-model' : 'other'; record.meta.error = i === 4 ? { kind: 'failed', reason: 'socket reset' } : null; await store.put('key_x', record); diff --git a/packages/gateway/__tests__/repo/expiration-sweep-integrity_test.ts b/packages/gateway/__tests__/repo/expiration-sweep-integrity_test.ts index 2d467cfbd7..43850b4a54 100644 --- a/packages/gateway/__tests__/repo/expiration-sweep-integrity_test.ts +++ b/packages/gateway/__tests__/repo/expiration-sweep-integrity_test.ts @@ -60,12 +60,17 @@ const insertApiKey = (raw: Awaited>, id: }; const insertDump = (raw: Awaited>, keyId: string, id: string): void => { + raw.run( + `INSERT INTO spilled_files (file_key, owner_kind, owner_key, state, collect_after) + VALUES (?, 'dump-response', ?, 'staged', 0)`, + [`dumps/v1/${keyId}/${id}.run.gz`, JSON.stringify([keyId, id])], + ); raw.run( `INSERT INTO dump_records (key_id, id, created_at, upstream_id, meta_json, request_headers_json, response_headers_json, request_body_descriptor, response_body_descriptor) - VALUES (?, ?, 1, NULL, '{}', '[]', NULL, NULL, NULL)`, - [keyId, id], + VALUES (?, ?, 1, NULL, '{}', '[]', NULL, NULL, ?)`, + [keyId, id, JSON.stringify({ key: `dumps/v1/${keyId}/${id}.run.gz`, type: 'run' })], ); }; diff --git a/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts b/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts index ebd4885bda..9a4c32f975 100644 --- a/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts +++ b/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts @@ -1,7 +1,6 @@ import { afterEach, expect, test, vi } from 'vitest'; import { initDumpStore } from '../../src/dump/registry.ts'; -import type { DumpWriteRecord } from '../../src/dump/types.ts'; import { FileDumpStore } from '../../src/repo/dump-store.ts'; import { initRepo } from '../../src/repo/index.ts'; import { quantizeOpenAIResponsesRefreshedAt, OPENAI_RESPONSES_REFRESH_GRANULARITY_MS } from '../../src/repo/openai-responses-retention.ts'; @@ -10,6 +9,7 @@ import type { ApiKey, StoredOpenAIResponsesItem } from '../../src/repo/types.ts' import { sweepExpirations } from '../../src/scheduled/expiration-sweeps.ts'; import { InMemoryRepo } from '../repo/memory.ts'; import { createSqliteTestDb, createSqlJsDatabase, migrationSqlByFilename, wrapSqlJsDatabase } from '../repo/test-sqlite.ts'; +import type { StoredDumpRecord } from '@floway-dev/dump/types'; import { initFileStore, MemoryFileStore } from '@floway-dev/platform'; afterEach(() => vi.useRealTimers()); @@ -37,8 +37,7 @@ const responseItem = (id: string, refreshedAt: number, apiKeyId = 'key-a'): Stor refreshedAt, }); -const dumpRecord = (id: string, completedAt: number): DumpWriteRecord => ({ - shape: 'edge', +const dumpRecord = (id: string, completedAt: number): StoredDumpRecord => ({ meta: { id, startedAt: completedAt - 1, @@ -50,18 +49,14 @@ const dumpRecord = (id: string, completedAt: number): DumpWriteRecord => ({ model: 'gpt-test', inputTokens: null, outputTokens: null, - requestBytes: 0, - responseBytes: 0, + requestBytes: 1, + responseBytes: 1, durationMs: 1, error: null, }, - request: { - method: 'POST', - path: '/v1/responses', - headers: [], - body: { encoding: 'identity', bytes: new Uint8Array(), decodedByteLength: 0 }, - }, - response: { status: 200, headers: [], body: { type: 'none' } }, + // One line is enough: what these exercise is the row, its file and the sweep that retires + // both, not what a run put in the stream. + events: new TextEncoder().encode('{"type":"stage.entered","stageId":1,"name":"serve","parentStageId":null}\n'), }); test('one fair driver drains bounded OpenAI Responses and dump backlogs', async () => { @@ -287,7 +282,7 @@ test('migration 0066 bounds existing-row discovery and tracks older dump files o } }); -test('expiration backfill rejects malformed legacy dump descriptors with row context', async () => { +test('expiration backfill rejects malformed run dump descriptors with row context', async () => { const db = await createSqlJsDatabase(); try { for (const [filename, sql] of migrationSqlByFilename) { @@ -304,13 +299,11 @@ test('expiration backfill rejects malformed legacy dump descriptors with row con db.run( `INSERT INTO dump_records (key_id, id, created_at, upstream_id, meta_json, request_headers_json, response_headers_json, request_body_descriptor, response_body_descriptor) - VALUES (?, ?, ?, NULL, '{}', '[]', NULL, ?, NULL)`, - ['key-legacy', recordId, 1_000, JSON.stringify({ key: 'dumps/v1/key-legacy/old.req.gz', type: 'chunks' })], + VALUES (?, ?, ?, NULL, '{}', '[]', NULL, NULL, ?)`, + ['key-legacy', recordId, 1_000, JSON.stringify({ key: 123, type: 'run' })], ); - // Run 0066 and every migration after it so the schema reaches the present - // (incl. columns like `response_upstream_body_descriptor` that the backfill - // SQL selects). The malformed descriptor was seeded before the validate - // trigger existed, so it survives the migration run. + // This malformed run survives edge retirement. Backfill must validate its key before + // it can register a file owner. let started = false; for (const [filename, sql] of migrationSqlByFilename) { if (!started) { @@ -322,7 +315,7 @@ test('expiration backfill rejects malformed legacy dump descriptors with row con const repo = new SqlRepo(wrapSqlJsDatabase(db)); await expect(repo.expirationSweeps.backfillCleanupTracking(500)).rejects.toThrow( - new RegExp(`Invalid dump record key-legacy/${recordId} request body descriptor during expiration backfill.*type`, 'su'), + new RegExp(`Invalid dump record key-legacy/${recordId} response body descriptor during expiration backfill.*key`, 'su'), ); } finally { db.close(); @@ -398,18 +391,20 @@ test('bounded cleanup backfill tracks rows whose API key was hard-deleted', asyn const repo = new SqlRepo(db); await repo.apiKeys.save(key(now)); const recordId = '01K00000000000000000ORPH'; - const fileKey = `dumps/v1/key-a/1970010100/${recordId}.req.gz`; + const fileKey = `dumps/v1/key-a/1970010100/${recordId}.run.gz`; + await db.prepare( + `INSERT INTO spilled_files (file_key, owner_kind, owner_key, state, collect_after) + VALUES (?, 'dump-response', ?, 'staged', 0)`, + ).bind(fileKey, JSON.stringify(['key-a', recordId])).run(); await db.prepare( `INSERT INTO dump_records (key_id, id, created_at, upstream_id, meta_json, request_headers_json, response_headers_json, request_body_descriptor, response_body_descriptor) - VALUES ('key-a', ?, 1, NULL, '{}', '[]', NULL, ?, NULL)`, - ).bind(recordId, JSON.stringify({ key: fileKey, type: 'bytes' })).run(); + VALUES ('key-a', ?, 1, NULL, '{}', '[]', NULL, NULL, ?)`, + ).bind(recordId, JSON.stringify({ key: fileKey, type: 'run' })).run(); await db.prepare("DELETE FROM api_keys WHERE id = 'key-a'").run(); await db.prepare("DELETE FROM expiration_sweeps WHERE key_id = 'key-a'").run(); await db.prepare('DELETE FROM spilled_files WHERE file_key = ?').bind(fileKey).run(); - // Run 0082 and every migration after it so the backfill SQL finds the - // columns later migrations added (e.g. the upstream body descriptor from - // 0089). The orphan row was seeded before the validate trigger existed. + // The run survives later migrations even though its API key was hard-deleted. let started = false; for (const [filename, sql] of migrationSqlByFilename) { if (!started) { @@ -426,7 +421,7 @@ test('bounded cleanup backfill tracks rows whose API key was hard-deleted', asyn expect(await db.prepare( 'SELECT owner_kind, owner_key, state FROM spilled_files WHERE file_key = ?', ).bind(fileKey).first()).toEqual({ - owner_kind: 'dump-request', + owner_kind: 'dump-response', owner_key: JSON.stringify(['key-a', recordId]), state: 'owned', }); diff --git a/packages/gateway/__tests__/scheduled_test.ts b/packages/gateway/__tests__/scheduled_test.ts index 82a2ca48e4..5f206ae70b 100644 --- a/packages/gateway/__tests__/scheduled_test.ts +++ b/packages/gateway/__tests__/scheduled_test.ts @@ -1,14 +1,14 @@ import { afterEach, expect, test, vi } from 'vitest'; +import { createSqliteTestDb } from './repo/test-sqlite.ts'; +import { setupAppTest } from './test-utils/app.ts'; import { initDumpStore } from '../src/dump/registry.ts'; -import type { DumpWriteRecord } from '../src/dump/types.ts'; import { FileDumpStore } from '../src/repo/dump-store.ts'; import { initRepo } from '../src/repo/index.ts'; import { SqlRepo } from '../src/repo/sql.ts'; import type { ApiKey } from '../src/repo/types.ts'; import { runScheduledMaintenance } from '../src/scheduled.ts'; -import { createSqliteTestDb } from './repo/test-sqlite.ts'; -import { setupAppTest } from './test-utils/app.ts'; +import type { StoredDumpRecord } from '@floway-dev/dump/types'; import { initFileStore, initImageCacheStore, MemoryFileStore } from '@floway-dev/platform'; afterEach(() => { @@ -29,8 +29,7 @@ const apiKey = (id: string, now: number, secretDigit: number): ApiKey => ({ openaiResponsesRetentionSeconds: 0, }); -const fileBackedDumpRecord = (id: string, completedAt: number): DumpWriteRecord => ({ - shape: 'edge', +const fileBackedDumpRecord = (id: string, completedAt: number): StoredDumpRecord => ({ meta: { id, startedAt: completedAt - 1, @@ -47,13 +46,9 @@ const fileBackedDumpRecord = (id: string, completedAt: number): DumpWriteRecord durationMs: 1, error: null, }, - request: { - method: 'POST', - path: '/v1/responses', - headers: [], - body: { encoding: 'identity', bytes: new Uint8Array([1]), decodedByteLength: 1 }, - }, - response: { status: 200, headers: [], body: { type: 'bytes', body: new Uint8Array([2]) } }, + // One line is enough: what these exercise is the row, its file and the sweep that retires + // both, not what a run put in the stream. + events: new TextEncoder().encode('{"type":"stage.entered","stageId":1,"name":"serve","parentStageId":null}\n'), }); test('scheduled maintenance isolates the shared expiration driver from later collectors', async () => { @@ -137,14 +132,15 @@ test('one maintenance tick collects every file retired by its four dump units', } const { results: ownedFiles } = await db.prepare('SELECT file_key FROM spilled_files ORDER BY file_key') .all<{ file_key: string }>(); - expect(ownedFiles).toHaveLength(400); + // One file per record — the run's own stream. + expect(ownedFiles).toHaveLength(200); await runScheduledMaintenance('TEST', () => {}); expect(await db.prepare('SELECT COUNT(*) AS count FROM dump_records').first<{ count: number }>()).toEqual({ count: 0 }); expect(await db.prepare('SELECT COUNT(*) AS count FROM spilled_files').first<{ count: number }>()).toEqual({ count: 0 }); expect(await Promise.all(ownedFiles.map(row => files.get(row.file_key)))) - .toEqual(Array.from({ length: 400 }, () => null)); + .toEqual(Array.from({ length: 200 }, () => null)); }); test('scheduled maintenance lease keeps overlapping ticks within one budget', async () => { diff --git a/packages/gateway/__tests__/test-utils/alpha-search-provider.ts b/packages/gateway/__tests__/test-utils/alpha-search-provider.ts new file mode 100644 index 0000000000..bc37eac7b6 --- /dev/null +++ b/packages/gateway/__tests__/test-utils/alpha-search-provider.ts @@ -0,0 +1,17 @@ +import { stubProviderPipeline } from './provider-pipeline.ts'; +import type { AlphaSearchDispatcher } from '../../src/data-plane/tools/web-search/alpha-search/upstream.ts'; +import type { ModelCandidate } from '@floway-dev/provider'; +import { stubModelCandidate } from '@floway-dev/test-utils'; + +export const stubAlphaSearchCandidate = (dispatch: AlphaSearchDispatcher): ModelCandidate => { + const base = stubModelCandidate(); + return { + ...base, provider: { + ...base.provider, pipelines: { + alphaSearch: stubProviderPipeline('alphaSearch', async (_model, payload, signal, options) => ({ + response: await dispatch(payload, signal, options.headers), modelKey: 'search-model', + })), + }, + }, + }; +}; diff --git a/packages/gateway/__tests__/test-utils/chat-provider-pipelines.ts b/packages/gateway/__tests__/test-utils/chat-provider-pipelines.ts new file mode 100644 index 0000000000..14a957b148 --- /dev/null +++ b/packages/gateway/__tests__/test-utils/chat-provider-pipelines.ts @@ -0,0 +1,25 @@ +import type { GatewayCtx } from '../../src/data-plane/shared/gateway-ctx.ts'; +import type { ProviderPipelines, ProviderChatServices } from '@floway-dev/provider'; +import { noopUpstreamCallOptions, stubChatProviderPipeline, type StubChatProviderCall } from '@floway-dev/test-utils'; + +export type ChatFixtureCalls = Partial<{ + callOpenAIChatCompletions: StubChatProviderCall<'openaiChatCompletions'>; + callOpenAIResponses: StubChatProviderCall<'openaiResponses'>; + callAnthropicMessages: StubChatProviderCall<'anthropicMessages'>; + callAnthropicMessagesCountTokens: StubChatProviderCall<'anthropicMessagesCountTokens'>; +}>; + +export const stubChatProviderPipelines = (calls: ChatFixtureCalls): ProviderPipelines => ({ + ...(calls.callOpenAIChatCompletions === undefined ? {} : { openaiChatCompletions: stubChatProviderPipeline('openaiChatCompletions', calls.callOpenAIChatCompletions) }), + ...(calls.callOpenAIResponses === undefined ? {} : { + openaiResponses: stubChatProviderPipeline('openaiResponses', calls.callOpenAIResponses), + openaiResponsesCompact: stubChatProviderPipeline('openaiResponsesCompact', calls.callOpenAIResponses), + }), + ...(calls.callAnthropicMessages === undefined ? {} : { anthropicMessages: stubChatProviderPipeline('anthropicMessages', calls.callAnthropicMessages) }), + ...(calls.callAnthropicMessagesCountTokens === undefined ? {} : { anthropicMessagesCountTokens: stubChatProviderPipeline('anthropicMessagesCountTokens', calls.callAnthropicMessagesCountTokens) }), +}); + +export const chatFixtureHttpServices = (gateway: GatewayCtx): ProviderChatServices => ({ + httpCall: () => ({ ...noopUpstreamCallOptions(), signal: gateway.abortSignal }), + recordProtocolFrames: frames => frames, +}); diff --git a/packages/gateway/__tests__/test-utils/gateway-ctx.ts b/packages/gateway/__tests__/test-utils/gateway-ctx.ts index 140fc28bb3..f819ea284e 100644 --- a/packages/gateway/__tests__/test-utils/gateway-ctx.ts +++ b/packages/gateway/__tests__/test-utils/gateway-ctx.ts @@ -4,8 +4,8 @@ import type { ChatGatewayCtx } from '../../src/data-plane/chat/shared/gateway-ct import type { GatewayCtx } from '../../src/data-plane/shared/gateway-ctx.ts'; import { stubModelCandidate } from '@floway-dev/test-utils'; -// Shared minimal GatewayCtx for tests that exercise serve / respond / -// interceptor code in isolation. Defaults satisfy every required field; pass +// Shared minimal GatewayCtx for tests that exercise serve / respond / stage +// code in isolation. Defaults satisfy every required field; pass // `overrides` to nudge what each test cares about. Callers that need a // downstream abort controller should construct one and spread // `{ abortSignal: controller.signal, downstreamAbortController: controller }` diff --git a/packages/gateway/__tests__/test-utils/provider-pipeline.ts b/packages/gateway/__tests__/test-utils/provider-pipeline.ts index ea11129195..8eb960821b 100644 --- a/packages/gateway/__tests__/test-utils/provider-pipeline.ts +++ b/packages/gateway/__tests__/test-utils/provider-pipeline.ts @@ -1,7 +1,8 @@ import type { GatewayCtx } from '../../src/data-plane/shared/gateway-ctx.ts'; import { takeHttpResponse } from '@floway-dev/http/pipeline'; import { compose, defineStage, isSecret, move, type Pipeline } from '@floway-dev/pipeline'; -import { directFetcher, identityWrapUpstreamCall, type ProviderCallResult, type ProviderModelFacts, type ProviderOperation, type ProviderOperationPayloads, type ProviderResponse, type ProviderRerankResponse, type ProviderRequest, type UpstreamCallOptions } from '@floway-dev/provider'; +import { directFetcher, identityWrapUpstreamCall, type ProviderModelFacts, type ProviderOperation, type ProviderOperationPayloads, type ProviderResponse, type ProviderRerankResponse, type ProviderRequest, type UpstreamCallOptions } from '@floway-dev/provider'; +import { type ProviderCallResult } from '@floway-dev/test-utils'; export const stubProviderPipeline = ( operation: O, diff --git a/packages/gateway/__tests__/vitest.setup.ts b/packages/gateway/__tests__/vitest.setup.ts index 0f06a9ce47..363aafbc2b 100644 --- a/packages/gateway/__tests__/vitest.setup.ts +++ b/packages/gateway/__tests__/vitest.setup.ts @@ -1,12 +1,10 @@ import { trackBackground } from './test-utils/background-tracker.ts'; import { initDumpBroker, initDumpStore } from '../src/dump/registry.ts'; -import type { DumpStore } from '../src/dump/store-contract.ts'; -import type { StoredDumpRecord } from '../src/dump/types.ts'; import { handleExecutionRequest } from '../src/execution/handler.ts'; import { initBackgroundSchedulerResolver } from '../src/runtime/background.ts'; import { initExecutionCellNamespace } from '../src/runtime/execution.ts'; import type { DumpBroker } from '@floway-dev/dump'; -import type { DumpMetadata, DumpRecordId } from '@floway-dev/dump/types'; +import type { StoredDumpRecord, DumpStore, DumpMetadata, DumpRecordId } from '@floway-dev/dump/types'; import { isReplayableBody } from '@floway-dev/http'; import { initEnv, initFetch, initRuntimeKind, initTimingSafeEqual, InProcessExecutionCellNamespace } from '@floway-dev/platform'; @@ -34,8 +32,7 @@ initExecutionCellNamespace(new InProcessExecutionCellNamespace(handleExecutionRe // independent of that subsystem. Dump-specific tests install real or recording // implementations. const noopStore: DumpStore = { - async prepareRequestBody(body) { return { encoding: 'identity', bytes: body, decodedByteLength: body.byteLength }; }, - async put(): Promise { /* noop */ }, + async put(): Promise {}, async list(): Promise { return []; }, async get(_keyId: string, _id: DumpRecordId): Promise { return null; }, async deleteExpiredBatch(): Promise { return 0; }, diff --git a/packages/gateway/migrations/0086_drop_edge_dump_records.sql b/packages/gateway/migrations/0086_drop_edge_dump_records.sql new file mode 100644 index 0000000000..492687cd2f --- /dev/null +++ b/packages/gateway/migrations/0086_drop_edge_dump_records.sql @@ -0,0 +1,20 @@ +-- Delete the dump records written as a turn's two edges. +-- +-- Every route opens its run through the prologue now, so nothing produces that shape any more: +-- a record is the whole run, every stage and both directions, as one NDJSON event stream. What +-- is left of the old shape is stored rows, and they are deleted rather than kept readable. An +-- edge record holds what the client sent and what it got back and nothing about what happened +-- between, so there is no run to rebuild it into — and a dump record is diagnostic and already +-- expires under its key's own `dump_retention_seconds`, so what this brings forward is a +-- deletion that was coming anyway. +-- +-- Which rows: the reader told the two shapes apart by the response body descriptor's own type, +-- because a run is one more body file under the same contract and says so. Anything that is not +-- a run is an edge record, including a row that stored no response body at all. +-- +-- The files go with them. `dump_records_retire_spilled_files` fires on delete and marks both +-- descriptors' keys retired with `collect_after = 0`, so the collector picks them up on its next +-- pass rather than leaving orphans in the file store. +DELETE FROM dump_records +WHERE response_body_descriptor IS NULL + OR json_extract(response_body_descriptor, '$.type') IS NOT 'run'; diff --git a/packages/gateway/package.json b/packages/gateway/package.json index 43daf15137..367e8533cd 100644 --- a/packages/gateway/package.json +++ b/packages/gateway/package.json @@ -20,9 +20,6 @@ "./control-plane/upstreams/types": { "types": "./src/control-plane/upstreams/types.ts" }, - "./dump-types": { - "types": "./src/dump/types.ts" - }, "./migrations-dir": { "import": "./src/migrations-dir.ts", "types": "./src/migrations-dir.ts" @@ -35,7 +32,6 @@ "@floway-dev/agent-setup": "workspace:*", "@floway-dev/dump": "workspace:*", "@floway-dev/http": "workspace:*", - "@floway-dev/interceptor": "workspace:*", "@floway-dev/pipeline": "workspace:*", "@floway-dev/platform": "workspace:*", "@floway-dev/protocols": "workspace:*", diff --git a/packages/gateway/src/control-plane/dump.ts b/packages/gateway/src/control-plane/dump.ts index 50463abc94..7f7d84ebd9 100644 --- a/packages/gateway/src/control-plane/dump.ts +++ b/packages/gateway/src/control-plane/dump.ts @@ -5,8 +5,8 @@ import { z } from 'zod'; import { ownedKeyForUser } from './shared/owned-key.ts'; import { getDumpBroker, getDumpStore } from '../dump/registry.ts'; -import { dumpRecordToWire } from '../dump/wire.ts'; import { zValidator } from '../middleware/zod-validator.ts'; +import { dumpRecordToWire } from '@floway-dev/dump'; const LIST_LIMIT_DEFAULT = 100; const LIST_LIMIT_MAX = 200; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/affinity/ingress.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/affinity/ingress.ts index c5c63df677..4322a3e39b 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/affinity/ingress.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/affinity/ingress.ts @@ -1,5 +1,6 @@ import { type AffinityCodec, type AffinityRequestAnalysis, type DecodedAffinityBlob, defineAffinityRequest, projectOptionalAffinityBlob } from '../../shared/affinity/index.ts'; import type { AnthropicMessagesAssistantContentBlock, AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import { withIndexesChanged, withKeysChanged } from '@floway-dev/protocols/common'; interface AnthropicMessagesBlobLocation { readonly messageIndex: number; @@ -31,11 +32,11 @@ export const analyzeAnthropicMessagesAffinity = async ( degrades: projections.some(item => item.projection.kind === 'remove' && item.projection.degrades), preferred: projections.every(item => item.projection.preferred), materialize: () => { - const candidatePayload = structuredClone(payload); const byMessage = Map.groupBy(projections, item => item.location.messageIndex); const emptiedByAffinity = new Set(); + const rewritten = new Map(); for (const [messageIndex, messageProjections] of byMessage) { - const message = candidatePayload.messages[messageIndex] as { role: 'assistant'; content: AnthropicMessagesAssistantContentBlock[] }; + const message = payload.messages[messageIndex] as { role: 'assistant'; content: AnthropicMessagesAssistantContentBlock[] }; const replacements = new Map(); for (const { location, projection } of messageProjections) { const block = message.content[location.blockIndex]; @@ -59,14 +60,14 @@ export const analyzeAnthropicMessagesAffinity = async ( ); } } - message.content = message.content.flatMap((block, blockIndex) => { - const replacement = replacements.get(blockIndex); - return replacement === undefined ? [block] : replacement === null ? [] : [replacement]; - }); - if (message.content.length === 0) emptiedByAffinity.add(messageIndex); + const content = withIndexesChanged(message.content, replacements); + if (content.length === 0) emptiedByAffinity.add(messageIndex); + rewritten.set(messageIndex, withKeysChanged(message, { content }) as AnthropicMessagesPayload['messages'][number]); } - candidatePayload.messages = candidatePayload.messages.filter((_message, messageIndex) => !emptiedByAffinity.has(messageIndex)); - return candidatePayload; + const messages = payload.messages + .map((message, messageIndex) => rewritten.get(messageIndex) ?? message) + .filter((_message, messageIndex) => !emptiedByAffinity.has(messageIndex)); + return withKeysChanged(payload, { messages }); }, }; }); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/answer-claude-code-probe.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/answer-claude-code-probe.ts new file mode 100644 index 0000000000..7cbc1b9615 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/answer-claude-code-probe.ts @@ -0,0 +1,76 @@ +import type { Fields } from './facts.ts'; +import type { ChatServices } from '../services.ts'; +import { isClaudeCodeProbe, probeFrames } from './claude-code-probe.ts'; +import { telemetryModelIdentity } from '../../shared/telemetry/attribution.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { providerModelOf } from '@floway-dev/provider'; + +/** + * Answers Claude Code's one-token probe itself, rather than spending an upstream turn on it. + * + * The CLI asks "is this model usable?" by generating one token against it and reporting the + * model unusable if the call throws. A one-token cap is not portable — OpenAI's Responses API + * floors `max_output_tokens` at 16 and rejects anything lower with a hard 400 — so every + * Anthropic-Messages-via-OpenAI-Responses candidate fails a probe that is asking nothing this gateway cannot + * answer. Resolution has already picked a real candidate by the time this runs, so an id no + * upstream serves still fails above with a 404; what is suppressed is only the generation. + * + * It answers rather than descending, which is why it carries the `return` trait — and why it + * has to live here rather than beside the shared rules: what it answers with is this + * family's own response keys. + */ +export const answerClaudeCodeProbe = defineStage< + Fields<'request.chat.anthropicMessages' | 'route.attempt' | 'ingress.http.headers'>, + Fields<'request.chat.anthropicMessages' | 'route.attempt' | 'ingress.http.headers'>, + Fields<'response.chat.anthropicMessages' | 'response.chat.anthropicMessages.streamedUsage' + | 'response.usage.billable' | 'response.http.headers' | 'response.http.body' | 'response.http.status'>, + Fields<'response.chat.anthropicMessages' | 'response.chat.anthropicMessages.streamedUsage' + | 'response.usage.billable' | 'response.http.headers' | 'response.http.body' | 'response.http.status'>, + Fields<'response.chat.anthropicMessages' | 'response.chat.anthropicMessages.streamedUsage' + | 'response.usage.billable' | 'response.http.headers' | 'response.http.body' | 'response.http.status'>, + ChatServices +>({ + name: 'answerClaudeCodeProbe', + through: { + request: { + needs: ['request.chat.anthropicMessages', 'route.attempt', 'ingress.http.headers'], + consumes: [], + provides: [], + }, + response: { + needs: ['response.chat.anthropicMessages', 'response.http.headers'], + consumes: [], + provides: [], + }, + }, + return: { + provides: [ + 'response.chat.anthropicMessages', + 'response.chat.anthropicMessages.streamedUsage', + 'response.usage.billable', + 'response.http.headers', 'response.http.body', 'response.http.status', + ], + }, + execute: async (facts, next, use) => { + const headers = new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value])); + if (!isClaudeCodeProbe(facts['request.chat.anthropicMessages'], headers)) return await next(facts); + + const candidate = use.resolveAttempt(facts['route.attempt']); + // The probe is answered *for* this candidate, so it is the one a follow-up turn carrying + // our own state must come back to — the same statement a dialled attempt makes. + use.selectAffinity(candidate); + use.gateway.dump?.success(telemetryModelIdentity(candidate, providerModelOf(candidate).id), null); + await use.log.debug('answering a Claude Code probe without dialling', { upstream: facts['route.attempt'].upstreamId }); + return move({ + ...facts, + 'response.chat.anthropicMessages': { + kind: 'stream' as const, + frames: probeFrames(facts['request.chat.anthropicMessages'].model), + }, + 'response.chat.anthropicMessages.streamedUsage': null, + 'response.usage.billable': [], + 'response.http.headers': [], + 'response.http.body': null, 'response.http.status': 200, + }); + }, +}); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/apply-role-compatibility.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/apply-role-compatibility.ts new file mode 100644 index 0000000000..91adb791e9 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/apply-role-compatibility.ts @@ -0,0 +1,46 @@ +import type { Chat } from '../facts.ts'; +import { mapKeepingIdentity } from '../shared/structural.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; + +// ── Anthropic Messages ──────────────────────────────────────────────────────────────────── + +/** + * The mid-conversation system rewrite, which on this protocol is every inline system message. + * + * Anthropic's top-level `payload.system` is the only first-position system slot, so a system + * message inside `messages` is by construction past the leading run — which is why this reads + * the same flag as the other protocols' fold but is not that fold: there is no leading run to + * cross here, and no developer role to trade with. + */ +export const applyRoleCompatibilityToAnthropicMessages = defineStage< + Chat<'request.chat.anthropicMessages' | 'route.attempt'>, + Chat<'request.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'> +>({ + name: 'applyRoleCompatibility', + through: { + request: { + needs: ['request.chat.anthropicMessages', 'route.attempt'], + consumes: [], + provides: ['request.chat.anthropicMessages'], + }, + response: { needs: ['response.chat.anthropicMessages'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.anthropicMessages' | 'route.attempt'>, + Chat<'request.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'> + >(() => ({ + request: facts => { + if (!facts['route.attempt'].flags.includes('rewrite-mid-conv-system-to-user')) return facts; + const payload = facts['request.chat.anthropicMessages']; + const messages = mapKeepingIdentity(payload.messages, message => + message.role === 'system' ? { role: 'user' as const, content: message.content } : message); + return messages === payload.messages + ? facts + : { ...facts, 'request.chat.anthropicMessages': move({ ...payload, messages }) }; + }, + })), +}); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/attempt.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/attempt.ts deleted file mode 100644 index 00537fdd69..0000000000 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/attempt.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { anthropicMessagesInterceptors, anthropicMessagesCountTokensInterceptors } from './interceptors/index.ts'; -import type { AnthropicMessagesInvocation } from './interceptors/types.ts'; -import { createAnthropicMessagesBillableUsageReader } from './usage.ts'; -import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; -import { openaiChatCompletionsAttempt } from '../openai-chat-completions/attempt.ts'; -import { openaiResponsesAttempt } from '../openai-responses/attempt.ts'; -import { applyRulesToUpstreamAnthropicMessages } from '../shared/alias-rules.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import { providerStreamResultToExecuteResult } from '../shared/provider-stream-result.ts'; -import { plainResultFromResponse } from '../shared/respond.ts'; -import { chatTargetPicker } from '../shared/target-picker.ts'; -import { captureFromDump, traverseTranslation } from '../shared/translate-traverse.ts'; -import { runInterceptors } from '@floway-dev/interceptor'; -import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ModelCandidate, ExecuteResult, AnthropicMessagesUpstreamCallOptions, PlainResult } from '@floway-dev/provider'; -import { providerModelOf } from '@floway-dev/provider'; -import { translateAnthropicMessagesViaOpenAIChatCompletions, translateAnthropicMessagesViaOpenAIResponses } from '@floway-dev/translate'; - -// `/v1/messages` generate prefers a native Anthropic Messages target, then the -// translated OpenAI Responses path, then the translated OpenAI Chat Completions path. -export const anthropicMessagesGenerateTarget = chatTargetPicker(['anthropicMessages', 'openaiResponses', 'openaiChatCompletions']); - -// `count_tokens` has no translation path — only a native Anthropic Messages target -// satisfies the operation. -export const anthropicMessagesCountTokensTarget = chatTargetPicker(['anthropicMessages']); - -export interface AnthropicMessagesAttemptArgs { - readonly payload: AnthropicMessagesPayload; - readonly ctx: ChatGatewayCtx; - readonly candidate: ModelCandidate; - readonly headers: Headers; - readonly anthropicBeta: readonly string[]; -} - -const buildAnthropicMessagesUpstreamCallOptions = ( - candidate: ModelCandidate, - ctx: ChatGatewayCtx, - headers: Headers, - anthropicBeta: readonly string[], -): AnthropicMessagesUpstreamCallOptions => ({ - ...buildUpstreamCallOptions(candidate, ctx, headers), - anthropicBeta, -}); - -export const anthropicMessagesAttempt = { - generate: async (args: AnthropicMessagesAttemptArgs): Promise>> => { - const { payload: sourcePayload, ctx, candidate, headers: sourceHeaders, anthropicBeta } = args; - const payload = { ...sourcePayload, model: candidate.model.id }; - const headers = new Headers(sourceHeaders); - headers.delete('anthropic-beta'); - const targetApi = anthropicMessagesGenerateTarget.pick(candidate.model.endpoints); - const invocation: AnthropicMessagesInvocation = { - payload, - candidate, - targetApi, - headers, - }; - return await runInterceptors(invocation, ctx, anthropicMessagesInterceptors, async () => { - if (targetApi === 'anthropicMessages') { - if (candidate.rules !== undefined) applyRulesToUpstreamAnthropicMessages(invocation.payload, candidate.rules); - const { model: _model, ...body } = invocation.payload; - const providerResult = await candidate.provider.instance.callAnthropicMessages( - providerModelOf(candidate), - body, - ctx.abortSignal, - buildAnthropicMessagesUpstreamCallOptions(candidate, ctx, invocation.headers, anthropicBeta), - ); - return await providerStreamResultToExecuteResult(providerResult, candidate, targetApi, ctx, createAnthropicMessagesBillableUsageReader()); - } - if (targetApi === 'openaiResponses') { - return await traverseTranslation( - invocation.payload, - p => translateAnthropicMessagesViaOpenAIResponses(p, { model: candidate.model.id }), - translated => openaiResponsesAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, - }), - captureFromDump(ctx.dump, targetApi), - ); - } - if (targetApi === 'openaiChatCompletions') { - return await traverseTranslation( - invocation.payload, - p => translateAnthropicMessagesViaOpenAIChatCompletions(p, { model: candidate.model.id }), - translated => openaiChatCompletionsAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, - }), - captureFromDump(ctx.dump, targetApi), - ); - } - throw new Error(`anthropicMessagesAttempt.generate: unexpected targetApi '${targetApi as string}'`); - }); - }, - - countTokens: async (args: AnthropicMessagesAttemptArgs): Promise => { - const { payload: sourcePayload, ctx, candidate, headers: sourceHeaders, anthropicBeta } = args; - const payload = { ...sourcePayload, model: candidate.model.id }; - const headers = new Headers(sourceHeaders); - headers.delete('anthropic-beta'); - // `pick` here is contractually total — serve filtered with - // `anthropicMessagesCountTokensTarget.canServe`, so a non-anthropic-messages candidate is - // a contract breach. - const targetApi = anthropicMessagesCountTokensTarget.pick(candidate.model.endpoints); - const invocation: AnthropicMessagesInvocation = { - payload, - candidate, - targetApi, - headers, - }; - const response = await runInterceptors(invocation, ctx, anthropicMessagesCountTokensInterceptors, async () => { - if (candidate.rules !== undefined) applyRulesToUpstreamAnthropicMessages(invocation.payload, candidate.rules); - const { model: _model, ...body } = invocation.payload; - const { response } = await candidate.provider.instance.callAnthropicMessagesCountTokens( - providerModelOf(candidate), - body, - ctx.abortSignal, - buildAnthropicMessagesUpstreamCallOptions(candidate, ctx, invocation.headers, anthropicBeta), - ); - return response; - }); - return await plainResultFromResponse(response, candidate.provider.upstreamId); - }, -}; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/call-upstream.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/call-upstream.ts new file mode 100644 index 0000000000..7082022535 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/call-upstream.ts @@ -0,0 +1,44 @@ +import type { Fields } from './facts.ts'; +import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; +import { providerEntry } from '../../pipeline/provider-entry.ts'; +import { upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; +import type { ChatServices } from '../services.ts'; +import { applyRulesToUpstreamAnthropicMessages } from '../shared/alias-rules.ts'; +import { providerChatAnswer } from '../shared/provider-answer.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { parseAnthropicBetaHeader } from '@floway-dev/protocols/anthropic-messages'; +import type { ProviderOperationRequest, ProviderChatResponse } from '@floway-dev/provider'; + +export const callAnthropicMessagesUpstream = defineStage< + Fields<'request.chat.anthropicMessages' | 'route.attempt' | 'ingress.http.headers' | 'ingress.chat.sourceProtocol'>, + ProviderOperationRequest<'anthropicMessages'>, + ProviderChatResponse<'anthropicMessages'>, + Fields<'response.chat.anthropicMessages' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status' | 'response.http.body'>, + ChatServices +>({ + name: 'callAnthropicMessagesUpstream', + into: { + request: { needs: ['request.chat.anthropicMessages', 'route.attempt', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], consumes: [], provides: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers', 'request.provider.anthropicBeta'] }, + response: { needs: ['response.http.exchange', 'response.provider.output', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.http.body'], consumes: ['response.http.exchange', 'response.provider.output', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'], provides: ['response.chat.anthropicMessages', 'response.usage.billable', 'response.http.headers', 'response.http.status', 'response.http.body'] }, + }, + execute: async (facts, next, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); + const body = bodyForAttempt(facts['request.chat.anthropicMessages'], candidate, applyRulesToUpstreamAnthropicMessages); + const headers = new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value])); + const anthropicBeta = facts['ingress.chat.sourceProtocol'] === 'anthropicMessages' ? parseAnthropicBetaHeader(headers.get('anthropic-beta')) : []; + headers.delete('anthropic-beta'); + const pipeline = candidate.provider.pipelines.anthropicMessages; + if (pipeline === undefined) throw new Error(`Provider ${candidate.provider.kind} has no anthropicMessages pipeline`); + const back = await next(move({ ...providerEntry(facts, candidate, body, [...headers]), 'request.provider.anthropicBeta': anthropicBeta }), pipeline); + const { 'response.http.exchange': _exchange, 'response.provider.output': _output, 'response.provider.modelKey': _modelKey, 'response.provider.called': _called, 'response.provider.previousCalls': _previousCalls, ...rest } = back; + const reply = await providerChatAnswer(candidate, back); + const answer = reply.answer; + if ('kind' in answer) use.selectAffinity(candidate); + return move({ + ...rest, 'response.chat.anthropicMessages': answer, + 'response.usage.billable': reply.billable, 'response.http.headers': reply.headers, + 'response.http.status': reply.status, 'response.http.body': reply.body, + }); + }, +}); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/claude-code-probe.ts similarity index 77% rename from packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe.ts rename to packages/gateway/src/data-plane/chat/anthropic-messages/claude-code-probe.ts index 394e9040ed..8c3d236ad2 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/answer-claude-code-probe.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/claude-code-probe.ts @@ -1,8 +1,5 @@ -import type { AnthropicMessagesInterceptor } from './types.ts'; -import { telemetryModelIdentity } from '../../../shared/telemetry/attribution.ts'; import { generateAnthropicId, type AnthropicMessagesPayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import { eventResult, providerModelOf, type ExecuteResult, type ModelCandidate } from '@floway-dev/provider'; // Claude Code answers "is this model usable?" by generating one token against // it. `/model ` runs the CLI's `model_validation` side query — a @@ -22,7 +19,7 @@ import { eventResult, providerModelOf, type ExecuteResult, type ModelCandidate } // through `eie` — `quota` for the rate-limit preflight, `test` for credential // verification, `.` for the Bedrock / Vertex / Mantle reachability checks. // -// A one-token cap is not portable. OpenAI's OpenAI Responses API floors +// A one-token cap is not portable. OpenAI's Responses API floors // `max_output_tokens` at 16 and rejects anything lower with a hard 400, so // every Anthropic Messages→OpenAI Responses candidate fails the probe and Claude Code // concludes the model does not exist — the CLI surfaces the upstream envelope @@ -31,16 +28,21 @@ import { eventResult, providerModelOf, type ExecuteResult, type ModelCandidate } // "code":"invalid_request_body"}}`. // // The gateway answers the probe itself instead. That is honest for what the -// probe actually asks: by the time this interceptor runs, model resolution has +// probe actually asks: by the time the stage that answers it runs, model resolution has // already picked a real candidate for the requested id, so an id no upstream // serves still fails at the serve layer with a 404 and the CLI still reports // it not found. What we suppress is only the pointless one-token generation // behind it — no upstream call, and no tokens to bill. +// The CLI builds its agent string as +// `claude-cli/${VERSION} (external, ${CLAUDE_CODE_ENTRYPOINT ?? "cli"}…)`, read off the +// vendor's own shipped build — @anthropic-ai/claude-code-darwin-arm64@2.1.226, GIT_SHA +// e140b3281c1e8d834468889bd0a5c3fd2f15507c. +// https://registry.npmjs.org/@anthropic-ai/claude-code-darwin-arm64/-/claude-code-darwin-arm64-2.1.226.tgz const CLAUDE_CODE_USER_AGENT = /^claude-cli\/\d+\.\d+\.\d+/i; -// The probes we can answer truthfully, written exactly as 2.1.226 sends them: -// `Hi` is the `/model` validation probe, `test` the credential check, which -// reads nothing at all off the response. +// The probes we can answer truthfully, read verbatim off that same build: +// `Hi` is the `model_validation` probe and `test` the `verify_api_key` +// credential check, which reads nothing at all off the response. // // Two recorded prompts are deliberately absent. `quota` is the rate-limit // preflight, and its caller consumes the `anthropic-ratelimit-unified-*` @@ -74,7 +76,7 @@ const soleUserPromptOf = (payload: AnthropicMessagesPayload): string | null => { return block.type === 'text' ? block.text : null; }; -const isClaudeCodeProbe = (payload: AnthropicMessagesPayload, headers: Headers): boolean => { +export const isClaudeCodeProbe = (payload: AnthropicMessagesPayload, headers: Headers): boolean => { if (payload.max_tokens !== 1) return false; // Every real Claude Code turn ships the session's tools; the one-token // probes never do. @@ -90,7 +92,7 @@ const isClaudeCodeProbe = (payload: AnthropicMessagesPayload, headers: Headers): // usage block is load-bearing — the validation path dereferences // `usage.input_tokens` / `usage.output_tokens` unconditionally, and a missing // `usage` throws inside the CLI and reads as a failed probe. -const probeFrames = async function* (model: string): AsyncGenerator> { +export const probeFrames = async function* (model: string): AsyncGenerator> { yield eventFrame({ type: 'message_start', message: { @@ -108,15 +110,3 @@ const probeFrames = async function* (model: string): AsyncGenerator> => - eventResult(probeFrames(model), telemetryModelIdentity(candidate, providerModelOf(candidate).id)); - -export const answerClaudeCodeProbe: AnthropicMessagesInterceptor = async (ctx, _gatewayCtx, run) => - isClaudeCodeProbe(ctx.payload, ctx.headers) - ? probeResult(ctx.candidate, ctx.payload.model) - : await run(); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts new file mode 100644 index 0000000000..53f8feeeb3 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens.ts @@ -0,0 +1,27 @@ +import { materializeAttempt } from '../materialize-attempt.ts'; +import { resolveChatCandidates } from '../resolve-candidates.ts'; +import { emitAnthropicMessagesTokenCount } from './count-tokens/emit.ts'; +import type { AnthropicMessagesCountTokensEntry, AnthropicMessagesCountTokensExit } from './count-tokens/facts.ts'; +import { narrowing } from './count-tokens/target.ts'; +import { anthropicMessagesCountTokensWire } from './count-tokens/wire.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import { failover } from '../../pipeline/failover.ts'; +import { serializeClientJson } from '../../pipeline/serialize-client-json.ts'; +import { writeSettlement } from '../../pipeline/settlement.ts'; +import { compose, type Pipeline } from '@floway-dev/pipeline'; + +export const anthropicMessagesCountTokensPipeline = (): Pipeline => + compose('anthropicMessagesCountTokens', [ + // Native counts measure a request without generating AI usage; a failed reply must + // not recover a billable entity from the transport-call proof. + writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, undefined, false), + serializeClientJson('response.chat.anthropicMessages.rendered'), + emitAnthropicMessagesTokenCount, + resolveChatCandidates(narrowing), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.chat.anthropicMessages'?: unknown })['response.chat.anthropicMessages']), + owns: ['response.http.body'], + }), + materializeAttempt('request.chat.anthropicMessages'), + ...anthropicMessagesCountTokensWire, + ]); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/call-upstream.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/call-upstream.ts new file mode 100644 index 0000000000..5eea866b48 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/call-upstream.ts @@ -0,0 +1,41 @@ +import { bodyForAttempt } from '../../../pipeline/attempt-body.ts'; +import { providerEntry } from '../../../pipeline/provider-entry.ts'; +import type { ChatServices } from '../../services.ts'; +import { applyRulesToUpstreamAnthropicMessages } from '../../shared/alias-rules.ts'; +import { providerChatAnswer } from '../../shared/provider-answer.ts'; +import type { Fields } from '../facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { parseAnthropicBetaHeader } from '@floway-dev/protocols/anthropic-messages'; +import type { ProviderOperationRequest, ProviderChatResponse } from '@floway-dev/provider'; + +export const callAnthropicMessagesCountTokensUpstream = defineStage< + Fields<'request.chat.anthropicMessages' | 'route.attempt' | 'ingress.http.headers' | 'ingress.chat.sourceProtocol'>, + ProviderOperationRequest<'anthropicMessagesCountTokens'>, + ProviderChatResponse<'anthropicMessagesCountTokens'>, + Fields<'response.chat.anthropicMessages' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status' | 'response.http.body'>, + ChatServices +>({ + name: 'callAnthropicMessagesCountTokensUpstream', + into: { + request: { needs: ['request.chat.anthropicMessages', 'route.attempt', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], consumes: [], provides: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers', 'request.provider.anthropicBeta'] }, + response: { needs: ['response.http.exchange', 'response.provider.output', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.http.body'], consumes: ['response.http.exchange', 'response.provider.output', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'], provides: ['response.chat.anthropicMessages', 'response.usage.billable', 'response.http.headers', 'response.http.status', 'response.http.body'] }, + }, + execute: async (facts, next, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + const body = bodyForAttempt(facts['request.chat.anthropicMessages'], candidate, applyRulesToUpstreamAnthropicMessages); + const headers = new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value])); + const anthropicBeta = facts['ingress.chat.sourceProtocol'] === 'anthropicMessages' ? parseAnthropicBetaHeader(headers.get('anthropic-beta')) : []; + headers.delete('anthropic-beta'); + const pipeline = candidate.provider.pipelines.anthropicMessagesCountTokens; + if (pipeline === undefined) throw new Error(`Provider ${candidate.provider.kind} has no anthropicMessagesCountTokens pipeline`); + const back = await next(move({ ...providerEntry(facts, candidate, body, [...headers]), 'request.provider.anthropicBeta': anthropicBeta }), pipeline); + const { 'response.http.exchange': _exchange, 'response.provider.output': _output, 'response.provider.modelKey': _modelKey, 'response.provider.called': _called, 'response.provider.previousCalls': _previousCalls, ...rest } = back; + const reply = await providerChatAnswer(candidate, back); + const answer = reply.answer; + return move({ + ...rest, 'response.chat.anthropicMessages': answer, + 'response.usage.billable': [], 'response.http.headers': reply.headers, + 'response.http.status': reply.status, 'response.http.body': reply.body, + }); + }, +}); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/emit.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/emit.ts new file mode 100644 index 0000000000..2c0d17ceb3 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/emit.ts @@ -0,0 +1,54 @@ +import type { Counted, Fields } from './facts.ts'; +import { isFailure, renderFailure, mintedAs } from '../../../pipeline/facts.ts'; +import { isForwardableUpstreamHeader } from '../../../shared/upstream-response.ts'; +import type { ChatServices } from '../../services.ts'; +import { renderAnthropicMessagesError } from '../errors.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; + +/** + * The outermost edge. A measurement is one object, so there is nothing to fold and nothing + * to stream: what this decides is only whether the client is shown the upstream's counts or + * the refusal that stands in for them. + */ +export const emitAnthropicMessagesTokenCount = defineStage< + Record, + Record, + Counted<'response.chat.anthropicMessages'> & Fields<'response.http.headers' | 'response.http.status'>, + Fields<'response.chat.anthropicMessages.rendered' | 'response.http.status' | 'response.http.headers'>, + ChatServices +>({ + name: 'emitAnthropicMessagesTokenCount', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { + needs: ['response.chat.anthropicMessages', 'response.http.headers', 'response.http.status'], + consumes: ['response.chat.anthropicMessages', 'response.http.headers'], + provides: ['response.chat.anthropicMessages.rendered', 'response.http.status', 'response.http.headers'], + }, + }, + execute: async (facts, next) => { + const back = await next(facts); + const { 'response.chat.anthropicMessages': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); + + if (isFailure(answer)) { + const failure = renderFailure(answer, mintedAs(({ status, message }) => renderAnthropicMessagesError(status, message))); + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.anthropicMessages.rendered': move(failure.body), + 'response.http.status': failure.status, + }; + } + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.anthropicMessages.rendered': move(answer.body as Record), + 'response.http.status': back['response.http.status'], + }; + }, +}); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/facts.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/facts.ts new file mode 100644 index 0000000000..e02dd697c3 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/facts.ts @@ -0,0 +1,19 @@ +import type { Failure } from '../../../pipeline/facts.ts'; +import type { AnthropicMessagesFacts } from '../facts.ts'; + +/** What a measurement hands up. It rides at a protocol's own response key — which is what + * lets the request rules generation runs be the same stages here — and the arm is narrowed + * because nothing in a counting chain opens a stream. */ +export type TokenCountAnswer = { readonly kind: 'value'; readonly body: unknown } | Failure; + +export type Counted = { [P in K]: TokenCountAnswer }; + +export type Fields = { [P in K]: AnthropicMessagesFacts[P] }; + +export type AnthropicMessagesCountTokensEntry = Fields< + 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'request.chat.anthropicMessages' | 'serve.model' +>; + +export type AnthropicMessagesCountTokensExit = Fields< + 'response.chat.anthropicMessages.rendered' | 'response.http.status' | 'response.http.jsonBody' | 'response.http.headers' +>; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/prepare-web-search.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/prepare-web-search.ts new file mode 100644 index 0000000000..838b5e074b --- /dev/null +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/prepare-web-search.ts @@ -0,0 +1,59 @@ +import type { Fields, Counted } from './facts.ts'; +import type { ChatServices } from '../../services.ts'; +import { prepareAnthropicMessagesWebSearchShimRequest } from '../web-search-shim.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; + +/** + * Puts the request into the shape the web-search shim would actually send. + * + * Anthropic's native `web_search_*` hosted tool becomes an ordinary client tool on the wire, + * and prior turns' results are rewritten back into the history the upstream issued them + * with — so a count taken on the client's own body would measure a request no upstream is + * ever sent. Only the rewrite runs here: there is no stream to intercept and no search to + * execute, because nothing is generated. + * + * `anthropic-messages-web-search-shim` is the whole of the gate, where generation also engages the + * shim unconditionally on a translated wire: counting has only the native wire, so the + * structural half of that condition can never hold. + * + * A tool definition this protocol rejects is an answer it already holds, which is why it + * carries the `return` trait. + */ +export const prepareAnthropicMessagesWebSearchRequest = defineStage< + Fields<'request.chat.anthropicMessages' | 'route.attempt'>, + Fields<'request.chat.anthropicMessages'>, + Counted<'response.chat.anthropicMessages'>, + Counted<'response.chat.anthropicMessages'>, + Counted<'response.chat.anthropicMessages'> & Fields<'response.usage.billable' | 'response.http.headers' | 'response.http.body' | 'response.http.status'>, + ChatServices +>({ + name: 'prepareAnthropicMessagesWebSearchRequest', + through: { + request: { + needs: ['request.chat.anthropicMessages', 'route.attempt'], + consumes: [], + provides: ['request.chat.anthropicMessages'], + }, + response: { needs: ['response.chat.anthropicMessages'], consumes: [], provides: [] }, + }, + return: { provides: ['response.chat.anthropicMessages', 'response.usage.billable', 'response.http.headers', 'response.http.body', 'response.http.status'] }, + execute: async (facts, next) => { + if (!facts['route.attempt'].flags.includes('anthropic-messages-web-search-shim')) return await next(facts); + const prepared = prepareAnthropicMessagesWebSearchShimRequest(facts['request.chat.anthropicMessages']); + if (prepared.type === 'invalid-request') { + // Nothing was dialled, which is what an empty billed set and an empty header list say + // on the other two keys. + return move({ + ...facts, + 'response.chat.anthropicMessages': { status: 400, message: prepared.message }, + 'response.usage.billable': [], + 'response.http.headers': [], + 'response.http.body': null, 'response.http.status': 400, + }); + } + // A request that engaged nothing comes back by identity, so the record shows no change + // where none happened. + if (prepared.payload === facts['request.chat.anthropicMessages']) return await next(facts); + return await next({ ...facts, 'request.chat.anthropicMessages': move(prepared.payload) }); + }, +}); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/target.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/target.ts new file mode 100644 index 0000000000..0dd6517ec8 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/target.ts @@ -0,0 +1,18 @@ +import type { Counted } from './facts.ts'; +import type { ChatNarrowing } from '../../resolve-candidates.ts'; +import { chatTargetPicker } from '../../shared/target-picker.ts'; +import { analyzeAnthropicMessagesAffinity } from '../affinity/ingress.ts'; + +/** Counting has no translation path: only an upstream's own Anthropic Messages endpoint can answer the + * question, so a candidate that would serve generation over a translated wire cannot serve + * this. */ +export const anthropicMessagesCountTokensTarget = chatTargetPicker(['anthropicMessages']); + +export const narrowing: ChatNarrowing, 'request.chat.anthropicMessages'> = ({ + requestKey: 'request.chat.anthropicMessages', + canServe: candidate => anthropicMessagesCountTokensTarget.canServe(candidate.model.endpoints), + affinity: async (payload, gateway) => await analyzeAnthropicMessagesAffinity(payload, gateway.affinity.codec), + unsupported: model => `Model ${model} does not support the /messages/count_tokens endpoint.`, + refuse: (status, message) => ({ 'response.chat.anthropicMessages': { status, message } }), + refuses: ['response.chat.anthropicMessages'], +}); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/wire.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/wire.ts new file mode 100644 index 0000000000..2c44023b9e --- /dev/null +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/count-tokens/wire.ts @@ -0,0 +1,25 @@ +import { prepareAnthropicMessagesWebSearchRequest } from './prepare-web-search.ts'; +import { applyRoleCompatibilityToAnthropicMessages } from '../apply-role-compatibility.ts'; +import { disableReasoningOnForcedToolChoiceForAnthropicMessages } from '../disable-reasoning-on-forced-tool-choice.ts'; +import { stripBillingAttributionFromAnthropicMessages } from '../strip-billing-attribution.ts'; +import { callAnthropicMessagesCountTokensUpstream } from './call-upstream.ts'; +import type { Stage } from '@floway-dev/pipeline'; + +/** + * The Anthropic Messages count-tokens wire, as the chain that dials it. + * + * Every source protocol whose measurement reaches an upstream does it over this endpoint — + * Gemini's `:countTokens` has no wire of its own and arrives here through a translation — so + * a rule that speaks about *this* wire belongs here rather than beside a source's own + * strippers. All four do: the system-role rewrite and the reasoning sentinel state what an + * upstream's Anthropic Messages endpoint accepts, the billing-attribution scrub states what generation + * would have sent it, and the web-search preparation states the tool shape it would have + * seen. + */ +export const anthropicMessagesCountTokensWire: readonly Stage[] = [ + prepareAnthropicMessagesWebSearchRequest, + stripBillingAttributionFromAnthropicMessages, + disableReasoningOnForcedToolChoiceForAnthropicMessages, + applyRoleCompatibilityToAnthropicMessages, + callAnthropicMessagesCountTokensUpstream, +]; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/disable-reasoning-on-forced-tool-choice.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/disable-reasoning-on-forced-tool-choice.ts new file mode 100644 index 0000000000..df39a2f8c1 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/disable-reasoning-on-forced-tool-choice.ts @@ -0,0 +1,54 @@ +import type { Chat } from '../facts.ts'; +import { withoutKeys } from '../shared/structural.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; + +/** The reasoning sentinel a forced tool choice needs, in the shape this protocol has + * natively. `tool` and `any` are the forced choices; `auto` and `none` leave the model free + * to reason. */ +export const disableReasoningOnForcedToolChoiceForAnthropicMessages = defineStage< + Chat<'request.chat.anthropicMessages' | 'route.attempt'>, + Chat<'request.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'> +>({ + name: 'disableReasoningOnForcedToolChoice', + through: { + request: { + needs: ['request.chat.anthropicMessages', 'route.attempt'], + consumes: [], + provides: ['request.chat.anthropicMessages'], + }, + response: { needs: ['response.chat.anthropicMessages'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.anthropicMessages' | 'route.attempt'>, + Chat<'request.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'> + >(() => ({ + request: facts => { + if (!facts['route.attempt'].flags.includes('disable-reasoning-on-forced-tool-choice')) { + return facts; + } + const payload = facts['request.chat.anthropicMessages']; + const choice = payload.tool_choice?.type; + if (choice !== 'tool' && choice !== 'any') return facts; + return { ...facts, 'request.chat.anthropicMessages': move(withAnthropicMessagesReasoningDisabled(payload)) }; + }, + })), +}); + +/** Only the reasoning subfield goes with the sentinel: a forced tool choice composes fine + * with structured output on these upstreams, and it is thinking it does not compose with, so + * `output_config.format` has to survive. An `output_config` that held nothing but the effort + * is nothing at all once the effort is gone, so it goes rather than riding on empty. */ +const withAnthropicMessagesReasoningDisabled = (payload: AnthropicMessagesPayload): AnthropicMessagesPayload => { + const { output_config, ...rest } = payload; + const next: AnthropicMessagesPayload = { ...rest, thinking: { type: 'disabled' as const } }; + if (output_config !== undefined) { + const remaining = withoutKeys(output_config, ['effort']); + if (Object.keys(remaining).length > 0) next.output_config = remaining; + } + return next; +}; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts new file mode 100644 index 0000000000..0b7715813f --- /dev/null +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/emit.ts @@ -0,0 +1,123 @@ +import { wrapAnthropicMessagesAffinityEgress } from './affinity/egress.ts'; +import { renderAnthropicMessagesError } from './errors.ts'; +import type { Fields } from './facts.ts'; +import { isFailure, renderFailure, mintedAs } from '../../pipeline/facts.ts'; +import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; +import type { ChatServices } from '../services.ts'; +import { affinityEgressOptions } from '../shared/affinity/index.ts'; +import { bindClientRelease, collectClientFrames, framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; +import { recordStream } from '@floway-dev/dump'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { collectAnthropicMessagesProtocolEventsToResult, anthropicMessagesEventToSsePayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import { eventFrame, sseFrame, type EventFrame, type ProtocolFrame, type SseWritableFrame } from '@floway-dev/protocols/common'; +import { toInternalDebugError } from '@floway-dev/provider'; + +/** + * The outermost edge. An Anthropic Messages answer is always a stream by the time it reaches here — + * the upstream speaks SSE whatever the client asked for — so what this decides is whether + * the client sees the frames or the one message they add up to. + * + * Collecting is therefore the edge's own work and not a second reading of the upstream: the + * same frames that would have gone out are folded here instead, by the protocol's own + * reassembly. Neither shape can answer with half a message: the ending fails a stream that + * ran out before `message_stop`, whichever of the two the client asked for. + */ +export const emitAnthropicMessages = defineStage< + Fields<'ingress.chat.anthropicMessages.wantsStream'>, + Fields<'ingress.chat.anthropicMessages.wantsStream'>, + Fields<'ingress.chat.anthropicMessages.wantsStream' | 'response.chat.anthropicMessages' | 'response.http.headers' | 'response.http.status' | 'response.chat.anthropicMessages.streamedUsage'>, + Fields<'response.chat.anthropicMessages.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.clientFrames' | 'response.chat.anthropicMessages.streamedUsage'>, + ChatServices +>({ + name: 'emitAnthropicMessages', + through: { + request: { needs: ['ingress.chat.anthropicMessages.wantsStream'], consumes: [], provides: [] }, + response: { + needs: ['response.chat.anthropicMessages', 'response.http.headers', 'response.http.status', 'response.chat.anthropicMessages.streamedUsage'], + consumes: ['response.chat.anthropicMessages', 'response.http.headers'], + provides: ['response.chat.clientFrames', 'response.chat.anthropicMessages.rendered', 'response.http.status', 'response.http.headers', 'response.chat.anthropicMessages.streamedUsage'], + }, + }, + execute: async (facts, next, use) => { + const back = await next(facts); + const { 'response.chat.anthropicMessages': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); + + if (isFailure(answer)) { + const failure = renderFailure(answer, mintedAs(({ status, message }) => renderAnthropicMessagesError(status, message))); + return { + ...rest, + 'response.chat.clientFrames': null, + 'response.http.headers': forClient, + 'response.chat.anthropicMessages.rendered': move(failure.body), + 'response.http.status': failure.status, + }; + } + if (answer.kind === 'value') { + return { + ...rest, + 'response.chat.clientFrames': null, + 'response.http.headers': forClient, + 'response.chat.anthropicMessages.rendered': move(answer.body as Record), + 'response.http.status': back['response.http.status'], + }; + } + + // The turn's own state, written back into the frames the client is handed: a follow-up + // carrying it comes back to the upstream that issued it. This is the other half of the + // affinity the resolver read on the way down, and it has to sit here because it rewrites + // the frames — below the fold, and there would be nothing left to rewrite. + const egress = wrapAnthropicMessagesAffinityEgress( + answer.frames as AsyncIterable>, + affinityEgressOptions(use.gateway), + ); + if (!back['ingress.chat.anthropicMessages.wantsStream']) { + const frames = recordStream(egress, use.gateway.dump); + return { + ...rest, + 'response.chat.clientFrames': move(frames), + 'response.http.headers': forClient, + 'response.chat.anthropicMessages.rendered': move( + await collectClientFrames(frames, collectAnthropicMessagesProtocolEventsToResult) as unknown as Record, + ), + 'response.http.status': back['response.http.status'], + }; + } + const completed = framedClientStream( + clientFrames(egress), + frame => sseFrame(JSON.stringify(frame.event), frame.event.type), + error => [eventFrame({ type: 'error' as const, error: toInternalDebugError(error, 'anthropicMessages') })], + use.gateway.dump, + ); + bindClientRelease(back, completed.release); + return { + ...rest, + 'response.chat.clientFrames': move(completed.frames), + 'response.http.headers': forClient, + 'response.chat.anthropicMessages.rendered': move(completed.rendered), + 'response.chat.anthropicMessages.streamedUsage': move(withClientVerdict(back['response.chat.anthropicMessages.streamedUsage'], completed.failed)), + 'response.http.status': back['response.http.status'], + }; + }, +}); + +type ClientEvent = ReturnType; + +// SSE citations have their wire spelling before recording; the protocol's synthetic done +// frame has no Anthropic client form. Nonstream reassembly retains its canonical events. +const clientFrames = (frames: AsyncIterable>): AsyncIterable> => ({ + [Symbol.asyncIterator]: () => (async function* () { + for await (const frame of frames) { + if (frame.type === 'event') yield eventFrame(anthropicMessagesEventToSsePayload(frame.event)); + } + })(), +}); + +/** What this protocol writes on an idle connection. Anthropic defines a `ping` event and + * clients read it, so an SSE comment — invisible by design — is not the same wire byte. + * The route that serves this chain hands it to the seam alongside the frames. */ +export const anthropicMessagesKeepAlive: SseWritableFrame = sseFrame(JSON.stringify({ type: 'ping' }), 'ping'); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/errors.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/errors.ts index 20170e0f61..a3f2400c2d 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/errors.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/errors.ts @@ -1,59 +1,29 @@ -import { appendFailedUpstreams } from '../../shared/failed-upstreams.ts'; -import type { ChatServeFailure } from '../shared/errors.ts'; -import { generateAnthropicId, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult, PerformanceTelemetryContext } from '@floway-dev/provider'; -import type { TranslatorInputError } from '@floway-dev/translate'; +import { generateAnthropicId } from '@floway-dev/protocols/anthropic-messages'; -// Anthropic Messages error envelope used to render pre-stream -// `ChatServeFailure`s. These are gateway-synthesized rather than received -// from any upstream — `source: 'gateway'` so the dump labels them as such. -// Byte-shape matches Anthropic-direct: `{type:'error', error:{type, message}, -// request_id}` with `request_id` at the top level (alongside `error`, not -// nested inside it) and key order load-bearing for byte-faithfulness. -const anthropicErrorResult = ( - status: number, - type: string, - message: string, - performance?: PerformanceTelemetryContext, -): ExecuteResult> => ({ - type: 'api-error', - source: 'gateway', - status, - headers: new Headers({ 'content-type': 'application/json' }), - body: new TextEncoder().encode(JSON.stringify({ - type: 'error', - error: { type, message }, - request_id: generateAnthropicId('req'), - })), - ...(performance ? { performance } : {}), +// Anthropic's own error envelope. The byte shape matches Anthropic-direct: +// `{type:'error', error:{type, message}, request_id}` with `request_id` at the +// top level — alongside `error`, not nested inside it — and the key order is +// load-bearing for byte-faithfulness. +/** What an Anthropic Messages client is sent when a turn produced no content. The envelope is + * Anthropic's own — `type` at the top, the error's own `type` inside, and a `request_id` a + * caller quotes back in a support thread — so it is the protocol's shape rather than the + * gateway's, and an OpenAI-shaped one would carry none of those fields. */ +export const renderAnthropicMessagesError = (status: number, message: string): Record => ({ + type: 'error', + error: { type: anthropicErrorTypeForStatus(status), message }, + request_id: generateAnthropicId('req'), }); -// Translator surfaced a caller-input violation (unsupported content part, -// disallowed role, missing required field, etc.). Render as a 400 -// invalid_request_error so the caller sees a protocol-shaped failure -// instead of the internal-error 502 envelope. `performance` carries the -// throwing candidate's telemetry attribution when the throw fired -// mid-attempt (see AttemptState.telemetry). -export const translatorInputErrorResult = ( - error: TranslatorInputError, - performance?: PerformanceTelemetryContext, -): ExecuteResult> => - anthropicErrorResult(400, 'invalid_request_error', error.message, performance); - -// `endpoint` selects between `/messages` and `/messages/count_tokens` only in -// the `model-unsupported` message string. -export const renderAnthropicMessagesFailure = ( - failure: ChatServeFailure, - endpoint: 'generate' | 'countTokens', -): ExecuteResult> => { - const endpointPath = endpoint === 'countTokens' ? '/messages/count_tokens' : '/messages'; - switch (failure.kind) { - case 'routing-unavailable': - return anthropicErrorResult(400, 'invalid_request_error', failure.message); - case 'model-missing': - return anthropicErrorResult(404, 'not_found_error', appendFailedUpstreams(`Model ${failure.model} is not available on any configured upstream.`, failure.failedUpstreams)); - case 'model-unsupported': - return anthropicErrorResult(400, 'invalid_request_error', appendFailedUpstreams(`Model ${failure.model} does not support the ${endpointPath} endpoint.`, failure.failedUpstreams)); +/** The type name each status carries in this protocol's envelope. */ +const anthropicErrorTypeForStatus = (status: number): string => { + switch (status) { + case 400: return 'invalid_request_error'; + case 401: return 'authentication_error'; + case 403: return 'permission_error'; + case 404: return 'not_found_error'; + case 413: return 'request_too_large'; + case 429: return 'rate_limit_error'; + case 529: return 'overloaded_error'; + default: return status >= 500 ? 'api_error' : 'invalid_request_error'; } }; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/facts.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/facts.ts new file mode 100644 index 0000000000..f03c0fa4cf --- /dev/null +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/facts.ts @@ -0,0 +1,29 @@ +import type { StreamOutcome } from '../../pipeline/serve.ts'; +import type { ChatFacts } from '../facts.ts'; +import type { Deferred } from '@floway-dev/pipeline'; +import type { SseFrame } from '@floway-dev/protocols/common'; + +/** What this family adds to the chat space. */ +export interface AnthropicMessagesFacts extends ChatFacts { + /** What the client is actually sent — an object when it asked for one, SSE frames when it + * asked to stream. The edge provides it, so a dump shows what the client received. */ + 'response.chat.anthropicMessages.rendered': Record | AsyncIterable; + /** What the upstream will have reported once the frames run out, and `null` when nothing + * streamed. Settling from this is the epilogue's job, after the drain. */ + 'response.chat.anthropicMessages.streamedUsage': Deferred | null; +} + +export type Fields = { [P in K]: AnthropicMessagesFacts[P] }; + +/** This family's own reading, which every wire under it hands up. */ +export const STREAMED_USAGE = 'response.chat.anthropicMessages.streamedUsage'; + +export type AnthropicMessagesServeEntry = Fields< + 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'ingress.chat.anthropicMessages.wantsStream' + | 'request.chat.anthropicMessages' | 'serve.model' +>; + +export type AnthropicMessagesServeExit = Fields< + 'response.chat.anthropicMessages.rendered' | 'response.chat.anthropicMessages.streamedUsage' + | 'response.http.status' | 'response.http.jsonBody' | 'response.http.headers' | 'response.usage.billable' +>; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts index 2489a30d3f..b409edc6b1 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/http.ts @@ -1,17 +1,22 @@ -import { translatorInputErrorResult } from './errors.ts'; -import { respondAnthropicMessages } from './respond.ts'; -import { anthropicMessagesServe } from './serve.ts'; +// POST /v1/messages and POST /v1/messages/count_tokens, both served through their own +// pipeline. Generation is one chain; counting is a second operation over the same protocol +// and is a chain of its own rather than another wire under the first. +// +// Each entry is a prologue and an epilogue around that chain: read what the client sent, +// hand it over, and turn what the run answered with into a response. Everything between is +// stages. + +import { anthropicMessagesCountTokensPipeline } from './count-tokens.ts'; +import { anthropicMessagesKeepAlive } from './emit.ts'; +import { renderAnthropicMessagesError } from './errors.ts'; +import { anthropicMessagesServePipeline } from './pipeline.ts'; import type { AuthedContext } from '../../../middleware/auth.ts'; -import { backgroundSchedulerFromContext } from '../../../runtime/background.ts'; -import { createGatewayCtxFromHono, finalizeGatewayResponse, type GatewayCtx } from '../../shared/gateway-ctx.ts'; -import { inboundHeaders } from '../../shared/inbound-headers.ts'; -import { readRequestBody, takeRequestBody, type RequestBody } from '../../shared/request-body.ts'; +import { isFrames, openPrologue, readIngress, serveThrough, type Ingress } from '../../pipeline/serve.ts'; +import { finalizeGatewayResponse } from '../../shared/gateway-ctx.ts'; import { createNonOpenAIResponsesSourceStore } from '../openai-responses/items/store.ts'; -import { createChatGatewayCtxFromHono, type ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import { providerModelsUnavailableResponse } from '../shared/upstream-models-error.ts'; +import { openChatPrologue } from '../prologue.ts'; +import { move } from '@floway-dev/pipeline'; import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; -import { internalErrorResult, toInternalDebugError } from '@floway-dev/provider'; -import { TranslatorInputError } from '@floway-dev/translate'; // Reject `anthropic_beta` / `betas` in the body; the Anthropic Messages protocol carries // them via the `anthropic-beta` HTTP header. @@ -35,69 +40,93 @@ const rejectBodyBetaResponse = (payload: AnthropicMessagesPayload): Response | n ); }; -// Surfaces a pre-stream throw (malformed JSON body, an interceptor crash, -// etc.) as an Anthropic-Messages-shaped 502 with the same internal-error envelope the -// in-flow `internal-error` ExecuteResult produces. The caller passes its -// outer `ctx` when one was already constructed (so the dump row preserves -// the model attribution the request-time `requestedModel` stamped, and the -// throwing-candidate telemetry stamped in serve.ts survives onto the error -// row); a fresh ctx is minted only for pre-parse failures where no payload -// was available to read model from. -const respondWithInternalError = async (c: AuthedContext, error: unknown, requestBody: RequestBody, ctx?: GatewayCtx): Promise => { - const verbatim = providerModelsUnavailableResponse(error); - if (verbatim !== null) return verbatim; - const effectiveCtx = ctx ?? createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - const result = internalErrorResult(502, toInternalDebugError(error), effectiveCtx.attempt.telemetry); - const response = await respondAnthropicMessages(c, result, false, effectiveCtx); - return finalizeGatewayResponse(effectiveCtx, response); +/** A body the gateway could not read is reported in the words this protocol's own clients + * parse, rather than as a fault of the gateway's. */ +const readRequest = (bytes: Uint8Array): { type: 'ok'; payload: AnthropicMessagesPayload } | { type: 'invalid'; message: string } => { + try { + return { type: 'ok', payload: JSON.parse(new TextDecoder().decode(bytes)) as AnthropicMessagesPayload }; + } catch (error) { + return { type: 'invalid', message: error instanceof Error ? error.message : String(error) }; + } }; -// Pre-stream caller-input failure raised by a translator → Anthropic-Messages-shaped -// 400 invalid_request_error envelope. Anything else falls through to the -// internal-error 502 path. -const respondToThrow = async (c: AuthedContext, error: unknown, requestBody: RequestBody, ctx?: GatewayCtx): Promise => { - if (!(error instanceof TranslatorInputError)) return await respondWithInternalError(c, error, requestBody, ctx); - const effectiveCtx = ctx ?? createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - const response = await respondAnthropicMessages(c, translatorInputErrorResult(error, effectiveCtx.attempt.telemetry), false, effectiveCtx); - return finalizeGatewayResponse(effectiveCtx, response); +/** A request the gateway refused before it reached a pipeline: there is no model to resolve + * and no attempt to make, so there is nothing for a run to record beyond the refusal. */ +const refuse = (c: AuthedContext, ingress: Ingress, response: Response): Response => { + const refused = openPrologue(c, ingress, { wantsStream: false }); + refused.gateway.dump?.error('gateway'); + return finalizeGatewayResponse(refused.gateway, response); }; -const parsePayload = (requestBody: RequestBody): AnthropicMessagesPayload => - JSON.parse(new TextDecoder().decode(requestBody.bytes)) as AnthropicMessagesPayload; - export const anthropicMessagesHttp = { generate: async (c: AuthedContext): Promise => { - const requestBody = await readRequestBody(c); - let ctx: ChatGatewayCtx | undefined; - try { - const payload = parsePayload(requestBody); - const rejected = rejectBodyBetaResponse(payload); - if (rejected) return rejected; - - const wantsStream = payload.stream === true; - ctx = createChatGatewayCtxFromHono(c, { wantsStream, requestBody: takeRequestBody(requestBody), model: payload.model, backgroundScheduler: backgroundSchedulerFromContext(c) }, apiKey => createNonOpenAIResponsesSourceStore(apiKey.id)); - const result = await anthropicMessagesServe.generate({ payload, ctx, headers: inboundHeaders(c) }); - const response = await respondAnthropicMessages(c, result, wantsStream, ctx); - return finalizeGatewayResponse(ctx, response); - } catch (error) { - return await respondToThrow(c, error, requestBody, ctx); + const ingress = await readIngress(c); + const request = readRequest(ingress.body.bytes); + if (request.type === 'invalid') { + return refuse(c, ingress, Response.json(renderAnthropicMessagesError(400, request.message), { status: 400 })); } + + const { payload } = request; + const rejected = rejectBodyBetaResponse(payload); + if (rejected !== null) return refuse(c, ingress, rejected); + + const wantsStream = payload.stream === true; + const prologue = openChatPrologue(c, ingress, { + wantsStream, + model: payload.model, + storeFactory: apiKey => createNonOpenAIResponsesSourceStore(apiKey.id), + }); + + return await serveThrough( + c, + prologue, + anthropicMessagesServePipeline(), + move({ + 'ingress.http.headers': prologue.headers, + 'ingress.chat.sourceProtocol': 'anthropicMessages', + 'ingress.chat.anthropicMessages.wantsStream': wantsStream, + 'request.chat.anthropicMessages': payload, + 'serve.model': payload.model, + }) as never, + facts => { + const rendered = facts['response.chat.anthropicMessages.rendered']; + // Anthropic defines a `ping` event and its clients read one, so an idle connection is + // held open with that rather than with a comment no client sees. + if (isFrames(rendered)) return { frames: rendered, keepAlive: anthropicMessagesKeepAlive }; + return { body: facts['response.http.jsonBody']!, contentType: 'application/json' }; + }, + ); }, countTokens: async (c: AuthedContext): Promise => { - const requestBody = await readRequestBody(c); - let ctx: ChatGatewayCtx | undefined; - try { - const payload = parsePayload(requestBody); - const rejected = rejectBodyBetaResponse(payload); - if (rejected) return rejected; - - ctx = createChatGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), model: payload.model, backgroundScheduler: backgroundSchedulerFromContext(c) }, apiKey => createNonOpenAIResponsesSourceStore(apiKey.id)); - const result = await anthropicMessagesServe.countTokens({ payload, ctx, headers: inboundHeaders(c) }); - const response = await respondAnthropicMessages(c, result, false, ctx); - return finalizeGatewayResponse(ctx, response); - } catch (error) { - return await respondToThrow(c, error, requestBody, ctx); + const ingress = await readIngress(c); + const request = readRequest(ingress.body.bytes); + if (request.type === 'invalid') { + return refuse(c, ingress, Response.json(renderAnthropicMessagesError(400, request.message), { status: 400 })); } + + const { payload } = request; + const rejected = rejectBodyBetaResponse(payload); + if (rejected !== null) return refuse(c, ingress, rejected); + + const prologue = openChatPrologue(c, ingress, { + wantsStream: false, + model: payload.model, + storeFactory: apiKey => createNonOpenAIResponsesSourceStore(apiKey.id), + }); + + return await serveThrough( + c, + prologue, + anthropicMessagesCountTokensPipeline(), + move({ + 'ingress.http.headers': prologue.headers, + 'ingress.chat.sourceProtocol': 'anthropicMessages', + 'request.chat.anthropicMessages': payload, + 'serve.model': payload.model, + }) as never, + // A measurement is one body however the turn went, so there is never a stream to write. + facts => ({ body: facts['response.http.jsonBody']!, contentType: 'application/json' }), + ); }, }; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/apply-role-compatibility.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/apply-role-compatibility.ts deleted file mode 100644 index 302556c8ae..0000000000 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/apply-role-compatibility.ts +++ /dev/null @@ -1,15 +0,0 @@ -import type { AnthropicMessagesPayloadInterceptor } from './types.ts'; -import { providerModelOf } from '@floway-dev/provider'; - -export const withRoleCompatibilityApplied: AnthropicMessagesPayloadInterceptor = (ctx, _gatewayCtx, run) => { - if (ctx.targetApi !== 'anthropicMessages') return run(); - if (!providerModelOf(ctx.candidate).enabledFlags.has('rewrite-mid-conv-system-to-user')) return run(); - - ctx.payload = { - ...ctx.payload, - messages: ctx.payload.messages.map(message => - message.role === 'system' ? { role: 'user' as const, content: message.content } : message), - }; - - return run(); -}; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/disable-reasoning-on-forced-tool-choice.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/disable-reasoning-on-forced-tool-choice.ts deleted file mode 100644 index fa508f0006..0000000000 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/disable-reasoning-on-forced-tool-choice.ts +++ /dev/null @@ -1,32 +0,0 @@ - -import type { AnthropicMessagesPayloadInterceptor } from './types.ts'; -import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; -import { providerModelOf } from '@floway-dev/provider'; - -// Opt-in workaround for upstreams where forced `tool_choice` and enabled -// thinking do not compose. Anthropic Messages has a native `thinking: disabled` shape. -const hasForcedToolChoice = (payload: AnthropicMessagesPayload): boolean => { - const type = payload.tool_choice?.type; - return type === 'tool' || type === 'any'; -}; - -const disableAnthropicMessagesReasoning = (payload: AnthropicMessagesPayload): AnthropicMessagesPayload => { - // Strip only the reasoning subfield (`effort`) so structured-output - // `output_config.format` survives — forced tool choice composes fine with - // structured output on these upstreams, only with thinking does it not. - // If output_config becomes empty after the strip, omit it entirely. - const { output_config, ...rest } = payload; - const next: AnthropicMessagesPayload = { ...rest, thinking: { type: 'disabled' as const } }; - if (output_config) { - const { effort: _effort, ...remaining } = output_config; - if (Object.keys(remaining).length > 0) next.output_config = remaining; - } - return next; -}; - -export const withReasoningDisabledOnForcedToolChoice: AnthropicMessagesPayloadInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('disable-reasoning-on-forced-tool-choice')) return await run(); - if (!hasForcedToolChoice(ctx.payload)) return await run(); - ctx.payload = disableAnthropicMessagesReasoning(ctx.payload); - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/index.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/index.ts deleted file mode 100644 index 6c2346af03..0000000000 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/index.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { answerClaudeCodeProbe } from './answer-claude-code-probe.ts'; -import { withRoleCompatibilityApplied } from './apply-role-compatibility.ts'; -import { withReasoningDisabledOnForcedToolChoice } from './disable-reasoning-on-forced-tool-choice.ts'; -import { withEmptyToolsToolChoiceNormalized } from './normalize-empty-tools-tool-choice.ts'; -import { stripBillingAttribution } from './strip-billing-attribution.ts'; -import type { AnthropicMessagesCountTokensInterceptor, AnthropicMessagesInterceptor, AnthropicMessagesPayloadInterceptor } from './types.ts'; -import { withAnthropicMessagesWebSearchRequestPrepared, withAnthropicMessagesWebSearchShim } from './web-search-shim.ts'; - -// Unified Anthropic Messages generation chain. All entries are attached to every -// candidate; each interceptor decides whether to act — from the candidate's -// model flags and selected target, or, for the client-compatibility entry, -// from the inbound request itself. -// -// Translated requests re-enter the selected target protocol's chain. The role -// compatibility entry therefore acts only when Anthropic Messages is the final target. -// -// - answerClaudeCodeProbe: registered ahead of everything else because it -// replaces the turn rather than shaping it — no payload transform below it -// can matter to a turn that never dials. Unconditional: whether a request -// is one of Claude Code's one-token probes is a property of the request, -// not of the candidate. -// - withAnthropicMessagesWebSearchShim: registered next so its request preparation, -// replay rewrite, and intercept loop wrap the rest of the generation chain. -// Unconditional for translated targets (OpenAI Responses / OpenAI Chat Completions cannot -// carry Anthropic server tools); gated by `anthropic-messages-web-search-shim` for -// native Anthropic Messages targets. count_tokens runs the same request preparation -// without the stream-response wrapper. -// - stripBillingAttribution: gated by `strip-billing-attribution` (default -// on for copilot/azure/custom, off for claude-code). On candidates -// where it runs, it scrubs Claude Code's `x-anthropic-billing-header` / -// `cch=` markers out of the source-shape system prompt so prompt-cache -// hits survive across requests; on claude-code, the block is left intact -// because Anthropic uses it for plan-tier billing. -// - withReasoningDisabledOnForcedToolChoice: gated by -// `disable-reasoning-on-forced-tool-choice`. -// - withEmptyToolsToolChoiceNormalized: gated by -// `empty-tools-tool-choice-none`. Makes an explicitly empty tool list use -// Messages' native `{ type: 'none' }` choice. -// - withRoleCompatibilityApplied: Anthropic's top-level `payload.system` is -// the only first-position system slot, so the mid-conversation-system flag -// rewrites every inline system message to user after Anthropic Messages is selected -// as the final target. -// -// The remaining three entries mutate only the request payload and are shared -// with count_tokens in the same order. Token counting therefore observes the -// same gateway-level billing-attribution, reasoning, and role shape as -// generation. -const anthropicMessagesPayloadInterceptors: readonly AnthropicMessagesPayloadInterceptor[] = [ - stripBillingAttribution, - withReasoningDisabledOnForcedToolChoice, - withEmptyToolsToolChoiceNormalized, - withRoleCompatibilityApplied, -]; - -export const anthropicMessagesInterceptors: readonly AnthropicMessagesInterceptor[] = [ - answerClaudeCodeProbe, - withAnthropicMessagesWebSearchShim, - ...anthropicMessagesPayloadInterceptors, -]; - -export const anthropicMessagesCountTokensInterceptors: readonly AnthropicMessagesCountTokensInterceptor[] = [ - withAnthropicMessagesWebSearchRequestPrepared, - ...anthropicMessagesPayloadInterceptors, -]; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/normalize-empty-tools-tool-choice.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/normalize-empty-tools-tool-choice.ts deleted file mode 100644 index d6c0c51862..0000000000 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/normalize-empty-tools-tool-choice.ts +++ /dev/null @@ -1,12 +0,0 @@ -import type { AnthropicMessagesPayloadInterceptor } from './types.ts'; -import { providerModelOf } from '@floway-dev/provider'; - -// Opt-in compatibility rewrite for upstreams that reject an active -// `tool_choice` when the caller explicitly supplies no tools. Anthropic Messages uses -// an object for its native no-tool choice. -export const withEmptyToolsToolChoiceNormalized: AnthropicMessagesPayloadInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('empty-tools-tool-choice-none')) return await run(); - if (!Array.isArray(ctx.payload.tools) || ctx.payload.tools.length !== 0) return await run(); - ctx.payload = { ...ctx.payload, tool_choice: { type: 'none' } }; - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/strip-billing-attribution.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/strip-billing-attribution.ts deleted file mode 100644 index 5d91dfb8f8..0000000000 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/strip-billing-attribution.ts +++ /dev/null @@ -1,42 +0,0 @@ -import type { AnthropicMessagesPayloadInterceptor } from './types.ts'; -import { providerModelOf } from '@floway-dev/provider'; - -// Claude Code clients seed every Anthropic Messages request with an -// `x-anthropic-billing-header: …` line carrying a per-turn `cch=` value. -// Two upstreams want opposite things from that block: -// -// - The Claude Code subscription endpoint (provider kind `claude-code`) -// reads the block to bill the request against the user's plan tier. If -// we strip it, the request silently falls off plan billing. -// - Every other upstream (copilot, azure, custom) treats the block as -// ordinary prompt text. The `cch=` hash flips per call, so the -// upstream's prompt-cache layer sees a "different" prompt every turn -// and never reuses its cache, even when the real conversation prefix -// hasn't changed. -// -// Gating is owned by the `strip-billing-attribution` flag in the provider -// flag catalog; defaults are wired there per kind. -const BILLING_HEADER_LINE_RE = /x-anthropic-billing-header[^\n]*/g; -const CCH_HASH_RE = /cch=[0-9a-f]{5,};?/gi; - -const stripText = (text: string): string => text.replace(BILLING_HEADER_LINE_RE, '').replace(CCH_HASH_RE, '').trim(); - -export const stripBillingAttribution: AnthropicMessagesPayloadInterceptor = (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('strip-billing-attribution')) return run(); - - const { payload } = ctx; - - if (typeof payload.system === 'string') { - const system = stripText(payload.system); - const { system: _system, ...rest } = payload; - ctx.payload = system.length > 0 ? { ...rest, system } : rest; - } else if (Array.isArray(payload.system)) { - const system = payload.system - .map(block => ({ ...block, text: stripText(block.text) })) - .filter(block => block.text.length > 0); - const { system: _system, ...rest } = payload; - ctx.payload = system.length > 0 ? { ...rest, system } : rest; - } - - return run(); -}; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/types.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/types.ts deleted file mode 100644 index 845c191f54..0000000000 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/types.ts +++ /dev/null @@ -1,32 +0,0 @@ -import type { GatewayCtx } from '../../../shared/gateway-ctx.ts'; -import type { Interceptor, InterceptorRun } from '@floway-dev/interceptor'; -import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult, AnthropicMessagesInvocation } from '@floway-dev/provider'; - -export type { AnthropicMessagesInvocation }; - -export type AnthropicMessagesInterceptor = Interceptor< - AnthropicMessagesInvocation, - GatewayCtx, - ExecuteResult> ->; - -// count_tokens is a one-shot, non-streaming HTTP exchange whose terminal -// returns the raw upstream `Response`. Shared entries must therefore be pure -// header/payload mutators; post-run stream inspection is not portable to this -// result type. -export type AnthropicMessagesCountTokensInterceptor = Interceptor< - AnthropicMessagesInvocation, - GatewayCtx, - Response ->; - -// Payload-only transforms can run in both the streaming generation chain and -// the one-shot count_tokens chain. Keeping the result generic prevents those -// shared interceptors from accidentally inspecting either result shape. -export type AnthropicMessagesPayloadInterceptor = ( - ctx: AnthropicMessagesInvocation, - gatewayCtx: GatewayCtx, - run: InterceptorRun, -) => Promise; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/meter.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/meter.ts new file mode 100644 index 0000000000..0c89f2bc81 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/meter.ts @@ -0,0 +1,95 @@ +import type { StreamOutcome } from '../../pipeline/serve.ts'; +import { meterChatWire } from '../meter.ts'; +import { createAnthropicMessagesBillableUsageReader } from './usage.ts'; +import type { BillableEntity } from '../../pipeline/facts.ts'; +import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; +import { chatUsageMeasurement, type ChatBillableUsage } from '../shared/usage.ts'; +import { defer, type Deferred } from '@floway-dev/pipeline'; +import { ANTHROPIC_MESSAGES_MISSING_TERMINAL_MESSAGE, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import type { TelemetryModelIdentity } from '@floway-dev/provider'; + +export const meterUsage = (streamedUsage: string) => meterChatWire({ + wire: 'anthropicMessages', + answer: 'response.chat.anthropicMessages', + streamedUsage, + read: meterAnthropicMessages, +}); + +/** Reads the upstream's own usage off its own events as they pass, so the reading costs one + * pass and the client's stream is what drives it. Anthropic states input accounting on + * `message_start` and output accounting on `message_delta`, so the reader that merges the + * two is per-stream state and is made here rather than shared. */ +const meterAnthropicMessages = ( + source: AsyncIterable>, + identity: TelemetryModelIdentity, + attempt: { firstOutputTokenAt: number | null }, +): { readonly frames: AsyncIterable>; readonly outcome: Deferred } => { + const completion = Promise.withResolvers(); + const outcome = defer(completion.promise); + let settled = false; + // Running out without the terminal frame is what "it did not finish" means, and it is known + // at the same moment the usage is. + let sawTerminal = false; + let failed = false; + const readBillableUsage = createAnthropicMessagesBillableUsageReader(); + const generator = (async function* () { + let reported: ChatBillableUsage | undefined; + const finish = (): void => { + if (settled) return; + settled = true; + try { completion.resolve({ billable: [billedEntity(reported, identity)], failed: failed || !sawTerminal }); } catch (error) { + completion.reject(error); + throw error; + } + }; + try { + for await (const frame of source) { + // Time to first token is measured where the token is, which is the only place that + // knows a frame carries generated content rather than the envelope around it. + if (attempt.firstOutputTokenAt === null && isFirstOutputTokenFrame(frame, 'anthropicMessages')) { + attempt.firstOutputTokenAt = performance.now(); + } + if (frame.type === 'event') { + const usage = readBillableUsage(frame.event); + if (usage !== null) reported = usage; + } + if (isAnthropicMessagesTerminalFrame(frame)) { + sawTerminal = true; + failed = frame.type === 'event' && frame.event.type === 'error'; + finish(); + } + yield frame; + // The turn is over, so there is nothing further to read. An upstream that holds the + // connection open past `message_stop` would otherwise hold the client's stream open + // with it; returning here closes the read, which cancels the upstream. + if (isAnthropicMessagesTerminalFrame(frame)) return; + } + // Frames ran out with no terminal event, which is a turn nobody can answer from: the + // message was never stopped and never failed. + throw new Error(ANTHROPIC_MESSAGES_MISSING_TERMINAL_MESSAGE); + } finally { + // Reached however the frames ended — the terminal event, a client that stopped + // reading, or a broken upstream — because tokens the upstream already metered are + // billable whatever happened to the downstream half. + finish(); + } + })(); + return { frames: { [Symbol.asyncIterator]: () => generator }, outcome }; +}; + +/** What ends an Anthropic Messages turn. Anthropic's own stream terminator is an event rather than a + * transport sentinel, and a stream that failed mid-turn says so with `error` in place of the + * `message_stop` that will now never come. */ +const isAnthropicMessagesTerminalFrame = (frame: ProtocolFrame): boolean => + frame.type === 'event' && (frame.event.type === 'message_stop' || frame.event.type === 'error'); + +/** What one attempt is billable for. An upstream that reported nothing leaves no quantities + * at all, which is a different statement from reporting zero — and a rate can depend on the + * service tier and on how much input there was, so both travel as pricing facts rather than + * being folded into the quantities. */ +const billedEntity = (usage: ChatBillableUsage | undefined, identity: TelemetryModelIdentity): BillableEntity => { + if (usage === undefined) return { identity, quantities: {} }; + const measurement = chatUsageMeasurement(usage); + return { identity, quantities: measurement.quantities, pricingFacts: measurement.pricingFacts }; +}; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/normalize-empty-tools-tool-choice.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/normalize-empty-tools-tool-choice.ts new file mode 100644 index 0000000000..2132ea04b7 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/normalize-empty-tools-tool-choice.ts @@ -0,0 +1,4 @@ +import { normalizeEmptyTools } from '../shared/normalize-empty-tools.ts'; + +export const normalizeEmptyToolsForAnthropicMessages = normalizeEmptyTools('request.chat.anthropicMessages', payload => + payload.tool_choice?.type === 'none' ? payload : { ...payload, tool_choice: { type: 'none' } }); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts new file mode 100644 index 0000000000..92e5d6c2ab --- /dev/null +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/pipeline.ts @@ -0,0 +1,42 @@ +import { STREAMED_USAGE, type AnthropicMessagesServeEntry, type AnthropicMessagesServeExit } from './facts.ts'; +import { composeChat as compose } from '../compose.ts'; +import { answerClaudeCodeProbe } from './answer-claude-code-probe.ts'; +import { emitAnthropicMessages } from './emit.ts'; +import { normalizeEmptyToolsForAnthropicMessages } from './normalize-empty-tools-tool-choice.ts'; +import { narrowing, anthropicMessagesTarget } from './target.ts'; +import { runAnthropicMessagesWebSearchTool } from './web-search-tool.ts'; +import { anthropicMessagesWireFor } from './wires.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import { failover } from '../../pipeline/failover.ts'; +import { serializeClientJson } from '../../pipeline/serialize-client-json.ts'; +import { writeSettlement } from '../../pipeline/settlement.ts'; +import { dialChatWire } from '../dial-wire.ts'; +import { materializeAttempt } from '../materialize-attempt.ts'; +import { resolveChatCandidates } from '../resolve-candidates.ts'; +import type { Pipeline } from '@floway-dev/pipeline'; + +export const anthropicMessagesServePipeline = (): Pipeline => + compose('anthropicMessagesServe', [ + writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, STREAMED_USAGE), + serializeClientJson('response.chat.anthropicMessages.rendered'), + emitAnthropicMessages, + resolveChatCandidates(narrowing), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.chat.anthropicMessages'?: unknown })['response.chat.anthropicMessages']), + owns: ['response.http.body'], + pendingUsage: STREAMED_USAGE, + }), + materializeAttempt('request.chat.anthropicMessages'), + answerClaudeCodeProbe, + // Below the probe, because a probe turn declares no tools; above the dial, because what it + // rewrites is the body that dial sends and the frames that dial hands back. + runAnthropicMessagesWebSearchTool({ targetOf: candidate => anthropicMessagesTarget.pick(candidate.model.endpoints) }), + normalizeEmptyToolsForAnthropicMessages, + dialChatWire({ + source: 'request.chat.anthropicMessages', + needs: ['request.chat.anthropicMessages', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], + provides: ['response.chat.anthropicMessages', STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], + pick: endpoints => anthropicMessagesTarget.pick(endpoints), + wire: anthropicMessagesWireFor, + }), + ]); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/respond.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/respond.ts deleted file mode 100644 index 98ad801d17..0000000000 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/respond.ts +++ /dev/null @@ -1,132 +0,0 @@ -import type { Context } from 'hono'; -import { streamSSE } from 'hono/streaming'; - -import { wrapAnthropicMessagesAffinityEgress } from './affinity/egress.ts'; -import type { GatewayCtx } from '../../shared/gateway-ctx.ts'; -import { type StreamCompletion, writeSSEFrames } from '../../shared/sse.ts'; -import { recordFailedRequest } from '../../shared/telemetry/performance.ts'; -import { settle } from '../../shared/telemetry/settle.ts'; -import { tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; -import { forwardUpstreamHeaders, mergeForwardedUpstreamHeaders } from '../../shared/upstream-response.ts'; -import { affinityEgressOptions } from '../shared/affinity/index.ts'; -import { SourceStreamState, eventResultMetadata, plainResultToResponse } from '../shared/respond.ts'; -import { anthropicMessagesProtocolFrameToSSEFrame, ANTHROPIC_MESSAGES_MISSING_TERMINAL_MESSAGE, collectAnthropicMessagesProtocolEventsToResult } from '@floway-dev/protocols/anthropic-messages'; -import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import { eventFrame, type ProtocolFrame, sseFrame } from '@floway-dev/protocols/common'; -import { type ExecuteResult, type PlainResult, type InternalDebugError, toInternalDebugError } from '@floway-dev/provider'; -import { apiErrorToResponse } from '@floway-dev/provider'; - -// Renders an upstream Anthropic Messages result into the client HTTP/SSE response. An -// error-typed result is a pre-stream failure and always answers as HTTP; an -// events result drains to one JSON body (non-streaming) or is proxied frame by -// frame (streaming). -export const respondAnthropicMessages = async ( - c: Context, - result: ExecuteResult> | PlainResult, - wantsStream: boolean, - ctx: GatewayCtx, -): Promise => { - if (result.type === 'api-error') { - recordFailedRequest(ctx, result.performance); - ctx.dump?.error(result.source, result.upstreamId); - return apiErrorToResponse(result); - } - - if (result.type === 'internal-error') { - recordFailedRequest(ctx, result.performance); - ctx.dump?.failed(result.error.message); - return internalAnthropicMessagesErrorResponse(result.status, result.error); - } - - if (result.type === 'plain') { - if (result.status >= 400) { - ctx.dump?.error(result.upstreamId !== undefined ? 'upstream' : 'gateway', result.upstreamId); - } - return plainResultToResponse(result); - } - - const state = new SourceStreamState(); - const observed = observeAnthropicMessagesFrames(result.events, state, ctx); - const frames = wrapAnthropicMessagesAffinityEgress(observed, affinityEgressOptions(ctx)); - - if (!wantsStream) { - try { - const response = await collectAnthropicMessagesProtocolEventsToResult(frames); - const metadata = await eventResultMetadata(result); - const usage = tokenUsageFromBillableUsage(metadata.billableUsage); - ctx.dump?.success(metadata.modelIdentity, usage); - settle(ctx, metadata.performance, metadata.modelIdentity, usage, state.failed); - return Response.json(response, { headers: mergeForwardedUpstreamHeaders(undefined, result.headers) }); - } catch (error) { - recordFailedRequest(ctx, result.performance); - ctx.dump?.failed(error); - return internalAnthropicMessagesErrorResponse(502, toInternalDebugError(error)); - } - } - - forwardUpstreamHeaders(c, result.headers); - return streamSSE(c, async stream => { - let completion: StreamCompletion = 'error'; - try { - completion = await writeSSEFrames(stream, anthropicMessagesSseFrames(frames, state, ctx), { - keepAlive: { frame: sseFrame(JSON.stringify({ type: 'ping' }), 'ping') }, - ...(ctx.downstreamAbortController !== undefined ? { downstreamAbortController: ctx.downstreamAbortController } : {}), - }); - } finally { - const metadata = await eventResultMetadata(result); - const failed = state.failedAfter(completion); - if (failed) { - ctx.dump?.failed(`messages stream failed (completion=${completion}, source-failed=${state.failed})`); - } else { - ctx.dump?.success(metadata.modelIdentity, tokenUsageFromBillableUsage(metadata.billableUsage)); - } - settle(ctx, metadata.performance, metadata.modelIdentity, tokenUsageFromBillableUsage(metadata.billableUsage), failed); - } - }); -}; - -const internalAnthropicMessagesErrorPayload = (error: InternalDebugError) => ({ - type: 'error', - error: { - type: error.type, - name: error.name, - message: error.message, - stack: error.stack, - cause: error.cause, - target_api: error.target_api, - }, -}); - -const internalAnthropicMessagesErrorResponse = (status: number, error: InternalDebugError): Response => Response.json(internalAnthropicMessagesErrorPayload(error), { status }); - -const isAnthropicMessagesTerminalFrame = (frame: ProtocolFrame) => frame.type === 'event' && (frame.event.type === 'message_stop' || frame.event.type === 'error'); - -const observeAnthropicMessagesFrames = async function* ( - frames: AsyncIterable>, - state: SourceStreamState, - ctx: GatewayCtx, -) { - for await (const frame of frames) { - await ctx.dump?.frame(frame); - const failed = frame.type === 'event' && frame.event.type === 'error'; - if (failed) state.failed = true; - if (isAnthropicMessagesTerminalFrame(frame) && !failed) state.completed = true; - yield frame; - if (isAnthropicMessagesTerminalFrame(frame)) return; - } - throw new Error(ANTHROPIC_MESSAGES_MISSING_TERMINAL_MESSAGE); -}; - -const anthropicMessagesSseFrames = async function* (frames: AsyncIterable>, state: SourceStreamState, ctx: GatewayCtx) { - try { - for await (const frame of frames) { - const sse = anthropicMessagesProtocolFrameToSSEFrame(frame); - if (sse) yield sse; - } - } catch (error) { - state.failed = true; - const event = internalAnthropicMessagesErrorPayload(toInternalDebugError(error)) as unknown as AnthropicMessagesStreamEvent; - await ctx.dump?.frame(eventFrame(event)); - yield sseFrame(JSON.stringify(event), 'error'); - } -}; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/serve.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/serve.ts deleted file mode 100644 index e0707abf47..0000000000 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/serve.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { analyzeAnthropicMessagesAffinity } from './affinity/ingress.ts'; -import { anthropicMessagesAttempt, anthropicMessagesGenerateTarget, anthropicMessagesCountTokensTarget } from './attempt.ts'; -import { renderAnthropicMessagesFailure } from './errors.ts'; -import { enumerateModelCandidates } from '../../providers/resolution.ts'; -import { iterateCandidates } from '../../shared/iterate-candidates.ts'; -import { selectAffinityCandidates } from '../shared/affinity/index.ts'; -import { noViableCandidateFailure } from '../shared/errors.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import { parseAnthropicBetaHeader, type AnthropicMessagesPayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult, PlainResult } from '@floway-dev/provider'; - -export interface AnthropicMessagesServeGenerateArgs { - readonly payload: AnthropicMessagesPayload; - readonly ctx: ChatGatewayCtx; - readonly headers: Headers; -} - -export interface AnthropicMessagesServeCountTokensArgs { - readonly payload: AnthropicMessagesPayload; - readonly ctx: ChatGatewayCtx; - readonly headers: Headers; -} - -export const anthropicMessagesServe = { - generate: async (args: AnthropicMessagesServeGenerateArgs): Promise>> => { - const { payload, ctx, headers } = args; - const anthropicBeta = parseAnthropicBetaHeader(headers.get('anthropic-beta')); - const { candidates: enumerated, sawModel, failedUpstreams } = await enumerateModelCandidates({ - upstreamIds: ctx.upstreamIds, - model: payload.model, - kind: 'chat', - scheduler: ctx.backgroundScheduler, - runtimeLocation: ctx.runtimeLocation, - }); - const affinity = await analyzeAnthropicMessagesAffinity(payload, ctx.affinity.codec); - const viable = enumerated.filter(c => anthropicMessagesGenerateTarget.canServe(c.model.endpoints)); - const selection = selectAffinityCandidates(viable, affinity); - if ('kind' in selection) return renderAnthropicMessagesFailure(selection, 'generate'); - if (selection.candidates.length === 0) return renderAnthropicMessagesFailure(noViableCandidateFailure(sawModel, payload.model, failedUpstreams), 'generate'); - - // Try each affinity-selected candidate in order. A successful attempt (SSE - // stream opened) is the final answer; an api-error or internal-error - // from one candidate falls through to the next so the gateway absorbs - // transient 5xx/429/network failures. When the list is exhausted, the - // most recent failure is forwarded verbatim. Each attempt stamps its - // private payload clone with the candidate's canonical model id. - return await iterateCandidates( - selection.candidates, - 'anthropicMessagesServe.generate', - ctx, - 'chat', - async candidate => { - const result = await anthropicMessagesAttempt.generate({ payload: selection.payloadFor(candidate), ctx, candidate, headers, anthropicBeta }); - if (result.type === 'events') ctx.affinity.select(candidate); - return result; - }, - ); - }, - - countTokens: async (args: AnthropicMessagesServeCountTokensArgs): Promise> | PlainResult> => { - const { payload, ctx, headers } = args; - const anthropicBeta = parseAnthropicBetaHeader(headers.get('anthropic-beta')); - const { candidates: enumerated, sawModel, failedUpstreams } = await enumerateModelCandidates({ - upstreamIds: ctx.upstreamIds, - model: payload.model, - kind: 'chat', - scheduler: ctx.backgroundScheduler, - runtimeLocation: ctx.runtimeLocation, - }); - const affinity = await analyzeAnthropicMessagesAffinity(payload, ctx.affinity.codec); - const viable = enumerated.filter(c => anthropicMessagesCountTokensTarget.canServe(c.model.endpoints)); - const selection = selectAffinityCandidates(viable, affinity); - if ('kind' in selection) return renderAnthropicMessagesFailure(selection, 'countTokens'); - if (selection.candidates.length === 0) return renderAnthropicMessagesFailure(noViableCandidateFailure(sawModel, payload.model, failedUpstreams), 'countTokens'); - - return await iterateCandidates( - selection.candidates, - 'anthropicMessagesServe.countTokens', - ctx, - 'chat', - candidate => anthropicMessagesAttempt.countTokens({ payload: selection.payloadFor(candidate), ctx, candidate, headers, anthropicBeta }), - ); - }, -}; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/strip-billing-attribution.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/strip-billing-attribution.ts new file mode 100644 index 0000000000..b7d64fac8c --- /dev/null +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/strip-billing-attribution.ts @@ -0,0 +1,90 @@ +import type { Chat } from '../facts.ts'; +import { withoutKeys } from '../shared/structural.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; + +/** + * Scrubs Claude Code's billing-attribution block out of the system prompt. + * + * The block leads every Claude Code turn as `system[0]`: + * `x-anthropic-billing-header: cc_version=.; cc_entrypoint=;` + * with optional `cch=`, `cc_workload=` and `cc_prev_req=` segments after it. Two of those parts + * vary per call — the version's trailing fingerprint is derived from the first user message, + * and `cch=` was a per-request nonce on builds through 2.1.177 — so an upstream reading the + * block as ordinary prompt text sees a different prompt every turn and reuses nothing. One + * report measured stripping the line as taking the prefix cache from 0% to ~99.7%. + * https://github.com/anthropics/claude-code/issues/68900 + * https://github.com/anthropics/claude-code/issues/50085 + * + * Anthropic's own endpoint wants it intact, which is what the flag is for: it is on for the + * upstreams that treat the block as text and off for the one that reads it. What that endpoint + * does with it is not documented anywhere primary — the observable part is that the CLI adds + * *more* of these fields on its first-party path, so it reads as a client-attribution signal. + */ +export const stripBillingAttributionFromAnthropicMessages = defineStage< + Chat<'request.chat.anthropicMessages' | 'route.attempt'>, + Chat<'request.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'> +>({ + name: 'stripBillingAttribution', + through: { + request: { + needs: ['request.chat.anthropicMessages', 'route.attempt'], + consumes: [], + provides: ['request.chat.anthropicMessages'], + }, + response: { needs: ['response.chat.anthropicMessages'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.anthropicMessages' | 'route.attempt'>, + Chat<'request.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'>, + Chat<'response.chat.anthropicMessages'> + >(() => ({ + request: facts => { + if (!facts['route.attempt'].flags.includes('strip-billing-attribution')) return facts; + const payload = facts['request.chat.anthropicMessages']; + const system = scrubbedSystemPrompt(payload.system); + if (system === payload.system) return facts; + const withoutSystem = withoutKeys(payload, ['system']); + return { + ...facts, + 'request.chat.anthropicMessages': move(system === undefined ? withoutSystem : { ...withoutSystem, system }), + }; + }, + })), +}); + +const BILLING_HEADER_LINE = /x-anthropic-billing-header[^\n]*/g; + +// Swept separately from the line because a `cch=` also reaches the prompt replayed inside +// message content, where the line pattern cannot see it. Every value captured in the wild is +// five hex digits with a trailing semicolon. +// https://github.com/anthropics/claude-code/issues/40652 +const BILLING_CACHE_HASH = /cch=[0-9a-f]{5,};?/gi; + +const scrubText = (text: string): string => + text.replace(BILLING_HEADER_LINE, '').replace(BILLING_CACHE_HASH, '').trim(); + +/** What is left of the system prompt once the block is out of it: the value that came in when + * there was nothing to scrub, and `undefined` when nothing at all is left — an empty system + * prompt is not a system prompt, so the field goes rather than riding on empty. */ +const scrubbedSystemPrompt = (system: AnthropicMessagesPayload['system']): AnthropicMessagesPayload['system'] => { + if (system === undefined) return undefined; + if (typeof system === 'string') { + const scrubbed = scrubText(system); + if (scrubbed.length === 0) return undefined; + return scrubbed === system ? system : scrubbed; + } + let changed = false; + const blocks = system.flatMap(block => { + const text = scrubText(block.text); + if (text.length === 0) { changed = true; return []; } + if (text === block.text) return [block]; + changed = true; + return [{ ...block, text }]; + }); + if (!changed) return system; + return blocks.length > 0 ? blocks : undefined; +}; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/target.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/target.ts new file mode 100644 index 0000000000..a1f06e4ce7 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/target.ts @@ -0,0 +1,22 @@ +import type { ChatNarrowing } from '../resolve-candidates.ts'; +import type { Fields } from './facts.ts'; +import { chatTargetPicker } from '../shared/target-picker.ts'; +import { analyzeAnthropicMessagesAffinity } from './affinity/ingress.ts'; + +/** `/v1/messages` prefers its own wire, then the translated OpenAI Responses path, then the + * translated OpenAI Chat Completions path. */ +export const anthropicMessagesTarget = chatTargetPicker(['anthropicMessages', 'openaiResponses', 'openaiChatCompletions']); + +export const narrowing: ChatNarrowing, 'request.chat.anthropicMessages'> = ({ + requestKey: 'request.chat.anthropicMessages', + canServe: candidate => anthropicMessagesTarget.canServe(candidate.model.endpoints), + affinity: async (payload, gateway) => await analyzeAnthropicMessagesAffinity(payload, gateway.affinity.codec), + unsupported: model => `Model ${model} does not support the /messages endpoint.`, + refuse: (status, message) => ({ + 'response.chat.anthropicMessages': { status, message }, + // A refusal never opened a stream, so there is nothing still to read — which is what + // lets settlement write its row here rather than wait for numbers that never come. + 'response.chat.anthropicMessages.streamedUsage': null, + }), + refuses: ['response.chat.anthropicMessages', 'response.chat.anthropicMessages.streamedUsage'], +}); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/usage.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/usage.ts index d0dbcaab14..d62790b6fb 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/usage.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/usage.ts @@ -1,25 +1,26 @@ +import type { ChatBillableUsage } from '../shared/usage.ts'; import { mergeAnthropicMessagesUsageSnapshot, anthropicMessagesUsageSnapshot, splitAnthropicMessagesCacheCreationTokens, type AnthropicMessagesStreamEvent, type AnthropicMessagesUsageSnapshot } from '@floway-dev/protocols/anthropic-messages'; -import { billableServiceTier, type BillableUsage } from '@floway-dev/protocols/common'; +import { billableServiceTier } from '@floway-dev/protocols/common'; // Anthropic reports `input_tokens` exclusive of both cache buckets already, // and splits cache creation by TTL — the two rates we are billed at. -export const billableUsageFromAnthropicMessagesUsage = (usage: AnthropicMessagesUsageSnapshot): BillableUsage | null => { +export const billableUsageFromAnthropicMessagesUsage = (usage: AnthropicMessagesUsageSnapshot): ChatBillableUsage | null => { if (usage.input_tokens === undefined && usage.output_tokens === undefined) return null; const { cacheWrite, cacheWrite1h } = splitAnthropicMessagesCacheCreationTokens(usage); const tier = billableServiceTier(usage.speed) ?? billableServiceTier(usage.service_tier); return { - input: usage.input_tokens ?? 0, + ...(usage.input_tokens === undefined ? {} : { input: usage.input_tokens }), cacheRead: usage.cache_read_input_tokens ?? 0, cacheWrite, cacheWrite1h, - output: usage.output_tokens ?? 0, + ...(usage.output_tokens === undefined ? {} : { output: usage.output_tokens }), ...(tier !== null ? { tier } : {}), }; }; // Anthropic reports input accounting on `message_start` and output accounting // on `message_delta`, so the running figure is merged across both. -export const createAnthropicMessagesBillableUsageReader = (): (event: AnthropicMessagesStreamEvent) => BillableUsage | null => { +export const createAnthropicMessagesBillableUsageReader = (): (event: AnthropicMessagesStreamEvent) => ChatBillableUsage | null => { let merged = anthropicMessagesUsageSnapshot(); return event => { const usage = event.type === 'message_start' ? event.message.usage diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/web-search-shim.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-shim.ts similarity index 85% rename from packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/web-search-shim.ts rename to packages/gateway/src/data-plane/chat/anthropic-messages/web-search-shim.ts index 69dcd922f6..8e3bde6fd9 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/interceptors/web-search-shim.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-shim.ts @@ -1,11 +1,10 @@ -import type { AnthropicMessagesCountTokensInterceptor, AnthropicMessagesInterceptor, AnthropicMessagesInvocation } from './types.ts'; -import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../../shared/base64url-json.ts'; -import { isJsonObject } from '../../../../shared/json-helpers.ts'; -import { loadWebSearchConfig } from '../../../tools/web-search/config.ts'; -import { resolveConfiguredWebSearchProvider } from '../../../tools/web-search/provider.ts'; -import { runWebSearchAndRecordUsage } from '../../../tools/web-search/search.ts'; -import type { WebSearchProvider, WebSearchProviderName, WebSearchProviderRequest, WebSearchProviderResult } from '../../../tools/web-search/types.ts'; +import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../shared/base64url-json.ts'; +import { isJsonObject } from '../../../shared/json-helpers.ts'; +import { loadWebSearchConfig } from '../../tools/web-search/config.ts'; +import { resolveConfiguredWebSearchProvider } from '../../tools/web-search/provider.ts'; +import { runWebSearchAndRecordUsage } from '../../tools/web-search/search.ts'; +import type { WebSearchProvider, WebSearchProviderName, WebSearchProviderRequest, WebSearchProviderResult } from '../../tools/web-search/types.ts'; import type { AnthropicMessagesAssistantContentBlock, AnthropicMessagesAssistantInputContentBlock, @@ -25,7 +24,8 @@ import type { } from '@floway-dev/protocols/anthropic-messages'; import { ANTHROPIC_MESSAGES_WEB_SEARCH_ERROR_CODES } from '@floway-dev/protocols/anthropic-messages'; import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import { providerModelOf, internalErrorResult, toInternalDebugError } from '@floway-dev/provider'; +import type { AnthropicMessagesInvocation } from '@floway-dev/provider'; +import { providerModelOf } from '@floway-dev/provider'; const MAX_QUERY_LENGTH = 1000; const WEB_SEARCH_TOOL_NAME = 'web_search'; @@ -176,7 +176,7 @@ const isWebSearchToolResultError = (value: unknown): value is AnthropicMessagesW const toUpstreamToolUseId = (toolUseId: string): string => (toolUseId.startsWith('srvtoolu_') ? `toolu_${toolUseId.slice('srvtoolu_'.length)}` : toolUseId); -const toNativeServerToolUseId = (toolUseId: string): string => (toolUseId.startsWith('toolu_') ? `srvtoolu_${toolUseId.slice('toolu_'.length)}` : toolUseId); +const toNativeHostedToolUseId = (toolUseId: string): string => (toolUseId.startsWith('toolu_') ? `srvtoolu_${toolUseId.slice('toolu_'.length)}` : toolUseId); const buildUpstreamSearchResultBlock = (result: AnthropicMessagesWebSearchResultBlock, decoded: NonNullable>): AnthropicMessagesSearchResultBlock => ({ type: 'search_result', @@ -188,14 +188,14 @@ const buildUpstreamSearchResultBlock = (result: AnthropicMessagesWebSearchResult const buildNativeWebSearchErrorResultBlock = (toolUseId: string, errorCode: AnthropicMessagesWebSearchErrorCode): Extract => ({ type: 'web_search_tool_result', - tool_use_id: toNativeServerToolUseId(toolUseId), + tool_use_id: toNativeHostedToolUseId(toolUseId), content: { type: 'web_search_tool_result_error', error_code: errorCode }, caller: { type: 'direct' }, }); -const buildNativeWebSearchServerToolUseBlock = (toolUseId: string, query: string): Extract => ({ +const buildNativeWebSearchHostedToolUseBlock = (toolUseId: string, query: string): Extract => ({ type: 'server_tool_use', - id: toNativeServerToolUseId(toolUseId), + id: toNativeHostedToolUseId(toolUseId), name: WEB_SEARCH_TOOL_NAME, input: { query }, }); @@ -213,12 +213,12 @@ const buildNativeWebSearchResultBlock = (result: Extract => { - const pairedServerToolUseIds = new Set(content.flatMap(block => (block.type === 'server_tool_use' && block.name === WEB_SEARCH_TOOL_NAME ? [block.id] : []))); - const ownedReplayResultsByServerToolUseId = new Map(); +const collectOwnedReplayResultsByHostedToolUseId = (content: AnthropicMessagesAssistantInputContentBlock[]): Map => { + const pairedHostedToolUseIds = new Set(content.flatMap(block => (block.type === 'server_tool_use' && block.name === WEB_SEARCH_TOOL_NAME ? [block.id] : []))); + const ownedReplayResultsByHostedToolUseId = new Map(); for (const block of content) { - if (block.type !== 'web_search_tool_result' || !pairedServerToolUseIds.has(block.tool_use_id)) { + if (block.type !== 'web_search_tool_result' || !pairedHostedToolUseIds.has(block.tool_use_id)) { continue; } @@ -227,10 +227,10 @@ const collectOwnedReplayResultsByServerToolUseId = (content: AnthropicMessagesAs continue; } - ownedReplayResultsByServerToolUseId.set(block.tool_use_id, ownedReplayResult); + ownedReplayResultsByHostedToolUseId.set(block.tool_use_id, ownedReplayResult); } - return ownedReplayResultsByServerToolUseId; + return ownedReplayResultsByHostedToolUseId; }; const messageHasOwnedReplayMarkers = (message: AnthropicMessagesMessage): boolean => { @@ -239,7 +239,7 @@ const messageHasOwnedReplayMarkers = (message: AnthropicMessagesMessage): boolea } return ( - collectOwnedReplayResultsByServerToolUseId(message.content).size > 0 || + collectOwnedReplayResultsByHostedToolUseId(message.content).size > 0 || message.content.some(block => { if (block.type !== 'text' || !block.citations) { return false; @@ -372,7 +372,7 @@ const prepareAnthropicMessagesWebSearchReplay = (messages: AnthropicMessagesMess // System messages with array content pass through unchanged: the // remaining rewrite path below assumes assistant-shape content // (server_tool_use, web_search_tool_result, citations) and finalizes - // with `role: 'assistant'`, which would silently corrupt a + // with `role: 'assistant'`, which would silently corrupt an // AnthropicMessagesSystemMessage carrying AnthropicMessagesTextBlock[] into an assistant // turn. System messages never own web-search replay markers. if (message.role === 'system') { @@ -380,15 +380,15 @@ const prepareAnthropicMessagesWebSearchReplay = (messages: AnthropicMessagesMess continue; } - const ownedReplayResultsByServerToolUseId = collectOwnedReplayResultsByServerToolUseId(message.content); + const ownedReplayResultsByHostedToolUseId = collectOwnedReplayResultsByHostedToolUseId(message.content); - for (const ownedReplayResult of ownedReplayResultsByServerToolUseId.values()) { + for (const ownedReplayResult of ownedReplayResultsByHostedToolUseId.values()) { priorSearchUseCount += 1; pendingOwnedReplayToolResults.push(ownedReplayResult); } const rewrittenContent = message.content.flatMap((block): AnthropicMessagesAssistantInputContentBlock[] => { - if (block.type === 'server_tool_use' && ownedReplayResultsByServerToolUseId.has(block.id)) { + if (block.type === 'server_tool_use' && ownedReplayResultsByHostedToolUseId.has(block.id)) { return [ { type: 'tool_use', @@ -399,7 +399,7 @@ const prepareAnthropicMessagesWebSearchReplay = (messages: AnthropicMessagesMess ]; } - if (block.type === 'web_search_tool_result' && ownedReplayResultsByServerToolUseId.has(block.tool_use_id)) { + if (block.type === 'web_search_tool_result' && ownedReplayResultsByHostedToolUseId.has(block.tool_use_id)) { return []; } @@ -556,7 +556,7 @@ const buildNativeWebSearchResultBlockFromProviderResult = (result: WebSearchProv return { type: 'web_search_tool_result', - tool_use_id: toNativeServerToolUseId(toolUseId), + tool_use_id: toNativeHostedToolUseId(toolUseId), content: result.results.map(buildNativeWebSearchResultBlock), caller: { type: 'direct' }, }; @@ -571,7 +571,7 @@ type ActiveBlock = | { kind: 'web-search-tool-use'; upstreamToolUseId: string; - serverToolUseIndex: number; + hostedToolUseIndex: number; resultIndex: number; inputJson: string; }; @@ -631,12 +631,13 @@ const rewriteContentBlockDeltaCitations = ( // Synthesised events use the canonical Anthropic Messages SSE shape for `server_tool_use` // and `web_search_tool_result` blocks (input baked into the start event, no // `input_json_delta`) so downstream clients see the same bytes Anthropic would -// emit for native server tools. +// emit for native hosted tools. const runWebSearchStopHandler = async function* ( block: Extract, shimState: ShimStreamingState, state: Extract, provider: ActiveAnthropicMessagesWebSearchProvider, + reportFailure: (error: unknown) => Promise, ): AsyncGenerator> { const parsedInput = (() => { if (block.inputJson === '') return null; @@ -654,10 +655,10 @@ const runWebSearchStopHandler = async function* ( yield eventFrame({ type: 'content_block_start', - index: block.serverToolUseIndex, - content_block: buildNativeWebSearchServerToolUseBlock(block.upstreamToolUseId, query ?? ''), + index: block.hostedToolUseIndex, + content_block: buildNativeWebSearchHostedToolUseBlock(block.upstreamToolUseId, query ?? ''), }); - yield eventFrame({ type: 'content_block_stop', index: block.serverToolUseIndex }); + yield eventFrame({ type: 'content_block_stop', index: block.hostedToolUseIndex }); const resultBlock = await (async () => { if (state.maxUses !== undefined && shimState.currentSearchUseCount >= state.maxUses) { @@ -684,8 +685,8 @@ const runWebSearchStopHandler = async function* ( }; const providerResult = await runWebSearchAndRecordUsage({ provider: provider.impl, providerName: provider.providerName, keyId: provider.apiKeyId, request }); return buildNativeWebSearchResultBlockFromProviderResult(providerResult, block.upstreamToolUseId); - } catch { - // TODO: Add gateway-side recent web-search error-log storage so operators can inspect detailed provider/runtime failures even though the client-visible native error intentionally collapses them to `unavailable`. + } catch (error) { + await reportFailure(error); return buildNativeWebSearchErrorResultBlock(block.upstreamToolUseId, 'unavailable'); } })(); @@ -707,7 +708,8 @@ const runWebSearchStopHandler = async function* ( export const rewriteAnthropicMessagesWebSearchEventsToNative = async function* ( frames: AsyncIterable>, state: AnthropicMessagesWebSearchShimState, - provider?: ActiveAnthropicMessagesWebSearchProvider, + provider: ActiveAnthropicMessagesWebSearchProvider | undefined, + reportFailure: (error: unknown) => Promise, ): AsyncGenerator> { if (state.mode === 'inactive') { yield* frames; @@ -747,7 +749,7 @@ export const rewriteAnthropicMessagesWebSearchEventsToNative = async function* ( activeBlock = { kind: 'web-search-tool-use', upstreamToolUseId: event.content_block.id, - serverToolUseIndex: downstreamBase, + hostedToolUseIndex: downstreamBase, resultIndex: downstreamBase + 1, inputJson: '', }; @@ -800,7 +802,7 @@ export const rewriteAnthropicMessagesWebSearchEventsToNative = async function* ( throw new Error('web-search shim entered intercept path without active state.'); } - yield* runWebSearchStopHandler(activeBlock, shimState, state, provider!); + yield* runWebSearchStopHandler(activeBlock, shimState, state, provider!, reportFailure); activeBlock = undefined; continue; } @@ -839,7 +841,10 @@ export const rewriteAnthropicMessagesWebSearchEventsToNative = async function* ( } }; -const anthropicMessagesWebSearchInvalidRequestBody = (message: string) => ({ +/** The body Anthropic Messages states an unusable tool declaration with. A refusal the gateway + * writes is a refusal in this protocol's own words, which is what makes it readable by the + * client that sent the declaration. */ +export const anthropicMessagesWebSearchInvalidRequestBody = (message: string): Record => ({ type: 'error', error: { type: 'invalid_request_error', @@ -847,41 +852,23 @@ const anthropicMessagesWebSearchInvalidRequestBody = (message: string) => ({ }, }); -const buildSyntheticInvalidRequestUpstreamError = (message: string) => ({ - type: 'api-error' as const, - source: 'gateway' as const, - status: 400, - headers: new Headers({ 'content-type': 'application/json' }), - body: new TextEncoder().encode(JSON.stringify(anthropicMessagesWebSearchInvalidRequestBody(message))), -}); - -const buildInvalidRequestResponse = (message: string): Response => - Response.json(anthropicMessagesWebSearchInvalidRequestBody(message), { status: 400 }); - -const resolveActiveAnthropicMessagesWebSearchProvider = async (apiKeyId: string): Promise<{ type: 'ok'; provider: ActiveAnthropicMessagesWebSearchProvider } | ReturnType> => { +/** + * The backend a search that will actually run needs. + * + * An operator who configured none has made this turn unanswerable, and that is this gateway's + * own fault rather than a refusal to fail over — so it raises, with the sentence naming which + * of the two states the configuration is in. + */ +export const resolveActiveAnthropicMessagesWebSearchProvider = async (apiKeyId: string): Promise => { const webSearchConfig = await loadWebSearchConfig(); const configuredProvider = resolveConfiguredWebSearchProvider(webSearchConfig); - if (configuredProvider.type === 'enabled') { - return { - type: 'ok', - provider: { - providerName: configuredProvider.provider, - impl: configuredProvider.impl, - apiKeyId, - }, - }; + return { providerName: configuredProvider.provider, impl: configuredProvider.impl, apiKeyId }; } - - return internalErrorResult( - 500, - toInternalDebugError( - new Error( - configuredProvider.type === 'disabled' - ? 'Native Anthropic Messages web search requires an enabled search provider.' - : `Native Anthropic Messages web search is missing the configured ${configuredProvider.provider} credential.`, - ), - ), + throw new Error( + configuredProvider.type === 'disabled' + ? 'Native Anthropic Messages web search requires an enabled search provider.' + : `Native Anthropic Messages web search is missing the configured ${configuredProvider.provider} credential.`, ); }; @@ -892,7 +879,7 @@ type PrepareAnthropicMessagesWebSearchInvocationResult = | { type: 'invalid-request'; message: string } | { type: 'prepared'; state: PreparedAnthropicMessagesWebSearchShimState }; -const prepareAnthropicMessagesWebSearchInvocation = (ctx: AnthropicMessagesInvocation): PrepareAnthropicMessagesWebSearchInvocationResult => { +export const prepareAnthropicMessagesWebSearchInvocation = (ctx: AnthropicMessagesInvocation): PrepareAnthropicMessagesWebSearchInvocationResult => { if (ctx.targetApi === 'anthropicMessages' && !providerModelOf(ctx.candidate).enabledFlags.has('anthropic-messages-web-search-shim')) { return { type: 'inactive' }; } @@ -906,39 +893,15 @@ const prepareAnthropicMessagesWebSearchInvocation = (ctx: AnthropicMessagesInvoc }; /** - * Anthropic exposes native `web_search_*` server tools, but non-Anthropic-Messages - * targets cannot run Anthropic server tools. This shim rewrites the native tool + * Anthropic exposes native `web_search_*` hosted tools, but non-Anthropic-Messages + * targets cannot run Anthropic hosted tools. This shim rewrites the native tool * definition into an ordinary client `web_search` tool, executes each search * the model issues using the gateway's configured provider, and rewrites the * response back to the Anthropic native `server_tool_use` / * `web_search_tool_result` / `web_search_result_location` shape. * * The shim is unconditional for non-native Anthropic Messages targets (OpenAI Responses / - * OpenAI Chat Completions cannot carry Anthropic server tools), and gated by the + * OpenAI Chat Completions cannot carry Anthropic hosted tools), and gated by the * `anthropic-messages-web-search-shim` flag for native Anthropic Messages targets (the upstream * may or may not be able to serve web_search natively). */ -export const withAnthropicMessagesWebSearchShim: AnthropicMessagesInterceptor = async (ctx, gatewayCtx, run) => { - const prepared = prepareAnthropicMessagesWebSearchInvocation(ctx); - if (prepared.type === 'inactive') return await run(); - if (prepared.type === 'invalid-request') return buildSyntheticInvalidRequestUpstreamError(prepared.message); - - const provider = prepared.state.mode === 'active' ? await resolveActiveAnthropicMessagesWebSearchProvider(gatewayCtx.apiKeyId) : { type: 'ok' as const, provider: undefined }; - if (provider.type !== 'ok') return provider; - - const result = await run(); - if (result.type !== 'events') return result; - - return { - ...result, - events: rewriteAnthropicMessagesWebSearchEventsToNative(result.events, prepared.state, provider.provider), - }; -}; - -// count_tokens has no stream to rewrite, but it must measure the same client- -// tool and replay-history request shape that generation sends upstream. -export const withAnthropicMessagesWebSearchRequestPrepared: AnthropicMessagesCountTokensInterceptor = async (ctx, _gatewayCtx, run) => { - const prepared = prepareAnthropicMessagesWebSearchInvocation(ctx); - if (prepared.type === 'invalid-request') return buildInvalidRequestResponse(prepared.message); - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts new file mode 100644 index 0000000000..f348fe75d0 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/web-search-tool.ts @@ -0,0 +1,128 @@ +// Anthropic's web-search hosted tool, as a stage. +// +// Anthropic exposes `web_search_*` as native hosted tools. An upstream that cannot serve them — +// every non-Anthropic-Messages target, and a native one whose operator says so — gets the tool +// rewritten into an ordinary client tool on the way down, each search the model issues executed +// by this gateway, and the answer rewritten back into Anthropic's own +// `server_tool_use` / `web_search_tool_result` shape on the way up. +// +// One descent, unlike the OpenAI Responses hosted-tool loop: Anthropic's protocol carries the tool result +// inside the same turn, so there is nothing to ask again for. What this stage is, then, is a +// request rewrite and a response rewrite around one dial. + +import type { Fields } from './facts.ts'; +import { + anthropicMessagesWebSearchInvalidRequestBody, + prepareAnthropicMessagesWebSearchInvocation, + resolveActiveAnthropicMessagesWebSearchProvider, + rewriteAnthropicMessagesWebSearchEventsToNative, +} from './web-search-shim.ts'; +import type { Failure } from '../../pipeline/facts.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import type { ChatAnswer, ChatFacts } from '../facts.ts'; +import type { ChatServices } from '../services.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import type { ChatTargetApi, ModelCandidate } from '@floway-dev/provider'; + +type M = { [P in K]: ChatFacts[P] }; + +type Answered = Fields<'response.chat.anthropicMessages' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status' | 'response.chat.anthropicMessages.streamedUsage'> & Record; + +/** The gateway's own refusal, in Anthropic's words. It is written here rather than dialled, + * because a tool declaration this gateway cannot execute would reach the upstream as a body this + * gateway wrote. The envelope is stated as itself: a refusal this gateway authored has no + * upstream bytes behind it, so there is nothing to serialize and read back. */ +const refusal = (message: string): Failure => ({ + status: 400, + message, + envelope: anthropicMessagesWebSearchInvalidRequestBody(message), +}); + +/** What the stage is told rather than reaching for: which wire this candidate is reachable on, + * which is what decides whether the upstream can carry Anthropic's hosted tools at all. */ +export interface WebSearchWiring { + readonly targetOf: (candidate: ModelCandidate) => ChatTargetApi; +} + +export const runAnthropicMessagesWebSearchTool = (wiring: WebSearchWiring) => defineStage< + M<'request.chat.anthropicMessages' | 'route.attempt'>, + M<'request.chat.anthropicMessages' | 'route.attempt'>, + Answered, + Answered, + Answered, + ChatServices +>({ + name: 'runAnthropicMessagesWebSearchTool', + through: { + request: { + needs: ['request.chat.anthropicMessages', 'route.attempt'], + consumes: [], + provides: ['request.chat.anthropicMessages'], + }, + response: { + needs: ['response.chat.anthropicMessages'], + consumes: [], + provides: ['response.chat.anthropicMessages'], + }, + }, + // A declaration this gateway cannot execute, and a search backend the operator has not + // configured, are both answered here: neither is a body an upstream should be asked about. + return: { + provides: ['response.chat.anthropicMessages', 'response.usage.billable', 'response.http.headers', 'response.http.body', 'response.http.status', 'response.chat.anthropicMessages.streamedUsage'], + }, + execute: async (facts, next, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + // The stage's own view of the turn, which is what the preparation reads and shapes. It is + // built here and handed nowhere, so the payload it ends up holding is what descends. + const invocation = { + payload: facts['request.chat.anthropicMessages'], + candidate, + targetApi: wiring.targetOf(candidate), + headers: new Headers(), + }; + + const prepared = prepareAnthropicMessagesWebSearchInvocation(invocation); + if (prepared.type === 'inactive') return await next(facts); + if (prepared.type === 'invalid-request') { + await use.log.debug('refusing a web-search declaration this gateway cannot execute'); + return move({ + ...facts, + 'response.chat.anthropicMessages': refusal(prepared.message), + 'response.chat.anthropicMessages.streamedUsage': null, + 'response.usage.billable': [], + 'response.http.headers': [], + 'response.http.body': null, 'response.http.status': 400, + }) as never; + } + + // A search that will actually run needs a backend; one the operator never configured + // raises, because a turn nobody can answer is this gateway's own fault to surface. + const backend = prepared.state.mode === 'active' + ? await resolveActiveAnthropicMessagesWebSearchProvider(use.gateway.apiKeyId) + : undefined; + + const back = await next(move({ + ...facts, + 'request.chat.anthropicMessages': move(invocation.payload as AnthropicMessagesPayload), + })) as Answered; + + const answer = back['response.chat.anthropicMessages'] as ChatAnswer; + if (isFailure(answer) || answer.kind !== 'stream') return back as never; + return { + ...back, + 'response.chat.anthropicMessages': move({ + kind: 'stream' as const, + frames: { + [Symbol.asyncIterator]: () => rewriteAnthropicMessagesWebSearchEventsToNative( + answer.frames as AsyncIterable>, + prepared.state, + backend, + async error => { await use.log.error('web search failed', { error }); }, + ), + }, + }), + } as never; + }, +}); diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/wire.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/wire.ts new file mode 100644 index 0000000000..1cf19be292 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/wire.ts @@ -0,0 +1,28 @@ +import { applyRoleCompatibilityToAnthropicMessages } from './apply-role-compatibility.ts'; +import { callAnthropicMessagesUpstream } from './call-upstream.ts'; +import { disableReasoningOnForcedToolChoiceForAnthropicMessages } from './disable-reasoning-on-forced-tool-choice.ts'; +import { meterUsage } from './meter.ts'; +import { normalizeEmptyToolsForAnthropicMessages } from './normalize-empty-tools-tool-choice.ts'; +import { stripBillingAttributionFromAnthropicMessages } from './strip-billing-attribution.ts'; +import type { Stage } from '@floway-dev/pipeline'; + +/** + * The Anthropic Messages wire, as the chain that dials it. + * + * Every source protocol that reaches an upstream over this endpoint runs this, whether the + * client spoke Anthropic Messages or a handoff arrived here — which is what makes a rule that + * speaks about *this* wire belong here. The system-role rewrite states what an upstream's Anthropic Messages + * endpoint accepts, so it applies to whatever body this wire actually sends and to nothing + * that leaves for another protocol. + * + * The meter is above every one of them, which is what makes the figure it bills the one the + * client is shown. + */ +export const anthropicMessagesWire = (streamedUsage: string): readonly Stage[] => [ + normalizeEmptyToolsForAnthropicMessages, + meterUsage(streamedUsage), + stripBillingAttributionFromAnthropicMessages, + disableReasoningOnForcedToolChoiceForAnthropicMessages, + applyRoleCompatibilityToAnthropicMessages, + callAnthropicMessagesUpstream, +]; diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/wires.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/wires.ts new file mode 100644 index 0000000000..7756e88de3 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/wires.ts @@ -0,0 +1,38 @@ +import { composeChat as compose } from '../compose.ts'; +import type { ChatWire } from '../wire.ts'; +import { STREAMED_USAGE } from './facts.ts'; +import { anthropicMessagesWire } from './wire.ts'; +import { handOff } from '../handoff.ts'; +import { openaiChatCompletionsWire } from '../openai-chat-completions/wire.ts'; +import { projectOpenAIResponsesCollaboration } from '../openai-responses/collaboration-shim.ts'; +import { openaiResponsesWire } from '../openai-responses/wire.ts'; +import type { ChatTargetApi, ModelCandidate } from '@floway-dev/provider'; +import { translateAnthropicMessagesViaOpenAIResponses, translateAnthropicMessagesViaOpenAIChatCompletions } from '@floway-dev/translate'; + +/** The three wires `/v1/messages` can be served on. Its own is the bare wire; each translated + * one is a handoff and then the target protocol's own wire. */ +export const anthropicMessagesWireFor = (target: ChatTargetApi, candidate: ModelCandidate): ChatWire => { + switch (target) { + case 'anthropicMessages': + return compose('anthropicMessagesNative', anthropicMessagesWire(STREAMED_USAGE)); + case 'openaiResponses': + return compose('anthropicMessagesViaOpenAIResponses', [ + handOff({ + from: { request: 'request.chat.anthropicMessages', response: 'response.chat.anthropicMessages' }, + to: { request: 'request.chat.openaiResponses', response: 'response.chat.openaiResponses' }, + trip: async payload => await translateAnthropicMessagesViaOpenAIResponses(payload, { model: candidate.model.id }), + }), + projectOpenAIResponsesCollaboration, + ...openaiResponsesWire(STREAMED_USAGE), + ]); + case 'openaiChatCompletions': + return compose('anthropicMessagesViaOpenAIChatCompletions', [ + handOff({ + from: { request: 'request.chat.anthropicMessages', response: 'response.chat.anthropicMessages' }, + to: { request: 'request.chat.openaiChatCompletions', response: 'response.chat.openaiChatCompletions' }, + trip: async payload => await translateAnthropicMessagesViaOpenAIChatCompletions(payload, { model: candidate.model.id }), + }), + ...openaiChatCompletionsWire(STREAMED_USAGE), + ]); + } +}; diff --git a/packages/gateway/src/data-plane/chat/compose.ts b/packages/gateway/src/data-plane/chat/compose.ts new file mode 100644 index 0000000000..05a695b9d1 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/compose.ts @@ -0,0 +1,41 @@ +import type { ChatAnswer } from './facts.ts'; +import type { ChatServices } from './services.ts'; +import { isFailure } from '../pipeline/facts.ts'; +import { recordStream } from '@floway-dev/dump'; +import { compose, isStreamFact, move, type Stage, type Use } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; + +const answerKeys = new Set([ + 'response.chat.openaiChatCompletions', + 'response.chat.openaiResponses', + 'response.chat.anthropicMessages', + 'response.chat.geminiGenerateContent', +]); + +// Record at publication so a downstream translation cannot consume the only reference to +// the upstream protocol. Observers retain the reference; content rewrites publish a new one. +const recordingStage = (stage: Stage): Stage => { + const keys = [...new Set([ + ...stage.return?.provides ?? [], + ...(stage.through ?? stage.into)?.response.provides ?? [], + ])].filter(key => answerKeys.has(key)); + if (keys.length === 0) return stage; + return { + ...stage, + execute: async (facts, ...args) => { + const back = await stage.execute(facts, ...args); + const use = args[args.length - 1] as Use; + if (use.gateway.dump === null) return back; + let recorded = back; + for (const key of keys) { + if (!(key in back)) continue; + const answer = back[key] as ChatAnswer; + if (isFailure(answer) || answer.kind !== 'stream' || isStreamFact(answer.frames)) continue; + recorded = { ...recorded, [key]: move({ ...answer, frames: recordStream(answer.frames as AsyncIterable>, use.gateway.dump) }) }; + } + return recorded; + }, + }; +}; + +export const composeChat: typeof compose = (name, stages) => compose(name, stages.map(recordingStage)); diff --git a/packages/gateway/src/data-plane/chat/dial-wire.ts b/packages/gateway/src/data-plane/chat/dial-wire.ts new file mode 100644 index 0000000000..6dc22d376a --- /dev/null +++ b/packages/gateway/src/data-plane/chat/dial-wire.ts @@ -0,0 +1,57 @@ +import type { ChatServices } from './services.ts'; +import type { RequestKey, ChatWire } from './wire.ts'; +import type { AttemptSelector } from '../pipeline/facts.ts'; +import { defineStage, type Use } from '@floway-dev/pipeline'; +import type { ModelEndpoints } from '@floway-dev/protocols/common'; +import type { ChatTargetApi, ModelCandidate } from '@floway-dev/provider'; + +/** How a source protocol reaches an upstream. */ +export interface ChatWiring { + /** The request key of the protocol the client spoke. It names the stage, so a dump says + * which chain forked without the name having to be written twice. */ + readonly source: RequestKey; + /** What the chain below this stage reads. A wire is built against the candidate it will + * dial, so assembly cannot ask one what it needs — the family says it here, and that is + * what puts those keys in the serve pipeline's entry contract. */ + readonly needs: readonly string[]; + /** What comes back through here, whichever wire ran. Every wire hands up this family's own + * keys, which is what makes them interchangeable. */ + readonly provides: readonly string[]; + /** Which wire this candidate is reachable on. Total by contract: serve narrowed the + * candidates with the same picker's `canServe`, so a candidate that reached here has one + * of the endpoints the preference list names. */ + readonly pick: (endpoints: ModelEndpoints) => ChatTargetApi; + /** The chain for one wire, built against the candidate that will be dialled — which is what + * lets a translation close over the model it is translating for. */ + readonly wire: (target: ChatTargetApi, candidate: ModelCandidate, use: Use) => ChatWire; +} + +/** + * Picks this candidate's wire and hands into the chain for it. + * + * It is last, which is what earns it the right to name a target at all, and it holds no + * state across candidates: what re-decides for the next candidate is failover re-running the + * suffix, which re-runs this stage. + */ +export const dialChatWire = (wiring: ChatWiring) => defineStage< + { 'route.attempt': AttemptSelector } & Record, + Record, + Record, + Record, + ChatServices +>({ + name: `dialChatWire:${wiring.source}`, + into: { + request: { needs: ['route.attempt', ...wiring.needs], consumes: [], provides: [] }, + // Nothing is read on the way back — a wire hands up this family's own keys and they ride + // through — but this is where they enter the chain, so this is the stage that provides + // them and the runner checks that the wire delivered. + response: { needs: [], consumes: [], provides: ['response.http.status', 'response.http.body', ...wiring.provides] }, + }, + execute: async (facts, next, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + const target = wiring.pick(candidate.model.endpoints); + await use.log.debug('dialling', { upstream: facts['route.attempt'].upstreamId, wire: target }); + return await next(facts, wiring.wire(target, candidate, use)); + }, +}); diff --git a/packages/gateway/src/data-plane/chat/facts.ts b/packages/gateway/src/data-plane/chat/facts.ts new file mode 100644 index 0000000000..8eb90606ad --- /dev/null +++ b/packages/gateway/src/data-plane/chat/facts.ts @@ -0,0 +1,70 @@ +// The chat family's fact space. Four source protocols, ten translation directions between +// them, and one shared array of stages that every chain runs. +// +// Two things the key layout says that the old context object could not. Each protocol has +// its own request key and its own response key, so a translation *consumes* the source and +// *provides* the target, and which protocol a stage is looking at is a declared need rather +// than an ambient field on a mutable context. That is visibility and checkability, not the +// deletion of the guards themselves — each is still made, by position where it is about a wire +// and by this key where it is about what the client spoke. And `ingress.*` is what the client asked for: it survives +// the switch to whatever protocol the upstream turned out to speak, which is why +// `wantsStream` is four disjoint keys rather than one shared one. + +import type { Failure, GatewayFacts } from '../pipeline/facts.ts'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import type { GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; +import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; +import type { OpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; + +/** The four protocols a client can speak here. Gemini generateContent is source-only: nothing translates + * into it, so by the ruling it contributes one pipeline rather than two — a source role + * and no target role. */ +export type ChatSourceProtocol = 'openaiChatCompletions' | 'anthropicMessages' | 'openaiResponses' | 'geminiGenerateContent'; + +export type ChatRequestKey = 'request.chat.openaiChatCompletions' | 'request.chat.anthropicMessages' | 'request.chat.openaiResponses' | 'request.chat.geminiGenerateContent'; + +export interface ChatFacts extends GatewayFacts { + /** Which protocol the client spoke. Read by the stages that must know — a vendor + * normalizer genuinely does — rather than reached for through an ambient field. + * + * This is what `ctx.targetApi` was, read through `needs` rather than reached for. A stage + * that must not run on a request which re-entered its protocol still says so itself, by + * needing this key. What changed is that the thing it tests is declared. */ + 'ingress.chat.sourceProtocol': ChatSourceProtocol; + /** What the client asked for, per protocol and disjoint, because a translated request + * must not inherit the target protocol's answer to a question the client never asked. */ + 'ingress.chat.openaiChatCompletions.wantsStream': boolean; + 'ingress.chat.anthropicMessages.wantsStream': boolean; + 'ingress.chat.openaiResponses.wantsStream': boolean; + 'ingress.chat.geminiGenerateContent.wantsStream': boolean; + /** Whether the client wants the usage chunk it would otherwise never see. The gateway + * always asks the upstream for one; this decides who is shown it. */ + 'ingress.chat.openaiChatCompletions.wantsUsageChunk': boolean; + + 'request.chat.candidatePayloads': Readonly>; + + /** One key per protocol. A translation consumes one and provides another, which is the + * whole of what a protocol handoff is. */ + 'request.chat.openaiChatCompletions': OpenAIChatCompletionsPayload; + 'request.chat.anthropicMessages': AnthropicMessagesPayload; + 'request.chat.openaiResponses': OpenAIResponsesPayload; + 'request.chat.geminiGenerateContent': GeminiGenerateContentPayload; + + /** A stream, a value and a failure sit at one key, so telling them apart is reading a + * value and no declaration ever mentions which arm is there. */ + 'response.chat.clientFrames': AsyncIterable | null; + 'response.chat.openaiChatCompletions': ChatAnswer; + 'response.chat.anthropicMessages': ChatAnswer; + 'response.chat.openaiResponses': ChatAnswer; + 'response.chat.geminiGenerateContent': ChatAnswer; +} + +/** What comes back at a protocol's response key. The frame type is the protocol's own; the + * three arms are the same shape for every protocol, which is why no stage dispatches on + * the arm in its declaration. */ +export type ChatAnswer = + | { readonly kind: 'stream'; readonly frames: AsyncIterable } + | { readonly kind: 'value'; readonly body: unknown } + | Failure; + +export type Chat = { [P in K]: ChatFacts[P] }; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/affinity/egress.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/affinity/egress.ts index 919b1eee5c..2275a66b2d 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/affinity/egress.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/affinity/egress.ts @@ -1,5 +1,5 @@ import type { AffinityEgressOptions } from '../../shared/affinity/index.ts'; -import { captureExtras, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; +import { captureExtras, eventFrame, withIndexesChanged, withKeysChanged, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { GeminiGenerateContentCandidate, GeminiGenerateContentPart, GeminiGenerateContentResult, GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; import { GEMINI_GENERATE_CONTENT_CANDIDATE_KEYS, GEMINI_GENERATE_CONTENT_RESULT_KEYS } from '@floway-dev/protocols/gemini-generate-content'; @@ -30,8 +30,12 @@ export const wrapGeminiGenerateContentAffinityEgress = async function* ( options: AffinityEgressOptions, ): AsyncGenerator> { const anchoredCandidates = new Set(); - const suppressedEvents = new WeakSet(); - let pending: GeminiGenerateContentResult | undefined; + // The one event held back, and what the pass that produced it decided about emitting it. + // Both used to be recorded against the event object — the verdict in a `WeakSet`, the + // rewrites by assignment — which only works while an event can be written in place. Nothing + // here is written in place any more, so a pass hands back new events and the verdict travels + // beside the one it is about rather than being looked up by identity. + let pending: PendingEvent | undefined; const iterator = frames[Symbol.asyncIterator](); let sourceCompleted = false; @@ -42,15 +46,8 @@ export const wrapGeminiGenerateContentAffinityEgress = async function* ( result = await iterator.next(); } catch (error) { if (pending !== undefined) { - const wrapped = await wrapGeminiGenerateContentEventAffinity( - pending, - undefined, - anchoredCandidates, - suppressedEvents, - options, - false, - ); - if (!suppressedEvents.has(wrapped)) yield eventFrame(wrapped); + const flushed = await flushPending(pending, anchoredCandidates, options, false); + if (flushed !== undefined) yield eventFrame(flushed); pending = undefined; } throw error; @@ -62,15 +59,8 @@ export const wrapGeminiGenerateContentAffinityEgress = async function* ( const frame = result.value; if (frame.type !== 'event') { if (pending !== undefined) { - const wrapped = await wrapGeminiGenerateContentEventAffinity( - pending, - undefined, - anchoredCandidates, - suppressedEvents, - options, - frame.type === 'done', - ); - if (!suppressedEvents.has(wrapped)) yield eventFrame(wrapped); + const flushed = await flushPending(pending, anchoredCandidates, options, frame.type === 'done'); + if (flushed !== undefined) yield eventFrame(flushed); pending = undefined; } yield frame; @@ -78,72 +68,118 @@ export const wrapGeminiGenerateContentAffinityEgress = async function* ( } if ('error' in frame.event) { if (pending !== undefined) { - const wrapped = await wrapGeminiGenerateContentEventAffinity( - pending, - undefined, - anchoredCandidates, - suppressedEvents, - options, - false, - ); - if (!suppressedEvents.has(wrapped)) yield eventFrame(wrapped); + const flushed = await flushPending(pending, anchoredCandidates, options, false); + if (flushed !== undefined) yield eventFrame(flushed); } yield frame; return; } - const next = cloneGeminiGenerateContentEvent(frame.event); - if (pending !== undefined) { - const wrapped = await wrapGeminiGenerateContentEventAffinity( - pending, - next, - anchoredCandidates, - suppressedEvents, - options, - false, - ); - if (!suppressedEvents.has(wrapped)) yield eventFrame(wrapped); + if (pending === undefined) { + pending = { event: frame.event, suppressed: false }; + continue; } - pending = next; + const pass = await passOverEventPair(pending, frame.event, anchoredCandidates, options, false); + if (!pass.current.suppressed) yield eventFrame(pass.current.event); + pending = pass.next; } } finally { if (!sourceCompleted) await iterator.return?.(); } if (pending !== undefined) { - const wrapped = await wrapGeminiGenerateContentEventAffinity( - pending, - undefined, - anchoredCandidates, - suppressedEvents, - options, - false, - ); - if (!suppressedEvents.has(wrapped)) yield eventFrame(wrapped); + const flushed = await flushPending(pending, anchoredCandidates, options, false); + if (flushed !== undefined) yield eventFrame(flushed); } }; -const cloneGeminiGenerateContentEvent = (event: GeminiGenerateContentResult): GeminiGenerateContentResult => structuredClone(event); +/** One event, and whether the pass that produced it already decided it is not to be emitted. */ +interface PendingEvent { + readonly event: GeminiGenerateContentResult; + readonly suppressed: boolean; +} + +/** The last pass an event gets: there is no successor, so nothing can move forward out of it. + * Returns what to emit, or nothing where the event was suppressed. */ +const flushPending = async ( + pending: PendingEvent, + anchoredCandidates: Set, + options: AffinityEgressOptions, + successfulTerminalBoundary: boolean, +): Promise => { + const pass = await passOverEventPair(pending, undefined, anchoredCandidates, options, successfulTerminalBoundary); + return pass.current.suppressed ? undefined : pass.current.event; +}; + +/** What one relocation hands on: the pair as it now stands, and whether either side was left + * with no Parts at all — which is the pair's own statement that the candidate should not be + * emitted. This used to be a `WeakSet` of the candidates a relocation emptied; a rebuilt + * candidate is a different object, so the verdict is carried rather than looked up. */ +interface CandidatePair { + readonly current: GeminiGenerateContentCandidate; + readonly next: GeminiGenerateContentCandidate | undefined; + readonly currentRemoved: boolean; + readonly nextRemoved: boolean; +} + +/** A candidate with different Parts, or the same candidate where they did not change. */ +const withParts = ( + candidate: GeminiGenerateContentCandidate, + parts: readonly GeminiGenerateContentPart[], +): GeminiGenerateContentCandidate => + parts === candidate.content.parts + ? candidate + : { ...candidate, content: { ...candidate.content, parts: parts as GeminiGenerateContentPart[] } }; + +/** An event with different candidates. An event that declared none keeps declaring none: the + * absent key and the empty array say different things to a client. */ +const withCandidates = ( + event: GeminiGenerateContentResult, + candidates: readonly GeminiGenerateContentCandidate[], +): GeminiGenerateContentResult => + event.candidates === undefined || candidates === event.candidates + ? event + : { ...event, candidates: candidates as GeminiGenerateContentCandidate[] }; -const wrapGeminiGenerateContentEventAffinity = async ( - current: GeminiGenerateContentResult, - next: GeminiGenerateContentResult | undefined, +/** + * One pass of the sliding window: the held event, the one behind it, and what becomes of both. + * + * Every rewrite here is a rebuild. A frame this transducer was handed belongs to the layer that + * produced it, and the events it hands on are new objects that share everything the pass did not + * touch — so a Part no rule spoke about is the same Part, and the pass costs what it changed + * rather than a copy of the event. + */ +const passOverEventPair = async ( + pending: PendingEvent, + nextEvent: GeminiGenerateContentResult | undefined, anchoredCandidates: Set, - suppressedEvents: WeakSet, options: AffinityEgressOptions, successfulTerminalBoundary: boolean, -): Promise => { +): Promise<{ readonly current: PendingEvent; readonly next: PendingEvent | undefined }> => { + const current = pending.event; const currentHadCandidates = (current.candidates?.length ?? 0) > 0; - const nextHadCandidates = (next?.candidates?.length ?? 0) > 0; - const removedCandidates = new WeakSet(); - - for (const candidate of current.candidates ?? []) { - const nextCandidate = next?.candidates?.find(nextCandidate => nextCandidate.index === candidate.index); - normalizeElementSignatures(candidate); - if (nextCandidate !== undefined) normalizeElementSignatures(nextCandidate); - relocateSignatureOnlyForward(candidate, nextCandidate, removedCandidates); - relocateSignatureOnlyBackward(candidate, nextCandidate, removedCandidates); - relocateContinuationSignature(candidate, nextCandidate); + const nextHadCandidates = (nextEvent?.candidates?.length ?? 0) > 0; + + // The two arrays under construction. Their elements are replaced, never written into: what + // goes back is whatever the rules built, and an untouched candidate goes back as itself. + const builtCurrent = [...(current.candidates ?? [])]; + const builtNext = nextEvent === undefined ? undefined : [...(nextEvent.candidates ?? [])]; + const removedCurrent = new Set(); + const removedNext = new Set(); + + for (let position = 0; position < builtCurrent.length; position += 1) { + const nextPosition = builtNext?.findIndex(entry => entry.index === builtCurrent[position].index) ?? -1; + const relocated = relocateContinuationSignature(relocateSignatureOnlyBackward(relocateSignatureOnlyForward({ + current: normalizeElementSignatures(builtCurrent[position]), + next: nextPosition >= 0 ? normalizeElementSignatures(builtNext![nextPosition]) : undefined, + currentRemoved: false, + nextRemoved: false, + }))); + let candidate = relocated.current; + const nextCandidate = relocated.next; + if (relocated.currentRemoved) removedCurrent.add(position); + if (relocated.nextRemoved && nextPosition >= 0) removedNext.add(nextPosition); + const firstIndexes = firstElementIndexes(candidate.content.parts); const firstHasNatural = firstIndexes.some(index => candidate.content.parts[index].thoughtSignature !== undefined); const signatureOnlyNatural = firstIndexes.length === 0 @@ -159,150 +195,192 @@ const wrapGeminiGenerateContentEventAffinity = async ( && candidate.content.parts.slice(lastFirstContentIndex + 1).some(hasPartContent); const firstElementClosed = startsAnotherElementInCurrentEvent || candidate.finishReason !== undefined - || (next !== undefined && nextCandidate === undefined) - || (nextCandidate !== undefined && nextContent === undefined && !removedCandidates.has(nextCandidate)) + || (nextEvent !== undefined && nextCandidate === undefined) + || (nextCandidate !== undefined && nextContent === undefined && !relocated.nextRemoved) || (nextCandidate !== undefined && nextContent !== undefined && !continuesInNextEvent) - || (next === undefined && successfulTerminalBoundary); + || (nextEvent === undefined && successfulTerminalBoundary); + const wrapped: GeminiGenerateContentPart[] = []; for (const part of candidate.content.parts) { - if (part.thoughtSignature === undefined) continue; - part.thoughtSignature = await options.codec.wrap(part.thoughtSignature, options.affinity, 'gemini-generate-content.part.thoughtSignature'); + wrapped.push(part.thoughtSignature === undefined ? part : { + ...part, + thoughtSignature: await options.codec.wrap(part.thoughtSignature, options.affinity, 'gemini-generate-content.part.thoughtSignature'), + }); } + candidate = withParts(candidate, wrapped); - if (anchoredCandidates.has(candidate.index)) continue; - if (firstHasNatural || signatureOnlyNatural) { - anchoredCandidates.add(candidate.index); - } else if (firstContentIndex !== undefined && firstElementClosed) { - candidate.content.parts[firstContentIndex] = { - ...candidate.content.parts[firstContentIndex], - thoughtSignature: await options.codec.wrap(undefined, options.affinity, 'gemini-generate-content.part.thoughtSignature'), - }; - anchoredCandidates.add(candidate.index); - } else if (candidate.finishReason !== undefined) { - candidate.content.parts.push({ - thoughtSignature: await options.codec.wrap( - undefined, - options.affinity, - 'gemini-generate-content.part.thoughtSignature', - ), - }); - anchoredCandidates.add(candidate.index); + if (!anchoredCandidates.has(candidate.index)) { + if (firstHasNatural || signatureOnlyNatural) { + anchoredCandidates.add(candidate.index); + } else if (firstContentIndex !== undefined && firstElementClosed) { + candidate = withParts(candidate, withIndexesChanged(candidate.content.parts, new Map([[firstContentIndex, { + ...candidate.content.parts[firstContentIndex], + thoughtSignature: await options.codec.wrap(undefined, options.affinity, 'gemini-generate-content.part.thoughtSignature'), + }]]))); + anchoredCandidates.add(candidate.index); + } else if (candidate.finishReason !== undefined) { + candidate = withParts(candidate, [...candidate.content.parts, { + thoughtSignature: await options.codec.wrap(undefined, options.affinity, 'gemini-generate-content.part.thoughtSignature'), + }]); + anchoredCandidates.add(candidate.index); + } } - } - if (next?.candidates !== undefined) { - next.candidates = next.candidates.filter(candidate => !removedCandidates.has(candidate)); - } - if (current.candidates !== undefined) { - current.candidates = current.candidates.filter(candidate => !removedCandidates.has(candidate)); + builtCurrent[position] = candidate; + if (nextPosition >= 0 && nextCandidate !== undefined) builtNext![nextPosition] = nextCandidate; } - const currentEmptied = currentHadCandidates && current.candidates?.length === 0; - const nextEmptied = nextHadCandidates && next?.candidates?.length === 0; - if (currentEmptied && next !== undefined && !nextEmptied) { - mergeEventMetadata(current, next, next); - clearEventMetadata(current); - suppressedEvents.add(current); - } else if (nextEmptied && next !== undefined && !currentEmptied) { - mergeEventMetadata(current, next, current); - clearEventMetadata(next); - suppressedEvents.add(next); + + const keptCurrent = builtCurrent.filter((_candidate, position) => !removedCurrent.has(position)); + const keptNext = builtNext?.filter((_candidate, position) => !removedNext.has(position)); + const currentEmptied = currentHadCandidates && keptCurrent.length === 0; + const nextEmptied = nextHadCandidates && keptNext?.length === 0; + + let currentOut = withCandidates(current, keptCurrent); + let nextOut = nextEvent === undefined ? undefined : withCandidates(nextEvent, keptNext ?? []); + let currentSuppressed = pending.suppressed; + let nextSuppressed = false; + + // An event whose every candidate moved into its neighbour has nothing left to say on its own, + // so what it was carrying beside them moves too and the event itself is not emitted. + if (currentEmptied && nextOut !== undefined && !nextEmptied) { + nextOut = mergedEventMetadata(currentOut, nextOut, nextOut); + currentOut = withoutEventMetadata(currentOut); + currentSuppressed = true; + } else if (nextEmptied && nextOut !== undefined && !currentEmptied) { + currentOut = mergedEventMetadata(currentOut, nextOut, currentOut); + nextOut = withoutEventMetadata(nextOut); + nextSuppressed = true; } - return current; + + return { + current: { event: currentOut, suppressed: currentSuppressed }, + next: nextOut === undefined ? undefined : { event: nextOut, suppressed: nextSuppressed }, + }; }; -const normalizeElementSignatures = (candidate: GeminiGenerateContentCandidate): void => { +/** Every signature a logical element carries, collapsed onto that element's first + * content-bearing Part, and the signature-only Parts that leaves behind dropped. */ +const normalizeElementSignatures = (candidate: GeminiGenerateContentCandidate): GeminiGenerateContentCandidate => { const parts = candidate.content.parts; - const signatureParts = new Set(parts.filter(part => part.thoughtSignature !== undefined)); - for (const indexes of logicalElementGroups(parts)) normalizeElementSignature(parts, indexes); - removeRelocatedSignatureParts(candidate, signatureParts); + const carriedSignature = new Set(parts.flatMap((part, index) => part.thoughtSignature === undefined ? [] : [index])); + // The groups are index lists read off the Parts as they arrived, and collapsing a signature + // never changes how many Parts there are, so they stay valid across the fold. + let folded: readonly GeminiGenerateContentPart[] = parts; + for (const indexes of logicalElementGroups(parts)) folded = normalizeElementSignature(folded, indexes); + return withParts(candidate, keptAfterRelocation(folded, carriedSignature)); }; -const normalizeElementSignature = (parts: GeminiGenerateContentPart[], indexes: readonly number[]): void => { +const normalizeElementSignature = ( + parts: readonly GeminiGenerateContentPart[], + indexes: readonly number[], +): readonly GeminiGenerateContentPart[] => { const signatures = indexes.flatMap(index => { const signature = parts[index].thoughtSignature; return signature === undefined ? [] : [signature]; }); - if (signatures.length === 0) return; + if (signatures.length === 0) return parts; const targetIndex = indexes.find(index => hasPartContent(parts[index])) ?? indexes.at(-1); if (targetIndex === undefined) throw new Error('Gemini generateContent signature group has no target Part'); - for (const index of indexes) delete parts[index].thoughtSignature; - parts[targetIndex].thoughtSignature = signatures[signatures.length - 1]; + const rewrites = new Map( + indexes.map(index => [index, withKeysChanged(parts[index], { thoughtSignature: undefined })]), + ); + rewrites.set(targetIndex, withKeysChanged(parts[targetIndex], { thoughtSignature: signatures[signatures.length - 1] })); + return parts.map((part, index) => rewrites.get(index) ?? part); }; -const relocateSignatureOnlyForward = ( - current: GeminiGenerateContentCandidate, - next: GeminiGenerateContentCandidate | undefined, - removedCandidates: WeakSet, -): void => { +const relocateSignatureOnlyForward = (pair: CandidatePair): CandidatePair => { + const { current, next } = pair; if ( next === undefined || current.finishReason !== undefined || current.content.parts.some(hasPartContent) - ) return; - const relocated = new Set(current.content.parts.filter(part => part.thoughtSignature !== undefined)); - const signature = relocated.values().next().value?.thoughtSignature; + ) return pair; + const relocated = new Set(current.content.parts.flatMap((part, index) => part.thoughtSignature === undefined ? [] : [index])); + const signature = current.content.parts.find(part => part.thoughtSignature !== undefined)?.thoughtSignature; const targetIndex = firstElementIndexes(next.content.parts).find(index => hasPartContent(next.content.parts[index])); - if (signature === undefined || targetIndex === undefined) return; - next.content.parts[targetIndex].thoughtSignature ??= signature; - for (const part of relocated) delete part.thoughtSignature; - removeRelocatedSignatureParts(current, relocated, removedCandidates); - if (current.content.parts.length === 0) transferCandidateMetadataForward(current, next); + if (signature === undefined || targetIndex === undefined) return pair; + + // A target that already carries one keeps it: this moves a signature into a gap, never over + // an upstream's own. + const carried = next.content.parts[targetIndex].thoughtSignature !== undefined ? next : withParts( + next, + withIndexesChanged(next.content.parts, new Map([[targetIndex, withKeysChanged(next.content.parts[targetIndex], { thoughtSignature: signature })]])), + ); + const strippedParts = keptAfterRelocation( + current.content.parts.map(part => part.thoughtSignature === undefined ? part : withKeysChanged(part, { thoughtSignature: undefined })), + relocated, + ); + const stripped = withParts(current, strippedParts); + return strippedParts.length > 0 + ? { ...pair, current: stripped, next: carried } + : { ...pair, current: stripped, next: withForwardedCandidateMetadata(stripped, carried), currentRemoved: true }; }; -const relocateSignatureOnlyBackward = ( - current: GeminiGenerateContentCandidate, - next: GeminiGenerateContentCandidate | undefined, - removedCandidates: WeakSet, -): void => { +const relocateSignatureOnlyBackward = (pair: CandidatePair): CandidatePair => { + const { current, next } = pair; if ( next === undefined || current.finishReason !== undefined || next.content.parts.some(hasPartContent) - ) return; + ) return pair; const targetIndex = firstContentIndexOfLastElement(current.content.parts); - if (targetIndex === undefined) return; - const signatureOnly = next.content.parts.find(part => part.thoughtSignature !== undefined); - if (signatureOnly?.thoughtSignature === undefined) return; + if (targetIndex === undefined) return pair; + const signatureIndex = next.content.parts.findIndex(part => part.thoughtSignature !== undefined); + const signature = signatureIndex < 0 ? undefined : next.content.parts[signatureIndex].thoughtSignature; + if (signature === undefined) return pair; - current.content.parts[targetIndex] = { - ...current.content.parts[targetIndex], - thoughtSignature: signatureOnly.thoughtSignature, - }; - delete signatureOnly.thoughtSignature; - removeRelocatedSignatureParts(next, new Set([signatureOnly]), removedCandidates); - if (next.content.parts.length === 0) { - transferCandidateMetadata(current, next); - delete next.finishReason; - } + const anchored = withParts( + current, + withIndexesChanged(current.content.parts, new Map([[targetIndex, withKeysChanged(current.content.parts[targetIndex], { thoughtSignature: signature })]])), + ); + const strippedParts = keptAfterRelocation( + withIndexesChanged(next.content.parts, new Map([[signatureIndex, withKeysChanged(next.content.parts[signatureIndex], { thoughtSignature: undefined })]])), + new Set([signatureIndex]), + ); + const stripped = withParts(next, strippedParts); + return strippedParts.length > 0 + ? { ...pair, current: anchored, next: stripped } + : { + ...pair, + current: withCandidateMetadataFrom(anchored, stripped), + next: withKeysChanged(stripped, { finishReason: undefined }), + nextRemoved: true, + }; }; -const relocateContinuationSignature = ( - current: GeminiGenerateContentCandidate, - next: GeminiGenerateContentCandidate | undefined, -): void => { - if (next === undefined || current.finishReason !== undefined) return; +const relocateContinuationSignature = (pair: CandidatePair): CandidatePair => { + const { current, next } = pair; + if (next === undefined || current.finishReason !== undefined) return pair; const targetIndex = firstContentIndexOfLastElement(current.content.parts); - const nextPart = firstContentPart(next.content.parts); + const nextIndex = next.content.parts.findIndex(hasPartContent); + const nextPart = nextIndex < 0 ? undefined : next.content.parts[nextIndex]; if ( targetIndex === undefined || nextPart?.thoughtSignature === undefined || !sameLogicalElement(current.content.parts[targetIndex], nextPart) - ) return; - current.content.parts[targetIndex] = { - ...current.content.parts[targetIndex], - thoughtSignature: nextPart.thoughtSignature, + ) return pair; + return { + ...pair, + current: withParts( + current, + withIndexesChanged(current.content.parts, new Map([[targetIndex, withKeysChanged(current.content.parts[targetIndex], { thoughtSignature: nextPart.thoughtSignature })]])), + ), + next: withParts( + next, + withIndexesChanged(next.content.parts, new Map([[nextIndex, withKeysChanged(nextPart, { thoughtSignature: undefined })]])), + ), }; - delete nextPart.thoughtSignature; }; -const removeRelocatedSignatureParts = ( - candidate: GeminiGenerateContentCandidate, - relocated: ReadonlySet, - removedCandidates?: WeakSet, -): void => { - candidate.content.parts = candidate.content.parts.filter(part => - !relocated.has(part) || hasPartContent(part) || part.thoughtSignature !== undefined); - if (candidate.content.parts.length === 0) removedCandidates?.add(candidate); +/** What survives a relocation: a Part it emptied is dropped, and a Part it never touched stays + * whatever it was. Membership is by position rather than by object, because the Parts handed + * in here have already been rebuilt. */ +const keptAfterRelocation = ( + parts: readonly GeminiGenerateContentPart[], + relocated: ReadonlySet, +): readonly GeminiGenerateContentPart[] => { + const kept = parts.filter((part, index) => !relocated.has(index) || hasPartContent(part) || part.thoughtSignature !== undefined); + return kept.length === parts.length ? parts : kept; }; const hasPartContent = (part: GeminiGenerateContentPart): boolean => { @@ -360,51 +438,68 @@ const firstContentIndexOfLastElement = (parts: readonly GeminiGenerateContentPar const firstContentPart = (parts: readonly GeminiGenerateContentPart[]): GeminiGenerateContentPart | undefined => parts.find(hasPartContent); -const mergeEventMetadata = ( +/** Everything an event carries beside its candidates, folded from the two into one — the later + * event's word wins where both have one. The result replaces `target`'s own metadata rather + * than adding to it, so a field neither event set is absent rather than stale. */ +const mergedEventMetadata = ( earlier: GeminiGenerateContentResult, later: GeminiGenerateContentResult, target: GeminiGenerateContentResult, -): void => { +): GeminiGenerateContentResult => { const extras: Record = {}; captureExtras(earlier as unknown as Record, GEMINI_GENERATE_CONTENT_RESULT_KEYS, extras); captureExtras(later as unknown as Record, GEMINI_GENERATE_CONTENT_RESULT_KEYS, extras); const usageMetadata = later.usageMetadata ?? earlier.usageMetadata; const modelVersion = later.modelVersion ?? earlier.modelVersion; const responseId = later.responseId ?? earlier.responseId; - for (const key of Object.keys(target)) if (key !== 'candidates') delete (target as Record)[key]; - Object.assign(target, { + return { + ...(target.candidates !== undefined ? { candidates: target.candidates } : {}), ...(usageMetadata !== undefined ? { usageMetadata } : {}), ...(modelVersion !== undefined ? { modelVersion } : {}), ...(responseId !== undefined ? { responseId } : {}), ...extras, - }); + } as GeminiGenerateContentResult; }; -const clearEventMetadata = (event: GeminiGenerateContentResult): void => { - for (const key of Object.keys(event)) if (key !== 'candidates') delete (event as Record)[key]; -}; +/** The candidates alone. What the event was carrying beside them has moved to its neighbour. */ +const withoutEventMetadata = (event: GeminiGenerateContentResult): GeminiGenerateContentResult => + (event.candidates !== undefined ? { candidates: event.candidates } : {}) as GeminiGenerateContentResult; -const mergeCandidateExtras = ( +/** The same fold one level down, over a candidate's non-standard keys. */ +const mergedCandidateExtras = ( earlier: GeminiGenerateContentCandidate, later: GeminiGenerateContentCandidate, target: GeminiGenerateContentCandidate, -): void => { +): GeminiGenerateContentCandidate => { const extras: Record = {}; captureExtras(earlier as unknown as Record, GEMINI_GENERATE_CONTENT_CANDIDATE_KEYS, extras); captureExtras(later as unknown as Record, GEMINI_GENERATE_CONTENT_CANDIDATE_KEYS, extras); - for (const key of Object.keys(target)) { - if (!GEMINI_GENERATE_CONTENT_CANDIDATE_KEYS.has(key as keyof GeminiGenerateContentCandidate)) delete (target as unknown as Record)[key]; - } - Object.assign(target, extras); + const known = Object.fromEntries( + Object.entries(target).filter(([key]) => GEMINI_GENERATE_CONTENT_CANDIDATE_KEYS.has(key as keyof GeminiGenerateContentCandidate)), + ); + return { ...known, ...extras } as GeminiGenerateContentCandidate; }; -const transferCandidateMetadata = (current: GeminiGenerateContentCandidate, next: GeminiGenerateContentCandidate): void => { - mergeCandidateExtras(current, next, current); - if (next.content.role !== undefined) current.content.role = next.content.role; - if (next.finishReason !== undefined) current.finishReason = next.finishReason; +/** `next` had everything taken out of it, so what it was saying about the turn is said by + * `current` instead. */ +const withCandidateMetadataFrom = ( + current: GeminiGenerateContentCandidate, + next: GeminiGenerateContentCandidate, +): GeminiGenerateContentCandidate => { + const merged = mergedCandidateExtras(current, next, current); + return withKeysChanged(merged, { + ...(next.content.role !== undefined ? { content: { ...merged.content, role: next.content.role } } : {}), + ...(next.finishReason !== undefined ? { finishReason: next.finishReason } : {}), + }); }; -const transferCandidateMetadataForward = (current: GeminiGenerateContentCandidate, next: GeminiGenerateContentCandidate): void => { - mergeCandidateExtras(current, next, next); - if (next.content.role === undefined && current.content.role !== undefined) next.content.role = current.content.role; +/** The same, in the other direction: `current` was emptied into `next`. */ +const withForwardedCandidateMetadata = ( + current: GeminiGenerateContentCandidate, + next: GeminiGenerateContentCandidate, +): GeminiGenerateContentCandidate => { + const merged = mergedCandidateExtras(current, next, next); + return next.content.role === undefined && current.content.role !== undefined + ? withKeysChanged(merged, { content: { ...merged.content, role: current.content.role } }) + : merged; }; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/affinity/ingress.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/affinity/ingress.ts index 92d412b239..46a01ff769 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/affinity/ingress.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/affinity/ingress.ts @@ -1,5 +1,6 @@ import { type AffinityCodec, type AffinityRequestAnalysis, type DecodedAffinityBlob, defineAffinityRequest, projectOptionalAffinityBlob } from '../../shared/affinity/index.ts'; -import type { GeminiGenerateContentPart, GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; +import { withIndexesChanged, withKeysChanged } from '@floway-dev/protocols/common'; +import type { GeminiGenerateContentContent, GeminiGenerateContentPart, GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; interface GeminiGenerateContentBlobLocation { readonly contentIndex: number; @@ -32,12 +33,13 @@ export const analyzeGeminiGenerateContentAffinity = async ( degrades: projections.some(item => item.projection.kind === 'remove' && item.projection.degrades), preferred: projections.every(item => item.projection.preferred), materialize: () => { - const candidatePayload = structuredClone(payload); - if (candidatePayload.contents === undefined) return candidatePayload; + const contents = payload.contents; + if (contents === undefined) return payload; const byContent = Map.groupBy(projections, item => item.location.contentIndex); const emptiedByAffinity = new Set(); + const rewritten = new Map(); for (const [contentIndex, contentProjections] of byContent) { - const content = candidatePayload.contents[contentIndex]; + const content = contents[contentIndex]; const replacements = new Map(); for (const { location, projection } of contentProjections) { const part = content.parts[location.partIndex]; @@ -45,19 +47,19 @@ export const analyzeGeminiGenerateContentAffinity = async ( if (projection.kind === 'preserve') { replacements.set(location.partIndex, { ...part, thoughtSignature: projection.value }); } else { - const replacement = { ...part }; - delete replacement.thoughtSignature; + const replacement = withKeysChanged(part, { thoughtSignature: undefined }); replacements.set(location.partIndex, hasPartContent(replacement) ? replacement : null); } } - content.parts = content.parts.flatMap((part, partIndex) => { - const replacement = replacements.get(partIndex); - return replacement === undefined ? [part] : replacement === null ? [] : [replacement]; - }); - if (content.parts.length === 0) emptiedByAffinity.add(contentIndex); + const parts = withIndexesChanged(content.parts, replacements); + if (parts.length === 0) emptiedByAffinity.add(contentIndex); + rewritten.set(contentIndex, withKeysChanged(content, { parts })); } - candidatePayload.contents = candidatePayload.contents.filter((_content, contentIndex) => !emptiedByAffinity.has(contentIndex)); - return candidatePayload; + return withKeysChanged(payload, { + contents: contents + .map((content, contentIndex) => rewritten.get(contentIndex) ?? content) + .filter((_content, contentIndex) => !emptiedByAffinity.has(contentIndex)), + }); }, }; }); diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/attempt.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/attempt.ts deleted file mode 100644 index 53a8f85154..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/attempt.ts +++ /dev/null @@ -1,173 +0,0 @@ -import { geminiGenerateContentStatusForHttpStatus } from './errors.ts'; -import { geminiGenerateContentCountTokensInterceptors, geminiGenerateContentInterceptors } from './interceptors/index.ts'; -import { stripUnsupportedPartFieldsFromPayload } from './interceptors/strip-unsupported-part-fields.ts'; -import { stripUnsupportedToolsFromPayload } from './interceptors/strip-unsupported-tools.ts'; -import { anthropicMessagesAttempt } from '../anthropic-messages/attempt.ts'; -import { openaiChatCompletionsAttempt } from '../openai-chat-completions/attempt.ts'; -import { openaiResponsesAttempt } from '../openai-responses/attempt.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import { chatTargetPicker } from '../shared/target-picker.ts'; -import { captureFromDump, traverseTranslation } from '../shared/translate-traverse.ts'; -import { runInterceptors } from '@floway-dev/interceptor'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiGenerateContentPayload, GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; -import { type ModelCandidate, plainResult, type ExecuteResult, type GeminiGenerateContentInvocation, type PlainResult } from '@floway-dev/provider'; -import { translateGeminiGenerateContentViaOpenAIChatCompletions, translateGeminiGenerateContentViaAnthropicMessages, translateGeminiGenerateContentViaOpenAIResponses } from '@floway-dev/translate'; - -// Gemini generateContent has no native upstream target in the provider API; prefer OpenAI Chat Completions -// Completions, then Anthropic Messages, then OpenAI Responses for generate. countTokens has -// no translation path beyond native Anthropic Messages count_tokens. -export const geminiGenerateContentGenerateTarget = chatTargetPicker(['openaiChatCompletions', 'anthropicMessages', 'openaiResponses']); -export const geminiGenerateContentCountTokensTarget = chatTargetPicker(['anthropicMessages']); - -export interface GeminiGenerateContentAttemptGenerateArgs { - readonly payload: GeminiGenerateContentPayload; - readonly ctx: ChatGatewayCtx; - readonly candidate: ModelCandidate; - readonly headers: Headers; -} - -export interface GeminiGenerateContentAttemptCountTokensArgs { - readonly payload: GeminiGenerateContentPayload; - readonly ctx: ChatGatewayCtx; - readonly candidate: ModelCandidate; - readonly headers: Headers; -} - -export const geminiGenerateContentAttempt = { - generate: async (args: GeminiGenerateContentAttemptGenerateArgs): Promise>> => { - const { payload: sourcePayload, ctx, candidate, headers: sourceHeaders } = args; - const payload = structuredClone(sourcePayload); - const headers = new Headers(sourceHeaders); - const targetApi = geminiGenerateContentGenerateTarget.pick(candidate.model.endpoints); - const invocation: GeminiGenerateContentInvocation = { payload, candidate, targetApi, headers }; - return await runInterceptors(invocation, ctx, geminiGenerateContentInterceptors, async () => { - // Gemini generateContent has no native upstream target today — every targetApi we - // pick is reached via translation. The dispatch threads each branch - // through `traverseTranslation` so each inner attempt owns its own - // interceptor chain and rewrite. - const transCtx = { - model: candidate.model.id, - fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, - }; - if (targetApi === 'anthropicMessages') { - return await traverseTranslation( - invocation.payload, - p => translateGeminiGenerateContentViaAnthropicMessages(p, transCtx), - translated => anthropicMessagesAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, anthropicBeta: [], - }), - captureFromDump(ctx.dump, targetApi), - ); - } - if (targetApi === 'openaiResponses') { - return await traverseTranslation( - invocation.payload, - p => translateGeminiGenerateContentViaOpenAIResponses(p, transCtx), - translated => openaiResponsesAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, - }), - captureFromDump(ctx.dump, targetApi), - ); - } - if (targetApi === 'openaiChatCompletions') { - return await traverseTranslation( - invocation.payload, - p => translateGeminiGenerateContentViaOpenAIChatCompletions(p, transCtx), - translated => openaiChatCompletionsAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, - }), - captureFromDump(ctx.dump, targetApi), - ); - } - throw new Error(`geminiGenerateContentAttempt.generate: unexpected targetApi '${targetApi as string}'`); - }); - }, - - countTokens: async (args: GeminiGenerateContentAttemptCountTokensArgs): Promise => { - const { payload: sourcePayload, ctx, candidate, headers: sourceHeaders } = args; - const payload = structuredClone(sourcePayload); - const headers = new Headers(sourceHeaders); - const targetApi = geminiGenerateContentCountTokensTarget.pick(candidate.model.endpoints); - const invocation: GeminiGenerateContentInvocation = { payload, candidate, targetApi, headers }; - return await runInterceptors(invocation, ctx, geminiGenerateContentCountTokensInterceptors, async () => { - // Gemini generateContent countTokens has no native upstream; translate to Anthropic Messages and - // delegate to `anthropicMessagesAttempt.countTokens`, then reshape the Anthropic Messages - // count_tokens reply into the Gemini generateContent `{ totalTokens }` envelope. The - // shipped Gemini generateContent interceptors that mutate the payload pre-dispatch - // cannot run via the countTokens interceptor list — the post-`run()` - // ones inspect event streams the result type cannot carry — so the - // payload-mutators are applied inline here before translation; the - // attempt-owned payload clone keeps the caller's source intact. - const transCtx = { - model: candidate.model.id, - fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, - }; - const cleaned = invocation.payload; - stripUnsupportedPartFieldsFromPayload(cleaned); - stripUnsupportedToolsFromPayload(cleaned); - delete cleaned.safetySettings; - const trip = await translateGeminiGenerateContentViaAnthropicMessages(cleaned, transCtx); - const { stream: _stream, ...target } = trip.target; - const anthropicMessagesResult = await anthropicMessagesAttempt.countTokens({ - payload: target, ctx, candidate, headers: invocation.headers, anthropicBeta: [], - }); - return reshapeAnthropicMessagesCountAsGeminiGenerateContent(anthropicMessagesResult); - }); - }, -}; - -// Reshape the Anthropic Messages count_tokens body into the Gemini generateContent `{ totalTokens }` -// envelope. The upstream body shape is provider-specific: Anthropic emits -// `{ input_tokens }`, Copilot's translated count emits `{ total_tokens }`; -// either is accepted. A missing or non-numeric figure is surfaced as a -// 502 Google-RPC error so the caller sees a typed Gemini generateContent failure rather -// than a passthrough of the upstream shape. -const reshapeAnthropicMessagesCountAsGeminiGenerateContent = (anthropicMessagesResult: PlainResult): PlainResult => { - if (anthropicMessagesResult.status !== 200) { - // Empty upstream bodies fall back to a fixed message so the Google-RPC envelope is never empty. - const text = new TextDecoder().decode(anthropicMessagesResult.body); - return geminiGenerateContentErrorPlainResult(anthropicMessagesResult.status, text || 'Upstream token counting request failed.', anthropicMessagesResult.upstreamId); - } - let decoded: unknown; - try { decoded = JSON.parse(new TextDecoder().decode(anthropicMessagesResult.body)); } catch {} - const upstreamTokenCounts = decoded && typeof decoded === 'object' - ? decoded as { input_tokens?: unknown; total_tokens?: unknown } - : {}; - const totalTokens = typeof upstreamTokenCounts.input_tokens === 'number' - ? upstreamTokenCounts.input_tokens - : typeof upstreamTokenCounts.total_tokens === 'number' - ? upstreamTokenCounts.total_tokens - : null; - if (totalTokens === null) { - return geminiGenerateContentInternalPlainResult(502, new Error('Invalid upstream token counting response.')); - } - return plainResult( - 200, - new Headers({ 'content-type': 'application/json' }), - new TextEncoder().encode(JSON.stringify({ totalTokens })), - anthropicMessagesResult.upstreamId, - ); -}; - -const geminiGenerateContentErrorPlainResult = (status: number, message: string, upstream?: string): PlainResult => plainResult( - status, - new Headers({ 'content-type': 'application/json' }), - new TextEncoder().encode(JSON.stringify({ error: { code: status, message, status: geminiGenerateContentStatusForHttpStatus(status) } })), - upstream, -); - -const geminiGenerateContentInternalPlainResult = (status: number, error: Error): PlainResult => plainResult( - status, - new Headers({ 'content-type': 'application/json' }), - new TextEncoder().encode(JSON.stringify({ - error: { - code: status, - message: error.message, - status: geminiGenerateContentStatusForHttpStatus(status), - type: 'internal_error', - name: error.name, - stack: error.stack, - }, - })), -); diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts new file mode 100644 index 0000000000..02e62fc4c3 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens.ts @@ -0,0 +1,35 @@ +import { emitGeminiGenerateContentTokenCount } from './count-tokens/emit.ts'; +import type { GeminiGenerateContentCountTokensEntry, GeminiGenerateContentCountTokensExit } from './count-tokens/facts.ts'; +import { serializeClientJson } from '../../pipeline/serialize-client-json.ts'; +import { materializeAttempt } from '../materialize-attempt.ts'; +import { resolveChatCandidates } from '../resolve-candidates.ts'; +import { measureGeminiGenerateContentAsAnthropicMessages } from './count-tokens/measure-via-anthropic-messages.ts'; +import { narrowing } from './count-tokens/target.ts'; +import { stripSafetySettingsFromGeminiGenerateContent } from './strip-safety-settings.ts'; +import { stripUnsupportedPartFieldsFromGeminiGenerateContent } from './strip-unsupported-part-fields.ts'; +import { stripUnsupportedToolsFromGeminiGenerateContent } from './strip-unsupported-tools.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import { failover } from '../../pipeline/failover.ts'; +import { writeSettlement } from '../../pipeline/settlement.ts'; +import { anthropicMessagesCountTokensWire } from '../anthropic-messages/count-tokens/wire.ts'; +import { compose, type Pipeline } from '@floway-dev/pipeline'; + +export const geminiGenerateContentCountTokensPipeline = (): Pipeline => + compose('geminiGenerateContentCountTokens', [ + // Native counts measure a request without generating AI usage; a failed reply must + // not recover a billable entity from the transport-call proof. + writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, undefined, false), + serializeClientJson('response.chat.geminiGenerateContent.rendered'), + emitGeminiGenerateContentTokenCount, + resolveChatCandidates(narrowing), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.chat.geminiGenerateContent'?: unknown })['response.chat.geminiGenerateContent']), + owns: ['response.http.body'], + }), + materializeAttempt('request.chat.geminiGenerateContent'), + stripUnsupportedPartFieldsFromGeminiGenerateContent, + stripUnsupportedToolsFromGeminiGenerateContent, + stripSafetySettingsFromGeminiGenerateContent, + measureGeminiGenerateContentAsAnthropicMessages, + ...anthropicMessagesCountTokensWire, + ]); diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/emit.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/emit.ts new file mode 100644 index 0000000000..6341d169cf --- /dev/null +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/emit.ts @@ -0,0 +1,54 @@ +import type { Counted, Fields } from './facts.ts'; +import { isFailure, renderFailure } from '../../../pipeline/facts.ts'; +import { isForwardableUpstreamHeader } from '../../../shared/upstream-response.ts'; +import type { ChatServices } from '../../services.ts'; +import { mintGeminiGenerateContentFailure } from '../errors.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; + +/** + * The outermost edge. A measurement is one object, so there is nothing to fold and nothing to + * stream: what this decides is only whether the client is shown the count or the refusal that + * stands in for it. + */ +export const emitGeminiGenerateContentTokenCount = defineStage< + Record, + Record, + Counted<'response.chat.geminiGenerateContent'> & Fields<'response.http.headers' | 'response.http.status'>, + Fields<'response.chat.geminiGenerateContent.rendered' | 'response.http.status' | 'response.http.headers'>, + ChatServices +>({ + name: 'emitGeminiGenerateContentTokenCount', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { + needs: ['response.chat.geminiGenerateContent', 'response.http.headers', 'response.http.status'], + consumes: ['response.chat.geminiGenerateContent', 'response.http.headers'], + provides: ['response.chat.geminiGenerateContent.rendered', 'response.http.status', 'response.http.headers'], + }, + }, + execute: async (facts, next) => { + const back = await next(facts); + const { 'response.chat.geminiGenerateContent': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); + + if (isFailure(answer)) { + const failure = renderFailure(answer, mintGeminiGenerateContentFailure); + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.geminiGenerateContent.rendered': move(failure.body), + 'response.http.status': failure.status, + }; + } + return { + ...rest, + 'response.http.headers': forClient, + 'response.chat.geminiGenerateContent.rendered': move(answer.body as Record), + 'response.http.status': back['response.http.status'], + }; + }, +}); diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/facts.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/facts.ts new file mode 100644 index 0000000000..b6df90a4fa --- /dev/null +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/facts.ts @@ -0,0 +1,14 @@ +import type { TokenCountAnswer } from '../../anthropic-messages/count-tokens/facts.ts'; +import type { GeminiGenerateContentFacts } from '../facts.ts'; + +export type Counted = { [P in K]: TokenCountAnswer }; + +export type Fields = { [P in K]: GeminiGenerateContentFacts[P] }; + +export type GeminiGenerateContentCountTokensEntry = Fields< + 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'request.chat.geminiGenerateContent' | 'serve.model' +>; + +export type GeminiGenerateContentCountTokensExit = Fields< + 'response.chat.geminiGenerateContent.rendered' | 'response.http.status' | 'response.http.jsonBody' | 'response.http.headers' +>; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/measure-via-anthropic-messages.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/measure-via-anthropic-messages.ts new file mode 100644 index 0000000000..e20df4a70c --- /dev/null +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/measure-via-anthropic-messages.ts @@ -0,0 +1,100 @@ +import type { Fields, Counted } from './facts.ts'; +import { asJsonObject, readJsonNumber } from '../../../../shared/json-helpers.ts'; +import { isFailure } from '../../../pipeline/facts.ts'; +import type { TokenCountAnswer } from '../../anthropic-messages/count-tokens/facts.ts'; +import type { ChatServices } from '../../services.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { translateGeminiGenerateContentViaAnthropicMessages, TranslatorInputError } from '@floway-dev/translate'; + +/** + * Asks the question in Anthropic Messages, and reads the answer back out as Google's. + * + * It consumes this protocol's request key and provides the target's, exactly as a handoff + * does, so the wire below sees only Anthropic Messages and cannot tell it was reached by translation. + * What differs is the way back: there are no frames to map, only counts. + * + * It carries the `return` trait for the same reason a handoff does — a body the target + * protocol cannot represent is answered here rather than measured. + */ +export const measureGeminiGenerateContentAsAnthropicMessages = defineStage< + Fields<'request.chat.geminiGenerateContent' | 'route.attempt'>, + Fields<'request.chat.anthropicMessages'>, + Counted<'response.chat.anthropicMessages'>, + Counted<'response.chat.geminiGenerateContent'>, + Counted<'response.chat.geminiGenerateContent'> & Fields<'response.usage.billable' | 'response.http.headers' | 'response.http.body' | 'response.http.status'>, + ChatServices +>({ + name: 'measureGeminiGenerateContentAsAnthropicMessages', + through: { + request: { + needs: ['request.chat.geminiGenerateContent', 'route.attempt'], + consumes: ['request.chat.geminiGenerateContent'], + provides: ['request.chat.anthropicMessages'], + }, + response: { + needs: ['response.chat.anthropicMessages'], + consumes: ['response.chat.anthropicMessages'], + provides: ['response.chat.geminiGenerateContent'], + }, + }, + return: { provides: ['response.chat.geminiGenerateContent', 'response.usage.billable', 'response.http.headers', 'response.http.body', 'response.http.status'] }, + execute: async (facts, next, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + const { 'request.chat.geminiGenerateContent': asked, ...down } = facts; + + let trip; + try { + trip = await translateGeminiGenerateContentViaAnthropicMessages(asked, { + model: candidate.model.id, + fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, + }); + } catch (error) { + // Input the counting wire's protocol cannot represent is the client's fault, and the + // translator is what knows which part of the body was at fault. Answering rather than + // throwing is also what keeps it a *candidate's* verdict, so the fork can try the next + // one. Anything else raised here is a fault of this gateway's and rides up as one. + if (!(error instanceof TranslatorInputError)) throw error; + return move({ + ...down, + 'response.chat.geminiGenerateContent': { status: 400, message: error.message }, + // Nothing was measured, which is what an empty billed set and an empty header list say + // on the other two keys. There is no streamed reading to leave behind: a measurement + // never opens a stream, which is why this chain carries no such key at all. + 'response.usage.billable': [], + 'response.http.headers': [], + 'response.http.body': null, 'response.http.status': 400, + }); + } + + // The translator writes a turn, and a turn says how its answer is delivered. There is no + // answer to deliver here, so the field does not travel. + const { stream: _stream, ...target } = trip.target; + + const back = await next({ ...down, 'request.chat.anthropicMessages': move(target) }); + const { 'response.chat.anthropicMessages': counted, ...up } = back; + return { ...up, 'response.chat.geminiGenerateContent': move(asGeminiGenerateContentTokenCount(counted)) }; + }, +}); + +/** + * Anthropic's counts, as Google states them. + * + * The upstream body is provider-specific: Anthropic's own endpoint answers `input_tokens`, + * and a Copilot upstream's translated count answers `total_tokens`. Either is the number this + * protocol calls `totalTokens`; a body carrying neither is an answer this gateway cannot + * read, which is the gateway failing to serve rather than anything the client can fix. + */ +const asGeminiGenerateContentTokenCount = (counted: TokenCountAnswer): TokenCountAnswer => { + if (isFailure(counted)) { + return { + ...counted, + // An upstream that refused with an empty body left nothing to quote, and a refusal that + // says nothing at all is not one a caller can act on. + message: counted.message === '' ? 'Upstream token counting request failed.' : counted.message, + }; + } + const body = asJsonObject(counted.body); + const totalTokens = readJsonNumber(body?.input_tokens) ?? readJsonNumber(body?.total_tokens); + if (totalTokens === null) return { status: 502, message: 'Invalid upstream token counting response.' }; + return { kind: 'value', body: { totalTokens } }; +}; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/target.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/target.ts new file mode 100644 index 0000000000..e1bec55b7c --- /dev/null +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/count-tokens/target.ts @@ -0,0 +1,17 @@ +import type { Counted } from './facts.ts'; +import type { ChatNarrowing } from '../../resolve-candidates.ts'; +import { chatTargetPicker } from '../../shared/target-picker.ts'; +import { analyzeGeminiGenerateContentAffinity } from '../affinity/ingress.ts'; + +/** Counting is reachable only over an upstream's own Anthropic Messages endpoint: no other protocol + * answers the question, and no translation invents an answer. */ +export const geminiGenerateContentCountTokensTarget = chatTargetPicker(['anthropicMessages']); + +export const narrowing: ChatNarrowing, 'request.chat.geminiGenerateContent'> = ({ + requestKey: 'request.chat.geminiGenerateContent', + canServe: candidate => geminiGenerateContentCountTokensTarget.canServe(candidate.model.endpoints), + affinity: async (payload, gateway) => await analyzeGeminiGenerateContentAffinity(payload, gateway.affinity.codec), + unsupported: model => `Model ${model} does not support countTokens.`, + refuse: (status, message) => ({ 'response.chat.geminiGenerateContent': { status, message } }), + refuses: ['response.chat.geminiGenerateContent'], +}); diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts new file mode 100644 index 0000000000..c39c17a2b5 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/emit.ts @@ -0,0 +1,121 @@ +import { wrapGeminiGenerateContentAffinityEgress } from './affinity/egress.ts'; +import { mintGeminiGenerateContentFailure } from './errors.ts'; +import type { Fields } from './facts.ts'; +import { isFailure, renderFailure } from '../../pipeline/facts.ts'; +import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; +import type { ChatServices } from '../services.ts'; +import { affinityEgressOptions } from '../shared/affinity/index.ts'; +import { bindClientRelease, collectClientFrames, framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; +import { recordStream } from '@floway-dev/dump'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { eventFrame, sseFrame, type EventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; +import { collectGeminiGenerateContentProtocolEventsToResult, type GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; +import { toInternalDebugError } from '@floway-dev/provider'; + +/** + * The outermost edge. A Gemini generateContent answer is always a stream by the time it reaches + * here — the wire below speaks SSE whatever the client asked for, and the translation back into Gemini generateContent + * is itself a stream of frames — so what this decides is whether the client sees the frames + * or the one object they add up to. + * + * Collecting is therefore the edge's own work and not a second reading of the upstream: the + * same frames that would have gone out are folded here instead. + */ +export const emitGeminiGenerateContent = defineStage< + Fields<'ingress.chat.geminiGenerateContent.wantsStream'>, + Fields<'ingress.chat.geminiGenerateContent.wantsStream'>, + Fields<'ingress.chat.geminiGenerateContent.wantsStream' | 'response.chat.geminiGenerateContent' | 'response.http.headers' | 'response.http.status' | 'response.chat.geminiGenerateContent.streamedUsage'>, + Fields<'response.chat.geminiGenerateContent.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.clientFrames' | 'response.chat.geminiGenerateContent.streamedUsage'>, + ChatServices +>({ + name: 'emitGeminiGenerateContent', + through: { + request: { + needs: ['ingress.chat.geminiGenerateContent.wantsStream'], + consumes: [], + provides: [], + }, + response: { + needs: ['response.chat.geminiGenerateContent', 'response.http.headers', 'response.http.status', 'response.chat.geminiGenerateContent.streamedUsage'], + consumes: ['response.chat.geminiGenerateContent', 'response.http.headers'], + provides: ['response.chat.clientFrames', 'response.chat.geminiGenerateContent.rendered', 'response.http.status', 'response.http.headers', 'response.chat.geminiGenerateContent.streamedUsage'], + }, + }, + execute: async (facts, next, use) => { + const back = await next(facts); + const { 'response.chat.geminiGenerateContent': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); + + if (isFailure(answer)) { + const failure = renderFailure(answer, mintGeminiGenerateContentFailure); + return { + ...rest, + 'response.chat.clientFrames': null, + 'response.http.headers': forClient, + 'response.chat.geminiGenerateContent.rendered': move(failure.body), + 'response.http.status': failure.status, + }; + } + if (answer.kind === 'value') { + return { + ...rest, + 'response.chat.clientFrames': null, + 'response.http.headers': forClient, + 'response.chat.geminiGenerateContent.rendered': move(answer.body as Record), + 'response.http.status': back['response.http.status'], + }; + } + + // The turn's own state, written back into the frames the client is handed: a follow-up + // carrying it comes back to the upstream that issued it. This is the other half of the + // affinity the resolver read on the way down, and it has to sit here because it rewrites + // the frames — below the fold, and there would be nothing left to rewrite. + const egress = wrapGeminiGenerateContentAffinityEgress( + answer.frames as AsyncIterable>, + affinityEgressOptions(use.gateway), + ); + if (!back['ingress.chat.geminiGenerateContent.wantsStream']) { + const frames = recordStream(egress, use.gateway.dump); + return { + ...rest, + 'response.chat.clientFrames': move(frames), + 'response.http.headers': forClient, + 'response.chat.geminiGenerateContent.rendered': move( + await collectClientFrames(frames, collectGeminiGenerateContentProtocolEventsToResult) as unknown as Record, + ), + 'response.http.status': back['response.http.status'], + }; + } + const completed = framedClientStream( + clientFrames(egress), + frame => sseFrame(JSON.stringify(frame.event)), + error => [eventFrame(geminiStreamError(error))], + use.gateway.dump, + ); + bindClientRelease(back, completed.release); + return { + ...rest, + 'response.chat.clientFrames': move(completed.frames), + 'response.http.headers': forClient, + 'response.chat.geminiGenerateContent.rendered': move(completed.rendered), + 'response.chat.geminiGenerateContent.streamedUsage': move(withClientVerdict(back['response.chat.geminiGenerateContent.streamedUsage'], completed.failed)), + 'response.http.status': back['response.http.status'], + }; + }, +}); + +const clientFrames = (frames: AsyncIterable>): AsyncIterable> => ({ + [Symbol.asyncIterator]: () => (async function* () { + for await (const frame of frames) if (frame.type === 'event') yield frame; + })(), +}); + +const geminiStreamError = (error: unknown): GeminiGenerateContentStreamEvent => { + const debug = toInternalDebugError(error, 'geminiGenerateContent'); + const envelope = mintGeminiGenerateContentFailure({ status: 500, message: debug.message }).body; + return { error: { ...(envelope.error as { code: number; message: string; status: string }), ...debug } }; +}; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/errors.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/errors.ts index 0a29203fb3..49167b5469 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/errors.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/errors.ts @@ -1,13 +1,45 @@ -import { appendFailedUpstreams } from '../../shared/failed-upstreams.ts'; -import type { ChatServeFailure } from '../shared/errors.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; -import type { ExecuteResult, PerformanceTelemetryContext } from '@floway-dev/provider'; -import type { TranslatorInputError } from '@floway-dev/translate'; - // Google RPC Status envelope, used by Gemini's `error` channel everywhere // (HTTP body, SSE-tunnelled error event). -export const geminiGenerateContentStatusForHttpStatus = (status: number): string => { + +/** + * What a Gemini generateContent client is sent when this gateway is the one refusing: the + * envelope, and the status it is sent with. + * + * `error.status` is the Google-RPC name and clients read it, so the shape is the protocol's + * rather than the gateway's — an OpenAI-shaped envelope would leave that field undefined. + * + * The two are decided together because the envelope states its code as well as naming it, and + * this protocol has a name for only nine codes: everything else is `INTERNAL`, which belongs + * to 500 alone. Minting from some other status would say two different things about one + * failure — `code: 418` beside `status: "INTERNAL"`, sent as a 418 — so a status this protocol + * cannot name is sent as the one that name belongs to. Deciding them apart is what let that + * disagreement exist, which is why nothing here renders an envelope without also saying what + * to send it with. + * + * A forwarded upstream body and a refusal's own envelope keep their status untouched: they + * already say what they mean, and neither is this gateway's to reword. + */ +export const mintGeminiGenerateContentFailure = (failure: { readonly status: number; readonly message: string }): { + readonly body: Record; + readonly status: number; +} => { + const status = geminiGenerateContentStatusForHttpStatus(failure.status) === 'INTERNAL' ? 500 : failure.status; + return { + body: { error: { code: status, message: failure.message, status: geminiGenerateContentStatusForHttpStatus(status) } }, + status, + }; +}; + +/** The same refusal as an HTTP response, for the edges that answer before a pipeline runs. */ +export const geminiGenerateContentErrorResponse = (status: number, message: string): Response => { + const failure = mintGeminiGenerateContentFailure({ status, message }); + return Response.json(failure.body, { status: failure.status }); +}; + +/** The Google-RPC name for an HTTP status. Private, because a caller holding only the name + * can write an envelope whose code contradicts it — which is the whole reason the two are + * minted together above. */ +const geminiGenerateContentStatusForHttpStatus = (status: number): string => { switch (status) { case 400: return 'INVALID_ARGUMENT'; @@ -28,42 +60,3 @@ export const geminiGenerateContentStatusForHttpStatus = (status: number): string return 'INTERNAL'; } }; - -const geminiGenerateContentRpcErrorResult = (status: number, message: string, performance?: PerformanceTelemetryContext): ExecuteResult> => ({ - type: 'api-error', - source: 'gateway', - status, - headers: new Headers({ 'content-type': 'application/json' }), - body: new TextEncoder().encode(JSON.stringify({ - error: { code: status, message, status: geminiGenerateContentStatusForHttpStatus(status) }, - })), - ...(performance ? { performance } : {}), -}); - -// Translator surfaced a caller-input violation (unsupported content part, -// disallowed role, missing required field, etc.). Render as a 400 -// INVALID_ARGUMENT envelope so the caller sees a Gemini-generateContent-shaped failure -// instead of the internal-error 500 envelope. `performance` carries the -// throwing candidate's telemetry attribution when the throw fired -// mid-attempt (see AttemptState.telemetry). -export const translatorInputErrorResult = ( - error: TranslatorInputError, - performance?: PerformanceTelemetryContext, -): ExecuteResult> => - geminiGenerateContentRpcErrorResult(400, error.message, performance); - -// `endpoint` selects between `:generateContent` and `:countTokens` only in -// the `model-unsupported` message string. -export const renderGeminiGenerateContentFailure = ( - failure: ChatServeFailure, - endpoint: 'generate' | 'countTokens', -): ExecuteResult> => { - switch (failure.kind) { - case 'routing-unavailable': - return geminiGenerateContentRpcErrorResult(400, failure.message); - case 'model-missing': - return geminiGenerateContentRpcErrorResult(404, appendFailedUpstreams(`Model ${failure.model} is not available on any configured upstream.`, failure.failedUpstreams)); - case 'model-unsupported': - return geminiGenerateContentRpcErrorResult(400, appendFailedUpstreams(`Model ${failure.model} does not support ${endpoint === 'countTokens' ? 'countTokens' : 'the Gemini generateContent endpoint'}.`, failure.failedUpstreams)); - } -}; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/facts.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/facts.ts new file mode 100644 index 0000000000..ef7c4a0945 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/facts.ts @@ -0,0 +1,29 @@ +import type { StreamOutcome } from '../../pipeline/serve.ts'; +import type { ChatFacts } from '../facts.ts'; +import type { Deferred } from '@floway-dev/pipeline'; +import type { SseFrame } from '@floway-dev/protocols/common'; + +/** What this family adds to the chat space. */ +export interface GeminiGenerateContentFacts extends ChatFacts { + /** What the client is actually sent — an object when it asked for one, SSE frames when it + * asked to stream. The edge provides it, so a dump shows what the client received. */ + 'response.chat.geminiGenerateContent.rendered': Record | AsyncIterable; + /** What the upstream will have reported once the frames run out, and `null` when nothing + * streamed. Settling from this is the epilogue's job, after the drain. */ + 'response.chat.geminiGenerateContent.streamedUsage': Deferred | null; +} + +export type Fields = { [P in K]: GeminiGenerateContentFacts[P] }; + +/** This family's own reading, which every wire under it hands up. */ +export const STREAMED_USAGE = 'response.chat.geminiGenerateContent.streamedUsage'; + +export type GeminiGenerateContentServeEntry = Fields< + 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'ingress.chat.geminiGenerateContent.wantsStream' + | 'request.chat.geminiGenerateContent' | 'serve.model' +>; + +export type GeminiGenerateContentServeExit = Fields< + 'response.chat.geminiGenerateContent.rendered' | 'response.chat.geminiGenerateContent.streamedUsage' + | 'response.http.status' | 'response.http.jsonBody' | 'response.http.headers' | 'response.usage.billable' +>; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts index 36cb533b07..81ccd6ed3b 100644 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/http.ts @@ -1,16 +1,21 @@ -import { translatorInputErrorResult } from './errors.ts'; -import { geminiGenerateContentInternalRpcErrorResponse, geminiGenerateContentRpcErrorResponse, respondGeminiGenerateContent } from './respond.ts'; -import { geminiGenerateContentServe } from './serve.ts'; +// `/v1beta/models/:modelAction`, whose three actions are each served through a pipeline. +// `:generateContent` and `:streamGenerateContent` are one chain; `:countTokens` is a second +// operation over the same protocol and is a chain of its own rather than another wire under +// the first. +// +// Each entry is a prologue and an epilogue around that chain: read what the client sent, hand +// it over, and turn what the run answered with into a response. Everything between is stages. + +import { geminiGenerateContentCountTokensPipeline } from './count-tokens.ts'; +import { geminiGenerateContentErrorResponse } from './errors.ts'; +import { geminiGenerateContentServePipeline } from './pipeline.ts'; import type { AuthedContext } from '../../../middleware/auth.ts'; -import { backgroundSchedulerFromContext } from '../../../runtime/background.ts'; +import { isFrames, openPrologue, readIngress, serveThrough, type Ingress } from '../../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../../shared/gateway-ctx.ts'; -import { inboundHeaders } from '../../shared/inbound-headers.ts'; -import { readRequestBody, takeRequestBody, type RequestBody } from '../../shared/request-body.ts'; import { createNonOpenAIResponsesSourceStore } from '../openai-responses/items/store.ts'; -import { createChatGatewayCtxFromHono, type ChatGatewayCtx } from '../shared/gateway-ctx.ts'; +import { openChatPrologue } from '../prologue.ts'; +import { move } from '@floway-dev/pipeline'; import type { GeminiGenerateContentContent, GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; -import { internalErrorResult, ProviderModelsUnavailableError, toInternalDebugError } from '@floway-dev/provider'; -import { TranslatorInputError } from '@floway-dev/translate'; interface GeminiGenerateContentModelAction { readonly model: string; @@ -23,10 +28,14 @@ interface GeminiGenerateContentModelAction { // we split on the trailing `:` here so each entry sees just the action and // the resolved model id (with a leading `models/` prefix tolerated, as Google // SDKs send it). +/** A refusal about the route itself. It is answered before a model is resolved, so there is no + * attempt to record and nothing for a run to hold beyond the refusal. */ +const unknownAction = (message: string): Response => geminiGenerateContentErrorResponse(404, message); + const parseGeminiGenerateContentModelAction = (modelAction: string | undefined): GeminiGenerateContentModelAction | Response => { - if (!modelAction) return geminiGenerateContentRpcErrorResponse(404, 'Missing Gemini model action.'); + if (!modelAction) return unknownAction('Missing Gemini model action.'); const separator = modelAction.lastIndexOf(':'); - if (separator <= 0 || separator === modelAction.length - 1) return geminiGenerateContentRpcErrorResponse(404, `Unknown Gemini model action: ${modelAction}`); + if (separator <= 0 || separator === modelAction.length - 1) return unknownAction(`Unknown Gemini model action: ${modelAction}`); return { model: modelAction.slice(0, separator).replace(/^models\//, ''), action: modelAction.slice(separator + 1) }; }; @@ -38,53 +47,6 @@ const parseGeminiGenerateContentCountTokensPayload = (body: unknown): GeminiGene return shape.generateContentRequest ?? { contents: shape.contents }; }; -const parseGeminiGenerateContentBodyBytes = (requestBody: RequestBody, project: (body: unknown) => T): T | Response => { - try { - const raw = JSON.parse(new TextDecoder().decode(requestBody.bytes)) as unknown; - return project(raw); - } catch (error) { - return geminiGenerateContentInternalRpcErrorResponse(500, error); - } -}; - -// Surfaces a pre-stream throw as a Gemini-generateContent-RPC envelope, routing through -// `respondGeminiGenerateContent` so the dump records the failure exactly as the sibling -// HTTP handlers do. `TranslatorInputError` renders a 400 INVALID_ARGUMENT -// envelope (caller-input violation). A `ProviderModelsUnavailableError` -// carrying an upstream HTTP body relays that body through the `api-error` -// path with `source: 'upstream'`; everything else collapses to an -// `internal-error` result rendered as the Gemini generateContent internal-error envelope -// (status, code, message, stack, cause, target_api). The throwing- -// candidate telemetry stamped in serve.ts survives onto the error row via -// `ctx.attempt.telemetry` so a mid-attempt throw still lands in -// performance_summary against the throwing upstream. -const respondWithGeminiGenerateContentError = async ( - c: AuthedContext, - error: unknown, - ctx: ChatGatewayCtx, - wantsStream: boolean, -): Promise => { - if (error instanceof TranslatorInputError) { - const response = await respondGeminiGenerateContent(c, translatorInputErrorResult(error, ctx.attempt.telemetry), wantsStream, ctx); - return finalizeGatewayResponse(ctx, response); - } - if (error instanceof ProviderModelsUnavailableError && error.httpResponse) { - const { status, headers, body } = error.httpResponse; - const apiErrorResult = { - type: 'api-error' as const, - source: 'upstream' as const, - status, - headers: new Headers(headers), - body: new TextEncoder().encode(body), - }; - const response = await respondGeminiGenerateContent(c, apiErrorResult, wantsStream, ctx); - return finalizeGatewayResponse(ctx, response); - } - const internalResult = internalErrorResult(500, toInternalDebugError(error), ctx.attempt.telemetry); - const response = await respondGeminiGenerateContent(c, internalResult, wantsStream, ctx); - return finalizeGatewayResponse(ctx, response); -}; - // Single entry for `/v1beta/models/:modelAction`. Splits the model and action // once, then dispatches to the matching sub-handler. Keeping the parse here // means the sub-handlers see a validated `(model, action)` pair and never @@ -96,35 +58,84 @@ export const geminiGenerateContentHttp = async (c: AuthedContext): Promise => { - const requestBody = await readRequestBody(c); - const payload = parseGeminiGenerateContentBodyBytes(requestBody, body => body as GeminiGenerateContentPayload); - if (payload instanceof Response) return payload; - - const ctx = createChatGatewayCtxFromHono(c, { wantsStream, requestBody: takeRequestBody(requestBody), model, backgroundScheduler: backgroundSchedulerFromContext(c) }, apiKey => createNonOpenAIResponsesSourceStore(apiKey.id)); +/** A body the gateway could not read is reported in the words this protocol's own clients + * parse, rather than as a fault of the gateway's. Each action reads the same bytes into its + * own shape, which is why the projection is the caller's. */ +const readRequest = (bytes: Uint8Array, project: (body: unknown) => T): { type: 'ok'; payload: T } | { type: 'invalid'; message: string } => { try { - const result = await geminiGenerateContentServe.generate({ payload, ctx, model, headers: inboundHeaders(c) }); - const response = await respondGeminiGenerateContent(c, result, wantsStream, ctx); - return finalizeGatewayResponse(ctx, response); + return { type: 'ok', payload: project(JSON.parse(new TextDecoder().decode(bytes)) as unknown) }; } catch (error) { - return await respondWithGeminiGenerateContentError(c, error, ctx, wantsStream); + return { type: 'invalid', message: error instanceof Error ? error.message : String(error) }; } }; +/** A request the gateway refused before it reached a pipeline: there is no model to resolve + * and no attempt to make, so there is nothing for a run to record beyond the refusal. */ +const refuse = (c: AuthedContext, ingress: Ingress, message: string): Response => { + const refused = openPrologue(c, ingress, { wantsStream: false }); + refused.gateway.dump?.error('gateway'); + return finalizeGatewayResponse(refused.gateway, geminiGenerateContentErrorResponse(400, message)); +}; + +const runGeminiGenerateContentGenerate = async (c: AuthedContext, model: string, wantsStream: boolean): Promise => { + const ingress = await readIngress(c); + const request = readRequest(ingress.body.bytes, (body): GeminiGenerateContentPayload => body as GeminiGenerateContentPayload); + if (request.type === 'invalid') return refuse(c, ingress, request.message); + + const { payload } = request; + const prologue = openChatPrologue(c, ingress, { + wantsStream, + model, + storeFactory: apiKey => createNonOpenAIResponsesSourceStore(apiKey.id), + }); + + return await serveThrough( + c, + prologue, + geminiGenerateContentServePipeline(), + move({ + 'ingress.http.headers': prologue.headers, + 'ingress.chat.sourceProtocol': 'geminiGenerateContent', + 'ingress.chat.geminiGenerateContent.wantsStream': wantsStream, + 'request.chat.geminiGenerateContent': payload, + // Gemini generateContent carries the model in the path rather than the body, so the id the run + // resolves against is the one the route split off. + 'serve.model': model, + }) as never, + facts => { + const rendered = facts['response.chat.geminiGenerateContent.rendered']; + if (isFrames(rendered)) return { frames: rendered }; + return { body: facts['response.http.jsonBody']!, contentType: 'application/json' }; + }, + ); +}; + const runGeminiGenerateContentCountTokens = async (c: AuthedContext, model: string): Promise => { - const requestBody = await readRequestBody(c); - const payload = parseGeminiGenerateContentBodyBytes(requestBody, parseGeminiGenerateContentCountTokensPayload); - if (payload instanceof Response) return payload; + const ingress = await readIngress(c); + const request = readRequest(ingress.body.bytes, parseGeminiGenerateContentCountTokensPayload); + if (request.type === 'invalid') return refuse(c, ingress, request.message); - const ctx = createChatGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), model, backgroundScheduler: backgroundSchedulerFromContext(c) }, apiKey => createNonOpenAIResponsesSourceStore(apiKey.id)); - try { - const result = await geminiGenerateContentServe.countTokens({ payload, ctx, model, headers: inboundHeaders(c) }); - const response = await respondGeminiGenerateContent(c, result, false, ctx); - return finalizeGatewayResponse(ctx, response); - } catch (error) { - return await respondWithGeminiGenerateContentError(c, error, ctx, false); - } + const { payload } = request; + const prologue = openChatPrologue(c, ingress, { + wantsStream: false, + model, + storeFactory: apiKey => createNonOpenAIResponsesSourceStore(apiKey.id), + }); + + return await serveThrough( + c, + prologue, + geminiGenerateContentCountTokensPipeline(), + move({ + 'ingress.http.headers': prologue.headers, + 'ingress.chat.sourceProtocol': 'geminiGenerateContent', + 'request.chat.geminiGenerateContent': payload, + 'serve.model': model, + }) as never, + // A measurement is one body however the turn went, so there is never a stream to write. + facts => ({ body: facts['response.http.jsonBody']!, contentType: 'application/json' }), + ); }; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/index.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/index.ts deleted file mode 100644 index d3cea7c5cb..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/index.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { stripSafetySettings } from './strip-safety-settings.ts'; -import { stripUnsupportedPartFields } from './strip-unsupported-part-fields.ts'; -import { stripUnsupportedTools } from './strip-unsupported-tools.ts'; -import { suppressThoughtParts } from './suppress-thought-parts.ts'; -import type { GeminiGenerateContentCountTokensInterceptor, GeminiGenerateContentInterceptor } from './types.ts'; - -// Unified Gemini generateContent interceptor list for `generate`. All four entries below are -// unconditional protocol-shape cleanups required because Gemini-generateContent-shape requests -// cannot ride verbatim through other targets, plus the post-stream thought -// suppression that hides Gemini-generateContent-native thought parts unless the caller opted -// in. There is no target-side companion list — Gemini generateContent has no native upstream -// in our provider API, so everything happens on the source side regardless of -// the chosen target. -export const geminiGenerateContentInterceptors: readonly GeminiGenerateContentInterceptor[] = [ - stripUnsupportedPartFields, - stripUnsupportedTools, - stripSafetySettings, - suppressThoughtParts, -]; - -// countTokens always translates Gemini generateContent → Anthropic Messages and calls the Anthropic Messages -// count_tokens upstream, which returns a raw `Response` rather than an event -// stream. The shipped Gemini generateContent interceptors all either mutate the payload pre- -// dispatch (acceptable) or wrap the post-`run()` event stream (incompatible -// with the count-tokens result shape). `geminiGenerateContentAttempt.countTokens` applies -// the payload-mutators inline before handing the translated payload to the -// Anthropic Messages count_tokens path, so this list stays empty; the chain still runs -// so the count-tokens path keeps the same invocation envelope as generate. -export const geminiGenerateContentCountTokensInterceptors: readonly GeminiGenerateContentCountTokensInterceptor[] = []; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/strip-safety-settings.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/strip-safety-settings.ts deleted file mode 100644 index ee46d9eb43..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/strip-safety-settings.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { GeminiGenerateContentInterceptor } from './types.ts'; - -/** - * Gemini generateContent safety controls are source-specific and have no matching control on - * every target path. Drop them so we don't pretend to enforce a policy we - * cannot honor end-to-end. - */ -export const stripSafetySettings: GeminiGenerateContentInterceptor = (ctx, _gatewayCtx, run) => { - delete ctx.payload.safetySettings; - return run(); -}; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/strip-unsupported-part-fields.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/strip-unsupported-part-fields.ts deleted file mode 100644 index 0903d58900..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/strip-unsupported-part-fields.ts +++ /dev/null @@ -1,29 +0,0 @@ -import type { GeminiGenerateContentInterceptor } from './types.ts'; -import type { GeminiGenerateContentPayload, GeminiGenerateContentPart } from '@floway-dev/protocols/gemini-generate-content'; - -/** - * Gemini generateContent file/code parts have no current equivalent in the upstream target - * graph. Drop them at source so every Gemini generateContent route target sees translatable - * parts. - */ -const stripPartFields = (parts: GeminiGenerateContentPart[]): GeminiGenerateContentPart[] => - parts.filter(part => { - delete part.fileData; - delete part.executableCode; - delete part.codeExecutionResult; - return Object.keys(part).length > 0; - }); - -export const stripUnsupportedPartFieldsFromPayload = (payload: GeminiGenerateContentPayload): void => { - payload.contents?.forEach(content => { - content.parts = stripPartFields(content.parts); - }); - if (payload.systemInstruction) { - payload.systemInstruction.parts = stripPartFields(payload.systemInstruction.parts); - } -}; - -export const stripUnsupportedPartFields: GeminiGenerateContentInterceptor = (ctx, _gatewayCtx, run) => { - stripUnsupportedPartFieldsFromPayload(ctx.payload); - return run(); -}; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/strip-unsupported-tools.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/strip-unsupported-tools.ts deleted file mode 100644 index c04f95aecf..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/strip-unsupported-tools.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { GeminiGenerateContentInterceptor } from './types.ts'; -import type { GeminiGenerateContentPayload, GeminiGenerateContentToolGroup } from '@floway-dev/protocols/gemini-generate-content'; - -/** - * Only function declarations are currently translatable from Gemini generateContent tool - * groups. Strip the rest after target planning so target emitters never see - * unsupported tool capabilities. - */ -const stripToolCapabilities = (tool: GeminiGenerateContentToolGroup): void => { - delete tool.googleSearch; - delete tool.googleSearchRetrieval; - delete tool.codeExecution; - delete tool.computerUse; - delete tool.urlContext; - delete tool.fileSearch; - delete tool.mcpServers; - delete tool.googleMaps; -}; - -export const stripUnsupportedToolsFromPayload = (payload: GeminiGenerateContentPayload): void => { - if (!payload.tools) return; - - const tools = payload.tools.filter(tool => { - stripToolCapabilities(tool); - return tool.functionDeclarations && tool.functionDeclarations.length > 0; - }); - - if (tools.length === 0) { - delete payload.tools; - } else { - payload.tools = tools; - } -}; - -export const stripUnsupportedTools: GeminiGenerateContentInterceptor = (ctx, _gatewayCtx, run) => { - stripUnsupportedToolsFromPayload(ctx.payload); - return run(); -}; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/suppress-thought-parts.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/suppress-thought-parts.ts deleted file mode 100644 index ce01af3d67..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/suppress-thought-parts.ts +++ /dev/null @@ -1,49 +0,0 @@ -import type { GeminiGenerateContentInterceptor } from './types.ts'; -import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; - -const hasEventPayload = (event: GeminiGenerateContentStreamEvent): boolean => { - if ('error' in event) return true; - return (event.candidates?.length ?? 0) > 0 || event.usageMetadata !== undefined || event.modelVersion !== undefined || event.responseId !== undefined; -}; - -const suppressThoughtPartsFromFrames = async function* (frames: AsyncIterable>): AsyncGenerator> { - for await (const frame of frames) { - if (frame.type !== 'event' || 'error' in frame.event) { - yield frame; - continue; - } - - const candidates = frame.event.candidates?.flatMap(candidate => { - const parts = candidate.content.parts.filter(part => part.thought !== true); - if (!parts.length && candidate.finishReason === undefined) return []; - - return [ - { - ...candidate, - content: { ...candidate.content, parts }, - }, - ]; - }); - - const event: GeminiGenerateContentStreamEvent = { - ...frame.event, - ...(candidates !== undefined ? { candidates } : {}), - }; - - if (hasEventPayload(event)) yield eventFrame(event); - } -}; - -/** - * Hide Gemini generateContent thought-summary parts unless the caller explicitly opted in via - * `generationConfig.thinkingConfig.includeThoughts === true`. - */ -export const suppressThoughtParts: GeminiGenerateContentInterceptor = async (ctx, _gatewayCtx, run) => { - const result = await run(); - if (result.type !== 'events' || ctx.payload.generationConfig?.thinkingConfig?.includeThoughts === true) { - return result; - } - - return { ...result, events: suppressThoughtPartsFromFrames(result.events) }; -}; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/types.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/types.ts deleted file mode 100644 index e81d2e52f0..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/interceptors/types.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { GatewayCtx } from '../../../shared/gateway-ctx.ts'; -import type { Interceptor } from '@floway-dev/interceptor'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; -import type { ExecuteResult, GeminiGenerateContentInvocation, PlainResult } from '@floway-dev/provider'; - -export type GeminiGenerateContentInterceptor = Interceptor< - GeminiGenerateContentInvocation, - GatewayCtx, - ExecuteResult> ->; - -// countTokens is a one-shot, non-streaming HTTP exchange — the terminal -// returns a `PlainResult` carrying the reshaped Gemini generateContent envelope, not an event -// stream. The interceptor chain still runs against a `GeminiGenerateContentInvocation` so -// payload-shaped reads stay symmetric with the generate path. Interceptors -// registered here MUST be pure header/payload mutators; post-`run()` result -// inspection is not portable to this result type. -export type GeminiGenerateContentCountTokensInterceptor = Interceptor< - GeminiGenerateContentInvocation, - GatewayCtx, - PlainResult ->; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts new file mode 100644 index 0000000000..24d8f8ab81 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/pipeline.ts @@ -0,0 +1,44 @@ +import { STREAMED_USAGE, type GeminiGenerateContentServeEntry, type GeminiGenerateContentServeExit } from './facts.ts'; +import { composeChat as compose } from '../compose.ts'; +import { emitGeminiGenerateContent } from './emit.ts'; +import { serializeClientJson } from '../../pipeline/serialize-client-json.ts'; +import { resolveChatCandidates } from '../resolve-candidates.ts'; +import { requireGeminiGenerateContentTerminal } from './require-terminal.ts'; +import { narrowing, geminiGenerateContentTarget } from './target.ts'; +import { failover } from '../../pipeline/failover.ts'; +import { materializeAttempt } from '../materialize-attempt.ts'; +import { stripSafetySettingsFromGeminiGenerateContent } from './strip-safety-settings.ts'; +import { stripUnsupportedPartFieldsFromGeminiGenerateContent } from './strip-unsupported-part-fields.ts'; +import { stripUnsupportedToolsFromGeminiGenerateContent } from './strip-unsupported-tools.ts'; +import { suppressThoughtPartsFromGeminiGenerateContent } from './suppress-thought-parts.ts'; +import { dialChatWire } from '../dial-wire.ts'; +import { geminiGenerateContentWireFor } from './wires.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import { writeSettlement } from '../../pipeline/settlement.ts'; +import type { Pipeline } from '@floway-dev/pipeline'; + +export const geminiGenerateContentServePipeline = (): Pipeline => + compose('geminiGenerateContentServe', [ + writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, STREAMED_USAGE), + serializeClientJson('response.chat.geminiGenerateContent.rendered'), + emitGeminiGenerateContent, + resolveChatCandidates(narrowing), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.chat.geminiGenerateContent'?: unknown })['response.chat.geminiGenerateContent']), + owns: ['response.http.body'], + pendingUsage: STREAMED_USAGE, + }), + materializeAttempt('request.chat.geminiGenerateContent'), + stripUnsupportedPartFieldsFromGeminiGenerateContent, + stripUnsupportedToolsFromGeminiGenerateContent, + stripSafetySettingsFromGeminiGenerateContent, + suppressThoughtPartsFromGeminiGenerateContent, + requireGeminiGenerateContentTerminal, + dialChatWire({ + source: 'request.chat.geminiGenerateContent', + needs: ['request.chat.geminiGenerateContent', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], + provides: ['response.chat.geminiGenerateContent', STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], + pick: endpoints => geminiGenerateContentTarget.pick(endpoints), + wire: geminiGenerateContentWireFor, + }), + ]); diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/require-terminal.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/require-terminal.ts new file mode 100644 index 0000000000..b079d758e3 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/require-terminal.ts @@ -0,0 +1,64 @@ +import type { Fields } from './facts.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import { isGeminiGenerateContentTerminalEvent, GEMINI_GENERATE_CONTENT_MISSING_TERMINAL_MESSAGE, type GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; + +/** + * Says where a Gemini generateContent turn ends, on the frames the client is handed rather than on the wire + * below them. + * + * The two are not the same statement. An OpenAI Chat Completions stream that closed cleanly without + * ever reporting a finish reason ends its own dialect properly and translates into candidates + * that never finish, and serving those would report a truncated answer as a whole one. So the + * turn's end is read here, where the protocol the client speaks is, and the wire's end is read + * on the wire. + * + * It sits below the thought suppressor and above the fork: what it truncates is what the rules + * above it then work on, which is the order the fused ending had. + */ +export const requireGeminiGenerateContentTerminal = defineStage< + Record, + Record, + Fields<'response.chat.geminiGenerateContent'>, + Fields<'response.chat.geminiGenerateContent'> +>({ + name: 'requireGeminiGenerateContentTerminal', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { needs: ['response.chat.geminiGenerateContent'], consumes: [], provides: ['response.chat.geminiGenerateContent'] }, + }, + execute: transform< + Record, + Record, + Fields<'response.chat.geminiGenerateContent'>, + Fields<'response.chat.geminiGenerateContent'> + >(() => ({ + response: facts => { + const answer = facts['response.chat.geminiGenerateContent']; + // A refusal and a collected body never opened a stream, so there is no end to find. + if (isFailure(answer) || answer.kind !== 'stream') return facts; + const frames = answer.frames as AsyncIterable>; + return { + ...facts, + 'response.chat.geminiGenerateContent': move({ + kind: 'stream' as const, + frames: { [Symbol.asyncIterator]: () => framesUntilTerminal(frames) }, + }), + }; + }, + })), +}); + +/** Stops at the frame that ends the turn — there is nothing further to read, and returning + * here closes the translation and the wire under it — and fails a stream that ran out before + * one arrived. */ +const framesUntilTerminal = async function* ( + frames: AsyncIterable>, +): AsyncGenerator> { + for await (const frame of frames) { + yield frame; + if (frame.type === 'done' || isGeminiGenerateContentTerminalEvent(frame.event)) return; + } + throw new Error(GEMINI_GENERATE_CONTENT_MISSING_TERMINAL_MESSAGE); +}; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/respond.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/respond.ts deleted file mode 100644 index 8b26d9b39c..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/respond.ts +++ /dev/null @@ -1,215 +0,0 @@ -import type { Context } from 'hono'; -import { streamSSE } from 'hono/streaming'; - -import { wrapGeminiGenerateContentAffinityEgress } from './affinity/egress.ts'; -import { geminiGenerateContentStatusForHttpStatus } from './errors.ts'; -import type { GatewayCtx } from '../../shared/gateway-ctx.ts'; -import { type StreamCompletion, writeSSEFrames } from '../../shared/sse.ts'; -import { recordFailedRequest } from '../../shared/telemetry/performance.ts'; -import { settle } from '../../shared/telemetry/settle.ts'; -import { tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; -import { forwardUpstreamHeaders, mergeForwardedUpstreamHeaders } from '../../shared/upstream-response.ts'; -import { affinityEgressOptions } from '../shared/affinity/index.ts'; -import { SourceStreamState, eventResultMetadata, plainResultToResponse } from '../shared/respond.ts'; -import { eventFrame, type ProtocolFrame, sseCommentFrame, sseFrame } from '@floway-dev/protocols/common'; -import { geminiGenerateContentProtocolFrameToSSEFrame, GEMINI_GENERATE_CONTENT_MISSING_TERMINAL_MESSAGE, isGeminiGenerateContentErrorEvent, isGeminiGenerateContentTerminalEvent, collectGeminiGenerateContentProtocolEventsToResult } from '@floway-dev/protocols/gemini-generate-content'; -import type { GeminiGenerateContentErrorResponse, GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; -import { type ExecuteResult, type PlainResult, type ApiErrorResult, type InternalDebugError, toInternalDebugError, decodeApiErrorBody } from '@floway-dev/provider'; - -// Renders an upstream Gemini generateContent result into the client HTTP/SSE response, in the -// Google-RPC error envelope. An error-typed result is a pre-stream failure and -// always answers as HTTP; an events result drains to one JSON body -// (non-streaming) or is proxied frame by frame (streaming). -export const respondGeminiGenerateContent = async ( - c: Context, - result: ExecuteResult> | PlainResult, - wantsStream: boolean, - ctx: GatewayCtx, -): Promise => { - if (result.type === 'api-error') { - recordFailedRequest(ctx, result.performance); - ctx.dump?.error(result.source, result.upstreamId); - return geminiGenerateContentApiErrorResponse(result); - } - - if (result.type === 'internal-error') { - recordFailedRequest(ctx, result.performance); - ctx.dump?.failed(result.error.message); - return geminiGenerateContentErrorResponse(result.status, result.error.message, internalDebugFields(result.error)); - } - - if (result.type === 'plain') { - if (result.status >= 400) { - ctx.dump?.error(result.upstreamId !== undefined ? 'upstream' : 'gateway', result.upstreamId); - } - return plainResultToResponse(result); - } - - const state = new SourceStreamState(); - const observed = observeGeminiGenerateContentFrames(result.events, state, ctx); - const frames = wrapGeminiGenerateContentAffinityEgress(observed, affinityEgressOptions(ctx)); - - if (!wantsStream) { - try { - const response = await collectGeminiGenerateContentProtocolEventsToResult(frames); - const metadata = await eventResultMetadata(result); - const usage = tokenUsageFromBillableUsage(metadata.billableUsage); - ctx.dump?.success(metadata.modelIdentity, usage); - settle(ctx, metadata.performance, metadata.modelIdentity, usage, state.failed); - return Response.json(response, { headers: mergeForwardedUpstreamHeaders(undefined, result.headers) }); - } catch (error) { - recordFailedRequest(ctx, result.performance); - ctx.dump?.failed(error); - return geminiGenerateContentCollectErrorResponse(error); - } - } - - forwardUpstreamHeaders(c, result.headers); - return streamSSE(c, async stream => { - let completion: StreamCompletion = 'error'; - try { - completion = await writeSSEFrames(stream, geminiGenerateContentSseFrames(frames, state, ctx), { - keepAlive: { frame: sseCommentFrame('keepalive') }, - ...(ctx.downstreamAbortController !== undefined ? { downstreamAbortController: ctx.downstreamAbortController } : {}), - }); - } finally { - const metadata = await eventResultMetadata(result); - const failed = state.failedAfter(completion); - if (failed) { - ctx.dump?.failed(`gemini stream failed (completion=${completion}, source-failed=${state.failed})`); - } else { - ctx.dump?.success(metadata.modelIdentity, tokenUsageFromBillableUsage(metadata.billableUsage)); - } - settle(ctx, metadata.performance, metadata.modelIdentity, tokenUsageFromBillableUsage(metadata.billableUsage), failed); - } - }); -}; - -// --- error rendering: Google-RPC envelope --- - -type GeminiGenerateContentErrorDebugFields = Partial> & { target_api?: string }; - -type GeminiGenerateContentErrorStatusPayload = { - error: GeminiGenerateContentErrorResponse['error'] & GeminiGenerateContentErrorDebugFields; -}; - -const googleRpcHttpStatusCode = (status: number): number => (Number.isInteger(status) && status >= 400 && status <= 599 ? status : 500); - -const geminiGenerateContentRpcErrorPayload = (status: number, message: string, debug: GeminiGenerateContentErrorDebugFields = {}): GeminiGenerateContentErrorStatusPayload => { - const code = googleRpcHttpStatusCode(status); - return { - error: { code, message, status: geminiGenerateContentStatusForHttpStatus(code), ...debug }, - }; -}; - -const internalDebugFields = (error: InternalDebugError): GeminiGenerateContentErrorDebugFields => ({ - type: error.type, - name: error.name, - stack: error.stack, - cause: error.cause, - ...(error.target_api ? { target_api: error.target_api } : {}), -}); - -const geminiGenerateContentInternalRpcErrorPayload = (status: number, error: unknown): GeminiGenerateContentErrorStatusPayload => { - const debug = toInternalDebugError(error); - return geminiGenerateContentRpcErrorPayload(status, debug.message, internalDebugFields(debug)); -}; - -// Response builders. The count_tokens path under `http.ts` reuses them -// alongside the error renderer for its synthesized JSON envelope. -export const geminiGenerateContentRpcErrorResponse = (status: number, message: string): Response => { - const payload = geminiGenerateContentRpcErrorPayload(status, message); - return Response.json(payload, { status: payload.error.code }); -}; - -export const geminiGenerateContentInternalRpcErrorResponse = (status: number, error: unknown): Response => { - const payload = geminiGenerateContentInternalRpcErrorPayload(status, error); - return Response.json(payload, { status: payload.error.code }); -}; - -const geminiGenerateContentErrorResponse = (status: number, message: string, debug: GeminiGenerateContentErrorDebugFields = {}): Response => { - // For gateway-minted errors, a non-500 that maps to INTERNAL is coerced to 500. - const code = geminiGenerateContentStatusForHttpStatus(status) === 'INTERNAL' && status !== 500 ? 500 : status; - return Response.json({ error: { code, message, status: geminiGenerateContentStatusForHttpStatus(code), ...debug } }, { status: code }); -}; - -const geminiGenerateContentApiErrorResponse = (error: ApiErrorResult): Response => googleRpcErrorPassthroughResponse(error) ?? geminiGenerateContentErrorResponse(error.status, apiErrorMessage(error)); - -const geminiGenerateContentCollectErrorResponse = (error: unknown): Response => { - const geminiGenerateContentError = caughtGeminiGenerateContentErrorEvent(error); - return geminiGenerateContentError - ? Response.json(geminiGenerateContentError, { status: googleRpcHttpStatusCode(geminiGenerateContentError.error.code) }) - : geminiGenerateContentInternalRpcErrorResponse(502, error); -}; - -// Recognizing / extracting an upstream-shaped Gemini generateContent error from a raw body or a -// thrown cause, so a native Google error is forwarded verbatim rather than -// re-wrapped. -const parseJson = (value: string): unknown => { - try { - return JSON.parse(value); - } catch { - return undefined; - } -}; - -const isGeminiGenerateContentErrorResponse = (value: unknown): value is GeminiGenerateContentErrorResponse => { - if (!value || typeof value !== 'object' || !('error' in value)) return false; - const error = (value as { error?: unknown }).error; - if (!error || typeof error !== 'object') return false; - const payload = error as Partial; - return typeof payload.code === 'number' && typeof payload.message === 'string' && typeof payload.status === 'string'; -}; - -const googleRpcErrorPassthroughResponse = (error: ApiErrorResult): Response | null => { - const parsed = parseJson(decodeApiErrorBody(error).trim()); - if (!isGeminiGenerateContentErrorResponse(parsed)) return null; - - return new Response(error.body.slice(), { - status: googleRpcHttpStatusCode(parsed.error.code), - headers: new Headers(error.headers), - }); -}; - -const apiErrorMessage = (error: ApiErrorResult): string => { - const body = decodeApiErrorBody(error).trim(); - return body || 'Upstream Gemini generateContent request failed.'; -}; - -const caughtGeminiGenerateContentErrorEvent = (error: unknown): GeminiGenerateContentErrorResponse | null => { - if (!(error instanceof Error)) return null; - return isGeminiGenerateContentErrorResponse(error.cause) ? error.cause : null; -}; - -const geminiGenerateContentStreamErrorEvent = (error: unknown): GeminiGenerateContentStreamEvent => - (caughtGeminiGenerateContentErrorEvent(error) ?? geminiGenerateContentInternalRpcErrorPayload(500, error)) as unknown as GeminiGenerateContentStreamEvent; - -// --- frame observation --- - -const isGeminiGenerateContentTerminalFrame = (frame: ProtocolFrame): boolean => frame.type === 'done' || (frame.type === 'event' && isGeminiGenerateContentTerminalEvent(frame.event)); - -const observeGeminiGenerateContentFrames = async function* (frames: AsyncIterable>, state: SourceStreamState, ctx: GatewayCtx) { - for await (const frame of frames) { - await ctx.dump?.frame(frame); - const failed = frame.type === 'event' && isGeminiGenerateContentErrorEvent(frame.event); - if (failed) state.failed = true; - if (isGeminiGenerateContentTerminalFrame(frame) && !failed) state.completed = true; - yield frame; - if (isGeminiGenerateContentTerminalFrame(frame)) return; - } - throw new Error(GEMINI_GENERATE_CONTENT_MISSING_TERMINAL_MESSAGE); -}; - -const geminiGenerateContentSseFrames = async function* (frames: AsyncIterable>, state: SourceStreamState, ctx: GatewayCtx) { - try { - for await (const frame of frames) { - const sse = geminiGenerateContentProtocolFrameToSSEFrame(frame); - if (sse) yield sse; - } - } catch (error) { - state.failed = true; - const event = geminiGenerateContentStreamErrorEvent(error); - await ctx.dump?.frame(eventFrame(event)); - yield sseFrame(JSON.stringify(event)); - } -}; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/serve.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/serve.ts deleted file mode 100644 index 9ca2706a9b..0000000000 --- a/packages/gateway/src/data-plane/chat/gemini-generate-content/serve.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { analyzeGeminiGenerateContentAffinity } from './affinity/ingress.ts'; -import { geminiGenerateContentAttempt, geminiGenerateContentCountTokensTarget, geminiGenerateContentGenerateTarget } from './attempt.ts'; -import { renderGeminiGenerateContentFailure } from './errors.ts'; -import { enumerateModelCandidates } from '../../providers/resolution.ts'; -import { iterateCandidates } from '../../shared/iterate-candidates.ts'; -import { selectAffinityCandidates } from '../shared/affinity/index.ts'; -import { noViableCandidateFailure } from '../shared/errors.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { GeminiGenerateContentPayload, GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; -import type { ExecuteResult, PlainResult } from '@floway-dev/provider'; - -export interface GeminiGenerateContentServeGenerateArgs { - readonly payload: GeminiGenerateContentPayload; - readonly ctx: ChatGatewayCtx; - // Per-request model id (Gemini generateContent carries it in the URL path, not the body), - // resolved by the HTTP entry and threaded through here so candidate - // enumeration and failure rendering all see the same value. - readonly model: string; - readonly headers: Headers; -} - -export interface GeminiGenerateContentServeCountTokensArgs { - readonly payload: GeminiGenerateContentPayload; - readonly ctx: ChatGatewayCtx; - readonly model: string; - readonly headers: Headers; -} - -export const geminiGenerateContentServe = { - generate: async (args: GeminiGenerateContentServeGenerateArgs): Promise>> => { - const { payload, ctx, model, headers } = args; - const { candidates: enumerated, sawModel, failedUpstreams } = await enumerateModelCandidates({ - upstreamIds: ctx.upstreamIds, - model, - kind: 'chat', - scheduler: ctx.backgroundScheduler, - runtimeLocation: ctx.runtimeLocation, - }); - const affinity = await analyzeGeminiGenerateContentAffinity(payload, ctx.affinity.codec); - const viable = enumerated.filter(c => geminiGenerateContentGenerateTarget.canServe(c.model.endpoints)); - const selection = selectAffinityCandidates(viable, affinity); - if ('kind' in selection) return renderGeminiGenerateContentFailure(selection, 'generate'); - if (selection.candidates.length === 0) return renderGeminiGenerateContentFailure(noViableCandidateFailure(sawModel, model, failedUpstreams), 'generate'); - - // Gemini generateContent carries the requested model in its URL, so affinity preparation - // owns each candidate payload while dispatch uses the candidate's canonical model. - return await iterateCandidates( - selection.candidates, - 'geminiGenerateContentServe.generate', - ctx, - 'chat', - async candidate => { - const result = await geminiGenerateContentAttempt.generate({ payload: selection.payloadFor(candidate), ctx, candidate, headers }); - if (result.type === 'events') ctx.affinity.select(candidate); - return result; - }, - ); - }, - - countTokens: async (args: GeminiGenerateContentServeCountTokensArgs): Promise> | PlainResult> => { - const { payload, ctx, model, headers } = args; - const { candidates: enumerated, sawModel, failedUpstreams } = await enumerateModelCandidates({ - upstreamIds: ctx.upstreamIds, - model, - kind: 'chat', - scheduler: ctx.backgroundScheduler, - runtimeLocation: ctx.runtimeLocation, - }); - const affinity = await analyzeGeminiGenerateContentAffinity(payload, ctx.affinity.codec); - const viable = enumerated.filter(c => geminiGenerateContentCountTokensTarget.canServe(c.model.endpoints)); - const selection = selectAffinityCandidates(viable, affinity); - if ('kind' in selection) return renderGeminiGenerateContentFailure(selection, 'countTokens'); - if (selection.candidates.length === 0) return renderGeminiGenerateContentFailure(noViableCandidateFailure(sawModel, model, failedUpstreams), 'countTokens'); - - return await iterateCandidates( - selection.candidates, - 'geminiGenerateContentServe.countTokens', - ctx, - 'chat', - candidate => geminiGenerateContentAttempt.countTokens({ payload: selection.payloadFor(candidate), ctx, candidate, headers }), - ); - }, -}; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/strip-safety-settings.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/strip-safety-settings.ts new file mode 100644 index 0000000000..50cc5e6662 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/strip-safety-settings.ts @@ -0,0 +1,31 @@ +import type { Chat } from '../facts.ts'; +import { withoutKeys } from '../shared/structural.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; + +/** Gemini generateContent safety controls are source-specific and have no matching control on every target + * path, so they go rather than have us pretend to enforce a policy we cannot honor + * end to end. */ +export const stripSafetySettingsFromGeminiGenerateContent = defineStage< + Chat<'request.chat.geminiGenerateContent'>, + Chat<'request.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'> +>({ + name: 'stripSafetySettings', + through: { + request: { needs: ['request.chat.geminiGenerateContent'], consumes: [], provides: ['request.chat.geminiGenerateContent'] }, + response: { needs: ['response.chat.geminiGenerateContent'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.geminiGenerateContent'>, + Chat<'request.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'> + >(() => ({ + request: facts => { + const payload = facts['request.chat.geminiGenerateContent']; + const stripped = withoutKeys(payload, ['safetySettings']); + return stripped === payload ? facts : { ...facts, 'request.chat.geminiGenerateContent': move(stripped) }; + }, + })), +}); diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/strip-unsupported-part-fields.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/strip-unsupported-part-fields.ts new file mode 100644 index 0000000000..7c70c02e06 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/strip-unsupported-part-fields.ts @@ -0,0 +1,74 @@ +import type { Chat } from '../facts.ts'; +import { mapKeepingIdentity, withoutKeys } from '../shared/structural.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; +import type { GeminiGenerateContentPayload, GeminiGenerateContentContent } from '@floway-dev/protocols/gemini-generate-content'; + +// ── Gemini generateContent ──────────────────────────────────────────────────────────────── +// +// The three strippers below are unconditional rather than flag-gated, and that is a statement +// about the target graph rather than about any upstream: Gemini generateContent has no wire of its own here, +// so every turn is served through a translation, and what no translation can carry cannot be +// sent whichever candidate answers. Stripping is a rewrite rather than a deletion — the record +// is frozen, so a stage that deleted a key in place would throw rather than strip. + +/** Gemini generateContent file and code parts have no equivalent anywhere the translations reach, so they go + * at source and every target sees translatable parts. A part left holding nothing goes with + * them: it is not a part any more. */ +export const stripUnsupportedPartFieldsFromGeminiGenerateContent = defineStage< + Chat<'request.chat.geminiGenerateContent'>, + Chat<'request.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'> +>({ + name: 'stripUnsupportedPartFields', + through: { + request: { needs: ['request.chat.geminiGenerateContent'], consumes: [], provides: ['request.chat.geminiGenerateContent'] }, + response: { needs: ['response.chat.geminiGenerateContent'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.geminiGenerateContent'>, + Chat<'request.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'> + >(() => ({ + request: facts => { + const payload = facts['request.chat.geminiGenerateContent']; + const stripped = withUnsupportedPartFieldsStripped(payload); + return stripped === payload ? facts : { ...facts, 'request.chat.geminiGenerateContent': move(stripped) }; + }, + })), +}); + +/** `fileData` addresses a Google-hosted file the API will not hand back — "you can use the API + * to get metadata about the files, but you can't download the files" — and the two code parts + * are the transcript of the server-side `codeExecution` tool that the stage below strips. + * None survives a translation, so they go at source. + * https://ai.google.dev/gemini-api/docs/files + * https://ai.google.dev/gemini-api/docs/code-execution */ +const UNSUPPORTED_GEMINI_PART_FIELDS = ['fileData', 'executableCode', 'codeExecutionResult'] as const; + +const withUnsupportedPartFieldsStripped = (payload: GeminiGenerateContentPayload): GeminiGenerateContentPayload => { + const contents = payload.contents === undefined + ? undefined + : mapKeepingIdentity(payload.contents, stripContentParts); + const systemInstruction = payload.systemInstruction === undefined + ? undefined + : stripContentParts(payload.systemInstruction); + if (contents === payload.contents && systemInstruction === payload.systemInstruction) return payload; + return { + ...payload, + ...(contents === undefined ? {} : { contents }), + ...(systemInstruction === undefined ? {} : { systemInstruction }), + }; +}; + +const stripContentParts = (content: GeminiGenerateContentContent): GeminiGenerateContentContent => { + let changed = false; + const parts = content.parts.flatMap(part => { + const stripped = withoutKeys(part, UNSUPPORTED_GEMINI_PART_FIELDS); + if (stripped === part) return [part]; + changed = true; + return Object.keys(stripped).length > 0 ? [stripped] : []; + }); + return changed ? { ...content, parts } : content; +}; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/strip-unsupported-tools.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/strip-unsupported-tools.ts new file mode 100644 index 0000000000..6d10d5a116 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/strip-unsupported-tools.ts @@ -0,0 +1,66 @@ +import type { Chat } from '../facts.ts'; +import { withoutKeys } from '../shared/structural.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; +import type { GeminiGenerateContentPayload, GeminiGenerateContentToolGroup } from '@floway-dev/protocols/gemini-generate-content'; + +/** Only function declarations translate out of a Gemini generateContent tool group, so the rest of a group's + * capabilities go — and a group left declaring no function goes with them, because a target + * emitter offered an empty group would be offered a tool that does nothing. `tools` itself + * goes when no group survived: an empty tool list is a different request from no tools. */ +export const stripUnsupportedToolsFromGeminiGenerateContent = defineStage< + Chat<'request.chat.geminiGenerateContent'>, + Chat<'request.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'> +>({ + name: 'stripUnsupportedTools', + through: { + request: { needs: ['request.chat.geminiGenerateContent'], consumes: [], provides: ['request.chat.geminiGenerateContent'] }, + response: { needs: ['response.chat.geminiGenerateContent'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.geminiGenerateContent'>, + Chat<'request.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'> + >(() => ({ + request: facts => { + const payload = facts['request.chat.geminiGenerateContent']; + const stripped = withUnsupportedToolsStripped(payload); + return stripped === payload ? facts : { ...facts, 'request.chat.geminiGenerateContent': move(stripped) }; + }, + })), +}); + +/** Every field a Gemini generateContent tool group can carry other than `functionDeclarations` — the + * capabilities Google executes inside its own generation loop, which no target wire can be + * asked to run. `functionDeclarations` is the one that survives because the model "does not + * execute the function"; it asks the caller to. Re-derive against the `Tool` schema whenever + * it gains a field, which it still does. + * https://generativelanguage.googleapis.com/$discovery/rest?version=v1beta + * https://ai.google.dev/api/generate-content */ +const UNSUPPORTED_GEMINI_TOOL_CAPABILITIES = [ + 'googleSearch', + 'googleSearchRetrieval', + 'codeExecution', + 'computerUse', + 'urlContext', + 'fileSearch', + 'mcpServers', + 'googleMaps', +] as const; + +const withUnsupportedToolsStripped = (payload: GeminiGenerateContentPayload): GeminiGenerateContentPayload => { + const { tools } = payload; + if (tools === undefined) return payload; + let changed = false; + const kept: GeminiGenerateContentToolGroup[] = []; + for (const tool of tools) { + const stripped = withoutKeys(tool, UNSUPPORTED_GEMINI_TOOL_CAPABILITIES); + if (stripped !== tool) changed = true; + if (stripped.functionDeclarations !== undefined && stripped.functionDeclarations.length > 0) kept.push(stripped); + else changed = true; + } + if (kept.length === 0) return withoutKeys(payload, ['tools']); + return changed ? { ...payload, tools: kept } : payload; +}; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/suppress-thought-parts.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/suppress-thought-parts.ts new file mode 100644 index 0000000000..f8be359a8d --- /dev/null +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/suppress-thought-parts.ts @@ -0,0 +1,92 @@ +import { isFailure } from '../../pipeline/facts.ts'; +import type { Chat } from '../facts.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; +import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; +import type { GeminiGenerateContentStreamEvent } from '@floway-dev/protocols/gemini-generate-content'; + +/** + * Hides Gemini generateContent thought-summary parts from a caller who did not ask for them. + * + * The opt-in is `generationConfig.thinkingConfig.includeThoughts`, which is something the + * client sent — a request-direction reading — and the rule that uses it runs on the way back. + * So it rides in this stage's own closure rather than as a declaration: a response-side + * `needs` can only name what the ending provides, which is the answer and not the turn. + */ +export const suppressThoughtPartsFromGeminiGenerateContent = defineStage< + Chat<'request.chat.geminiGenerateContent'>, + Chat<'request.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'> +>({ + name: 'suppressThoughtParts', + through: { + request: { needs: ['request.chat.geminiGenerateContent'], consumes: [], provides: [] }, + response: { needs: ['response.chat.geminiGenerateContent'], consumes: [], provides: ['response.chat.geminiGenerateContent'] }, + }, + execute: transform< + Chat<'request.chat.geminiGenerateContent'>, + Chat<'request.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'>, + Chat<'response.chat.geminiGenerateContent'> + >(() => { + // Assigned on the way down and read on the way back, which is the order `transform` runs + // the two halves in. + let includeThoughts!: boolean; + return { + request: facts => { + includeThoughts = facts['request.chat.geminiGenerateContent'].generationConfig?.thinkingConfig?.includeThoughts === true; + return facts; + }, + response: facts => { + if (includeThoughts) return facts; + const answer = facts['response.chat.geminiGenerateContent']; + // A refusal and a collected body have no thought parts to hide, and telling the three + // apart is reading a value rather than dispatching on a declaration. + if (isFailure(answer) || answer.kind !== 'stream') return facts; + const frames = answer.frames as AsyncIterable>; + return { + ...facts, + 'response.chat.geminiGenerateContent': move({ + kind: 'stream' as const, + frames: { [Symbol.asyncIterator]: () => withoutThoughtParts(frames) }, + }), + }; + }, + }; + }), +}); + +/** A candidate that has nothing left to say and has not finished is not a candidate, and an + * event left carrying no candidate, no usage, no model and no id is not an event — dropping + * both is what keeps a stream of pure thought from reaching the client as a stream of + * empties. */ +const withoutThoughtParts = async function* ( + frames: AsyncIterable>, +): AsyncGenerator> { + for await (const frame of frames) { + if (frame.type !== 'event' || 'error' in frame.event) { + yield frame; + continue; + } + + const candidates = frame.event.candidates?.flatMap(candidate => { + const parts = candidate.content.parts.filter(part => part.thought !== true); + if (!parts.length && candidate.finishReason === undefined) return []; + return [{ ...candidate, content: { ...candidate.content, parts } }]; + }); + + const event: GeminiGenerateContentStreamEvent = { + ...frame.event, + ...(candidates === undefined ? {} : { candidates }), + }; + if (hasGeminiGenerateContentEventPayload(event)) yield eventFrame(event); + } +}; + +const hasGeminiGenerateContentEventPayload = (event: GeminiGenerateContentStreamEvent): boolean => { + if ('error' in event) return true; + return (event.candidates?.length ?? 0) > 0 + || event.usageMetadata !== undefined + || event.modelVersion !== undefined + || event.responseId !== undefined; +}; diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/target.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/target.ts new file mode 100644 index 0000000000..065e738ad9 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/target.ts @@ -0,0 +1,22 @@ +import type { ChatNarrowing } from '../resolve-candidates.ts'; +import type { Fields } from './facts.ts'; +import { chatTargetPicker } from '../shared/target-picker.ts'; +import { analyzeGeminiGenerateContentAffinity } from './affinity/ingress.ts'; + +/** `:generateContent` has no wire of its own, so the whole preference list is translated: + * OpenAI Chat Completions first, then Anthropic Messages, then OpenAI Responses. */ +export const geminiGenerateContentTarget = chatTargetPicker(['openaiChatCompletions', 'anthropicMessages', 'openaiResponses']); + +export const narrowing: ChatNarrowing, 'request.chat.geminiGenerateContent'> = ({ + requestKey: 'request.chat.geminiGenerateContent', + canServe: candidate => geminiGenerateContentTarget.canServe(candidate.model.endpoints), + affinity: async (payload, gateway) => await analyzeGeminiGenerateContentAffinity(payload, gateway.affinity.codec), + unsupported: model => `Model ${model} does not support the Gemini generateContent endpoint.`, + refuse: (status, message) => ({ + 'response.chat.geminiGenerateContent': { status, message }, + // A refusal never opened a stream, so there is nothing still to read — which is what + // lets settlement write its row here rather than wait for numbers that never come. + 'response.chat.geminiGenerateContent.streamedUsage': null, + }), + refuses: ['response.chat.geminiGenerateContent', 'response.chat.geminiGenerateContent.streamedUsage'], +}); diff --git a/packages/gateway/src/data-plane/chat/gemini-generate-content/wires.ts b/packages/gateway/src/data-plane/chat/gemini-generate-content/wires.ts new file mode 100644 index 0000000000..690b4a2540 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/gemini-generate-content/wires.ts @@ -0,0 +1,46 @@ +import { composeChat as compose } from '../compose.ts'; +import { handOff } from '../handoff.ts'; +import { STREAMED_USAGE } from './facts.ts'; +import { anthropicMessagesWire } from '../anthropic-messages/wire.ts'; +import { openaiChatCompletionsWire } from '../openai-chat-completions/wire.ts'; +import { projectOpenAIResponsesCollaboration } from '../openai-responses/collaboration-shim.ts'; +import { openaiResponsesWire } from '../openai-responses/wire.ts'; +import type { ChatWire } from '../wire.ts'; +import type { ChatTargetApi, ModelCandidate } from '@floway-dev/provider'; +import { translateGeminiGenerateContentViaOpenAIChatCompletions, translateGeminiGenerateContentViaAnthropicMessages, translateGeminiGenerateContentViaOpenAIResponses } from '@floway-dev/translate'; + +/** The three wires `:generateContent` can be served on, and all three are translated ones: + * the provider surface has no Gemini generateContent call, so a handoff is how every candidate is reached. */ +export const geminiGenerateContentWireFor = (target: ChatTargetApi, candidate: ModelCandidate): ChatWire => { + const context = { model: candidate.model.id, fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens }; + switch (target) { + case 'openaiChatCompletions': + return compose('geminiGenerateContentViaOpenAIChatCompletions', [ + handOff({ + from: { request: 'request.chat.geminiGenerateContent', response: 'response.chat.geminiGenerateContent' }, + to: { request: 'request.chat.openaiChatCompletions', response: 'response.chat.openaiChatCompletions' }, + trip: async payload => await translateGeminiGenerateContentViaOpenAIChatCompletions(payload, context), + }), + ...openaiChatCompletionsWire(STREAMED_USAGE), + ]); + case 'anthropicMessages': + return compose('geminiGenerateContentViaAnthropicMessages', [ + handOff({ + from: { request: 'request.chat.geminiGenerateContent', response: 'response.chat.geminiGenerateContent' }, + to: { request: 'request.chat.anthropicMessages', response: 'response.chat.anthropicMessages' }, + trip: async payload => await translateGeminiGenerateContentViaAnthropicMessages(payload, context), + }), + ...anthropicMessagesWire(STREAMED_USAGE), + ]); + case 'openaiResponses': + return compose('geminiGenerateContentViaOpenAIResponses', [ + handOff({ + from: { request: 'request.chat.geminiGenerateContent', response: 'response.chat.geminiGenerateContent' }, + to: { request: 'request.chat.openaiResponses', response: 'response.chat.openaiResponses' }, + trip: async payload => await translateGeminiGenerateContentViaOpenAIResponses(payload, context), + }), + projectOpenAIResponsesCollaboration, + ...openaiResponsesWire(STREAMED_USAGE), + ]); + } +}; diff --git a/packages/gateway/src/data-plane/chat/handoff.ts b/packages/gateway/src/data-plane/chat/handoff.ts new file mode 100644 index 0000000000..0fd82acf0c --- /dev/null +++ b/packages/gateway/src/data-plane/chat/handoff.ts @@ -0,0 +1,164 @@ +import type { ChatFacts, ChatAnswer } from './facts.ts'; +import type { RequestKey } from './wire.ts'; +import { isFailure } from '../pipeline/facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import { TranslatorInputError, type TranslateTripResult } from '@floway-dev/translate'; + +type ResponseKey = Extract; + +export interface Handoff { + /** The two request keys, and the two response keys they answer at. Naming all four is what + * lets the declaration be written; a pair is not derivable from one half. */ + readonly from: { readonly request: Source; readonly response: ResponseKey }; + readonly to: { readonly request: Target; readonly response: ResponseKey }; + /** The pair function, already closed over whatever context it reads. It returns the target + * payload together with the closure that maps the target's frames back — trip-scoped + * state (synthetic ids, custom-tool name sets) lives in that closure and the source chain + * never sees it. + * + * Only the *target's* event type is named. It is what the pair's mapping closure is handed, + * so leaving it open would make the frames coming off the wire unassignable to it; what the + * closure produces goes straight to the source's response key, which no declaration here + * looks inside. */ + readonly trip: (source: ChatFacts[Source]) => Promise>; +} + +/** + * What a source chain holds when the translation itself refused. + * + * The four keys are what "no upstream was called" looks like on this family's slice: nothing + * billed, no upstream headers to carry, and nothing still streaming for settlement to wait on. + * The streamed-usage key is the response key's own sub-key by the fact space's naming rule, so + * naming it here costs no call site a parameter. A family that broke the convention is caught by + * the runner rather than by assembly: `handOff` is the first stage of every wire, so nothing + * above it inside that composition needs the key, and it is `dialChatWire` — which declares the + * whole set the wire owes — whose provides check fails when the wire hands up a key by another + * name. + * + * No envelope is written. Each family renders a failure in its own protocol at its own edge, + * which is the only place that knows what that protocol's clients read. + */ +const refuseTranslation = (response: ResponseKey, message: string): Record => ({ + 'response.usage.billable': [], + 'response.http.headers': [], + 'response.http.body': null, 'response.http.status': 400, + [`${response}.streamedUsage`]: null, + [response]: { status: 400, message }, +}); + +/** + * Translates on the way down and back on the way up. + * + * An upstream refusal gets the pair's own rewrite where it has one: the canonical case is a + * context-window error becoming the Anthropic `prompt is too long:` shape that Claude Code + * reads for auto-compaction. Where a pair declares no rewrite, the status and the sentence go up + * and the object they arrived in does not — it is the target protocol's, and the client is not + * reading that one. + * + * It carries the `return` trait too, because a body the target protocol cannot represent is + * answered here rather than dialled. + */ +export const handOff = ( + handoff: Handoff, +) => defineStage< + { [K in Source]: ChatFacts[Source] }, + { [K in Target]: ChatFacts[Target] }, + { [K in ResponseKey]?: ChatAnswer } & Record, + { [K in ResponseKey]?: ChatAnswer } & Record +>({ + name: `handOff:${handoff.from.request}→${handoff.to.request}`, + return: { + provides: [ + handoff.from.response, + `${handoff.from.response}.streamedUsage`, + 'response.usage.billable', + 'response.http.headers', 'response.http.body', 'response.http.status', + ], + }, + through: { + request: { + needs: [handoff.from.request], + consumes: [handoff.from.request], + provides: [handoff.to.request], + }, + response: { + // The upstream's own headers are read, not rewritten: a pair that rewrites a refusal + // is handed what actually came back rather than a synthesized set. + needs: [handoff.to.response, 'response.http.headers', 'response.http.body'], + consumes: [handoff.to.response], + provides: [handoff.from.response], + }, + }, + execute: async (facts, next) => { + const record = facts as Record; + const { [handoff.from.request]: source, ...down } = record; + + let trip; + try { + trip = await handoff.trip(source as ChatFacts[Source]); + } catch (error) { + // Input this target protocol cannot represent is the client's fault, and the translator + // is what knows which part of the body was at fault. Answering rather than throwing is + // also what keeps it a *candidate's* verdict: the same body may translate cleanly for + // the next candidate, whose wire is a different protocol, and failover re-runs the + // suffix to find out. Anything else raised down here is a fault of this gateway's and + // rides up as one. + if (!(error instanceof TranslatorInputError)) throw error; + return move({ ...down, ...refuseTranslation(handoff.from.response, error.message) }) as never; + } + + const back = await next({ ...down, [handoff.to.request]: move(trip.target) } as never) as Record; + const { [handoff.to.response]: answered, ...up } = back; + const answer = answered as ChatAnswer; + + if (isFailure(answer)) { + // The pair works on the upstream's own bytes, which is what `message` holds: a refusal + // is read as text and parsed only for whoever wants the object. + const rewritten = trip.apiError?.({ + status: answer.status, + headers: new Headers((up['response.http.headers'] as readonly (readonly [string, string])[]).map(([name, value]): [string, string] => [name, value])), + body: new TextEncoder().encode(answer.message), + }); + // A pair that declares no rewrite has nothing that can carry the refusal across, so what + // crosses is the status and the sentence and not the object they came in. The object is + // the *target* protocol's envelope — an OpenAI `{error:{type,code}}` where the client is + // reading Anthropic, or anything at all where the client is Gemini generateContent and reads + // `error.status` — and handing it on would answer one protocol in another's words. The + // family's own edge renders what is left, in the protocol its client actually speaks. + // + // Nothing is lost to the record: the target's failure was already written at the target's + // own key, with its body, before this stage consumed it. + if (rewritten === undefined) { + const { body: _foreign, envelope: _foreignEnvelope, ...carried } = answer; + return { ...up, [handoff.from.response]: move(carried) } as never; + } + const text = new TextDecoder().decode(rewritten.body); + let parsed: unknown; + try { parsed = JSON.parse(text) as unknown; } catch { parsed = undefined; } + return { + ...up, + [handoff.from.response]: move({ + status: rewritten.status, + message: text, + ...(parsed === undefined ? {} : { body: parsed }), + }), + } as never; + } + if (answer.kind === 'value') { + // A pair maps frames and nothing else, so an answer that arrived as a value has no + // mapping to cross the protocol boundary with. Handing it on would give the client + // another protocol's object under its own key, so this fails rather than mistranslates. + throw new Error( + `${handoff.to.request} answered with a value, and no pair maps one into ${handoff.from.request}`, + ); + } + return { + ...up, + [handoff.from.response]: move({ + kind: 'stream', + frames: trip.events(answer.frames as AsyncIterable>), + }), + } as never; + }, +}); diff --git a/packages/gateway/src/data-plane/chat/materialize-attempt.ts b/packages/gateway/src/data-plane/chat/materialize-attempt.ts new file mode 100644 index 0000000000..f23e59adee --- /dev/null +++ b/packages/gateway/src/data-plane/chat/materialize-attempt.ts @@ -0,0 +1,22 @@ +import type { Chat } from './facts.ts'; +import type { ChatServices } from './services.ts'; +import type { Slice } from './stage-contracts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; + +// Select this attempt's affinity projection below the fork. Rules then rewrite that request +// fact, while the resolver's other candidate values remain shared for a later attempt. +export const materializeAttempt = (requestKey: string) => defineStage< + Slice<'route.attempt'> & Chat<'request.chat.candidatePayloads'>, + Record, + Record, + Record, + ChatServices +>({ + name: `materializeAttempt:${requestKey}`, + through: { + request: { needs: ['route.attempt', 'request.chat.candidatePayloads'], consumes: [], provides: [requestKey] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => + await next({ ...facts, [requestKey]: move(facts['request.chat.candidatePayloads'][facts['route.attempt'].candidateId]) }), +}); diff --git a/packages/gateway/src/data-plane/chat/meter.ts b/packages/gateway/src/data-plane/chat/meter.ts new file mode 100644 index 0000000000..7cf346f42e --- /dev/null +++ b/packages/gateway/src/data-plane/chat/meter.ts @@ -0,0 +1,117 @@ +// The meter, as a wire's outermost stage. +// +// A wire's rules rewrite the answer on the way up: a usage block gets moved onto the carrier +// chunk the spec puts it on, cache buckets an upstream reports alongside the input count get +// folded back into it, and a vendor's own spelling of both becomes OpenAI's. What the client +// reads is the result of all of that. +// +// So that is what the row must be written from. Reading the upstream's raw frames instead +// bills one number and shows another for the same turn, and the two disagree exactly where a +// normalizer had something to say — which is every upstream that needs one. Placing the meter +// **above** the rules rather than at the dial is what makes the two readings the same reading: +// the frames reach it having been through every rewrite, so there is no seam below it left +// where a figure could be taken early. +// +// It is also why the dial no longer names the streamed-usage key at all. A stage cannot report +// a figure it has no means to produce, and stopping the dial from producing one is what keeps +// the class from coming back the next time a rule is added. + +import type { ChatAnswer, ChatFacts } from './facts.ts'; +import type { ChatServices } from './services.ts'; +import type { BillableEntity } from '../pipeline/facts.ts'; +import { isFailure } from '../pipeline/facts.ts'; +import type { StreamOutcome } from '../pipeline/serve.ts'; +import { streamReferenceOf } from '@floway-dev/dump'; +import { defer, defineStage, move, setRelease, type Owned, type Deferred } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import type { TelemetryModelIdentity } from '@floway-dev/provider'; + +type ResponseKey = Extract; + +/** What one protocol contributes: reading its own events. Everything else about metering — + * where the reading is taken, which identity it is attributed to, what a run that never + * streamed reports — is the same for all of them and lives here. */ +export type ChatMeterReading = ( + frames: AsyncIterable>, + identity: TelemetryModelIdentity, + attempt: { firstOutputTokenAt: number | null }, +) => { readonly frames: AsyncIterable>; readonly outcome: Deferred }; + +/** Credential retries precede the final called entity. Only the final reply has frames + * to meter; earlier endpoint replies retain their own empty-quantity observations. */ +const attributedTo = (billable: readonly BillableEntity[], wire: string): TelemetryModelIdentity => { + if (billable.length === 0) { + throw new Error(`${wire} handed up a stream billable to ${billable.length} entities, and a metered stream requires a called model identity`); + } + return billable[billable.length - 1].identity; +}; + +/** + * Meters whatever the wire below hands up. + * + * A refusal and a collected body have no frames to read, and both are ordinary: the reading is + * `null`, which is how a family says settlement has nothing left to wait for and should write + * the row now. + */ +export const meterChatWire = (spec: { + /** The wire this meters, which names the stage — so a dump says which protocol's events the + * reading was taken off without the name being written twice. */ + readonly wire: string; + /** The answer this wire hands up, whose frames carry the upstream's own usage. */ + readonly answer: ResponseKey; + /** Where the reading lands. It is the *source* family's key rather than this protocol's, + * which is why the wire is told it rather than naming one. */ + readonly streamedUsage: string; + readonly read: ChatMeterReading; +}) => defineStage< + Record, + Record, + { [K in ResponseKey]?: ChatAnswer } & { 'response.usage.billable': readonly BillableEntity[] } & Record, + { [K in ResponseKey]?: ChatAnswer } & Record, + ChatServices +>({ + name: `meter:${spec.wire}`, + through: { + // Nothing on the way down. A meter is a reader of what came back, and naming a request key + // here would put it in the entry contract of every chain that dials this wire. + request: { needs: [], consumes: [], provides: [] }, + response: { + needs: [spec.answer, 'response.usage.billable'], + consumes: [], + provides: [spec.answer, spec.streamedUsage], + }, + }, + execute: async (facts, next, use) => { + const back = await next(facts); + const answer = back[spec.answer] as ChatAnswer; + if (isFailure(answer) || answer.kind !== 'stream') { + return move({ ...back, [spec.streamedUsage]: null }) as never; + } + + const metered = spec.read( + answer.frames as AsyncIterable>, + attributedTo(back['response.usage.billable'], spec.wire), + use.gateway.attempt.timing, + ); + const iterator = metered.frames[Symbol.asyncIterator](); + // Downstream transforms may stop after a terminal or cancellation. The body owner keeps + // this single reading alive so release can finish the upstream's billable usage. + const ownsBody = 'response.http.body' in back && back['response.http.body'] !== null; + const frames = { ...streamReferenceOf(answer.frames), [Symbol.asyncIterator]: () => ownsBody ? { next: () => iterator.next() } : iterator }; + if ('response.http.body' in back && back['response.http.body'] !== null) { + setRelease(back['response.http.body'] as ReadableStream & Owned, async () => { + let step = await iterator.next(); + while (!step.done) step = await iterator.next(); + }); + } + const prior = back['response.usage.billable'].slice(0, -1); + return move({ + ...back, + [spec.answer]: { kind: 'stream' as const, frames }, + [spec.streamedUsage]: prior.length === 0 && use.gateway.abortSignal === undefined ? metered.outcome : defer(metered.outcome.then(outcome => ({ + ...outcome, billable: prior.length === 0 ? outcome.billable : [...prior, ...outcome.billable], + failed: outcome.failed || use.gateway.abortSignal?.aborted === true, + }))), + }) as never; + }, +}); diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/affinity/ingress.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/affinity/ingress.ts index 3700de10d9..7b62d2f5a7 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/affinity/ingress.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/affinity/ingress.ts @@ -1,4 +1,5 @@ import { type AffinityCodec, type AffinityRequestAnalysis, type DecodedAffinityBlob, defineAffinityRequest, projectOptionalAffinityBlob } from '../../shared/affinity/index.ts'; +import { withKeysChanged } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; export const analyzeOpenAIChatCompletionsAffinity = async ( @@ -18,13 +19,14 @@ export const analyzeOpenAIChatCompletionsAffinity = async ( degrades: projections.some(item => item.projection.kind === 'remove' && item.projection.degrades), preferred: projections.every(item => item.projection.preferred), materialize: () => { - const candidatePayload = structuredClone(payload); - for (const { index, projection } of projections) { - const message = candidatePayload.messages[index]; - if (projection.kind === 'preserve') message.reasoning_opaque = projection.value; - else if (projection.kind === 'remove') delete message.reasoning_opaque; - } - return candidatePayload; + const rewritten = new Map(projections.map(({ index, projection }) => [ + index, + withKeysChanged(payload.messages[index], { + reasoning_opaque: projection.kind === 'preserve' ? projection.value : undefined, + }), + ])); + const messages = payload.messages.map((message, index) => rewritten.get(index) ?? message); + return withKeysChanged(payload, { messages }); }, }; }); diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/apply-role-compatibility.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/apply-role-compatibility.ts new file mode 100644 index 0000000000..c39c8da49f --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/apply-role-compatibility.ts @@ -0,0 +1,56 @@ +import type { Chat } from '../facts.ts'; +import { rolesFor, roleRewriter, type RoleRewrite } from '../shared/role-compatibility.ts'; +import { mapKeepingIdentity } from '../shared/structural.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; +import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; + +/** Role rewrites, in the fixed order `system → developer → system → user`. Later rewrites + * are authoritative when flags overlap, and the last step affects only a system message + * that appears after the leading run — which is what "mid-conversation" means. The fold + * itself is `roleRewriter`; what is here is the walk over this protocol's messages. */ +export const applyRoleCompatibilityToOpenAIChatCompletions = defineStage< + Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> +>({ + name: 'applyRoleCompatibility', + through: { + request: { + needs: ['request.chat.openaiChatCompletions', 'route.attempt'], + consumes: [], + // Declared even though it will not always be written: a stage that only modifies a + // field need not write on every run, and `provides \ needs` is the set that must be. + provides: ['request.chat.openaiChatCompletions'], + }, + response: { needs: ['response.chat.openaiChatCompletions'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> + >(() => ({ + request: facts => { + const rewrite = rolesFor(facts['route.attempt'].flags); + if (rewrite === null) return facts; // the free do-nothing path + const payload = facts['request.chat.openaiChatCompletions']; + const messages = rewriteOpenAIChatCompletionsRoles(payload.messages, rewrite); + // Written conditionally the whole way down, so a conversation this does not touch + // comes back by identity and the layer costs what the rewrite actually changed. + return messages === payload.messages + ? facts + : { ...facts, 'request.chat.openaiChatCompletions': move({ ...payload, messages }) }; + }, + })), +}); + +type OpenAIChatCompletionsMessages = OpenAIChatCompletionsPayload['messages']; + +const rewriteOpenAIChatCompletionsRoles = (messages: OpenAIChatCompletionsMessages, rewrite: RoleRewrite): OpenAIChatCompletionsMessages => { + const nextRole = roleRewriter(rewrite); + return mapKeepingIdentity(messages, message => { + const role = nextRole(message.role); + return role === message.role ? message : { ...message, role }; + }); +}; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/attempt.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/attempt.ts deleted file mode 100644 index 6e703dfc92..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/attempt.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { openaiChatCompletionsInterceptors } from './interceptors/index.ts'; -import type { OpenAIChatCompletionsInvocation } from './interceptors/types.ts'; -import { billableUsageFromOpenAIChatCompletionsEvent } from './usage.ts'; -import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; -import { anthropicMessagesAttempt } from '../anthropic-messages/attempt.ts'; -import { openaiResponsesAttempt } from '../openai-responses/attempt.ts'; -import { applyRulesToUpstreamOpenAIChatCompletions } from '../shared/alias-rules.ts'; -import { createExternalImageLoader } from '../shared/external-image-loader.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import { providerStreamResultToExecuteResult } from '../shared/provider-stream-result.ts'; -import { chatTargetPicker } from '../shared/target-picker.ts'; -import { captureFromDump, traverseTranslation } from '../shared/translate-traverse.ts'; -import { runInterceptors } from '@floway-dev/interceptor'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import { type ModelCandidate, type ExecuteResult, providerModelOf } from '@floway-dev/provider'; -import { translateOpenAIChatCompletionsViaAnthropicMessages, translateOpenAIChatCompletionsViaOpenAIResponses } from '@floway-dev/translate'; - -// `/v1/chat/completions` generate prefers a native OpenAI Chat Completions target, -// then the translated Anthropic Messages path, then the translated OpenAI Responses path. -export const openaiChatCompletionsTarget = chatTargetPicker(['openaiChatCompletions', 'anthropicMessages', 'openaiResponses']); - -export interface OpenAIChatCompletionsAttemptArgs { - readonly payload: OpenAIChatCompletionsPayload; - readonly ctx: ChatGatewayCtx; - readonly candidate: ModelCandidate; - readonly headers: Headers; -} - -export const openaiChatCompletionsAttempt = { - generate: async (args: OpenAIChatCompletionsAttemptArgs): Promise>> => { - const { payload: sourcePayload, ctx, candidate, headers: sourceHeaders } = args; - const payload = { ...sourcePayload, model: candidate.model.id }; - const headers = new Headers(sourceHeaders); - const targetApi = openaiChatCompletionsTarget.pick(candidate.model.endpoints); - const invocation: OpenAIChatCompletionsInvocation = { - payload, - candidate, - targetApi, - headers, - }; - return await runInterceptors(invocation, ctx, openaiChatCompletionsInterceptors, async () => { - if (targetApi === 'openaiChatCompletions') { - if (candidate.rules !== undefined) applyRulesToUpstreamOpenAIChatCompletions(invocation.payload, candidate.rules); - const { model: _model, ...body } = invocation.payload; - const providerResult = await candidate.provider.instance.callOpenAIChatCompletions( - providerModelOf(candidate), - body, - ctx.abortSignal, - buildUpstreamCallOptions(candidate, ctx, invocation.headers), - ); - return await providerStreamResultToExecuteResult(providerResult, candidate, 'openaiChatCompletions', ctx, billableUsageFromOpenAIChatCompletionsEvent); - } - if (targetApi === 'anthropicMessages') { - return await traverseTranslation( - invocation.payload, - p => translateOpenAIChatCompletionsViaAnthropicMessages(p, { - model: candidate.model.id, - fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, - loadRemoteImage: createExternalImageLoader(ctx.abortSignal), - }), - translated => anthropicMessagesAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, anthropicBeta: [], - }), - captureFromDump(ctx.dump, targetApi), - ); - } - if (targetApi === 'openaiResponses') { - return await traverseTranslation( - invocation.payload, - p => translateOpenAIChatCompletionsViaOpenAIResponses(p, { model: candidate.model.id }), - translated => openaiResponsesAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, - }), - captureFromDump(ctx.dump, targetApi), - ); - } - throw new Error(`openaiChatCompletionsAttempt.generate: unexpected targetApi '${targetApi as string}'`); - }); - }, -}; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/call-upstream.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/call-upstream.ts new file mode 100644 index 0000000000..c0f3b77b14 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/call-upstream.ts @@ -0,0 +1,40 @@ +import type { Fields } from './facts.ts'; +import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; +import { providerEntry } from '../../pipeline/provider-entry.ts'; +import { upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; +import type { ChatServices } from '../services.ts'; +import { applyRulesToUpstreamOpenAIChatCompletions } from '../shared/alias-rules.ts'; +import { providerChatAnswer } from '../shared/provider-answer.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { ProviderOperationRequest, ProviderChatResponse } from '@floway-dev/provider'; + +export const callOpenAIChatCompletionsUpstream = defineStage< + Fields<'request.chat.openaiChatCompletions' | 'route.attempt' | 'ingress.http.headers'>, + ProviderOperationRequest<'openaiChatCompletions'>, + ProviderChatResponse<'openaiChatCompletions'>, + Fields<'response.chat.openaiChatCompletions' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status' | 'response.http.body'>, + ChatServices +>({ + name: 'callOpenAIChatCompletionsUpstream', + into: { + request: { needs: ['request.chat.openaiChatCompletions', 'route.attempt', 'ingress.http.headers'], consumes: [], provides: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'] }, + response: { needs: ['response.http.exchange', 'response.provider.output', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.http.body'], consumes: ['response.http.exchange', 'response.provider.output', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'], provides: ['response.chat.openaiChatCompletions', 'response.usage.billable', 'response.http.headers', 'response.http.status', 'response.http.body'] }, + }, + execute: async (facts, next, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); + const body = bodyForAttempt(facts['request.chat.openaiChatCompletions'], candidate, applyRulesToUpstreamOpenAIChatCompletions); + const pipeline = candidate.provider.pipelines.openaiChatCompletions; + if (pipeline === undefined) throw new Error(`Provider ${candidate.provider.kind} has no openaiChatCompletions pipeline`); + const back = await next(providerEntry(facts, candidate, body), pipeline); + const { 'response.http.exchange': _exchange, 'response.provider.output': _output, 'response.provider.modelKey': _modelKey, 'response.provider.called': _called, 'response.provider.previousCalls': _previousCalls, ...rest } = back; + const reply = await providerChatAnswer(candidate, back); + const answer = reply.answer; + if ('kind' in answer) use.selectAffinity(candidate); + return move({ + ...rest, 'response.chat.openaiChatCompletions': answer, + 'response.usage.billable': reply.billable, 'response.http.headers': reply.headers, + 'response.http.status': reply.status, 'response.http.body': reply.body, + }); + }, +}); diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/disable-reasoning-on-forced-tool-choice.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/disable-reasoning-on-forced-tool-choice.ts new file mode 100644 index 0000000000..01038439a9 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/disable-reasoning-on-forced-tool-choice.ts @@ -0,0 +1,42 @@ +import type { Chat } from '../facts.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; +import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; + +/** The reasoning sentinel a forced tool choice needs. Gated by its flag, and the sentinel + * is the gateway's canonical form — putting it on the wire in a vendor's shape is the + * vendor normalizer's job, which is why this runs above them. */ +export const disableReasoningOnForcedToolChoiceForOpenAIChatCompletions = defineStage< + Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> +>({ + name: 'disableReasoningOnForcedToolChoice', + through: { + request: { + needs: ['request.chat.openaiChatCompletions', 'route.attempt'], + consumes: [], + provides: ['request.chat.openaiChatCompletions'], + }, + response: { needs: ['response.chat.openaiChatCompletions'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> + >(() => ({ + request: facts => { + if (!facts['route.attempt'].flags.includes('disable-reasoning-on-forced-tool-choice')) { + return facts; + } + const payload = facts['request.chat.openaiChatCompletions']; + if (!isForcedToolChoice(payload.tool_choice)) return facts; + return { ...facts, 'request.chat.openaiChatCompletions': move({ ...payload, reasoning_effort: 'none' }) }; + }, + })), +}); + +/** `required`, or a named function. `auto` and `none` leave the model free to reason. */ +const isForcedToolChoice = (choice: OpenAIChatCompletionsPayload['tool_choice']): boolean => + choice === 'required' || (typeof choice === 'object' && choice !== null); diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/emit.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/emit.ts new file mode 100644 index 0000000000..c005197940 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/emit.ts @@ -0,0 +1,119 @@ +import { wrapOpenAIChatCompletionsAffinityEgress } from './affinity/egress.ts'; +import type { Fields } from './facts.ts'; +import { isFailure, renderFailure, mintedErrorEnvelope } from '../../pipeline/facts.ts'; +import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; +import type { ChatServices } from '../services.ts'; +import { affinityEgressOptions } from '../shared/affinity/index.ts'; +import { bindClientRelease, collectClientFrames, framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; +import { recordStream } from '@floway-dev/dump'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { eventFrame, isOpenAIUsageOnlyEventShape, type ProtocolFrame } from '@floway-dev/protocols/common'; +import { collectOpenAIChatCompletionsProtocolEventsToResult, openaiChatCompletionsProtocolFrameToSSEFrame, type OpenAIChatCompletionsStreamEvent, type ClientOpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; +import { toInternalDebugError } from '@floway-dev/provider'; + +/** + * The outermost edge. A chat answer is always a stream by the time it reaches here — the + * upstream speaks SSE whatever the client asked for — so what this decides is whether the + * client sees the frames or the one object they add up to. + * + * Collecting is therefore the edge's own work and not a second reading of the upstream: the + * same frames that would have gone out are folded here instead. + */ +export const emitOpenAIChatCompletions = defineStage< + Fields<'ingress.chat.openaiChatCompletions.wantsStream' | 'ingress.chat.openaiChatCompletions.wantsUsageChunk'>, + Fields<'ingress.chat.openaiChatCompletions.wantsStream' | 'ingress.chat.openaiChatCompletions.wantsUsageChunk'>, + Fields<'ingress.chat.openaiChatCompletions.wantsStream' | 'ingress.chat.openaiChatCompletions.wantsUsageChunk' + | 'response.chat.openaiChatCompletions' | 'response.http.headers' | 'response.http.status' | 'response.chat.openaiChatCompletions.streamedUsage'>, + Fields<'response.chat.openaiChatCompletions.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.clientFrames' | 'response.chat.openaiChatCompletions.streamedUsage'>, + ChatServices +>({ + name: 'emitOpenAIChatCompletions', + through: { + request: { + needs: ['ingress.chat.openaiChatCompletions.wantsStream', 'ingress.chat.openaiChatCompletions.wantsUsageChunk'], + consumes: [], + provides: [], + }, + response: { + needs: ['response.chat.openaiChatCompletions', 'response.http.headers', 'response.http.status', 'response.chat.openaiChatCompletions.streamedUsage'], + consumes: ['response.chat.openaiChatCompletions', 'response.http.headers'], + provides: ['response.chat.clientFrames', 'response.chat.openaiChatCompletions.rendered', 'response.http.status', 'response.http.headers', 'response.chat.openaiChatCompletions.streamedUsage'], + }, + }, + execute: async (facts, next, use) => { + const back = await next(facts); + const { 'response.chat.openaiChatCompletions': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); + + if (isFailure(answer)) { + const failure = renderFailure(answer, mintedErrorEnvelope); + return { + ...rest, + 'response.chat.clientFrames': null, + 'response.http.headers': forClient, + 'response.chat.openaiChatCompletions.rendered': move(failure.body), + 'response.http.status': failure.status, + }; + } + if (answer.kind === 'value') { + return { + ...rest, + 'response.chat.clientFrames': null, + 'response.http.headers': forClient, + 'response.chat.openaiChatCompletions.rendered': move(answer.body as Record), + 'response.http.status': back['response.http.status'], + }; + } + + // The turn's own state, written back into the frames the client is handed: a follow-up + // carrying it comes back to the upstream that issued it. This is the other half of the + // affinity the resolver read on the way down, and it has to sit here because it rewrites + // the frames — below the fold, and there would be nothing left to rewrite. + const egress = wrapOpenAIChatCompletionsAffinityEgress( + answer.frames as AsyncIterable>, + affinityEgressOptions(use.gateway), + ); + if (!back['ingress.chat.openaiChatCompletions.wantsStream']) { + const frames = recordStream(egress, use.gateway.dump); + return { + ...rest, + 'response.chat.clientFrames': move(frames), + 'response.http.headers': forClient, + 'response.chat.openaiChatCompletions.rendered': move( + await collectClientFrames(frames, collectOpenAIChatCompletionsProtocolEventsToResult) as unknown as Record, + ), + 'response.http.status': back['response.http.status'], + }; + } + const completed = framedClientStream>( + clientFrames(egress, back['ingress.chat.openaiChatCompletions.wantsUsageChunk']), + frame => openaiChatCompletionsProtocolFrameToSSEFrame(frame, { includeUsageChunk: true }), + error => [eventFrame({ error: toInternalDebugError(error, 'openaiChatCompletions') })], + use.gateway.dump, + ); + bindClientRelease(back, completed.release); + return { + ...rest, + 'response.chat.clientFrames': move(completed.frames), + 'response.http.headers': forClient, + 'response.chat.openaiChatCompletions.rendered': move(completed.rendered), + 'response.chat.openaiChatCompletions.streamedUsage': move(withClientVerdict(back['response.chat.openaiChatCompletions.streamedUsage'], completed.failed)), + 'response.http.status': back['response.http.status'], + }; + }, +}); + +// Metering reads every usage chunk below this edge. A streaming client opts into seeing it, +// so that protocol selection precedes the record of what the client is served. +const clientFrames = (frames: AsyncIterable>, includeUsageChunk: boolean): AsyncIterable> => includeUsageChunk ? frames : ({ + [Symbol.asyncIterator]: () => (async function* () { + for await (const frame of frames) { + if (frame.type === 'event' && isOpenAIUsageOnlyEventShape(frame.event)) continue; + yield frame; + } + })(), +}); diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/errors.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/errors.ts deleted file mode 100644 index 8f4d29575c..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/errors.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { appendFailedUpstreams } from '../../shared/failed-upstreams.ts'; -import { openAiErrorResult, type ChatServeFailure } from '../shared/errors.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import type { ExecuteResult, PerformanceTelemetryContext } from '@floway-dev/provider'; -import type { TranslatorInputError } from '@floway-dev/translate'; - -// Translator surfaced a caller-input violation. Render as a 400 -// invalid_request_error so the caller sees a protocol-shaped failure -// instead of the internal-error 502 envelope. `param` falls back to -// `messages` (the OpenAI Chat Completions canonical messages field) when the -// translator did not carry a more specific path. `performance` carries the -// throwing candidate's telemetry attribution when the throw fired -// mid-attempt (see AttemptState.telemetry). -export const translatorInputErrorResult = ( - error: TranslatorInputError, - performance?: PerformanceTelemetryContext, -): ExecuteResult> => - openAiErrorResult(400, error.message, { param: error.param ?? 'messages', code: null }, performance); - -export const renderOpenAIChatCompletionsFailure = ( - failure: ChatServeFailure, -): ExecuteResult> => { - switch (failure.kind) { - case 'routing-unavailable': - return openAiErrorResult(400, failure.message, { param: 'input', code: 'responses_item_routing_unavailable' }); - case 'model-missing': - return openAiErrorResult(404, appendFailedUpstreams(`Model ${failure.model} is not available on any configured upstream.`, failure.failedUpstreams)); - case 'model-unsupported': - return openAiErrorResult(400, appendFailedUpstreams(`Model ${failure.model} does not support the /chat/completions endpoint.`, failure.failedUpstreams)); - } -}; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/facts.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/facts.ts new file mode 100644 index 0000000000..9a58297cc9 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/facts.ts @@ -0,0 +1,30 @@ +import type { StreamOutcome } from '../../pipeline/serve.ts'; +import type { ChatFacts } from '../facts.ts'; +import type { Deferred } from '@floway-dev/pipeline'; +import type { SseFrame } from '@floway-dev/protocols/common'; + +/** What this family adds to the chat space. */ +export interface OpenAIChatCompletionsFacts extends ChatFacts { + /** What the client is actually sent — an object when it asked for one, SSE frames when it + * asked to stream. The edge provides it, so a dump shows what the client received. */ + 'response.chat.openaiChatCompletions.rendered': Record | AsyncIterable; + /** What the upstream will have reported once the frames run out, and `null` when nothing + * streamed. Settling from this is the epilogue's job, after the drain. */ + 'response.chat.openaiChatCompletions.streamedUsage': Deferred | null; +} + +export type Fields = { [P in K]: OpenAIChatCompletionsFacts[P] }; + +/** This family's own reading, which every wire under it hands up. */ +export const STREAMED_USAGE = 'response.chat.openaiChatCompletions.streamedUsage'; + +export type OpenAIChatCompletionsServeEntry = Fields< + 'ingress.http.headers' | 'ingress.chat.sourceProtocol' + | 'ingress.chat.openaiChatCompletions.wantsStream' | 'ingress.chat.openaiChatCompletions.wantsUsageChunk' + | 'request.chat.openaiChatCompletions' | 'serve.model' +>; + +export type OpenAIChatCompletionsServeExit = Fields< + 'response.chat.openaiChatCompletions.rendered' | 'response.chat.openaiChatCompletions.streamedUsage' + | 'response.http.status' | 'response.http.jsonBody' | 'response.http.headers' | 'response.usage.billable' +>; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts index a1d936b749..dd344d895b 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/http.ts @@ -1,66 +1,73 @@ -import { translatorInputErrorResult } from './errors.ts'; -import { respondOpenAIChatCompletions } from './respond.ts'; -import { openaiChatCompletionsServe } from './serve.ts'; +// POST /v1/chat/completions, served through the pipeline. +// +// The handler is a prologue and an epilogue around `openaiChatCompletionsServePipeline`: read what +// the client sent, hand it over, and turn what the run answered with into a response. +// Everything between is stages. +// +// Two things are decided here because only the entry knows them, and both are read before any +// stage can rewrite the payload they are read from: whether the client asked to stream, which +// the run has to be opened knowing, and whether it asked to be shown the usage chunk that +// metering asks the upstream for on every streaming turn. + +import { openaiChatCompletionsServePipeline } from './pipeline.ts'; import type { AuthedContext } from '../../../middleware/auth.ts'; -import { backgroundSchedulerFromContext } from '../../../runtime/background.ts'; -import { createGatewayCtxFromHono, finalizeGatewayResponse, type GatewayCtx } from '../../shared/gateway-ctx.ts'; -import { inboundHeaders } from '../../shared/inbound-headers.ts'; -import { readRequestBody, takeRequestBody, type RequestBody } from '../../shared/request-body.ts'; +import { isFrames, openPrologue, readIngress, serveThrough } from '../../pipeline/serve.ts'; +import { finalizeGatewayResponse } from '../../shared/gateway-ctx.ts'; import { createNonOpenAIResponsesSourceStore } from '../openai-responses/items/store.ts'; -import { createChatGatewayCtxFromHono, type ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import { providerModelsUnavailableResponse } from '../shared/upstream-models-error.ts'; +import { openChatPrologue } from '../prologue.ts'; +import { move } from '@floway-dev/pipeline'; import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; -import { internalErrorResult, toInternalDebugError } from '@floway-dev/provider'; -import { TranslatorInputError } from '@floway-dev/translate'; - -// Surfaces a pre-stream throw (malformed JSON body, an interceptor crash, -// etc.) as an OpenAI Chat Completions-shaped 502 with the same internal-error -// envelope the in-flow `internal-error` ExecuteResult produces. A -// `ProviderModelsUnavailableError` carrying an upstream HTTP body relays -// that body verbatim — the upstream's `/models` 401 IS the diagnostic. The -// caller passes its outer `ctx` when one was already constructed (so the -// dump row preserves the model attribution the request-time -// `requestedModel` stamped, and the throwing-candidate telemetry stamped -// in serve.ts survives onto the error row); a fresh ctx is minted only -// for pre-parse failures where no payload was available to read model from. -const respondWithInternalError = async (c: AuthedContext, error: unknown, requestBody: RequestBody, ctx?: GatewayCtx): Promise => { - const verbatim = providerModelsUnavailableResponse(error); - if (verbatim !== null) return verbatim; - const effectiveCtx = ctx ?? createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - const result = internalErrorResult(502, toInternalDebugError(error), effectiveCtx.attempt.telemetry); - const response = await respondOpenAIChatCompletions(c, result, false, false, effectiveCtx); - return finalizeGatewayResponse(effectiveCtx, response); -}; -// Pre-stream caller-input failure raised by a translator → an OpenAI Chat -// Completions-shaped 400 invalid_request_error envelope. Anything else -// falls through to the internal-error 502 path. -const respondToThrow = async (c: AuthedContext, error: unknown, requestBody: RequestBody, ctx?: GatewayCtx): Promise => { - if (!(error instanceof TranslatorInputError)) return await respondWithInternalError(c, error, requestBody, ctx); - const effectiveCtx = ctx ?? createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - const response = await respondOpenAIChatCompletions(c, translatorInputErrorResult(error, effectiveCtx.attempt.telemetry), false, false, effectiveCtx); - return finalizeGatewayResponse(effectiveCtx, response); +/** A body the gateway could not read is reported in the words the protocol's own clients + * parse, rather than as a fault of the gateway's. */ +const readRequest = (bytes: Uint8Array): { type: 'ok'; payload: OpenAIChatCompletionsPayload } | { type: 'invalid'; message: string } => { + try { + return { type: 'ok', payload: JSON.parse(new TextDecoder().decode(bytes)) as OpenAIChatCompletionsPayload }; + } catch (error) { + return { type: 'invalid', message: error instanceof Error ? error.message : String(error) }; + } }; export const openaiChatCompletionsHttp = { generate: async (c: AuthedContext): Promise => { - const requestBody = await readRequestBody(c); - let ctx: ChatGatewayCtx | undefined; - try { - const payload = JSON.parse(new TextDecoder().decode(requestBody.bytes)) as OpenAIChatCompletionsPayload; - const wantsStream = payload.stream === true; - // Read the caller's intent BEFORE any interceptor mutates - // `payload.stream_options.include_usage`. Capturing it here means the - // downstream renderer never needs to consult per-request Hono context - // slots — the value lives in this http-entry closure for the duration of - // the request. - const includeUsageChunk = payload.stream_options?.include_usage === true; - ctx = createChatGatewayCtxFromHono(c, { wantsStream, requestBody: takeRequestBody(requestBody), model: payload.model, backgroundScheduler: backgroundSchedulerFromContext(c) }, apiKey => createNonOpenAIResponsesSourceStore(apiKey.id)); - const result = await openaiChatCompletionsServe.generate({ payload, ctx, headers: inboundHeaders(c) }); - const response = await respondOpenAIChatCompletions(c, result, wantsStream, includeUsageChunk, ctx); - return finalizeGatewayResponse(ctx, response); - } catch (error) { - return await respondToThrow(c, error, requestBody, ctx); + const ingress = await readIngress(c); + const request = readRequest(ingress.body.bytes); + if (request.type === 'invalid') { + // A request the gateway could not read never reaches a pipeline: there is no model to + // resolve and no attempt to make, so there is nothing for a run to record. + const refused = openPrologue(c, ingress, { wantsStream: false }); + refused.gateway.dump?.error('gateway'); + return finalizeGatewayResponse( + refused.gateway, + Response.json({ error: { message: request.message, type: 'invalid_request_error' } }, { status: 400 }), + ); } + + const { payload } = request; + const wantsStream = payload.stream === true; + const prologue = openChatPrologue(c, ingress, { + wantsStream, + model: payload.model, + storeFactory: apiKey => createNonOpenAIResponsesSourceStore(apiKey.id), + }); + + return await serveThrough( + c, + prologue, + openaiChatCompletionsServePipeline(), + move({ + 'ingress.http.headers': prologue.headers, + 'ingress.chat.sourceProtocol': 'openaiChatCompletions', + 'ingress.chat.openaiChatCompletions.wantsStream': wantsStream, + 'ingress.chat.openaiChatCompletions.wantsUsageChunk': payload.stream_options?.include_usage === true, + 'request.chat.openaiChatCompletions': payload, + 'serve.model': payload.model, + }) as never, + facts => { + const rendered = facts['response.chat.openaiChatCompletions.rendered']; + if (isFrames(rendered)) return { frames: rendered }; + return { body: facts['response.http.jsonBody']!, contentType: 'application/json' }; + }, + ); }, }; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/include-usage-stream-options.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/include-usage-stream-options.ts new file mode 100644 index 0000000000..efc41b6499 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/include-usage-stream-options.ts @@ -0,0 +1,57 @@ +import type { Chat } from '../facts.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; + +/** + * Asks the upstream for the usage chunk that billing is read off. + * + * OpenAI Chat Completions emits the final usage-only chunk only when `stream_options.include_usage` + * is on, and the gateway meters every stream from those frames — so what the client asked for + * and what the upstream is asked for differ here, which is the one thing this rule exists to + * say. Nothing downstream has to thread the client's original value through, because it is a + * fact of its own: `ingress.chat.openaiChatCompletions.wantsUsageChunk` describes the request that + * arrived and survives this rewrite, and the edge reads it to decide who is shown the chunk. + * + * It belongs to the wire rather than to the source chain because it names a field of *this* + * protocol's request. Every source protocol that reaches an upstream over this endpoint runs + * it — an Anthropic Messages or Gemini generateContent turn served here would otherwise be metered off a stream that was + * never asked to report anything — and a turn that leaves for another wire gets that wire's + * own rule for the same thing. + * + * Reference: https://platform.openai.com/docs/api-reference/chat/create + */ +export const includeUsageStreamOptionsForOpenAIChatCompletions = defineStage< + Chat<'request.chat.openaiChatCompletions'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> +>({ + name: 'includeUsageStreamOptions', + through: { + request: { + needs: ['request.chat.openaiChatCompletions'], + consumes: [], + provides: ['request.chat.openaiChatCompletions'], + }, + response: { needs: ['response.chat.openaiChatCompletions'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.openaiChatCompletions'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> + >(() => ({ + request: facts => { + const payload = facts['request.chat.openaiChatCompletions']; + if (payload.stream_options?.include_usage === true) return facts; + // Whatever else the client put on `stream_options` is its own and stays; only the one + // field the gateway has an interest in is written. + return { + ...facts, + 'request.chat.openaiChatCompletions': move({ + ...payload, + stream_options: { ...payload.stream_options, include_usage: true }, + }), + }; + }, + })), +}); diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/apply-role-compatibility.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/apply-role-compatibility.ts deleted file mode 100644 index f374cc0bef..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/apply-role-compatibility.ts +++ /dev/null @@ -1,30 +0,0 @@ -import type { OpenAIChatCompletionsInterceptor } from './types.ts'; -import type { OpenAIChatCompletionsMessage } from '@floway-dev/protocols/openai-chat-completions'; -import { providerModelOf } from '@floway-dev/provider'; - -export const withRoleCompatibilityApplied: OpenAIChatCompletionsInterceptor = (ctx, _gatewayCtx, run) => { - if (ctx.targetApi !== 'openaiChatCompletions') return run(); - - const flags = providerModelOf(ctx.candidate).enabledFlags; - const rewriteSystemToDeveloper = flags.has('rewrite-system-to-developer'); - const rewriteDeveloperToSystem = flags.has('rewrite-developer-to-system'); - const rewriteMidConvSystemToUser = flags.has('rewrite-mid-conv-system-to-user'); - if (!rewriteSystemToDeveloper && !rewriteDeveloperToSystem && !rewriteMidConvSystemToUser) return run(); - - let crossedLeadingSystemRun = false; - ctx.payload = { - ...ctx.payload, - messages: ctx.payload.messages.map(message => { - let mapped: OpenAIChatCompletionsMessage = message; - if (rewriteSystemToDeveloper && mapped.role === 'system') mapped = { ...mapped, role: 'developer' }; - if (rewriteDeveloperToSystem && mapped.role === 'developer') mapped = { ...mapped, role: 'system' }; - if (!crossedLeadingSystemRun && mapped.role !== 'system') crossedLeadingSystemRun = true; - if (rewriteMidConvSystemToUser && crossedLeadingSystemRun && mapped.role === 'system') { - mapped = { ...mapped, role: 'user' }; - } - return mapped; - }), - }; - - return run(); -}; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/disable-reasoning-on-forced-tool-choice.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/disable-reasoning-on-forced-tool-choice.ts deleted file mode 100644 index d39047981f..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/disable-reasoning-on-forced-tool-choice.ts +++ /dev/null @@ -1,24 +0,0 @@ - -import type { OpenAIChatCompletionsInterceptor } from './types.ts'; -import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; -import { providerModelOf } from '@floway-dev/provider'; - -// Opt-in workaround for upstreams where forced `tool_choice` and enabled -// reasoning do not compose. Sets the gateway's canonical "no reasoning" -// sentinel `reasoning_effort: 'none'` (also OpenAI's documented disable -// value). Any active Vendor: * flag's last-running normalizer then -// translates that into the vendor's wire form (DeepSeek -// `thinking: {type:'disabled'}`, Qwen `enable_thinking: false`, etc.). -const hasForcedToolChoice = (payload: OpenAIChatCompletionsPayload): boolean => { - const toolChoice = payload.tool_choice; - if (toolChoice === undefined || toolChoice === null) return false; - if (typeof toolChoice === 'string') return toolChoice === 'required'; - return true; -}; - -export const withReasoningDisabledOnForcedToolChoice: OpenAIChatCompletionsInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('disable-reasoning-on-forced-tool-choice')) return await run(); - if (!hasForcedToolChoice(ctx.payload)) return await run(); - ctx.payload = { ...ctx.payload, reasoning_effort: 'none' }; - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/include-usage-stream-options.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/include-usage-stream-options.ts deleted file mode 100644 index 6ff54cfe57..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/include-usage-stream-options.ts +++ /dev/null @@ -1,20 +0,0 @@ -import type { OpenAIChatCompletionsInterceptor } from './types.ts'; - -// OpenAI Chat Completions streaming only includes the final usage-only chunk when -// `stream_options.include_usage` is enabled. We force that on here because -// the gateway's source responders and usage tracking rely on those usage -// frames for both streaming passthrough and non-stream reassembly. -// -// The follow-on question is whether to surface the synthesized usage chunk to -// the client. The client's intent is whatever the caller put on -// `stream_options.include_usage` BEFORE this interceptor mutated it; the -// downstream SSE renderer reads that intent directly off the original payload -// the http entry parsed (see `openaiChatCompletionsHttp`), so this interceptor only -// has to flip the upstream-facing flag and never needs to thread the original -// value through a Hono context slot. -// -// Reference: https://platform.openai.com/docs/api-reference/chat/create -export const withUsageStreamOptionsIncluded: OpenAIChatCompletionsInterceptor = async (ctx, _gatewayCtx, run) => { - ctx.payload.stream_options = ctx.payload.stream_options ? { ...ctx.payload.stream_options, include_usage: true } : { include_usage: true }; - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/index.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/index.ts deleted file mode 100644 index 26fb4904a1..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/index.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { withRoleCompatibilityApplied } from './apply-role-compatibility.ts'; -import { withReasoningDisabledOnForcedToolChoice } from './disable-reasoning-on-forced-tool-choice.ts'; -import { withUsageStreamOptionsIncluded } from './include-usage-stream-options.ts'; -import { withEmptyToolsToolChoiceNormalized } from './normalize-empty-tools-tool-choice.ts'; -import { withExclusiveCachedTokensNormalized } from './normalize-exclusive-cached-tokens.ts'; -import { withUsageNormalized } from './normalize-usage.ts'; -import { withPromptCacheKeyStripped } from './strip-prompt-cache-key.ts'; -import type { OpenAIChatCompletionsInterceptor } from './types.ts'; -import { withVendorDeepSeekOpenAIChatCompletionsNormalize } from './vendor-deepseek-normalize.ts'; -import { withVendorKimiOpenAIChatCompletionsNormalize } from './vendor-kimi-normalize.ts'; -import { withVendorQwenOpenAIChatCompletionsNormalize } from './vendor-qwen-normalize.ts'; - -// Unified OpenAI Chat Completions interceptor list. All entries are attached to -// every candidate; each interceptor's body decides whether to act (flag-gated -// entries early-return on `providerModelOf(ctx.candidate).enabledFlags.has(flagId)`). -// -// Translated requests re-enter the selected target protocol's chain. The role -// compatibility entry therefore acts only when OpenAI Chat Completions is the final -// target, after pairwise translation has finished. -// -// - withUsageStreamOptionsIncluded, withUsageNormalized: unconditional. -// Both gate the gateway's usage-tracking pipeline. Turning either off -// would silently break per-key telemetry, so neither is surfaced as a flag. -// - withReasoningDisabledOnForcedToolChoice: gated by -// `disable-reasoning-on-forced-tool-choice`. Emits the gateway's canonical -// "no reasoning" sentinel only; vendor wire form is the vendor's job. -// - withEmptyToolsToolChoiceNormalized: gated by -// `empty-tools-tool-choice-none`. Makes an explicitly empty tool list use -// the target protocol's no-tool choice. -// - withRoleCompatibilityApplied: applies role flags in the fixed order -// `system → developer → system → user`; later rewrites are authoritative -// when flags overlap, and the final step affects only mid-conversation system. -// - withPromptCacheKeyStripped: gated by `strip-prompt-cache-key`. Drops -// the top-level `prompt_cache_key` field for upstreams that reject it as -// an unknown argument (e.g. Azure DeepSeek). Runs before vendor -// normalizers so vendor-specific translation sees the already-stripped -// canonical payload. -// - withExclusiveCachedTokensNormalized: unconditional on an OpenAI Chat Completions -// Completions target. Folds the cache buckets back into `prompt_tokens` -// whenever `total_tokens` witnesses that the upstream reports them -// alongside it, and consults `usage-exclusive-cached-tokens` for the -// responses whose totals witness nothing — so the flag is a declaration -// input rather than a gate. Registered before the vendor normalizers so -// that on the inbound path it runs after them, on usage whose cache fields -// already carry OpenAI's names. -// - withVendor*OpenAIChatCompletionsNormalize: gated by `vendor-`. Registered -// LAST so that on the outbound path each gets the final say on the wire -// body and on the inbound path each gets the first say on the upstream -// stream — the generic interceptors above only see OpenAI-canonical form. -// Vendor flags are mutually exclusive in practice, but the interceptors -// are independent and run in declared order if more than one is somehow -// enabled. -export const openaiChatCompletionsInterceptors: readonly OpenAIChatCompletionsInterceptor[] = [ - withUsageStreamOptionsIncluded, - withUsageNormalized, - withReasoningDisabledOnForcedToolChoice, - withEmptyToolsToolChoiceNormalized, - withRoleCompatibilityApplied, - withPromptCacheKeyStripped, - withExclusiveCachedTokensNormalized, - withVendorDeepSeekOpenAIChatCompletionsNormalize, - withVendorQwenOpenAIChatCompletionsNormalize, - withVendorKimiOpenAIChatCompletionsNormalize, -]; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/normalize-empty-tools-tool-choice.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/normalize-empty-tools-tool-choice.ts deleted file mode 100644 index 2f2f64fb19..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/normalize-empty-tools-tool-choice.ts +++ /dev/null @@ -1,12 +0,0 @@ -import type { OpenAIChatCompletionsInterceptor } from './types.ts'; -import { providerModelOf } from '@floway-dev/provider'; - -// Opt-in compatibility rewrite for upstreams that reject an active -// `tool_choice` when the caller explicitly supplies no tools. Keep `tools` as -// sent and make the caller's effective intent unambiguous on the target wire. -export const withEmptyToolsToolChoiceNormalized: OpenAIChatCompletionsInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('empty-tools-tool-choice-none')) return await run(); - if (!Array.isArray(ctx.payload.tools) || ctx.payload.tools.length !== 0) return await run(); - ctx.payload = { ...ctx.payload, tool_choice: 'none' }; - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/normalize-exclusive-cached-tokens.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/normalize-exclusive-cached-tokens.ts deleted file mode 100644 index 0bbe882c5a..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/normalize-exclusive-cached-tokens.ts +++ /dev/null @@ -1,97 +0,0 @@ -// Restores OpenAI's inclusive input-token contract on upstreams that report -// the cache buckets alongside `prompt_tokens` instead of inside it. -// -// OpenAI states the subset relationship outright — "Cached tokens here are -// counted as a subset of input tokens, meaning input tokens will include -// cached and uncached tokens" -// (https://github.com/openai/openai-openapi/blob/d4fb706e6e05d4cc9f1b33ca59b6e4f3e8edd439/openapi.yaml#L51043-L51049) -// — and every downstream consumer here subtracts on that basis. Anthropic -// takes the opposite convention: `input_tokens` counts only what was neither -// read from nor written to the cache, and the three buckets sum to the real -// input (https://platform.claude.com/docs/en/docs/build-with-claude/prompt-caching). -// -// A gateway that fronts an Anthropic-shaped upstream and projects it into -// OpenAI Chat Completions can carry the exclusive convention through to a wire that -// declares the inclusive one. Portkey does exactly that — it assigns -// Anthropic's `input_tokens` straight to `prompt_tokens` while summing -// `total_tokens` from all four buckets -// (https://github.com/Portkey-AI/gateway/blob/669825cbe89ee51569918b8f78a9db486fd69dd4/src/providers/anthropic/chatComplete.ts#L612-L627). -// Charm Hyper produces the same shape by subtracting the cached prefix out of -// the `prompt_tokens` it received: for one observed kimi-k3 turn it reported -// `prompt_tokens 479, cached_tokens 13312, completion_tokens 373, -// total_tokens 14164`, where 479 + 13312 + 373 = 14164. -// -// `foldsExclusiveCacheTokens` owns the decision and the two contradictions -// that must not pass silently; the `usage-exclusive-cached-tokens` flag is its -// declaration input. `total_tokens` itself is left alone: under the exclusive -// convention it already counts the real input, so the rewritten -// `prompt_tokens + completion_tokens` is what it was equal to all along. - -import type { OpenAIChatCompletionsInterceptor } from './types.ts'; -import { asJsonObject, type JsonObject, readJsonNumber } from '../../../../shared/json-helpers.ts'; -import { foldsExclusiveCacheTokens } from '../../../shared/telemetry/usage.ts'; -import { eventFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import { providerModelOf } from '@floway-dev/provider'; - -const rewriteInboundUsage = ( - chunk: OpenAIChatCompletionsStreamEvent, - declaredExclusive: boolean, - identity: string, -): OpenAIChatCompletionsStreamEvent => { - const usage = asJsonObject(chunk.usage); - if (!usage) return chunk; - const inputTokens = readJsonNumber(usage.prompt_tokens); - const outputTokens = readJsonNumber(usage.completion_tokens); - if (inputTokens == null || outputTokens == null) return chunk; - - const details = asJsonObject(usage.prompt_tokens_details); - const cacheRead = readJsonNumber(details?.cached_tokens) ?? 0; - const cacheWrite = readJsonNumber(details?.cache_creation_input_tokens) ?? readJsonNumber(details?.cache_write_tokens) ?? 0; - if (cacheRead === 0 && cacheWrite === 0) return chunk; - - const fold = foldsExclusiveCacheTokens(declaredExclusive, { - inputTokens, - outputTokens, - totalTokens: readJsonNumber(usage.total_tokens) ?? undefined, - cacheRead, - cacheWrite, - }, identity); - if (!fold) return chunk; - - const next: JsonObject = { ...usage, prompt_tokens: inputTokens + cacheRead + cacheWrite }; - return { ...chunk, usage: next as unknown as OpenAIChatCompletionsStreamEvent['usage'] }; -}; - -export const withExclusiveCachedTokensNormalized: OpenAIChatCompletionsInterceptor = async (ctx, _gatewayCtx, run) => { - // Everything this entry says is about one upstream's OpenAI Chat Completions wire: - // the flag it reads describes how that upstream writes its usage there, and - // the remedy its errors name is a setting for that upstream. A translated - // request re-enters its target's chain, where these events are a projection - // of some other wire — the flag answers a question about counts it does not - // describe, and telling an operator to set it would be advice that cannot - // help. Nothing is lost by standing down: a translator emits the canonical - // form, which is the one case the fold has nothing to do with. - if (ctx.targetApi !== 'openaiChatCompletions') return await run(); - - const model = providerModelOf(ctx.candidate); - const declaredExclusive = model.enabledFlags.has('usage-exclusive-cached-tokens'); - const identity = `${ctx.candidate.provider.upstreamId}/${model.id}`; - - const result = await run(); - if (result.type !== 'events') return result; - - return { - ...result, - events: (async function* () { - for await (const frame of result.events) { - if (frame.type !== 'event') { - yield frame; - continue; - } - const event = rewriteInboundUsage(frame.event, declaredExclusive, identity); - yield event === frame.event ? frame : eventFrame(event); - } - })(), - }; -}; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/normalize-usage.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/normalize-usage.ts deleted file mode 100644 index ca52980cdd..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/normalize-usage.ts +++ /dev/null @@ -1,44 +0,0 @@ -import type { OpenAIChatCompletionsInterceptor } from './types.ts'; -import { asJsonObject } from '../../../../shared/json-helpers.ts'; -import { eventFrame } from '@floway-dev/protocols/common'; - -// Spec-compliant OpenAI Chat Completions usage chunk shape. The OpenAI spec puts the -// final `usage` on a `choices: []` carrier chunk -// (https://platform.openai.com/docs/api-reference/chat-streaming). Some -// upstreams have been observed to attach `usage` to the same chunk that -// carries the final delta and `finish_reason`. We strip `usage` from such a -// chunk and re-emit it on a synthesized spec-compliant carrier chunk -// immediately after, so downstream consumers can rely on the standard shape. -// -// Vendor-specific cache-token field rewrites (DeepSeek -// `prompt_cache_hit_tokens` / `prompt_cache_miss_tokens`, Kimi -// `cached_tokens`) live on each vendor's own `vendor--normalize` -// interceptor and run before this one on the response path, so by the time -// the chunk reaches us its `usage.prompt_tokens_details.cached_tokens` is -// already in the OpenAI standard shape. - -export const withUsageNormalized: OpenAIChatCompletionsInterceptor = async (_ctx, _gatewayCtx, run) => { - const result = await run(); - if (result.type !== 'events') return result; - return { - ...result, - events: (async function* () { - for await (const frame of result.events) { - if (frame.type !== 'event') { - yield frame; - continue; - } - - const chunk = frame.event; - const usage = asJsonObject(chunk.usage); - if (!usage || chunk.choices.length === 0) { - yield frame; - continue; - } - const { usage: chunkUsage, ...withoutUsage } = chunk; - yield eventFrame(withoutUsage); - yield eventFrame({ ...withoutUsage, choices: [], usage: chunkUsage }); - } - })(), - }; -}; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/strip-prompt-cache-key.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/strip-prompt-cache-key.ts deleted file mode 100644 index ef99046db6..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/strip-prompt-cache-key.ts +++ /dev/null @@ -1,16 +0,0 @@ -import type { OpenAIChatCompletionsInterceptor } from './types.ts'; -import { providerModelOf } from '@floway-dev/provider'; - -// Opt-in workaround for upstreams that reject `prompt_cache_key` as an unknown -// request argument (e.g. Azure DeepSeek returns -// `unrecognized_request_argument`). Drop the top-level field before the -// request reaches the terminal. OpenAI-native and truly OpenAI-compatible -// upstreams accept it for prefix-cache attribution, so removal only happens -// under the flag. -export const withPromptCacheKeyStripped: OpenAIChatCompletionsInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('strip-prompt-cache-key')) return await run(); - if (ctx.payload.prompt_cache_key === undefined) return await run(); - const { prompt_cache_key: _stripped, ...rest } = ctx.payload; - ctx.payload = rest; - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/types.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/types.ts deleted file mode 100644 index 6d10cb8a2f..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/types.ts +++ /dev/null @@ -1,13 +0,0 @@ -import type { GatewayCtx } from '../../../shared/gateway-ctx.ts'; -import type { Interceptor } from '@floway-dev/interceptor'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import type { OpenAIChatCompletionsInvocation, ExecuteResult } from '@floway-dev/provider'; - -export type { OpenAIChatCompletionsInvocation }; - -export type OpenAIChatCompletionsInterceptor = Interceptor< - OpenAIChatCompletionsInvocation, - GatewayCtx, - ExecuteResult> ->; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/vendor-deepseek-normalize.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/vendor-deepseek-normalize.ts deleted file mode 100644 index db9e9be580..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/vendor-deepseek-normalize.ts +++ /dev/null @@ -1,143 +0,0 @@ -// DeepSeek wire-dialect normalizer for OpenAI Chat Completions. Always-attached; -// flag-gated by `vendor-deepseek`. Runs last among the gateway's interceptors -// so it has the final say on the outbound wire body and the first say on the -// inbound stream — the gateway's generic interceptors above it deal only in -// OpenAI-canonical form. -// -// Outbound (request → upstream): -// -// - `reasoning_effort: 'none'` is the gateway's canonical "no reasoning" -// sentinel (produced when an Anthropic Messages source had `thinking: { type: -// 'disabled' }`, when an OpenAI Chat Completions source sent it literally, etc.). DeepSeek -// doesn't accept 'none' in its `reasoning_effort` enum and instead uses -// a top-level `thinking: { type: 'disabled' }` field. We strip the -// sentinel and emit the DeepSeek form. -// - Assistant messages: rewrite `reasoning_text` → `reasoning_content` (and -// synthesise `reasoning_content` from `reasoning_items.summary` when the -// newer OpenAI shape is the only thing present). DeepSeek documents only -// the scalar `reasoning_content` field and reports 400s when it is -// omitted from the assistant-message replay of a multi-turn tool-call loop. -// - `response_format: { type: 'json_schema', … }` is downgraded to -// `response_format: { type: 'json_object' }`. DeepSeek's structured-output -// API supports only `json_object`; the schema body is dropped on the floor -// rather than rejected by the upstream. -// -// Inbound (stream → client): -// -// - Each delta: rewrite `reasoning_content` → `reasoning_text` so downstream -// gateway code sees the OpenAI shape. -// - Each usage chunk: remap `prompt_cache_hit_tokens` / -// `prompt_cache_miss_tokens` into OpenAI's -// `prompt_tokens_details.cached_tokens`. The remap is computed from -// `prompt_cache_hit_tokens` alone (DeepSeek's "hit" count is the cached -// prefix length); the "miss" field is dropped. -// -// References: -// - https://api-docs.deepseek.com/zh-cn/guides/thinking_mode -// - https://api-docs.deepseek.com/guides/kv_cache -// - https://api-docs.deepseek.com/quick_start/agent_integrations/oh_my_pi - -import type { OpenAIChatCompletionsInterceptor } from './types.ts'; -import { asJsonObject, type JsonObject, readJsonNumber } from '../../../../shared/json-helpers.ts'; -import { eventFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsStreamEvent, OpenAIChatCompletionsPayload, OpenAIChatCompletionsReasoningItem, OpenAIChatCompletionsMessage } from '@floway-dev/protocols/openai-chat-completions'; -import { providerModelOf } from '@floway-dev/provider'; - -const synthesizeFromItems = (items: OpenAIChatCompletionsReasoningItem[] | null | undefined): string | undefined => { - if (!items?.length) return undefined; - const parts = items.flatMap(item => item.summary?.map(s => s.text) ?? []); - return parts.length > 0 ? parts.join('') : undefined; -}; - -const rewriteOutboundMessage = (message: OpenAIChatCompletionsMessage): OpenAIChatCompletionsMessage => { - // `reasoning_opaque` is the OpenAI-canonical signature for cross-turn - // reasoning replay; DeepSeek doesn't accept it, so it's dropped on the - // floor when we project assistant messages onto `reasoning_content`. - const { reasoning_text, reasoning_opaque: _opaque, reasoning_items, ...rest } = message; - const text = typeof reasoning_text === 'string' ? reasoning_text : synthesizeFromItems(reasoning_items); - if (text === undefined) return rest as OpenAIChatCompletionsMessage; - return { ...rest, reasoning_content: text } as OpenAIChatCompletionsMessage; -}; - -const stripCanonicalReasoningSentinel = (payload: OpenAIChatCompletionsPayload): OpenAIChatCompletionsPayload => { - if (payload.reasoning_effort !== 'none') return payload; - const { reasoning_effort: _stripped, ...rest } = payload; - return { ...rest, thinking: { type: 'disabled' as const } } as OpenAIChatCompletionsPayload; -}; - -const downgradeJsonSchemaResponseFormat = (payload: OpenAIChatCompletionsPayload): OpenAIChatCompletionsPayload => { - const rf = payload.response_format; - if (rf?.type !== 'json_schema') return payload; - return { ...payload, response_format: { type: 'json_object' } }; -}; - -const rewriteOutboundPayload = (payload: OpenAIChatCompletionsPayload): OpenAIChatCompletionsPayload => { - const withDisable = stripCanonicalReasoningSentinel(payload); - const withResponseFormat = downgradeJsonSchemaResponseFormat(withDisable); - return { - ...withResponseFormat, - messages: withResponseFormat.messages.map(rewriteOutboundMessage), - }; -}; - -const rewriteInboundDeltas = (chunk: OpenAIChatCompletionsStreamEvent): OpenAIChatCompletionsStreamEvent => { - let changed = false; - const choices = chunk.choices.map(choice => { - const delta = choice.delta as OpenAIChatCompletionsStreamEvent['choices'][number]['delta']; - if (typeof delta.reasoning_content !== 'string') return choice; - - const { reasoning_content, ...rest } = delta; - changed = true; - return { - ...choice, - delta: { - ...rest, - ...(delta.reasoning_text === undefined ? { reasoning_text: reasoning_content } : {}), - }, - }; - }); - return changed ? { ...chunk, choices } : chunk; -}; - -const VENDOR_CACHE_FIELDS = ['prompt_cache_hit_tokens', 'prompt_cache_miss_tokens'] as const; - -const rewriteInboundUsage = (chunk: OpenAIChatCompletionsStreamEvent): OpenAIChatCompletionsStreamEvent => { - const usage = asJsonObject(chunk.usage); - if (!usage) return chunk; - const hit = readJsonNumber(usage.prompt_cache_hit_tokens); - const hasVendorField = VENDOR_CACHE_FIELDS.some(field => usage[field] !== undefined); - if (!hasVendorField) return chunk; - - const next: JsonObject = { ...usage }; - for (const field of VENDOR_CACHE_FIELDS) delete next[field]; - if (hit != null) { - next.prompt_tokens_details = { - ...(asJsonObject(usage.prompt_tokens_details) ?? {}), - cached_tokens: hit, - }; - } - return { ...chunk, usage: next as unknown as OpenAIChatCompletionsStreamEvent['usage'] }; -}; - -export const withVendorDeepSeekOpenAIChatCompletionsNormalize: OpenAIChatCompletionsInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('vendor-deepseek')) return await run(); - - ctx.payload = rewriteOutboundPayload(ctx.payload); - - const result = await run(); - if (result.type !== 'events') return result; - - return { - ...result, - events: (async function* () { - for await (const frame of result.events) { - if (frame.type !== 'event') { - yield frame; - continue; - } - const event = rewriteInboundUsage(rewriteInboundDeltas(frame.event)); - yield event === frame.event ? frame : eventFrame(event); - } - })(), - }; -}; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/vendor-kimi-normalize.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/vendor-kimi-normalize.ts deleted file mode 100644 index 0e63503f16..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/vendor-kimi-normalize.ts +++ /dev/null @@ -1,59 +0,0 @@ -// Kimi (Moonshot) wire-dialect normalizer for OpenAI Chat Completions. -// Always-attached; flag-gated by `vendor-kimi`. Runs last among the gateway's -// interceptors so it has the first say on the inbound stream. -// -// Outbound: nothing today — Kimi accepts the OpenAI-canonical request shape -// the gateway emits. -// -// Inbound (stream → client): -// -// - Each usage chunk: remap the flat `cached_tokens` field into OpenAI's -// `prompt_tokens_details.cached_tokens`. Generic interceptors upstream of -// this one then see the standard shape. -// -// Reference: -// - https://platform.kimi.com/docs/api/chat - -import type { OpenAIChatCompletionsInterceptor } from './types.ts'; -import { asJsonObject, type JsonObject, readJsonNumber } from '../../../../shared/json-helpers.ts'; -import { eventFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import { providerModelOf } from '@floway-dev/provider'; - -const rewriteInboundUsage = (chunk: OpenAIChatCompletionsStreamEvent): OpenAIChatCompletionsStreamEvent => { - const usage = asJsonObject(chunk.usage); - if (!usage) return chunk; - const cached = readJsonNumber(usage.cached_tokens); - if (cached == null) return chunk; - - const { cached_tokens: _stripped, ...rest } = usage; - const next: JsonObject = { - ...rest, - prompt_tokens_details: { - ...(asJsonObject(usage.prompt_tokens_details) ?? {}), - cached_tokens: cached, - }, - }; - return { ...chunk, usage: next as unknown as OpenAIChatCompletionsStreamEvent['usage'] }; -}; - -export const withVendorKimiOpenAIChatCompletionsNormalize: OpenAIChatCompletionsInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('vendor-kimi')) return await run(); - - const result = await run(); - if (result.type !== 'events') return result; - - return { - ...result, - events: (async function* () { - for await (const frame of result.events) { - if (frame.type !== 'event') { - yield frame; - continue; - } - const event = rewriteInboundUsage(frame.event); - yield event === frame.event ? frame : eventFrame(event); - } - })(), - }; -}; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/vendor-qwen-normalize.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/vendor-qwen-normalize.ts deleted file mode 100644 index cf00f43c1a..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/interceptors/vendor-qwen-normalize.ts +++ /dev/null @@ -1,30 +0,0 @@ -// Qwen wire-dialect normalizer for OpenAI Chat Completions. Always-attached; -// flag-gated by `vendor-qwen`. Runs last among the gateway's interceptors so -// it has the final say on the outbound wire body. -// -// Outbound (request → upstream): -// -// - `reasoning_effort: 'none'` is the gateway's canonical "no reasoning" -// sentinel. Qwen doesn't accept 'none' in its `reasoning_effort` enum and -// instead uses a top-level `enable_thinking: false` field. We strip the -// sentinel and emit the Qwen form. -// -// Inbound: Qwen's response shape matches OpenAI for the fields the gateway reads. -// -// Reference: -// - https://www.alibabacloud.com/help/en/model-studio/deep-thinking - -import type { OpenAIChatCompletionsInterceptor } from './types.ts'; -import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; -import { providerModelOf } from '@floway-dev/provider'; - -export const withVendorQwenOpenAIChatCompletionsNormalize: OpenAIChatCompletionsInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('vendor-qwen')) return await run(); - - if (ctx.payload.reasoning_effort === 'none') { - const { reasoning_effort: _stripped, ...rest } = ctx.payload; - ctx.payload = { ...rest, enable_thinking: false } as OpenAIChatCompletionsPayload; - } - - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/meter.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/meter.ts new file mode 100644 index 0000000000..f089199f65 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/meter.ts @@ -0,0 +1,88 @@ +import type { StreamOutcome } from '../../pipeline/serve.ts'; +import { meterChatWire } from '../meter.ts'; +import { billableUsageFromOpenAIChatCompletionsEvent } from './usage.ts'; +import type { BillableEntity } from '../../pipeline/facts.ts'; +import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; +import { chatUsageMeasurement, type ChatBillableUsage } from '../shared/usage.ts'; +import { defer, type Deferred } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import { openaiChatCompletionsErrorPayloadMessage, type OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; +import type { TelemetryModelIdentity } from '@floway-dev/provider'; + +export const meterUsage = (streamedUsage: string) => meterChatWire({ + wire: 'openaiChatCompletions', + answer: 'response.chat.openaiChatCompletions', + streamedUsage, + read: meterOpenAIChatCompletions, +}); + +/** Reads the upstream's own usage off its own events as they pass, so the reading costs one + * pass and the client's stream is what drives it. Only a report carrying real counts + * replaces the running figure, so a trailing empty usage frame cannot wipe a good one. */ +const meterOpenAIChatCompletions = ( + source: AsyncIterable>, + identity: TelemetryModelIdentity, + attempt: { firstOutputTokenAt: number | null }, +): { readonly frames: AsyncIterable>; readonly outcome: Deferred } => { + const completion = Promise.withResolvers(); + const outcome = defer(completion.promise); + let settled = false; + // A clean EOF completes Chat Completions even when no [DONE] frame was sent. + let completed = false; + let failed = false; + const generator = (async function* () { + let reported: ChatBillableUsage | undefined; + const finish = (): void => { + if (settled) return; + settled = true; + try { completion.resolve({ billable: [billedEntity(reported, identity)], failed: failed || !completed }); } catch (error) { + completion.reject(error); + throw error; + } + }; + try { + for await (const frame of source) { + // Time to first token is measured where the token is, which is the only place that + // knows a frame carries generated content rather than the envelope around it. + if (attempt.firstOutputTokenAt === null && isFirstOutputTokenFrame(frame, 'openaiChatCompletions')) { + attempt.firstOutputTokenAt = performance.now(); + } + if (frame.type === 'event') { + const usage = billableUsageFromOpenAIChatCompletionsEvent(frame.event); + if (usage !== null) reported = usage; + } + if (isTerminal(frame)) { + completed = true; + failed = frame.type === 'event' && 'error' in frame.event; + finish(); + } + yield frame; + // The terminator is written out before the read stops, because it is what the client + // reads as the end. Stopping here also drops anything an upstream sends after it. + if (isTerminal(frame)) return; + } + completed = true; + } finally { + // Reached however the frames ended — the terminal chunk, a client that stopped + // reading, or a broken upstream — because tokens the upstream already metered are + // billable whatever happened to the downstream half. + finish(); + } + })(); + return { frames: { [Symbol.asyncIterator]: () => generator }, outcome }; +}; + +/** The frame that says this turn is over: the transport's own terminator, or an error the + * upstream wrote into the stream instead of finishing it. */ +const isTerminal = (frame: ProtocolFrame): boolean => + frame.type === 'done' || (frame.type === 'event' && openaiChatCompletionsErrorPayloadMessage(frame.event) !== null); + +/** What one attempt is billable for. An upstream that reported nothing leaves no quantities + * at all, which is a different statement from reporting zero — and a rate can depend on the + * service tier and on how much input there was, so both travel as pricing facts rather than + * being folded into the quantities. */ +const billedEntity = (usage: ChatBillableUsage | undefined, identity: TelemetryModelIdentity): BillableEntity => { + if (usage === undefined) return { identity, quantities: {} }; + const measurement = chatUsageMeasurement(usage); + return { identity, quantities: measurement.quantities, pricingFacts: measurement.pricingFacts }; +}; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/normalize-empty-tools-tool-choice.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/normalize-empty-tools-tool-choice.ts new file mode 100644 index 0000000000..5c46dab165 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/normalize-empty-tools-tool-choice.ts @@ -0,0 +1,4 @@ +import { normalizeEmptyTools } from '../shared/normalize-empty-tools.ts'; + +export const normalizeEmptyToolsForOpenAIChatCompletions = normalizeEmptyTools('request.chat.openaiChatCompletions', payload => + payload.tool_choice === 'none' ? payload : { ...payload, tool_choice: 'none' }); diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/normalize-exclusive-cached-tokens.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/normalize-exclusive-cached-tokens.ts new file mode 100644 index 0000000000..9cb11e7b0f --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/normalize-exclusive-cached-tokens.ts @@ -0,0 +1,89 @@ +import { asJsonObject } from '../../../shared/json-helpers.ts'; +import type { AttemptSelector } from '../../pipeline/facts.ts'; +import type { Chat } from '../facts.ts'; +import { withCacheBucketsFolded, type CacheBucketNames } from '../shared/cache-buckets.ts'; +import { answerWithFrames, rewritingEvents } from '../shared/frames.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; +import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; + +/** + * Restores OpenAI's inclusive input-token contract on an upstream that reports the cache + * buckets alongside `prompt_tokens` instead of inside it. The fold, the evidence for the two + * conventions and the contradictions it raises are at `withCacheBucketsFolded`. + * + * Unconditional on this wire rather than flag-gated: `foldsExclusiveCacheTokens` reads + * `total_tokens` as the witness, and the `usage-exclusive-cached-tokens` flag is a declaration + * input for the responses whose totals witness nothing. + * + * Everything it says is about one upstream's OpenAI Chat Completions wire — the flag it reads + * describes how that upstream writes its usage there, and the remedy its errors name is a + * setting for that upstream. That is why it belongs to the wire: on any other wire these + * events are a projection of some other protocol, where the flag answers a question about + * counts it does not describe and telling an operator to set it would be advice that cannot + * help. Nothing is lost by standing down there — a translator emits the canonical form, which + * is the one case the fold has nothing to do with. + * + * It runs below the vendor normalizers, so on the way back it reads usage whose cache fields + * already carry OpenAI's names. + */ +export const normalizeExclusiveCachedTokensForOpenAIChatCompletions = defineStage< + Chat<'route.attempt'>, + Chat<'route.attempt'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> +>({ + name: 'normalizeExclusiveCachedTokens', + through: { + request: { needs: ['route.attempt'], consumes: [], provides: [] }, + response: { + needs: ['response.chat.openaiChatCompletions'], + consumes: [], + provides: ['response.chat.openaiChatCompletions'], + }, + }, + execute: transform< + Chat<'route.attempt'>, + Chat<'route.attempt'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> + >(() => { + // Which attempt this is gets read on the way down and spoken about on the way back, which + // is the order `transform` runs the two halves in. + let attempt!: AttemptSelector; + return { + request: facts => { + attempt = facts['route.attempt']; + return facts; + }, + response: facts => { + const declaredExclusive = attempt.flags.includes('usage-exclusive-cached-tokens'); + const identity = `${attempt.upstreamId}/${attempt.modelId}`; + const answer = answerWithFrames( + facts['response.chat.openaiChatCompletions'], + frames => rewritingEvents(frames, chunk => foldOpenAIChatCompletionsUsage(chunk, declaredExclusive, identity)), + ); + return answer === null ? facts : { ...facts, 'response.chat.openaiChatCompletions': move(answer) }; + }, + }; + }), +}); + +/** OpenAI Chat Completions carries usage on the chunk's own root and names the buckets + * `prompt_tokens` and `prompt_tokens_details.{cached_tokens, cache_creation_input_tokens}`. */ +const OPENAI_CHAT_COMPLETIONS_CACHE_BUCKETS: CacheBucketNames = { + input: 'prompt_tokens', + output: 'completion_tokens', + details: 'prompt_tokens_details', + cacheWrite: ['cache_creation_input_tokens', 'cache_write_tokens'], +}; + +const foldOpenAIChatCompletionsUsage = ( + chunk: OpenAIChatCompletionsStreamEvent, + declaredExclusive: boolean, + identity: string, +): OpenAIChatCompletionsStreamEvent => { + const usage = asJsonObject(chunk.usage); + if (usage === null) return chunk; + const folded = withCacheBucketsFolded(usage, OPENAI_CHAT_COMPLETIONS_CACHE_BUCKETS, declaredExclusive, identity); + return folded === usage ? chunk : { ...chunk, usage: folded as unknown as OpenAIChatCompletionsStreamEvent['usage'] }; +}; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/normalize-usage.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/normalize-usage.ts new file mode 100644 index 0000000000..542882f4f3 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/normalize-usage.ts @@ -0,0 +1,67 @@ +import { asJsonObject } from '../../../shared/json-helpers.ts'; +import type { Chat } from '../facts.ts'; +import { answerWithFrames } from '../shared/frames.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; +import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; +import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; + +/** + * Puts a usage block back on the carrier chunk the spec says it arrives on. + * + * OpenAI puts the final `usage` on a `choices: []` chunk of its own + * (https://platform.openai.com/docs/api-reference/chat-streaming), and some upstreams have + * been observed to attach it to the same chunk that carries the last delta and + * `finish_reason`. Such a chunk is split in two — the delta as it arrived, then a synthesized + * carrier holding the usage — so everything downstream can rely on the standard shape. + * + * It runs above the vendor normalizers, so on the way back it sees a usage block whose cache + * fields already carry OpenAI's names. + */ +export const normalizeUsageForOpenAIChatCompletions = defineStage< + Record, + Record, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> +>({ + name: 'normalizeUsage', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { + needs: ['response.chat.openaiChatCompletions'], + consumes: [], + provides: ['response.chat.openaiChatCompletions'], + }, + }, + execute: transform< + Record, + Record, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> + >(() => ({ + response: facts => { + const answer = answerWithFrames(facts['response.chat.openaiChatCompletions'], withUsageOnItsOwnCarrier); + return answer === null ? facts : { ...facts, 'response.chat.openaiChatCompletions': move(answer) }; + }, + })), +}); + +/** One chunk in, two out — and only for the chunk that carried both, which is why this is not + * a one-for-one event rewrite. */ +const withUsageOnItsOwnCarrier = async function* ( + frames: AsyncIterable>, +): AsyncGenerator> { + for await (const frame of frames) { + if (frame.type !== 'event') { + yield frame; + continue; + } + const chunk = frame.event; + if (asJsonObject(chunk.usage) === null || chunk.choices.length === 0) { + yield frame; + continue; + } + const { usage, ...withoutUsage } = chunk; + yield eventFrame(withoutUsage); + yield eventFrame({ ...withoutUsage, choices: [], usage }); + } +}; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts new file mode 100644 index 0000000000..ef1c2a484f --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/pipeline.ts @@ -0,0 +1,36 @@ +import { STREAMED_USAGE, type OpenAIChatCompletionsServeEntry, type OpenAIChatCompletionsServeExit } from './facts.ts'; +import { composeChat as compose } from '../compose.ts'; +import { emitOpenAIChatCompletions } from './emit.ts'; +import { normalizeEmptyToolsForOpenAIChatCompletions } from './normalize-empty-tools-tool-choice.ts'; +import { narrowing, openaiChatCompletionsTarget } from './target.ts'; +import { openaiChatCompletionsWireFor } from './wires.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import { failover } from '../../pipeline/failover.ts'; +import { serializeClientJson } from '../../pipeline/serialize-client-json.ts'; +import { writeSettlement } from '../../pipeline/settlement.ts'; +import { dialChatWire } from '../dial-wire.ts'; +import { materializeAttempt } from '../materialize-attempt.ts'; +import { resolveChatCandidates } from '../resolve-candidates.ts'; +import type { Pipeline } from '@floway-dev/pipeline'; + +export const openaiChatCompletionsServePipeline = (): Pipeline => + compose('openaiChatCompletionsServe', [ + writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, STREAMED_USAGE), + serializeClientJson('response.chat.openaiChatCompletions.rendered'), + emitOpenAIChatCompletions, + resolveChatCandidates(narrowing), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.chat.openaiChatCompletions'?: unknown })['response.chat.openaiChatCompletions']), + owns: ['response.http.body'], + pendingUsage: STREAMED_USAGE, + }), + materializeAttempt('request.chat.openaiChatCompletions'), + normalizeEmptyToolsForOpenAIChatCompletions, + dialChatWire({ + source: 'request.chat.openaiChatCompletions', + needs: ['request.chat.openaiChatCompletions', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], + provides: ['response.chat.openaiChatCompletions', STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], + pick: endpoints => openaiChatCompletionsTarget.pick(endpoints), + wire: openaiChatCompletionsWireFor, + }), + ]); diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/respond.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/respond.ts deleted file mode 100644 index 0f303b9761..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/respond.ts +++ /dev/null @@ -1,130 +0,0 @@ -import type { Context } from 'hono'; -import { streamSSE } from 'hono/streaming'; - -import { wrapOpenAIChatCompletionsAffinityEgress } from './affinity/egress.ts'; -import type { GatewayCtx } from '../../shared/gateway-ctx.ts'; -import { type StreamCompletion, writeSSEFrames } from '../../shared/sse.ts'; -import { recordFailedRequest } from '../../shared/telemetry/performance.ts'; -import { settle } from '../../shared/telemetry/settle.ts'; -import { tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; -import { forwardUpstreamHeaders, mergeForwardedUpstreamHeaders } from '../../shared/upstream-response.ts'; -import { affinityEgressOptions } from '../shared/affinity/index.ts'; -import { SourceStreamState, eventResultMetadata, plainResultToResponse } from '../shared/respond.ts'; -import { eventFrame, type ProtocolFrame, sseCommentFrame, sseFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import { openaiChatCompletionsProtocolFrameToSSEFrame, collectOpenAIChatCompletionsProtocolEventsToResult, openaiChatCompletionsErrorPayloadMessage } from '@floway-dev/protocols/openai-chat-completions'; -import { type ExecuteResult, type PlainResult, type InternalDebugError, toInternalDebugError } from '@floway-dev/provider'; -import { apiErrorToResponse } from '@floway-dev/provider'; - -export const respondOpenAIChatCompletions = async ( - c: Context, - result: ExecuteResult> | PlainResult, - wantsStream: boolean, - includeUsageChunk: boolean, - ctx: GatewayCtx, -): Promise => { - if (result.type === 'api-error') { - recordFailedRequest(ctx, result.performance); - ctx.dump?.error(result.source, result.upstreamId); - return apiErrorToResponse(result); - } - - if (result.type === 'internal-error') { - recordFailedRequest(ctx, result.performance); - ctx.dump?.failed(result.error.message); - return internalOpenAIChatCompletionsErrorResponse(result.status, result.error); - } - - if (result.type === 'plain') { - if (result.status >= 400) { - ctx.dump?.error(result.upstreamId !== undefined ? 'upstream' : 'gateway', result.upstreamId); - } - return plainResultToResponse(result); - } - - const state = new SourceStreamState(); - const observed = observeOpenAIChatCompletionsFrames(result.events, state, ctx); - const frames = wrapOpenAIChatCompletionsAffinityEgress(observed, affinityEgressOptions(ctx)); - - if (!wantsStream) { - try { - const response = await collectOpenAIChatCompletionsProtocolEventsToResult(frames); - const metadata = await eventResultMetadata(result); - const usage = tokenUsageFromBillableUsage(metadata.billableUsage); - ctx.dump?.success(metadata.modelIdentity, usage); - settle(ctx, metadata.performance, metadata.modelIdentity, usage, state.failed); - return Response.json(response, { headers: mergeForwardedUpstreamHeaders(undefined, result.headers) }); - } catch (error) { - recordFailedRequest(ctx, result.performance); - ctx.dump?.failed(error); - return internalOpenAIChatCompletionsErrorResponse(502, toInternalDebugError(error)); - } - } - - forwardUpstreamHeaders(c, result.headers); - return streamSSE(c, async stream => { - let completion: StreamCompletion = 'error'; - try { - completion = await writeSSEFrames(stream, openaiChatCompletionsSseFrames(frames, includeUsageChunk, state, ctx), { - keepAlive: { frame: sseCommentFrame('keepalive') }, - ...(ctx.downstreamAbortController !== undefined ? { downstreamAbortController: ctx.downstreamAbortController } : {}), - }); - } finally { - const metadata = await eventResultMetadata(result); - const failed = state.failedAfter(completion); - if (failed) { - ctx.dump?.failed(`openai-chat-completions stream failed (completion=${completion}, source-failed=${state.failed})`); - } else { - ctx.dump?.success(metadata.modelIdentity, tokenUsageFromBillableUsage(metadata.billableUsage)); - } - settle(ctx, metadata.performance, metadata.modelIdentity, tokenUsageFromBillableUsage(metadata.billableUsage), failed); - } - }); -}; - -// --- error rendering --- - -const internalOpenAIChatCompletionsErrorPayload = (error: InternalDebugError) => ({ - error: { - type: error.type, - name: error.name, - message: error.message, - stack: error.stack, - cause: error.cause, - target_api: error.target_api, - }, -}); - -const internalOpenAIChatCompletionsErrorResponse = (status: number, error: InternalDebugError): Response => Response.json(internalOpenAIChatCompletionsErrorPayload(error), { status }); - -// --- frame observation --- - -const isOpenAIChatCompletionsFailureFrame = (frame: ProtocolFrame) => frame.type === 'event' && openaiChatCompletionsErrorPayloadMessage(frame.event) !== null; - -const isOpenAIChatCompletionsTerminalFrame = (frame: ProtocolFrame) => frame.type === 'done' || isOpenAIChatCompletionsFailureFrame(frame); - -const observeOpenAIChatCompletionsFrames = async function* (frames: AsyncIterable>, state: SourceStreamState, ctx: GatewayCtx) { - for await (const frame of frames) { - await ctx.dump?.frame(frame); - const failed = isOpenAIChatCompletionsFailureFrame(frame); - if (failed) state.failed = true; - if (isOpenAIChatCompletionsTerminalFrame(frame) && !failed) state.completed = true; - yield frame; - if (isOpenAIChatCompletionsTerminalFrame(frame)) return; - } - state.completed = true; -}; - -const openaiChatCompletionsSseFrames = async function* (frames: AsyncIterable>, includeUsageChunk: boolean, state: SourceStreamState, ctx: GatewayCtx) { - try { - for await (const frame of frames) { - const sse = openaiChatCompletionsProtocolFrameToSSEFrame(frame, { includeUsageChunk }); - if (sse) yield sse; - } - } catch (error) { - state.failed = true; - const event = internalOpenAIChatCompletionsErrorPayload(toInternalDebugError(error)) as unknown as OpenAIChatCompletionsStreamEvent; - await ctx.dump?.frame(eventFrame(event)); - yield sseFrame(JSON.stringify(event), 'error'); - } -}; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/serve.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/serve.ts deleted file mode 100644 index 74d885748d..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/serve.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { analyzeOpenAIChatCompletionsAffinity } from './affinity/ingress.ts'; -import { openaiChatCompletionsAttempt, openaiChatCompletionsTarget } from './attempt.ts'; -import { renderOpenAIChatCompletionsFailure } from './errors.ts'; -import { enumerateModelCandidates } from '../../providers/resolution.ts'; -import { iterateCandidates } from '../../shared/iterate-candidates.ts'; -import { selectAffinityCandidates } from '../shared/affinity/index.ts'; -import { noViableCandidateFailure } from '../shared/errors.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import type { ExecuteResult } from '@floway-dev/provider'; - -export interface OpenAIChatCompletionsServeGenerateArgs { - readonly payload: OpenAIChatCompletionsPayload; - readonly ctx: ChatGatewayCtx; - readonly headers: Headers; -} - -export const openaiChatCompletionsServe = { - generate: async (args: OpenAIChatCompletionsServeGenerateArgs): Promise>> => { - const { payload, ctx, headers } = args; - const { candidates: enumerated, sawModel, failedUpstreams } = await enumerateModelCandidates({ - upstreamIds: ctx.upstreamIds, - model: payload.model, - kind: 'chat', - scheduler: ctx.backgroundScheduler, - runtimeLocation: ctx.runtimeLocation, - }); - const affinity = await analyzeOpenAIChatCompletionsAffinity(payload, ctx.affinity.codec); - const viable = enumerated.filter(c => openaiChatCompletionsTarget.canServe(c.model.endpoints)); - const selection = selectAffinityCandidates(viable, affinity); - if ('kind' in selection) return renderOpenAIChatCompletionsFailure(selection); - if (selection.candidates.length === 0) return renderOpenAIChatCompletionsFailure(noViableCandidateFailure(sawModel, payload.model, failedUpstreams)); - - // Try each affinity-selected candidate in order. A successful attempt (SSE - // stream opened) is the final answer; an api-error or internal-error - // from one candidate falls through to the next so the gateway absorbs - // transient 5xx/429/network failures. When the list is exhausted, the - // most recent failure is forwarded verbatim so the client still sees - // real upstream telemetry rather than a synthetic envelope. Each attempt - // stamps its private payload clone with the candidate's canonical model id - // so aliases and prefixed ids resolve without mutating the caller payload. - return await iterateCandidates( - selection.candidates, - 'openaiChatCompletionsServe.generate', - ctx, - 'chat', - async candidate => { - const result = await openaiChatCompletionsAttempt.generate({ payload: selection.payloadFor(candidate), ctx, candidate, headers }); - if (result.type === 'events') ctx.affinity.select(candidate); - return result; - }, - ); - }, -}; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/strip-prompt-cache-key.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/strip-prompt-cache-key.ts new file mode 100644 index 0000000000..012d6684a0 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/strip-prompt-cache-key.ts @@ -0,0 +1,35 @@ +import type { Chat } from '../facts.ts'; +import { withoutKeys } from '../shared/structural.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; + +/** Drops a field the upstream would reject as an unknown argument. It runs above the vendor + * normalizers so each of them sees the already-stripped canonical payload. */ +export const stripPromptCacheKeyForOpenAIChatCompletions = defineStage< + Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> +>({ + name: 'stripPromptCacheKey', + through: { + request: { + needs: ['request.chat.openaiChatCompletions', 'route.attempt'], + consumes: [], + provides: ['request.chat.openaiChatCompletions'], + }, + response: { needs: ['response.chat.openaiChatCompletions'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> + >(() => ({ + request: facts => { + if (!facts['route.attempt'].flags.includes('strip-prompt-cache-key')) return facts; + const payload = facts['request.chat.openaiChatCompletions']; + const stripped = withoutKeys(payload, ['prompt_cache_key']); + return stripped === payload ? facts : { ...facts, 'request.chat.openaiChatCompletions': move(stripped) }; + }, + })), +}); diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/target.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/target.ts new file mode 100644 index 0000000000..6048b68b96 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/target.ts @@ -0,0 +1,32 @@ +import type { ChatNarrowing } from '../resolve-candidates.ts'; +import type { Fields } from './facts.ts'; +import { chatTargetPicker } from '../shared/target-picker.ts'; +import { analyzeOpenAIChatCompletionsAffinity } from './affinity/ingress.ts'; + +/** `/v1/chat/completions` prefers its own wire, then the translated Anthropic Messages path, + * then the translated OpenAI Responses path. */ +export const openaiChatCompletionsTarget = chatTargetPicker(['openaiChatCompletions', 'anthropicMessages', 'openaiResponses']); + +export const narrowing: ChatNarrowing, 'request.chat.openaiChatCompletions'> = ({ + requestKey: 'request.chat.openaiChatCompletions', + canServe: candidate => openaiChatCompletionsTarget.canServe(candidate.model.endpoints), + affinity: async (payload, gateway) => await analyzeOpenAIChatCompletionsAffinity(payload, gateway.affinity.codec), + unsupported: model => `Model ${model} does not support the /chat/completions endpoint.`, + refuse: (status, message, reason) => ({ + 'response.chat.openaiChatCompletions.streamedUsage': null, + 'response.chat.openaiChatCompletions': { + status, + message, + // What an OpenAI client reads: the condition's own type, and for a turn whose carried + // state cannot be routed, the field at fault and the code that names it. + envelope: { + error: { + message, + type: 'invalid_request_error', + ...(reason === 'routing-unavailable' ? { param: 'input', code: 'responses_item_routing_unavailable' } : {}), + }, + }, + }, + }), + refuses: ['response.chat.openaiChatCompletions', 'response.chat.openaiChatCompletions.streamedUsage'], +}); diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/usage.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/usage.ts index e44ba4c97f..136a59ee34 100644 --- a/packages/gateway/src/data-plane/chat/openai-chat-completions/usage.ts +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/usage.ts @@ -1,4 +1,5 @@ -import { billableServiceTier, splitInclusiveInputTokens, type BillableUsage } from '@floway-dev/protocols/common'; +import type { ChatBillableUsage } from '../shared/usage.ts'; +import { billableServiceTier, splitInclusiveInputTokens } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; type OpenAIChatCompletionsUsage = NonNullable; @@ -6,7 +7,7 @@ type OpenAIChatCompletionsUsage = NonNullable { +): ChatBillableUsage => { const cacheWrite = usage.prompt_tokens_details?.cache_creation_input_tokens ?? usage.prompt_tokens_details?.cache_write_tokens ?? 0; @@ -17,15 +18,15 @@ export const billableUsageFromOpenAIChatCompletionsUsage = ( ); const tier = billableServiceTier(serviceTier); return { - input, + ...(usage.prompt_tokens === undefined ? {} : { input }), cacheRead, cacheWrite, // OpenAI Chat Completions has no cache-write TTL split. cacheWrite1h: 0, - output: usage.completion_tokens, + ...(usage.completion_tokens === undefined ? {} : { output: usage.completion_tokens }), ...(tier !== null ? { tier } : {}), }; }; -export const billableUsageFromOpenAIChatCompletionsEvent = (event: OpenAIChatCompletionsStreamEvent): BillableUsage | null => +export const billableUsageFromOpenAIChatCompletionsEvent = (event: OpenAIChatCompletionsStreamEvent): ChatBillableUsage | null => event.usage ? billableUsageFromOpenAIChatCompletionsUsage(event.usage, event.service_tier) : null; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/vendor-deep-seek-normalize.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/vendor-deep-seek-normalize.ts new file mode 100644 index 0000000000..c4668eeb65 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/vendor-deep-seek-normalize.ts @@ -0,0 +1,155 @@ +import { asJsonObject, readJsonNumber, type JsonObject } from '../../../shared/json-helpers.ts'; +import type { Chat } from '../facts.ts'; +import { answerWithFrames, rewritingEvents } from '../shared/frames.ts'; +import { withoutKeys, mapKeepingIdentity } from '../shared/structural.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; +import type { OpenAIChatCompletionsStreamEvent, OpenAIChatCompletionsPayload, OpenAIChatCompletionsMessage, OpenAIChatCompletionsReasoningItem } from '@floway-dev/protocols/openai-chat-completions'; + +// ── OpenAI Chat Completions vendor dialects ─────────────────────────────────────────────── +// +// Each of the three below is last among this wire's rewrites, which is what gives it the final +// say on the outbound body and the first say on the inbound stream: everything above deals +// only in OpenAI-canonical form. The vendor flags are mutually exclusive in practice, but the +// stages are independent and run in the order they are composed if more than one is somehow on. + +/** + * DeepSeek's wire dialect. + * + * Outbound: `reasoning_effort: 'none'` is the gateway's canonical "no reasoning" sentinel and + * is not in DeepSeek's own enum, which uses a top-level `thinking: { type: 'disabled' }` + * instead. Assistant messages carry their reasoning on the scalar `reasoning_content` DeepSeek + * documents — it is the only field it reads, and it reports 400s when the assistant-message + * replay of a multi-turn tool-call loop omits it. And `response_format: { type: 'json_schema' }` + * is downgraded to `json_object`, which is the only structured output DeepSeek supports; the + * schema body is dropped rather than rejected by the upstream. + * + * Inbound: `reasoning_content` deltas become `reasoning_text`, and the `prompt_cache_hit_tokens` + * / `prompt_cache_miss_tokens` pair becomes OpenAI's `prompt_tokens_details.cached_tokens` — + * computed from the hit count alone, which is the cached prefix length. + * + * References: + * - https://api-docs.deepseek.com/zh-cn/guides/thinking_mode + * - https://api-docs.deepseek.com/guides/kv_cache + * - https://api-docs.deepseek.com/quick_start/agent_integrations/oh_my_pi + */ +export const vendorDeepSeekNormalizeForOpenAIChatCompletions = defineStage< + Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> +>({ + name: 'vendorDeepSeekNormalize', + through: { + request: { + needs: ['request.chat.openaiChatCompletions', 'route.attempt'], + consumes: [], + provides: ['request.chat.openaiChatCompletions'], + }, + response: { + needs: ['response.chat.openaiChatCompletions'], + consumes: [], + provides: ['response.chat.openaiChatCompletions'], + }, + }, + execute: transform< + Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> + >(() => { + // Whether this vendor's dialect applies at all is read on the way down and acted on in + // both directions, which is the order `transform` runs the two halves in. + let enabled = false; + return { + request: facts => { + enabled = facts['route.attempt'].flags.includes('vendor-deepseek'); + if (!enabled) return facts; + const payload = facts['request.chat.openaiChatCompletions']; + const normalized = deepSeekOutbound(payload); + return normalized === payload ? facts : { ...facts, 'request.chat.openaiChatCompletions': move(normalized) }; + }, + response: facts => { + if (!enabled) return facts; + const answer = answerWithFrames( + facts['response.chat.openaiChatCompletions'], + frames => rewritingEvents(frames, chunk => deepSeekInboundUsage(deepSeekInboundDeltas(chunk))), + ); + return answer === null ? facts : { ...facts, 'response.chat.openaiChatCompletions': move(answer) }; + }, + }; + }), +}); + +const deepSeekOutbound = (payload: OpenAIChatCompletionsPayload): OpenAIChatCompletionsPayload => { + const withThinking = payload.reasoning_effort === 'none' + ? { ...withoutKeys(payload, ['reasoning_effort']), thinking: { type: 'disabled' } } satisfies OpenAIChatCompletionsPayloadWithDeepSeekThinking + : payload; + const withResponseFormat = withThinking.response_format?.type === 'json_schema' + ? { ...withThinking, response_format: { type: 'json_object' } } + : withThinking; + const messages = mapKeepingIdentity(withResponseFormat.messages, deepSeekAssistantReasoning); + return messages === withResponseFormat.messages ? withResponseFormat : { ...withResponseFormat, messages }; +}; + +/** The reasoning DeepSeek reads, and only that: `reasoning_opaque` is the OpenAI-canonical + * signature for cross-turn replay and DeepSeek does not accept it, so it goes with the two + * fields that are projected onto `reasoning_content`. */ +const deepSeekAssistantReasoning = (message: OpenAIChatCompletionsMessage): OpenAIChatCompletionsMessage => { + const stripped = withoutKeys(message, ['reasoning_text', 'reasoning_opaque', 'reasoning_items']); + if (stripped === message) return message; + const text = typeof message.reasoning_text === 'string' + ? message.reasoning_text + : deepSeekReasoningFromItems(message.reasoning_items); + if (text === undefined) return stripped; + const projected: OpenAIChatCompletionsMessageWithDeepSeekReasoning = { ...stripped, reasoning_content: text }; + return projected; +}; + +/** The newer OpenAI shape carries reasoning as summary items; DeepSeek documents only the + * scalar, so what summaries there are become it. */ +const deepSeekReasoningFromItems = (items: OpenAIChatCompletionsReasoningItem[] | null | undefined): string | undefined => { + const parts = items?.flatMap(item => item.summary?.map(summary => summary.text) ?? []) ?? []; + return parts.length > 0 ? parts.join('') : undefined; +}; + +const deepSeekInboundDeltas = (chunk: OpenAIChatCompletionsStreamEvent): OpenAIChatCompletionsStreamEvent => { + const choices = mapKeepingIdentity(chunk.choices, choice => { + const delta = choice.delta as OpenAIChatCompletionsDeltaWithDeepSeekReasoning; + if (typeof delta.reasoning_content !== 'string') return choice; + const stripped = withoutKeys(delta, ['reasoning_content']); + return { + ...choice, + delta: delta.reasoning_text === undefined ? { ...stripped, reasoning_text: delta.reasoning_content } : stripped, + }; + }); + return choices === chunk.choices ? chunk : { ...chunk, choices }; +}; + +/** DeepSeek's "hit" count is the cached prefix length, which is what OpenAI's + * `cached_tokens` means; the "miss" count is what is left of the input and is dropped. + * `prompt_tokens` is documented as exactly hit + miss, and the cache matches only a prefix. + * https://api-docs.deepseek.com/api/create-chat-completion + * https://api-docs.deepseek.com/guides/kv_cache */ +const DEEPSEEK_CACHE_FIELDS = ['prompt_cache_hit_tokens', 'prompt_cache_miss_tokens'] as const; + +const deepSeekInboundUsage = (chunk: OpenAIChatCompletionsStreamEvent): OpenAIChatCompletionsStreamEvent => { + const usage = asJsonObject(chunk.usage); + if (usage === null) return chunk; + const stripped = withoutKeys(usage, DEEPSEEK_CACHE_FIELDS); + if (stripped === usage) return chunk; + const hit = readJsonNumber(usage.prompt_cache_hit_tokens); + const next: JsonObject = hit == null + ? stripped + : { ...stripped, prompt_tokens_details: { ...(asJsonObject(usage.prompt_tokens_details) ?? {}), cached_tokens: hit } }; + return { ...chunk, usage: next as unknown as OpenAIChatCompletionsStreamEvent['usage'] }; +}; + +/** None of the three fields is an OpenAI Chat Completions field, so each is declared beside the vendor + * that reads it rather than widening the protocol's own types. */ +type OpenAIChatCompletionsPayloadWithDeepSeekThinking = Omit & { thinking: { type: string } }; + +type OpenAIChatCompletionsMessageWithDeepSeekReasoning = + Omit & { reasoning_content: string }; + +type OpenAIChatCompletionsDeltaWithDeepSeekReasoning = + OpenAIChatCompletionsStreamEvent['choices'][number]['delta'] & { reasoning_content?: unknown }; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/vendor-kimi-normalize.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/vendor-kimi-normalize.ts new file mode 100644 index 0000000000..6df6eb57c2 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/vendor-kimi-normalize.ts @@ -0,0 +1,61 @@ +import { asJsonObject, readJsonNumber, type JsonObject } from '../../../shared/json-helpers.ts'; +import type { Chat } from '../facts.ts'; +import { answerWithFrames, rewritingEvents } from '../shared/frames.ts'; +import { withoutKeys } from '../shared/structural.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; +import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; + +/** Kimi (Moonshot) reports the cached prefix on a flat `cached_tokens` beside the totals; the + * rules above this one read OpenAI's `prompt_tokens_details.cached_tokens`, so it is put + * there. Kimi accepts the canonical request shape, so there is nothing to do on the way out. + * https://platform.kimi.com/docs/api/chat */ +export const vendorKimiNormalizeForOpenAIChatCompletions = defineStage< + Chat<'route.attempt'>, + Chat<'route.attempt'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> +>({ + name: 'vendorKimiNormalize', + through: { + request: { needs: ['route.attempt'], consumes: [], provides: [] }, + response: { + needs: ['response.chat.openaiChatCompletions'], + consumes: [], + provides: ['response.chat.openaiChatCompletions'], + }, + }, + execute: transform< + Chat<'route.attempt'>, + Chat<'route.attempt'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> + >(() => { + let enabled = false; + return { + request: facts => { + enabled = facts['route.attempt'].flags.includes('vendor-kimi'); + return facts; + }, + response: facts => { + if (!enabled) return facts; + const answer = answerWithFrames( + facts['response.chat.openaiChatCompletions'], + frames => rewritingEvents(frames, kimiInboundUsage), + ); + return answer === null ? facts : { ...facts, 'response.chat.openaiChatCompletions': move(answer) }; + }, + }; + }), +}); + +const kimiInboundUsage = (chunk: OpenAIChatCompletionsStreamEvent): OpenAIChatCompletionsStreamEvent => { + const usage = asJsonObject(chunk.usage); + if (usage === null) return chunk; + const cached = readJsonNumber(usage.cached_tokens); + if (cached == null) return chunk; + const next: JsonObject = { + ...withoutKeys(usage, ['cached_tokens']), + prompt_tokens_details: { ...(asJsonObject(usage.prompt_tokens_details) ?? {}), cached_tokens: cached }, + }; + return { ...chunk, usage: next as unknown as OpenAIChatCompletionsStreamEvent['usage'] }; +}; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/vendor-qwen-normalize.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/vendor-qwen-normalize.ts new file mode 100644 index 0000000000..14c0bfd958 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/vendor-qwen-normalize.ts @@ -0,0 +1,44 @@ +import type { Chat } from '../facts.ts'; +import { withoutKeys } from '../shared/structural.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; +import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; + +/** Qwen says "no reasoning" with a top-level `enable_thinking: false` rather than with the + * canonical sentinel. Its response shape matches OpenAI for the fields the gateway reads, so + * there is nothing to do on the way back. + * https://www.alibabacloud.com/help/en/model-studio/deep-thinking */ +export const vendorQwenNormalizeForOpenAIChatCompletions = defineStage< + Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> +>({ + name: 'vendorQwenNormalize', + through: { + request: { + needs: ['request.chat.openaiChatCompletions', 'route.attempt'], + consumes: [], + provides: ['request.chat.openaiChatCompletions'], + }, + response: { needs: ['response.chat.openaiChatCompletions'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.openaiChatCompletions' | 'route.attempt'>, + Chat<'request.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'>, + Chat<'response.chat.openaiChatCompletions'> + >(() => ({ + request: facts => { + if (!facts['route.attempt'].flags.includes('vendor-qwen')) return facts; + const payload = facts['request.chat.openaiChatCompletions']; + if (payload.reasoning_effort !== 'none') return facts; + const normalized: OpenAIChatCompletionsPayloadWithQwenThinking = { + ...withoutKeys(payload, ['reasoning_effort']), + enable_thinking: false, + }; + return { ...facts, 'request.chat.openaiChatCompletions': move(normalized) }; + }, + })), +}); + +type OpenAIChatCompletionsPayloadWithQwenThinking = Omit & { enable_thinking: false }; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/wire.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/wire.ts new file mode 100644 index 0000000000..a081c8f508 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/wire.ts @@ -0,0 +1,52 @@ +import { applyRoleCompatibilityToOpenAIChatCompletions } from './apply-role-compatibility.ts'; +import { callOpenAIChatCompletionsUpstream } from './call-upstream.ts'; +import { disableReasoningOnForcedToolChoiceForOpenAIChatCompletions } from './disable-reasoning-on-forced-tool-choice.ts'; +import { includeUsageStreamOptionsForOpenAIChatCompletions } from './include-usage-stream-options.ts'; +import { meterUsage } from './meter.ts'; +import { normalizeEmptyToolsForOpenAIChatCompletions } from './normalize-empty-tools-tool-choice.ts'; +import { normalizeExclusiveCachedTokensForOpenAIChatCompletions } from './normalize-exclusive-cached-tokens.ts'; +import { normalizeUsageForOpenAIChatCompletions } from './normalize-usage.ts'; +import { stripPromptCacheKeyForOpenAIChatCompletions } from './strip-prompt-cache-key.ts'; +import { vendorDeepSeekNormalizeForOpenAIChatCompletions } from './vendor-deep-seek-normalize.ts'; +import { vendorKimiNormalizeForOpenAIChatCompletions } from './vendor-kimi-normalize.ts'; +import { vendorQwenNormalizeForOpenAIChatCompletions } from './vendor-qwen-normalize.ts'; +import type { Stage } from '@floway-dev/pipeline'; + +/** + * The OpenAI Chat Completions wire, as the chain that dials it. + * + * Every source protocol that reaches an upstream over this endpoint runs this, whether the + * client spoke OpenAI Chat Completions or a handoff arrived here — which is what makes a rule that + * speaks about *this* wire belong here. The role rewrite states what an upstream's Chat + * Completions endpoint accepts; the usage rules speak about the usage this wire asks for and + * this wire reports; the vendor dialects are how one upstream spells both. All of them apply + * to whatever body this wire actually sends and to nothing that leaves for another protocol. + * + * Requests descend through this array and responses return through it in reverse: + * a stage earlier in the array rewrites the request first and reads the answer last. So the + * usage chunk is asked for above everything, and coming back the vendor dialects have the + * first say — the generic rules above them then read a body already in OpenAI-canonical form, + * with the cache-bucket fold seeing cache fields under OpenAI's names and the carrier split + * seeing usage the fold has already settled. The meter is above all of them, which is what + * makes the figure it bills the one the client is shown. + * + * `disableReasoningOnForcedToolChoice` and `stripPromptCacheKey` are here rather than above + * the fork, because both speak about what an upstream's OpenAI Chat Completions endpoint accepts — + * so a turn that arrived over a translation gets them too. Both still precede every vendor + * dialect on the request path: the canonical sentinel is emitted before a vendor spells it, + * and the field an upstream would reject is gone before a vendor rewrites what is left. + */ +export const openaiChatCompletionsWire = (streamedUsage: string): readonly Stage[] => [ + normalizeEmptyToolsForOpenAIChatCompletions, + meterUsage(streamedUsage), + includeUsageStreamOptionsForOpenAIChatCompletions, + normalizeUsageForOpenAIChatCompletions, + disableReasoningOnForcedToolChoiceForOpenAIChatCompletions, + applyRoleCompatibilityToOpenAIChatCompletions, + stripPromptCacheKeyForOpenAIChatCompletions, + normalizeExclusiveCachedTokensForOpenAIChatCompletions, + vendorDeepSeekNormalizeForOpenAIChatCompletions, + vendorQwenNormalizeForOpenAIChatCompletions, + vendorKimiNormalizeForOpenAIChatCompletions, + callOpenAIChatCompletionsUpstream, +]; diff --git a/packages/gateway/src/data-plane/chat/openai-chat-completions/wires.ts b/packages/gateway/src/data-plane/chat/openai-chat-completions/wires.ts new file mode 100644 index 0000000000..f244f9102a --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-chat-completions/wires.ts @@ -0,0 +1,45 @@ +import { composeChat as compose } from '../compose.ts'; +import type { ChatWire } from '../wire.ts'; +import { STREAMED_USAGE } from './facts.ts'; +import { openaiChatCompletionsWire } from './wire.ts'; +import { anthropicMessagesWire } from '../anthropic-messages/wire.ts'; +import { handOff } from '../handoff.ts'; +import { projectOpenAIResponsesCollaboration } from '../openai-responses/collaboration-shim.ts'; +import { openaiResponsesWire } from '../openai-responses/wire.ts'; +import type { ChatServices } from '../services.ts'; +import { createExternalImageLoader } from '../shared/external-image-loader.ts'; +import type { Use } from '@floway-dev/pipeline'; +import type { ChatTargetApi, ModelCandidate } from '@floway-dev/provider'; +import { translateOpenAIChatCompletionsViaAnthropicMessages, translateOpenAIChatCompletionsViaOpenAIResponses } from '@floway-dev/translate'; + +/** The three wires `/v1/chat/completions` can be served on. Its own is the bare wire; each + * translated one is a handoff and then the target protocol's own wire. */ +export const openaiChatCompletionsWireFor = (target: ChatTargetApi, candidate: ModelCandidate, use: Use): ChatWire => { + switch (target) { + case 'openaiChatCompletions': + return compose('openaiChatCompletionsNative', openaiChatCompletionsWire(STREAMED_USAGE)); + case 'anthropicMessages': + return compose('openaiChatCompletionsViaAnthropicMessages', [ + handOff({ + from: { request: 'request.chat.openaiChatCompletions', response: 'response.chat.openaiChatCompletions' }, + to: { request: 'request.chat.anthropicMessages', response: 'response.chat.anthropicMessages' }, + trip: async payload => await translateOpenAIChatCompletionsViaAnthropicMessages(payload, { + model: candidate.model.id, + fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, + loadRemoteImage: createExternalImageLoader(use.gateway.abortSignal), + }), + }), + ...anthropicMessagesWire(STREAMED_USAGE), + ]); + case 'openaiResponses': + return compose('openaiChatCompletionsViaOpenAIResponses', [ + handOff({ + from: { request: 'request.chat.openaiChatCompletions', response: 'response.chat.openaiChatCompletions' }, + to: { request: 'request.chat.openaiResponses', response: 'response.chat.openaiResponses' }, + trip: async payload => await translateOpenAIChatCompletionsViaOpenAIResponses(payload, { model: candidate.model.id }), + }), + projectOpenAIResponsesCollaboration, + ...openaiResponsesWire(STREAMED_USAGE), + ]); + } +}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/affinity/egress.ts b/packages/gateway/src/data-plane/chat/openai-responses/affinity/egress.ts index 94cfc190ec..f90a1325ee 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/affinity/egress.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/affinity/egress.ts @@ -1,5 +1,5 @@ import type { AffinityEgressOptions } from '../../shared/affinity/index.ts'; -import { isOpenAIResponsesCompactShimItem } from '../interceptors/compact-shim.ts'; +import { isOpenAIResponsesCompactShimItem } from '../compact-shim.ts'; import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import { createRandomOpenAIResponsesItemId, type OpenAIResponsesOutputItem, type OpenAIResponsesOutputReasoning, type OpenAIResponsesResult, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/affinity/ingress.ts b/packages/gateway/src/data-plane/chat/openai-responses/affinity/ingress.ts index 8a90cad8a9..2cd8112870 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/affinity/ingress.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/affinity/ingress.ts @@ -10,7 +10,8 @@ import { projectOptionalAffinityBlob, projectRequiredAffinityBlob, } from '../../shared/affinity/index.ts'; -import { isOpenAIResponsesCompactShimItem } from '../interceptors/compact-shim.ts'; +import { isOpenAIResponsesCompactShimItem } from '../compact-shim.ts'; +import { withIndexesChanged, withKeysChanged } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesInputItem } from '@floway-dev/protocols/openai-responses'; import type { ModelCandidate } from '@floway-dev/provider'; @@ -137,36 +138,34 @@ const materializeOpenAIResponsesPayload = ( projectionsByItem: ReadonlyMap, ): CanonicalOpenAIResponsesPayload => { if (projectionsByItem.size === 0) return payload; + // Rebuilt the way its three sibling families are: an item no projection touches rides through + // by identity, and what one candidate is owed differs from the next by the slots a projection + // actually spoke about. const input = payload.input.flatMap((item, itemIndex): OpenAIResponsesInputItem[] => { const projections = projectionsByItem.get(itemIndex); if (projections === undefined) return [item]; if (projections === null) return []; - const replacement = { ...item } as OpenAIResponsesInputItem & Record; - for (const { location, projection } of projections) { - if (location.contentIndex !== undefined) continue; - if (projection.kind === 'preserve') replacement[location.slot] = projection.value; - else delete replacement[location.slot]; - } - - if (item.type === 'agent_message') { - const nested = new Map(projections.flatMap(projection => - projection.location.contentIndex === undefined ? [] : [[projection.location.contentIndex, projection] as const])); - if (nested.size > 0) { - const agentMessage = replacement as Extract; - agentMessage.content = agentMessage.content.flatMap((content, contentIndex) => { - const projected = nested.get(contentIndex); - if (projected === undefined) return [content]; - return projected.projection.kind === 'preserve' - ? [{ ...content, encrypted_content: projected.projection.value }] - : []; - }); - } - } - - return [replacement]; + const slots = Object.fromEntries(projections.flatMap(({ location, projection }) => + location.contentIndex !== undefined + ? [] + : [[location.slot, projection.kind === 'preserve' ? projection.value : undefined] as const])); + const rewritten = withKeysChanged(item, slots); + if (item.type !== 'agent_message') return [rewritten]; + + const content = (rewritten as Extract).content; + const nested = new Map(projections.flatMap(({ location, projection }) => + location.contentIndex === undefined + ? [] + : [[ + location.contentIndex, + projection.kind === 'preserve' + ? { ...content[location.contentIndex], encrypted_content: projection.value } + : null, + ] as const])); + return nested.size === 0 ? [rewritten] : [withKeysChanged(rewritten, { content: withIndexesChanged(content, nested) })]; }); - return { ...payload, input }; + return withKeysChanged(payload, { input }); }; const evaluateOpenAIResponsesCandidate = ( diff --git a/packages/gateway/src/data-plane/chat/openai-responses/answer-websocket-warmup.ts b/packages/gateway/src/data-plane/chat/openai-responses/answer-websocket-warmup.ts new file mode 100644 index 0000000000..5ae1dbb181 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/answer-websocket-warmup.ts @@ -0,0 +1,43 @@ +import type { OpenAIResponsesFacts } from './facts.ts'; +import { syntheticEventsFromResult } from './items/output.ts'; +import { telemetryModelIdentity } from '../../shared/telemetry/attribution.ts'; +import type { ChatServices } from '../services.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesResult } from '@floway-dev/protocols/openai-responses'; +import { providerModelOf } from '@floway-dev/provider'; + +type Request = Pick; +type Response = Pick; +const provides = ['response.chat.openaiResponses', 'response.chat.openaiResponses.streamedUsage', 'response.usage.billable', 'response.http.headers', 'response.http.body', 'response.http.status'] as const; + +// Codex continues from the response id of its WebSocket prewarm. The source chain has +// already staged its input; normal client egress commits that snapshot without inference. +// https://github.com/openai/codex/blob/6989c6548b3737f108e2bb5ae1171b1d2032e30c/codex-rs/core/src/client.rs#L2181-L2184 +export const answerOpenAIResponsesWebSocketWarmup = defineStage({ + name: 'answerOpenAIResponsesWebSocketWarmup', + through: { + request: { needs: ['request.chat.openaiResponses', 'route.attempt'], consumes: [], provides: [] }, + response: { needs: provides, consumes: [], provides: [] }, + }, + return: { provides }, + execute: async (facts, next, use) => { + const payload = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; + if (payload.generate !== false) return await next(facts); + const candidate = use.resolveAttempt(facts['route.attempt']); + use.selectAffinity(candidate); + use.gateway.dump?.success(telemetryModelIdentity(candidate, providerModelOf(candidate).id), null); + const response: OpenAIResponsesResult = { + id: '', object: 'response', model: candidate.model.id, status: 'completed', + output: [], error: null, incomplete_details: null, + usage: { input_tokens: 0, output_tokens: 0, total_tokens: 0 }, + }; + return move({ + ...facts, + 'response.chat.openaiResponses': { kind: 'stream', frames: syntheticEventsFromResult(response) }, + 'response.chat.openaiResponses.streamedUsage': null, + 'response.usage.billable': [], + 'response.http.headers': [], + 'response.http.body': null, 'response.http.status': 200, + }); + }, +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/apply-role-compatibility.ts b/packages/gateway/src/data-plane/chat/openai-responses/apply-role-compatibility.ts new file mode 100644 index 0000000000..1221ae84da --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/apply-role-compatibility.ts @@ -0,0 +1,54 @@ +import type { Chat } from '../facts.ts'; +import { rolesFor, roleRewriter } from '../shared/role-compatibility.ts'; +import { mapKeepingIdentity } from '../shared/structural.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesInputItem } from '@floway-dev/protocols/openai-responses'; + +// ── OpenAI Responses ────────────────────────────────────────────────────────────────────── + +/** Role rewrites, in the same fixed order and by the same fold as OpenAI Chat Completions; what is + * here is the walk over this protocol's input items. Only a message item carries a role, and + * an item that carries none still crosses the leading system run — a reasoning item between + * two system messages is what makes the second one mid-conversation. */ +export const applyRoleCompatibilityToOpenAIResponses = defineStage< + Chat<'request.chat.openaiResponses' | 'route.attempt'>, + Chat<'request.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> +>({ + name: 'applyRoleCompatibility', + through: { + request: { + needs: ['request.chat.openaiResponses', 'route.attempt'], + consumes: [], + provides: ['request.chat.openaiResponses'], + }, + response: { needs: ['response.chat.openaiResponses'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.openaiResponses' | 'route.attempt'>, + Chat<'request.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> + >(() => ({ + request: facts => { + const rewrite = rolesFor(facts['route.attempt'].flags); + if (rewrite === null) return facts; // the free do-nothing path + // The key holds what a client may send, whose `input` is a string or a list; this chain + // runs on the canonical form the entry normalized it to, which is the one a wire takes. + const payload = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; + const nextRole = roleRewriter(rewrite); + const input = mapKeepingIdentity(payload.input, (item): OpenAIResponsesInputItem => { + if (item.type !== 'message') { + nextRole(undefined); + return item; + } + const role = nextRole(item.role); + return role === item.role ? item : { ...item, role }; + }); + return input === payload.input + ? facts + : { ...facts, 'request.chat.openaiResponses': move({ ...payload, input }) }; + }, + })), +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/attempt.ts b/packages/gateway/src/data-plane/chat/openai-responses/attempt.ts deleted file mode 100644 index 2514569a3c..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-responses/attempt.ts +++ /dev/null @@ -1,246 +0,0 @@ -import { klona } from 'klona/json'; - -import { openaiResponsesInterceptors } from './interceptors/index.ts'; -import type { OpenAIResponsesAttemptResult, OpenAIResponsesInvocation } from './interceptors/types.ts'; -import { normalizeAssistantInputText } from './items/normalize-assistant-content.ts'; -import { syntheticEventsFromCompaction } from './items/output.ts'; -import { billableUsageFromOpenAIResponsesEvent, billableUsageFromOpenAIResponsesResult } from './usage.ts'; -import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; -import { tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; -import { buildUpstreamCallOptions } from '../../shared/upstream-call-options.ts'; -import { anthropicMessagesAttempt } from '../anthropic-messages/attempt.ts'; -import { openaiChatCompletionsAttempt } from '../openai-chat-completions/attempt.ts'; -import { applyRulesToUpstreamOpenAIResponses } from '../shared/alias-rules.ts'; -import { createExternalImageLoader } from '../shared/external-image-loader.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import { providerStreamResultToExecuteResult } from '../shared/provider-stream-result.ts'; -import { chatTargetPicker } from '../shared/target-picker.ts'; -import { captureFromDump, traverseTranslation } from '../shared/translate-traverse.ts'; -import { runInterceptors } from '@floway-dev/interceptor'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { collectOpenAIResponsesProtocolEventsToResult, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { type ModelCandidate, eventResult, readUpstreamApiError, providerModelOf, type ChatTargetApi, type ExecuteResult, type ProviderOpenAIResponsesResult, type OpenAIResponsesAction } from '@floway-dev/provider'; -import { translateOpenAIResponsesViaOpenAIChatCompletions, translateOpenAIResponsesViaAnthropicMessages } from '@floway-dev/translate'; - -// `/v1/responses` generate prefers the native OpenAI Responses target, then the -// translated Anthropic Messages path, then the translated OpenAI Chat Completions path. The -// same picker covers compact: every OpenAI Responses target is reachable via the -// shim, which pivots compact→generate inside the chain on non-openai-responses -// targets. -export const openaiResponsesTarget = chatTargetPicker(['openaiResponses', 'anthropicMessages', 'openaiChatCompletions']); - -interface OpenAIResponsesAttemptBaseArgs { - readonly action: OpenAIResponsesAction; - readonly payload: CanonicalOpenAIResponsesPayload; - readonly ctx: ChatGatewayCtx; - readonly candidate: ModelCandidate; - readonly headers: Headers; -} - -interface OpenAIResponsesSourceState { - readonly privatePayloads: ReadonlyMap; -} - -export type OpenAIResponsesAttemptInvokeArgs = OpenAIResponsesAttemptBaseArgs & { - readonly sourceState?: OpenAIResponsesSourceState; -}; -type OpenAIResponsesAttemptGenerateArgs = Omit; - -// Single entry point for both `action: 'generate'` and `action: 'compact'`. -// Envelope-drain branches on the caller's intent (`action` passed by value), -// not on `invocation.action`. Interceptors are free to mutate `ctx.action` -// to steer inner dispatch — and, by the project's interceptor convention, -// they do not restore on the way out — so post-chain `invocation.action` -// reflects whatever the last writer left it at. The shape of the result we -// hand back is the caller's contract; keying off the caller's value is the -// only place that contract lives. -// -// The module-boundary invariant `compact-shaped ⇒ targetApi='openaiResponses'` -// at dispatch time is enforced in two places, each at the layer that owns -// the corresponding piece of state: -// -// - `invocation.action === 'compact'` is caught inside `dispatchOpenAIResponses`'s -// `case 'messages'` / `case 'openai-chat-completions'` arms — action is a -// OpenAI-Responses-level metadata field that the translators never see. -// - A `compaction_trigger` (or any other compact-shaped) item in input is -// caught by the translator itself — the `openai-responses-via-anthropic-messages` and -// `openai-responses-via-openai-chat-completions` translators reject any input-item -// variant they do not handle, so a compaction_trigger that slipped past -// the shim surfaces as a translator-level error rather than a silent -// drop. -// -// Both safety nets fire pre-upstream-call, live inside the chain (not after -// the interceptor finally blocks), and stay independent of the shim's -// presence. -// -// OpenAI Responses state and client affinity both belong to the native source edge, -// outside this candidate attempt. Native serve passes the already-hydrated -// candidate payload plus any private source state; translated inner OpenAI Responses -// calls pass only their translated payload. Keeping this function free of -// affinity decoding, state hydration, and persistence -// prevents an inner OpenAI Responses target from owning another source protocol's -// client-visible state. -export const openaiResponsesAttempt = { - invoke: async (args: OpenAIResponsesAttemptInvokeArgs): Promise => { - const { action, ctx, candidate, headers: sourceHeaders } = args; - const headers = new Headers(sourceHeaders); - const targetApi = openaiResponsesTarget.pick(candidate.model.endpoints); - const payload = { ...klona(args.payload), model: candidate.model.id }; - ctx.store.beginAttempt(args.sourceState?.privatePayloads ?? new Map()); - // Copilot compaction and Azure-native compaction both emit assistant - // messages whose content blocks have `type: 'input_text'`, then refuse - // the same items echoed back as input on the next turn. Normalising - // here, after native serve has expanded stored `item_reference` items, - // catches both the direct-echo and store-replay paths in one place. - const normalized: CanonicalOpenAIResponsesPayload = { ...payload, input: normalizeAssistantInputText(payload.input) }; - - const invocation: OpenAIResponsesInvocation = { - payload: normalized, - action, - candidate, - targetApi, - headers, - }; - const chainResult = await runInterceptors(invocation, ctx, openaiResponsesInterceptors, async () => - await dispatchOpenAIResponses(invocation, ctx)); - - if (chainResult.type !== 'events') return chainResult; - - if (action === 'compact') { - const upstreamCompacted = await collectOpenAIResponsesProtocolEventsToResult(chainResult.events); - return { - type: 'result', - result: upstreamCompacted, - modelIdentity: chainResult.modelIdentity, - usage: tokenUsageFromBillableUsage((await chainResult.finalMetadata)?.billableUsage), - performance: chainResult.performance, - }; - } - return chainResult; - }, - - // Narrowing wrapper for cross-protocol translation callers - // (Anthropic Messages/Gemini generateContent/OpenAI Chat Completions translating into OpenAI Responses) and the - // native HTTP/WS generate entry — both always run in generate mode and - // want the ExecuteResult branch. The compact branch is a contract - // violation here; an interceptor that pivoted generate→compact would - // surface as a throw, not a silent shape mismatch. - generate: async (args: OpenAIResponsesAttemptGenerateArgs): Promise>> => { - const result = await openaiResponsesAttempt.invoke({ ...args, action: 'generate' }); - if (result.type === 'result') { - throw new Error('openaiResponsesAttempt.generate received a compact result; an interceptor pivoted generate→compact unexpectedly'); - } - return result; - }, -}; - -const dispatchOpenAIResponses = async ( - invocation: OpenAIResponsesInvocation, - ctx: ChatGatewayCtx, -): Promise>> => { - const { candidate, targetApi } = invocation; - switch (targetApi) { - case 'openaiResponses': { - if (candidate.rules !== undefined) applyRulesToUpstreamOpenAIResponses(invocation.payload, candidate.rules); - // Compact drops `stream` and `store` before hitting the wire: `store` is a - // gateway-only snapshot-persistence hint the upstream compact endpoint - // rejects, and `stream` is irrelevant on a non-streaming call. The generate - // branch leaves both fields on the body — every provider's streaming call - // forces stream=true anyway. - let body: Omit; - if (invocation.action === 'compact') { - const { model: _model, stream: _stream, store: _store, ...rest } = invocation.payload; - body = rest; - } else { - const { model: _model, ...rest } = invocation.payload; - body = rest; - } - const providerResult = await candidate.provider.instance.callOpenAIResponses( - providerModelOf(candidate), - body, - invocation.action, - ctx.abortSignal, - buildUpstreamCallOptions(candidate, ctx, invocation.headers), - ); - return await providerOpenAIResponsesResultToExecuteResult(providerResult, candidate, targetApi, ctx); - } - case 'anthropicMessages': - if (invocation.action === 'compact') { - // The openai-responses-compact-shim is structurally required on non-openai-responses - // targets and pivots ctx.action to 'generate' before reaching here; - // landing inside this case with action='compact' means the shim - // disengaged or was wired out of the chain. A compaction_trigger in - // input is caught one layer down by the translator's - // unexpected-input-item guard. - throw new Error(`openaiResponsesAttempt: action='compact' reached dispatch on targetApi='anthropicMessages' — the openai-responses-compact-shim must engage and pivot the action`); - } - return await traverseTranslation( - invocation.payload, - p => translateOpenAIResponsesViaAnthropicMessages(p, { - model: candidate.model.id, - fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, - loadRemoteImage: createExternalImageLoader(ctx.abortSignal), - }), - translated => anthropicMessagesAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, anthropicBeta: [], - }), - captureFromDump(ctx.dump, targetApi), - ); - case 'openaiChatCompletions': - if (invocation.action === 'compact') { - throw new Error(`openaiResponsesAttempt: action='compact' reached dispatch on targetApi='openaiChatCompletions' — the openai-responses-compact-shim must engage and pivot the action`); - } - return await traverseTranslation( - invocation.payload, - p => translateOpenAIResponsesViaOpenAIChatCompletions(p, { model: candidate.model.id }), - translated => openaiChatCompletionsAttempt.generate({ - payload: translated, ctx, candidate, headers: invocation.headers, - }), - captureFromDump(ctx.dump, targetApi), - ); - default: { - const exhaustive: never = targetApi; - throw new Error(`unexpected targetApi '${exhaustive as string}'`); - } - } -}; - -// Lowers a `ProviderOpenAIResponsesResult` into the chain's -// ExecuteResult> currency. The compact -// branch synthesizes SSE frames from the envelope so every downstream -// interceptor sees the same event-stream contract regardless of which action -// the provider executed. -const providerOpenAIResponsesResultToExecuteResult = async ( - providerResult: ProviderOpenAIResponsesResult, - candidate: ModelCandidate, - targetApi: ChatTargetApi, - ctx: ChatGatewayCtx, -): Promise>> => { - if (providerResult.action === 'generate') { - return await providerStreamResultToExecuteResult( - providerResult.ok - ? { ok: true, events: providerResult.events, modelKey: providerResult.modelKey, ...(providerResult.headers ? { headers: providerResult.headers } : {}) } - : { ok: false, response: providerResult.response, modelKey: providerResult.modelKey }, - candidate, - targetApi, - ctx, - billableUsageFromOpenAIResponsesEvent, - ); - } - const context = upstreamPerformanceContext(ctx, candidate, 'chat'); - if (!providerResult.ok) { - return { ...(await readUpstreamApiError(providerResult.response, candidate.provider.upstreamId)), performance: context }; - } - // A native compaction is a turn the upstream ran and charged for, and its - // body states the counts. - const modelIdentity = telemetryModelIdentity(candidate, providerResult.modelKey); - const billableUsage = billableUsageFromOpenAIResponsesResult(providerResult.result); - return eventResult( - syntheticEventsFromCompaction(providerResult.result), - modelIdentity, - { - performance: context, - ...(billableUsage === null ? {} : { finalMetadata: Promise.resolve({ modelIdentity, billableUsage }) }), - }, - ); -}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/begin-stored-attempt.ts b/packages/gateway/src/data-plane/chat/openai-responses/begin-stored-attempt.ts new file mode 100644 index 0000000000..efb706413d --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/begin-stored-attempt.ts @@ -0,0 +1,26 @@ +import type { Fields } from './facts.ts'; +import type { ChatServices } from '../services.ts'; +import { defineStage } from '@floway-dev/pipeline'; + +/** + * Each candidate starts with the hydrated private payloads. Hosted tools can add execution + * state during that attempt; reseeding here prevents a failed candidate's state from leaking + * into failover while preserving payloads when stored items are replayed. + */ +export const beginStoredAttempt = defineStage< + Fields<'request.chat.openaiResponses.privatePayloads'>, + Fields<'request.chat.openaiResponses.privatePayloads'>, + Record, + Record, + ChatServices +>({ + name: 'beginStoredAttempt', + through: { + request: { needs: ['request.chat.openaiResponses.privatePayloads'], consumes: [], provides: [] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next, use) => { + use.gateway.store.beginAttempt(new Map(facts['request.chat.openaiResponses.privatePayloads'])); + return await next(facts); + }, +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/call-upstream.ts b/packages/gateway/src/data-plane/chat/openai-responses/call-upstream.ts new file mode 100644 index 0000000000..04807f64e5 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/call-upstream.ts @@ -0,0 +1,44 @@ +import type { Fields } from './facts.ts'; +import { bodyForAttempt } from '../../pipeline/attempt-body.ts'; +import { providerEntry } from '../../pipeline/provider-entry.ts'; +import { upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; +import type { ChatServices } from '../services.ts'; +import { applyRulesToUpstreamOpenAIResponses } from '../shared/alias-rules.ts'; +import { providerChatAnswer } from '../shared/provider-answer.ts'; +import { syntheticEventsFromCompaction } from './items/output.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; +import type { ProviderOperationRequest, ProviderChatResponse } from '@floway-dev/provider'; + +export const callOpenAIResponsesUpstream = defineStage< + Fields<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers'>, + ProviderOperationRequest<'openaiResponses'>, + ProviderChatResponse<'openaiResponses'>, + Fields<'response.chat.openaiResponses' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status' | 'response.http.body'>, + ChatServices +>({ + name: 'callOpenAIResponsesUpstream', + into: { + request: { needs: ['request.chat.openaiResponses', 'route.attempt', 'ingress.http.headers'], consumes: [], provides: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'] }, + response: { needs: ['response.http.exchange', 'response.provider.output', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.http.body'], consumes: ['response.http.exchange', 'response.provider.output', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'], provides: ['response.chat.openaiResponses', 'response.usage.billable', 'response.http.headers', 'response.http.status', 'response.http.body'] }, + }, + execute: async (facts, next, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); + const body = bodyForAttempt(facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload, candidate, applyRulesToUpstreamOpenAIResponses); + const pipeline = candidate.provider.pipelines.openaiResponses; + if (pipeline === undefined) throw new Error(`Provider ${candidate.provider.kind} has no openaiResponses pipeline`); + const back = await next(providerEntry(facts, candidate, body), pipeline); + const { 'response.http.exchange': _exchange, 'response.provider.output': _output, 'response.provider.modelKey': _modelKey, 'response.provider.called': _called, 'response.provider.previousCalls': _previousCalls, ...rest } = back; + const reply = await providerChatAnswer(candidate, back); + const answer = 'kind' in reply.answer && reply.answer.kind === 'value' + ? { kind: 'stream' as const, frames: use.recordProtocolFrames(syntheticEventsFromCompaction(reply.answer.body as Parameters[0])) } + : reply.answer; + if ('kind' in answer) use.selectAffinity(candidate); + return move({ + ...rest, 'response.chat.openaiResponses': answer, + 'response.usage.billable': reply.billable, 'response.http.headers': reply.headers, + 'response.http.status': reply.status, 'response.http.body': reply.body, + }); + }, +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/client-output.ts b/packages/gateway/src/data-plane/chat/openai-responses/client-output.ts index 159af368a9..1b6ad0f423 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/client-output.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/client-output.ts @@ -33,14 +33,11 @@ export const wrapOpenAIResponsesStatefulOutput = ( // at the stateful half and completes that resource itself. export const wrapOpenAIResponsesClientEgress = ( frames: AsyncIterable>, - ctx: GatewayCtx, + ctx: ChatGatewayCtx, request: CanonicalOpenAIResponsesPayload, -): AsyncIterable> => { - if (!('affinity' in ctx) || !('store' in ctx)) throw new Error('OpenAI Responses output requires chat context'); - const chatCtx = ctx as ChatGatewayCtx; - return wrapResponseResourceCompletion(wrapOpenAIResponsesStatefulOutput(frames, chatCtx), { +): AsyncIterable> => + wrapResponseResourceCompletion(wrapOpenAIResponsesStatefulOutput(frames, ctx), { request, createdAt: openaiResponsesCreatedAt(ctx), - stored: chatCtx.store.writesState, + stored: ctx.store.writesState, }); -}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/collaboration-shim.ts b/packages/gateway/src/data-plane/chat/openai-responses/collaboration-shim.ts similarity index 84% rename from packages/gateway/src/data-plane/chat/openai-responses/interceptors/collaboration-shim.ts rename to packages/gateway/src/data-plane/chat/openai-responses/collaboration-shim.ts index bc0b8ca1f8..63a57307f9 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/collaboration-shim.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/collaboration-shim.ts @@ -1,4 +1,6 @@ -import type { OpenAIResponsesInterceptor } from './types.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import type { Chat } from '../facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; import { eventFrame } from '@floway-dev/protocols/common'; import { type CanonicalOpenAIResponsesPayload, @@ -9,7 +11,6 @@ import { type OpenAIResponsesTool, type OpenAIResponsesToolChoice, } from '@floway-dev/protocols/openai-responses'; -import { providerModelOf } from '@floway-dev/provider'; const CLIENT_NAMESPACE = 'collaboration'; // Shared ordinary namespace spelling used by deployed Codex gateways. @@ -321,45 +322,61 @@ const createClientEventRestorer = (upstreamNamespace: string, names: ReadonlySet // reserved identity and the plaintext marker before Codex dispatches it. // https://github.com/openai/codex/blob/c4f42d161ae44a8d696ee9fb595709661979d187/codex-rs/core/src/tools/router.rs#L31-L55 // https://github.com/openai/codex/blob/c4f42d161ae44a8d696ee9fb595709661979d187/codex-rs/core/tests/suite/subagent_notifications.rs#L1514-L1563 -// TODO: Stateful Responses must hydrate before alias allocation and persist -// client-facing snapshots with the same full-history mapping on continuation. -export const withOpenAIResponsesCollaborationShim: OpenAIResponsesInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('openai-responses-collaboration-shim')) return await run(); - const toolLists = toolInventories(ctx.payload); - if (!hasCollaborationNamespace(ctx.payload)) return await run(); +export const projectOpenAIResponsesCollaboration = defineStage< + Chat<'request.chat.openaiResponses' | 'route.attempt'>, + Chat<'request.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> +>({ + name: 'projectOpenAIResponsesCollaboration', + through: { + request: { needs: ['request.chat.openaiResponses', 'route.attempt'], consumes: [], provides: ['request.chat.openaiResponses'] }, + response: { needs: ['response.chat.openaiResponses'], consumes: [], provides: ['response.chat.openaiResponses'] }, + }, + execute: async (facts, next) => { + const payload = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; + if (!facts['route.attempt'].flags.includes('openai-responses-collaboration-shim')) return await next(facts); + const toolLists = toolInventories(payload); + if (!hasCollaborationNamespace(payload)) return await next(facts); - const names = new Set(); - const flatNames = new Set(); - for (const tools of toolLists) for (const tool of tools ?? []) { - if (tool.type === 'namespace' && tool.name === CLIENT_NAMESPACE) { - for (const child of tool.tools) names.add(child.name); - } else if (tool.type === 'function' || tool.type === 'custom') flatNames.add(tool.name); - } - for (const item of ctx.payload.input) { - if ((item.type === 'function_call' || item.type === 'custom_tool_call') && item.namespace === CLIENT_NAMESPACE) names.add(item.name); - } - const targetNamespace = upstreamNamespace(namespaceNames(ctx.payload)); - ctx.payload = { - ...ctx.payload, - tools: ctx.payload.tools == null ? ctx.payload.tools : rewriteTools(ctx.payload.tools, CLIENT_NAMESPACE, targetNamespace), - tool_choice: rewriteToolChoice(ctx.payload.tool_choice, CLIENT_NAMESPACE, targetNamespace, names, flatNames), - input: ctx.payload.input.map(item => requestItem(item, targetNamespace, names, flatNames)), - }; + const names = new Set(); + const flatNames = new Set(); + for (const tools of toolLists) for (const tool of tools ?? []) { + if (tool.type === 'namespace' && tool.name === CLIENT_NAMESPACE) { + for (const child of tool.tools) names.add(child.name); + } else if (tool.type === 'function' || tool.type === 'custom') flatNames.add(tool.name); + } + for (const item of payload.input) { + if ((item.type === 'function_call' || item.type === 'custom_tool_call') && item.namespace === CLIENT_NAMESPACE) names.add(item.name); + } + const targetNamespace = upstreamNamespace(namespaceNames(payload)); + const projected = move({ + ...payload, + tools: payload.tools == null ? payload.tools : rewriteTools(payload.tools, CLIENT_NAMESPACE, targetNamespace), + tool_choice: rewriteToolChoice(payload.tool_choice, CLIENT_NAMESPACE, targetNamespace, names, flatNames), + input: payload.input.map(item => requestItem(item, targetNamespace, names, flatNames)), + }); - const restoreEvent = createClientEventRestorer(targetNamespace, names); - const result = await run(); - if (result.type !== 'events') return result; - return { - ...result, - events: (async function* () { - for await (const frame of result.events) { - if (frame.type !== 'event') { - yield frame; - continue; - } - const event = restoreEvent(frame.event); - yield event === frame.event ? frame : eventFrame(event); - } - })(), - }; -}; + const restoreEvent = createClientEventRestorer(targetNamespace, names); + const result = await next({ ...facts, 'request.chat.openaiResponses': projected }); + const answer = result['response.chat.openaiResponses']; + if (isFailure(answer) || answer.kind !== 'stream') return result; + return { + ...result, + 'response.chat.openaiResponses': move({ + kind: 'stream' as const, frames: { + [Symbol.asyncIterator]: () => (async function* () { + for await (const frame of answer.frames as AsyncIterable>) { + if (frame.type !== 'event') { + yield frame; + continue; + } + const event = restoreEvent(frame.event); + yield event === frame.event ? frame : eventFrame(event); + } + })(), + }, + }), + }; + }, +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact-decrypt.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact-decrypt.ts new file mode 100644 index 0000000000..f51441acd9 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact-decrypt.ts @@ -0,0 +1,118 @@ +import { encodeShimCompactionPayload, EXACT_REPEAT_PREFIX, EXACT_REPEAT_SUFFIX, summaryTextFrom, sumResponseUsage } from './compact-shim.ts'; +import { internalErrorEnvelope } from './errors.ts'; +import { syntheticEventsFromCompaction, syntheticEventsFromResult } from './items/output.ts'; +import type { BillableEntity } from '../../pipeline/facts.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import { providerCalls } from '../../pipeline/provider-usage.ts'; +import type { StreamOutcome } from '../../pipeline/serve.ts'; +import type { ChatFacts, ChatAnswer } from '../facts.ts'; +import type { ChatServices } from '../services.ts'; +import type { ChatWire } from '../wire.ts'; +import { defer, defineStage, getFailureFacts, move, type Deferred } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import { + collectOpenAIResponsesProtocolEventsToResult, + isOpenAIResponsesCompactionItem, + type CanonicalOpenAIResponsesPayload, + type OpenAIResponsesOutputItem, + type OpenAIResponsesResult, + type OpenAIResponsesStreamEvent, +} from '@floway-dev/protocols/openai-responses'; + +type Request = Pick; +type Answer = Pick & Record; + +export const decryptNativeCompaction = (spec: { + native: ChatWire; + replay: ChatWire; + streamedUsage: string; + compactEndpoint: boolean; + asked: (payload: CanonicalOpenAIResponsesPayload) => boolean; +}) => defineStage({ + name: 'decryptNativeCompaction', + into: { + request: { needs: ['request.chat.openaiResponses', 'route.attempt', 'ingress.http.headers', 'ingress.chat.sourceProtocol', 'serve.usage.prior'], consumes: [], provides: ['request.chat.openaiResponses'] }, + response: { + needs: ['response.chat.openaiResponses', spec.streamedUsage, 'response.usage.billable', 'response.http.headers'], + consumes: ['response.http.body'], + provides: ['response.chat.openaiResponses', spec.streamedUsage, 'response.usage.billable', 'response.http.body'], + }, + }, + execute: async (facts, next, use) => { + const payload = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; + const initial = await next(facts, spec.native as never); + if (!facts['route.attempt'].flags.includes('openai-responses-compact-decrypt') || !spec.asked(payload)) return initial; + const answer = initial['response.chat.openaiResponses']; + if (isFailure(answer)) return initial; + + const billed: BillableEntity[] = []; + let failed = false; + const collect = async (back: Answer): Promise => { + const response = back['response.chat.openaiResponses'] as Exclude; + try { + if (response.kind === 'value') return response.body as OpenAIResponsesResult; + return await collectOpenAIResponsesProtocolEventsToResult(response.frames as AsyncIterable>); + } finally { + const pending = back[spec.streamedUsage] as Deferred | null; + if (pending === null) billed.push(...back['response.usage.billable']); + else { + const outcome = await pending; + billed.push(...outcome.billable); + failed ||= outcome.failed; + } + } + }; + const returned = (back: Answer, response: ChatAnswer, fault: boolean): Answer => ({ + ...back, + 'response.chat.openaiResponses': move(response), + 'response.usage.billable': move(billed), + [spec.streamedUsage]: isFailure(response) ? null : move(defer(Promise.resolve({ billable: billed, failed: failed || fault }))), + }); + + try { + const native = await collect(initial); + let usage = native.usage; + let sawCompaction = false; + const output: OpenAIResponsesOutputItem[] = []; + for (const item of native.output) { + if (!isOpenAIResponsesCompactionItem(item)) { output.push(item); continue; } + sawCompaction = true; + use.gateway.abortSignal?.throwIfAborted(); + const replayPayload: CanonicalOpenAIResponsesPayload = { + model: payload.model, + input: [ + { type: 'message', role: 'system', content: [{ type: 'input_text', text: EXACT_REPEAT_PREFIX }] }, + item, + { type: 'message', role: 'system', content: [{ type: 'input_text', text: EXACT_REPEAT_SUFFIX }] }, + ], + store: false, + }; + let replay: Answer; + try { + replay = await next({ ...facts, 'request.chat.openaiResponses': move(replayPayload), 'serve.usage.prior': move([...facts['serve.usage.prior'], ...billed]) }, spec.replay as never); + } catch (error) { + const atFailure = getFailureFacts(error); + if (atFailure !== undefined) billed.push(...providerCalls(use.resolveAttempt(facts['route.attempt']), atFailure)); + throw error; + } + const replayAnswer = replay['response.chat.openaiResponses']; + if (isFailure(replayAnswer)) { + billed.push(...replay['response.usage.billable']); + return returned(replay, replayAnswer, true); + } + const response = await collect(replay); + const text = summaryTextFrom(response.output); + if (text.length === 0) throw new Error('OpenAI Responses compact decryption: the replay turn closed no assistant text'); + usage = sumResponseUsage(usage, response.usage); + output.push({ ...item, encrypted_content: encodeShimCompactionPayload(text) }); + } + if (!sawCompaction) throw new Error('OpenAI Responses compact decryption: native compaction returned no compaction item'); + const result = { ...native, ...(!spec.compactEndpoint ? { object: 'response' as const } : {}), output, ...(usage === undefined ? {} : { usage }) }; + return returned(initial, { kind: 'stream', frames: spec.compactEndpoint ? syntheticEventsFromCompaction(result) : syntheticEventsFromResult(result) }, false); + } catch (error) { + use.gateway.dump?.failed(error); + const envelope = internalErrorEnvelope(error); + return returned(initial, { status: 500, message: envelope.error.message, envelope }, true); + } + }, +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/compact-shim.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts similarity index 54% rename from packages/gateway/src/data-plane/chat/openai-responses/interceptors/compact-shim.ts rename to packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts index 1926f54df8..b62db00e15 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/compact-shim.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact-shim.ts @@ -1,75 +1,40 @@ -// Compact-shim — simulates a `response.compaction` envelope against upstreams -// that have no native compaction wire. +// Compact-shim — what a `response.compaction` envelope is made of, where no upstream wire +// makes one. // -// Engagement is the OR of two conditions: -// 1. The per-upstream `openai-responses-compact-shim` flag is on. This is the -// operator-controlled opt-in for OpenAI-Responses-target upstreams that -// already answer a compact request themselves — natively through -// `/responses/compact` (codex / azure / custom), or by replaying -// `RemoteCompactionV2` over `/responses` (copilot) — but where we still -// want shim-synthesized envelopes. -// 2. The candidate's `targetApi` is not `responses`. When the upstream is -// Anthropic Messages or OpenAI Chat Completions, the translation layer has no concept -// of a `compaction` output item or a `compaction_trigger` input item. -// The shim is structurally required regardless of the flag — without -// it, a `compaction_trigger` item would reach the translator and -// crash on the unknown variant. +// The rules that use these are stages: `simulatesCompaction` says which candidates' +// compactions are the shim's to make, `expandShimCompactions` puts a blob of ours back into +// the items it stood for, and `summarizeForCompaction` sends the turn built here and packs +// what comes back (`openai-responses/pipeline.ts`). Both of this protocol's chains compose them, +// because a compaction the shim wrote is issued through one entry — `/v1/responses/compact`, +// or a generate turn whose input ends in a `compaction_trigger` — and echoed back into the +// other by the client that received it. // -// Inner compact-shape detection is also the OR of two conditions: -// - `invocation.action === 'compact'` (the native `/responses/compact` -// entry point), or -// - `invocation.payload.input` contains a `compaction_trigger` item -// (Codex CLI's RemoteCompactionV2 path: a `generate` call whose input -// ends in a control item that semantically requests compaction). +// What is here is the shim's own substance: // -// Every request first walks `payload.input` for `compaction` and -// `compaction_summary` items whose `encrypted_content` decodes as our -// base64url-JSON marker. Each match is replaced inline with the items it -// originally encoded. This normalization is independent of both flags because -// gateway-owned payloads are portable and must never be forwarded as if they -// were an upstream's opaque state. +// - `expandShimCompactionItems`: walk `payload.input` for `compaction` items whose +// `encrypted_content` decodes as our base64url-JSON marker, and replace each inline with +// the items it originally encoded — so a turn that echoes back a synthesized compaction +// is sent the summarized history rather than a blob. +// - `summarizationTurnFor`: the turn a compaction is simulated with. A role=system message +// carrying the SUMMARIZATION_PROMPT (vendored from openai/codex) at the head of the +// history, any `compaction_trigger` items stripped, a terminal user message appended +// where the history ends on a non-user item (Anthropic Messages rejects assistant +// prefill), and `store: false` so the ephemeral summarization turn does not pollute the +// upstream's conversation history. The caller's `instructions` field flows through +// untouched — native `/responses/compact` keeps SUMMARIZATION_PROMPT as a system-role +// prompt AND forwards the caller's instructions as a developer-role message alongside, +// and we mirror that shape. +// - `summaryTextFrom` and `buildCompactionEnvelope`: the summary the turn produced, and the +// synthetic `response.compaction` envelope carrying it as a blob only this gateway reads. // -// When simulation is engaged for a compact-shaped request: -// 1. Pivot the action to 'generate', prepend a role=system -// message carrying the SUMMARIZATION_PROMPT (vendored from -// openai/codex), strip any `compaction_trigger` items, append a -// terminal user message if the history ends on a non-user item -// (Anthropic Messages rejects assistant prefill), and force -// `store: false` so the ephemeral summarization turn does not -// pollute the upstream's conversation history. The caller's -// `instructions` field flows through untouched — native -// `/responses/compact` keeps SUMMARIZATION_PROMPT as a system-role -// prompt AND forwards the caller's instructions as a developer-role -// message alongside, and we mirror that shape. Call `run()` to -// drive the chain through the normal generate path; collect the -// resulting summary text; pack a single user-role message -// containing the summary into a synthetic `response.compaction` -// envelope. Mutations of `ctx.payload` / `ctx.action` are one-way -// per the project's interceptor convention; attempt.invoke does -// not consume the post-chain `ctx` for its result shape (it keys -// envelope-drain off the caller's intent action). -// -// Foreign-upstream blobs (opaque strings that fail base64url+JSON decoding -// or fail the array-of-objects-with-string-types schema below) round-trip -// untouched, so the operator can selectively turn the flag off for the -// codex / copilot / azure / custom upstreams that answer compact themselves. -// -// When `openai-responses-compact-decrypt` is enabled while the shim is off, -// native compaction runs first. Each returned opaque compaction item, including -// `compaction_summary`, is then replayed through the same model between two -// system-role instructions: the first identifies the following text and the -// second constrains the output. The recovered plaintext is packed into the -// same gateway-owned base64url envelope used by the shim, so later requests -// expand it before reaching the upstream. +// Foreign-upstream blobs (opaque strings that fail base64url+JSON decoding or fail the +// array-of-objects-with-string-types schema below) round-trip untouched, so the operator can +// selectively turn the flag off for the codex / copilot / azure / custom upstreams that +// answer compact themselves. -import type { OpenAIResponsesInterceptor, OpenAIResponsesInvocation } from './types.ts'; -import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../../shared/base64url-json.ts'; -import { isJsonObject } from '../../../../shared/json-helpers.ts'; -import type { ChatGatewayCtx } from '../../shared/gateway-ctx.ts'; -import { syntheticEventsFromCompaction, syntheticEventsFromResult } from '../items/output.ts'; -import { sumBillableUsage, type ProtocolFrame } from '@floway-dev/protocols/common'; -import { collectOpenAIResponsesProtocolEventsToResult, createRandomOpenAIResponsesItemId, isOpenAIResponsesCompactionItem, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesInputItem, type OpenAIResponsesOutputItem, type OpenAIResponsesResult, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { providerModelOf, type EventResultMetadata, type ExecuteResult } from '@floway-dev/provider'; +import { decodeBase64UrlJson, encodeBase64UrlJson } from '../../../shared/base64url-json.ts'; +import { isJsonObject } from '../../../shared/json-helpers.ts'; +import { isOpenAIResponsesCompactionItem, type OpenAIResponsesInputItem, type OpenAIResponsesOutputItem, type OpenAIResponsesResult, type CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; // The two vendored constants below (SUMMARIZATION_PROMPT and SUMMARY_PREFIX) // are the compactor system prompt and the handoff prefix openai/codex ships @@ -155,8 +120,8 @@ const SUMMARY_PREFIX export { SUMMARY_PREFIX }; -const EXACT_REPEAT_PREFIX = 'Repeat the following text exactly, which may contain a compaction summary, character for character.'; -const EXACT_REPEAT_SUFFIX = 'Output only the exact summary text, with no preface, explanation, markdown fence, or changes.'; +export const EXACT_REPEAT_PREFIX = 'Repeat the following text exactly, which may contain a compaction summary, character for character.'; +export const EXACT_REPEAT_SUFFIX = 'Output only the exact summary text, with no preface, explanation, markdown fence, or changes.'; // ── Inbound expansion ───────────────────────────────────────────────────────── @@ -174,7 +139,7 @@ export const isOpenAIResponsesCompactShimItem = ( && typeof item.encrypted_content === 'string' && isShimCompactionPayload(decodeBase64UrlJson(item.encrypted_content)); -const encodeShimCompactionPayload = (text: string): string => +export const encodeShimCompactionPayload = (text: string): string => encodeBase64UrlJson([{ type: 'message', role: 'user', @@ -209,49 +174,7 @@ export const expandShimCompactionItems = (payload: CanonicalOpenAIResponsesPaylo // ── Outbound summarization ──────────────────────────────────────────────────── -type ChainRun = () => Promise>>; - -type ResponseUsage = NonNullable; - -const sumResponseUsage = (left: ResponseUsage | null | undefined, right: ResponseUsage | null | undefined): ResponseUsage | undefined => { - if (left == null) return right ?? undefined; - if (right == null) return left; - return { - input_tokens: left.input_tokens + right.input_tokens, - output_tokens: left.output_tokens + right.output_tokens, - total_tokens: left.total_tokens + right.total_tokens, - ...(left.input_tokens_details !== undefined || right.input_tokens_details !== undefined - ? { - input_tokens_details: { - cached_tokens: (left.input_tokens_details?.cached_tokens ?? 0) + (right.input_tokens_details?.cached_tokens ?? 0), - ...( - left.input_tokens_details?.cache_write_tokens !== undefined - || right.input_tokens_details?.cache_write_tokens !== undefined - ? { cache_write_tokens: (left.input_tokens_details?.cache_write_tokens ?? 0) + (right.input_tokens_details?.cache_write_tokens ?? 0) } - : {} - ), - }, - } - : {}), - ...(left.output_tokens_details !== undefined || right.output_tokens_details !== undefined - ? { - output_tokens_details: { - reasoning_tokens: (left.output_tokens_details?.reasoning_tokens ?? 0) + (right.output_tokens_details?.reasoning_tokens ?? 0), - }, - } - : {}), - }; -}; - -const resultMetadata = async ( - result: Extract>, { type: 'events' }>, -): Promise => - await (result.finalMetadata ?? { - modelIdentity: result.modelIdentity, - ...(result.performance !== undefined ? { performance: result.performance } : {}), - }); - -const summaryTextFrom = (items: readonly OpenAIResponsesOutputItem[]): string => { +export const summaryTextFrom = (items: readonly OpenAIResponsesOutputItem[]): string => { const parts: string[] = []; for (const item of items) { if (item.type !== 'message') continue; @@ -262,20 +185,18 @@ const summaryTextFrom = (items: readonly OpenAIResponsesOutputItem[]): string => return parts.join(''); }; -const collectSummaryTurn = async ( - result: Extract>, { type: 'events' }>, -): Promise<{ response: OpenAIResponsesResult; text: string }> => { - const response = await collectOpenAIResponsesProtocolEventsToResult(result.events); - return { response, text: summaryTextFrom(response.output) }; -}; - -const buildCompactionEnvelope = (cmpId: string, summaryText: string, upstream: OpenAIResponsesResult): OpenAIResponsesResult => { +export const buildCompactionEnvelope = (cmpId: string, summaryText: string, upstream: OpenAIResponsesResult): OpenAIResponsesResult => { // The prefix lives inside the blob so it round-trips atomically with the // summary — a downstream LLM sees `${SUMMARY_PREFIX}\n${summaryText}` in // one message and reads it as "another LLM's handoff", not as the human // speaking. Encoding the prefix here rather than at expand-time keeps the // envelope's semantics complete regardless of who decodes it. - const encryptedContent = encodeShimCompactionPayload(`${SUMMARY_PREFIX}\n${summaryText}`); + const summaryItem: OpenAIResponsesInputItem = { + type: 'message', + role: 'user', + content: [{ type: 'input_text', text: `${SUMMARY_PREFIX}\n${summaryText}` }], + }; + const encryptedContent = encodeBase64UrlJson([summaryItem]); // Drop the SDK-only `output_text` alias that some upstreams emit — its // value is the upstream's summary plaintext, which has no place on a @@ -304,12 +225,15 @@ const buildCompactionEnvelope = (cmpId: string, summaryText: string, upstream: O }; }; -const simulateCompaction = async (ctx: OpenAIResponsesInvocation, gatewayCtx: ChatGatewayCtx, run: ChainRun): Promise>> => { - const originalPayload = ctx.payload; - +// The turn a compaction is simulated with: the compactor's own prompt, the +// history that is being compacted, and a nudge to produce the summary now. +// Exported because the pipeline's compaction chain sends the same turn — one +// definition is what keeps the simulated compaction identical whichever entry +// asked for it. +export const summarizationTurnFor = (payload: CanonicalOpenAIResponsesPayload): CanonicalOpenAIResponsesPayload => { // Strip compaction_trigger so the upstream sees a plain generate turn // against SUMMARIZATION_PROMPT. - const historyItems = originalPayload.input.filter(item => item.type !== 'compaction_trigger'); + const historyItems = payload.input.filter(item => item.type !== 'compaction_trigger'); // Anthropic Messages rejects assistant prefill — when the translated // conversation ends on an assistant message, the upstream returns 400 @@ -364,151 +288,51 @@ const simulateCompaction = async (ctx: OpenAIResponsesInvocation, gatewayCtx: Ch role: 'system', content: [{ type: 'input_text', text: SUMMARIZATION_PROMPT }], }; - const inputForSummarization = [compactorSystemMessage, ...historyItems, terminalUserMessage]; - - ctx.payload = { - ...originalPayload, - input: inputForSummarization, + return { + ...payload, + input: [compactorSystemMessage, ...historyItems, terminalUserMessage], // Do not persist the ephemeral summarization turn in the upstream's // conversation history. store: false, }; - // Pivot the action so the inner dispatch routes to the upstream's - // generate wire instead of its compact wire. The mutation is one-way: - // the project's interceptor convention is that every `ctx.*` write - // propagates downstream and is never restored on the way out. Post-chain - // consumers keep their own captured copies of inputs they care about. - ctx.action = 'generate'; - - const upstreamResult = await run(); - - if (upstreamResult.type !== 'events') { - // api-error / internal-error from the upstream propagate so the client - // learns the compaction failed rather than receiving a silent empty - // envelope. - return upstreamResult; - } - - const { response: collected, text: summaryText } = await collectSummaryTurn(upstreamResult); - // A compaction blob is the whole of what the next turn inherits, so an empty - // one silently discards the conversation. - if (summaryText.length === 0) { - throw new Error('OpenAI Responses compact shim: the summarization turn closed no assistant text to summarize'); - } - const cmpId = createRandomOpenAIResponsesItemId('compaction'); - const synthesized = buildCompactionEnvelope(cmpId, summaryText, collected); - - // A real generate turn backs this envelope, so its own `incomplete` or - // `failed` status must reach the terminal event. - return { - ...upstreamResult, - events: syntheticEventsFromResult(synthesized), - }; }; -const decryptNativeCompaction = async ( - ctx: OpenAIResponsesInvocation, - run: ChainRun, -): Promise>> => { - const callerAction = ctx.action; - const originalModel = ctx.payload.model; - const nativeResult = await run(); - if (nativeResult.type !== 'events') return nativeResult; - - const nativeResponse = await collectOpenAIResponsesProtocolEventsToResult(nativeResult.events); - let usage = nativeResponse.usage; - let metadata = await resultMetadata(nativeResult); - let sawCompaction = false; - const output: OpenAIResponsesOutputItem[] = []; - - for (const item of nativeResponse.output) { - if (!isOpenAIResponsesCompactionItem(item)) { - output.push(item); - continue; - } - sawCompaction = true; - ctx.action = 'generate'; - ctx.payload = { - model: originalModel, - input: [ - { - type: 'message', - role: 'system', - content: [{ type: 'input_text', text: EXACT_REPEAT_PREFIX }], - }, - item, - { - type: 'message', - role: 'system', - content: [{ type: 'input_text', text: EXACT_REPEAT_SUFFIX }], - }, - ], - store: false, - }; - - const replayResult = await run(); - if (replayResult.type !== 'events') return replayResult; - const { response: replayResponse, text } = await collectSummaryTurn(replayResult); - if (text.length === 0) { - throw new Error('OpenAI Responses compact decryption: the replay turn closed no assistant text'); - } - usage = sumResponseUsage(usage, replayResponse.usage); - const replayMetadata = await resultMetadata(replayResult); - const billableUsage = sumBillableUsage(metadata.billableUsage, replayMetadata.billableUsage); - metadata = { - modelIdentity: nativeResult.modelIdentity, - ...(nativeResult.performance !== undefined ? { performance: nativeResult.performance } : {}), - ...(billableUsage !== undefined ? { billableUsage } : {}), - }; - output.push({ - ...item, - encrypted_content: encodeShimCompactionPayload(text), - }); - } - - if (!sawCompaction) { - throw new Error('OpenAI Responses compact decryption: native compaction returned no compaction output item'); - } - - const decryptedResponse: OpenAIResponsesResult = { - ...nativeResponse, - output, - ...(usage === undefined ? {} : { usage }), - }; - return { - ...nativeResult, - events: callerAction === 'compact' - ? syntheticEventsFromCompaction(decryptedResponse) - : syntheticEventsFromResult(decryptedResponse), - finalMetadata: Promise.resolve(metadata), - }; -}; +// A summarization that closed no assistant text produced no summary, and a +// compaction blob is the whole of what the next turn inherits — so an empty one +// silently discards the conversation. +export const EMPTY_SUMMARY_MESSAGE = 'OpenAI Responses compact shim: the summarization turn closed no assistant text to summarize'; export const containsCompactionTrigger = (input: readonly OpenAIResponsesInputItem[]): boolean => input.some(item => item.type === 'compaction_trigger'); -export const withOpenAIResponsesCompactShim: OpenAIResponsesInterceptor = async (ctx, gatewayCtx, run) => { - // Gateway-owned compaction payloads are portable plaintext containers. They - // must be expanded before flag gating so a request may move to any eligible - // candidate without sending our private envelope to an upstream. - ctx.payload = expandShimCompactionItems(ctx.payload); - - // The shim is engaged when the operator turned it on for this upstream, - // OR when the upstream's targetApi is not OpenAI Responses (Anthropic Messages / - // OpenAI Chat Completions have no compaction wire and would crash on the - // unknown `compaction_trigger` input variant). - const flagOn = providerModelOf(ctx.candidate).enabledFlags.has('openai-responses-compact-shim'); - const decryptFlagOn = providerModelOf(ctx.candidate).enabledFlags.has('openai-responses-compact-decrypt'); - const structurallyRequired = ctx.targetApi !== 'openaiResponses'; - if (!flagOn && !structurallyRequired) { - const isCompactShaped = ctx.action === 'compact' || containsCompactionTrigger(ctx.payload.input); - return decryptFlagOn && isCompactShaped ? await decryptNativeCompaction(ctx, run) : await run(); - } - - // Compact-shaped requests are either the native `/responses/compact` - // action or a `generate` call whose input ends in a `compaction_trigger`. - const isCompactShaped = ctx.action === 'compact' || containsCompactionTrigger(ctx.payload.input); - if (!isCompactShaped) return await run(); +type ResponseUsage = NonNullable; - return await simulateCompaction(ctx, gatewayCtx, run); +export const sumResponseUsage = (left: ResponseUsage | null | undefined, right: ResponseUsage | null | undefined): ResponseUsage | undefined => { + if (left == null) return right ?? undefined; + if (right == null) return left; + return { + input_tokens: left.input_tokens + right.input_tokens, + output_tokens: left.output_tokens + right.output_tokens, + total_tokens: left.total_tokens + right.total_tokens, + ...(left.input_tokens_details !== undefined || right.input_tokens_details !== undefined + ? { + input_tokens_details: { + cached_tokens: (left.input_tokens_details?.cached_tokens ?? 0) + (right.input_tokens_details?.cached_tokens ?? 0), + ...( + left.input_tokens_details?.cache_write_tokens !== undefined + || right.input_tokens_details?.cache_write_tokens !== undefined + ? { cache_write_tokens: (left.input_tokens_details?.cache_write_tokens ?? 0) + (right.input_tokens_details?.cache_write_tokens ?? 0) } + : {} + ), + }, + } + : {}), + ...(left.output_tokens_details !== undefined || right.output_tokens_details !== undefined + ? { + output_tokens_details: { + reasoning_tokens: (left.output_tokens_details?.reasoning_tokens ?? 0) + (right.output_tokens_details?.reasoning_tokens ?? 0), + }, + } + : {}), + }; }; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact.ts new file mode 100644 index 0000000000..8fabee2927 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact.ts @@ -0,0 +1,49 @@ +import { failover } from '../../pipeline/failover.ts'; +import { resolveChatCandidates } from '../resolve-candidates.ts'; +import { OPENAI_RESPONSES_STREAMED_USAGE } from './facts.ts'; +import { materializeAttempt } from '../materialize-attempt.ts'; +import { beginStoredAttempt } from './begin-stored-attempt.ts'; +import { projectOpenAIResponsesCollaboration } from './collaboration-shim.ts'; +import { composeChat as compose } from '../compose.ts'; +import { dialOpenAIResponsesCompaction } from './compact/dial.ts'; +import { emitOpenAIResponsesCompaction } from './compact/emit.ts'; +import type { OpenAIResponsesCompactEntry, OpenAIResponsesCompactExit } from './compact/facts.ts'; +import { compactionWire, simulationWire } from './compact/wire.ts'; +import { expandShimCompactions } from './expand-compactions.ts'; +import { imageGenerationHostedTool } from './hosted-tools/image-generation.ts'; +import { webSearchHostedTool } from './hosted-tools/web-search.ts'; +import { hostedTools } from './hosted-tools.ts'; +import { hydrateStoredItems } from './hydrate-stored-items.ts'; +import { openaiResponsesNarrowing, openaiResponsesTarget } from './target.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import { serializeClientJson } from '../../pipeline/serialize-client-json.ts'; +import { writeSettlement } from '../../pipeline/settlement.ts'; +import type { Pipeline } from '@floway-dev/pipeline'; + +export const openaiResponsesCompactPipeline = (): Pipeline => { + return compose('openaiResponsesCompact', [ + writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, OPENAI_RESPONSES_STREAMED_USAGE), + serializeClientJson('response.chat.openaiResponses.rendered'), + emitOpenAIResponsesCompaction, + hydrateStoredItems, + resolveChatCandidates(openaiResponsesNarrowing), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.chat.openaiResponses'?: unknown })['response.chat.openaiResponses']), + owns: ['response.http.body'], + pendingUsage: OPENAI_RESPONSES_STREAMED_USAGE, + }), + materializeAttempt('request.chat.openaiResponses'), + beginStoredAttempt, + expandShimCompactions, + projectOpenAIResponsesCollaboration, + // A compaction declares no hosted tool of its own, so what this does here is the other half + // of the same rule: an `image_generation_call` or `web_search_call` echoed from an earlier + // turn is rewritten into something the upstream can read. The loop never runs — there is no + // hosted tool to dispatch — which is what keeps a summarization from calling out. + hostedTools([webSearchHostedTool, imageGenerationHostedTool], { + streamedUsage: OPENAI_RESPONSES_STREAMED_USAGE, + targetOf: candidate => openaiResponsesTarget.pick(candidate.model.endpoints), + }), + dialOpenAIResponsesCompaction({ native: compactionWire, simulated: simulationWire }), + ]); +}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact/call-upstream.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact/call-upstream.ts new file mode 100644 index 0000000000..bfa554ee45 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact/call-upstream.ts @@ -0,0 +1,45 @@ +import { bodyForAttempt } from '../../../pipeline/attempt-body.ts'; +import { providerEntry } from '../../../pipeline/provider-entry.ts'; +import { upstreamPerformanceContext } from '../../../shared/telemetry/attribution.ts'; +import type { ChatServices } from '../../services.ts'; +import { applyRulesToUpstreamOpenAIResponses } from '../../shared/alias-rules.ts'; +import { providerChatAnswer } from '../../shared/provider-answer.ts'; +import type { Fields } from '../facts.ts'; +import { syntheticEventsFromCompaction } from '../items/output.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; +import type { ProviderOperationRequest, ProviderChatResponse } from '@floway-dev/provider'; + +export const callOpenAIResponsesCompactUpstream = defineStage< + Fields<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers'>, + ProviderOperationRequest<'openaiResponsesCompact'>, + ProviderChatResponse<'openaiResponsesCompact'>, + Fields<'response.chat.openaiResponses' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status' | 'response.http.body' | 'response.chat.openaiResponses.streamedUsage'>, + ChatServices +>({ + name: 'callOpenAIResponsesCompactUpstream', + into: { + request: { needs: ['request.chat.openaiResponses', 'route.attempt', 'ingress.http.headers'], consumes: [], provides: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'] }, + response: { needs: ['response.http.exchange', 'response.provider.output', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.http.body'], consumes: ['response.http.exchange', 'response.provider.output', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'], provides: ['response.chat.openaiResponses', 'response.usage.billable', 'response.http.headers', 'response.http.status', 'response.http.body', 'response.chat.openaiResponses.streamedUsage'] }, + }, + execute: async (facts, next, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); + const { stream: _stream, store: _store, ...body } = bodyForAttempt(facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload, candidate, applyRulesToUpstreamOpenAIResponses); + const pipeline = candidate.provider.pipelines.openaiResponsesCompact; + if (pipeline === undefined) throw new Error(`Provider ${candidate.provider.kind} has no openaiResponsesCompact pipeline`); + const back = await next(providerEntry(facts, candidate, body), pipeline); + const { 'response.http.exchange': _exchange, 'response.provider.output': _output, 'response.provider.modelKey': _modelKey, 'response.provider.called': _called, 'response.provider.previousCalls': _previousCalls, ...rest } = back; + const reply = await providerChatAnswer(candidate, back); + const answer = 'kind' in reply.answer && reply.answer.kind === 'value' + ? { kind: 'stream' as const, frames: use.recordProtocolFrames(syntheticEventsFromCompaction(reply.answer.body as Parameters[0])) } + : reply.answer; + if ('kind' in answer) use.selectAffinity(candidate); + return move({ + ...rest, 'response.chat.openaiResponses': answer, + 'response.usage.billable': reply.billable, 'response.http.headers': reply.headers, + 'response.http.status': reply.status, 'response.http.body': reply.body, + 'response.chat.openaiResponses.streamedUsage': null, + }); + }, +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact/dial.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact/dial.ts new file mode 100644 index 0000000000..08ebc5085f --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact/dial.ts @@ -0,0 +1,54 @@ +import type { ChatServices } from '../../services.ts'; +import type { ChatWire } from '../../wire.ts'; +import { simulatesCompaction } from '../compaction-policy.ts'; +import type { Fields } from '../facts.ts'; +import { OPENAI_RESPONSES_STREAMED_USAGE } from '../facts.ts'; +import { defineStage } from '@floway-dev/pipeline'; + +/** + * Picks how this candidate's compaction is produced. + * + * An upstream whose own endpoint compacts is asked to, unless the operator asked for the + * simulation instead — which is what the `openai-responses-compact-shim` flag is: an opt-in for an + * OpenAI Responses upstream that would answer a compaction itself. Everything else is simulated, + * structurally rather than by choice: no translation carries a compaction, so an Anthropic + * Messages or OpenAI Chat Completions candidate has no compaction to dial. + * + * It is last, which is what earns it the right to name a target at all, and it holds no state + * across candidates: failover re-running the suffix re-runs this stage. + */ +export const dialOpenAIResponsesCompaction = (wires: { native: ChatWire; simulated: ChatWire }) => defineStage< + Fields<'route.attempt'> & Record, + Record, + Record, + Record, + ChatServices +>({ + name: 'dialOpenAIResponsesCompaction', + into: { + request: { + needs: ['route.attempt', 'request.chat.openaiResponses', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], + consumes: [], + provides: [], + }, + // Nothing is read on the way back — a wire hands up this family's own keys and they ride + // through — but this is where they enter the chain, so this is the stage that provides + // them and the runner checks that the wire delivered. + response: { + needs: [], + consumes: [], + provides: [ + 'response.chat.openaiResponses', + OPENAI_RESPONSES_STREAMED_USAGE, + 'response.usage.billable', + 'response.http.headers', 'response.http.status', 'response.http.body', + ], + }, + }, + execute: async (facts, next, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + const compacts = !simulatesCompaction(candidate, facts['route.attempt']); + await use.log.debug('compacting', { upstream: facts['route.attempt'].upstreamId, wire: compacts ? 'compact' : 'simulated' }); + return await next(facts, compacts ? wires.native : wires.simulated); + }, +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact/emit.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact/emit.ts new file mode 100644 index 0000000000..d9a0055d75 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact/emit.ts @@ -0,0 +1,121 @@ +import type { Fields } from '../facts.ts'; +import type { Compacted } from './facts.ts'; +import { isFailure, renderFailure, mintedErrorEnvelope } from '../../../pipeline/facts.ts'; +import type { StreamOutcome } from '../../../pipeline/serve.ts'; +import { isForwardableUpstreamHeader } from '../../../shared/upstream-response.ts'; +import type { ChatServices } from '../../services.ts'; +import { collectClientFrames } from '../../shared/client-stream.ts'; +import { wrapOpenAIResponsesStatefulOutput, openaiResponsesCreatedAt } from '../client-output.ts'; +import { completeOpenAIResponsesCompaction } from '../compaction-resource.ts'; +import { internalErrorEnvelope } from '../errors.ts'; +import { recordStream } from '@floway-dev/dump'; +import { defineStage, move, defer, type Deferred } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import { collectOpenAIResponsesProtocolEventsToResult, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; + +/** + * The outermost edge. + * + * A compaction answers with one resource, so the frames never reach a client: they are read + * here, which is also what stores each item the turn emitted under this gateway's own + * response id. The resource that comes out is `CompactResource` and not the response + * resource, so it is completed here rather than by the generate chain's egress — the fields + * the two require are different, and a compaction decorated with the twenty-odd keys a + * response resource declares would answer with a shape the spec does not give it. + */ +export const emitOpenAIResponsesCompaction = defineStage< + Record, + Record, + Compacted<'response.chat.openaiResponses'> & Fields<'response.http.headers' | 'response.http.status' | 'response.chat.openaiResponses.streamedUsage'>, + Fields<'response.chat.openaiResponses.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.openaiResponses.streamedUsage' | 'response.chat.clientFrames'>, + ChatServices +>({ + name: 'emitOpenAIResponsesCompaction', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { + needs: ['response.chat.openaiResponses', 'response.http.headers', 'response.http.status', 'response.chat.openaiResponses.streamedUsage'], + consumes: ['response.chat.openaiResponses', 'response.http.headers'], + provides: [ + 'response.chat.clientFrames', + 'response.chat.openaiResponses.rendered', + 'response.http.status', + 'response.http.headers', + 'response.chat.openaiResponses.streamedUsage', + ], + }, + }, + execute: async (facts, next, use) => { + const back = await next(facts); + const { 'response.chat.openaiResponses': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); + + if (isFailure(answer)) { + const failure = renderFailure(answer, mintedErrorEnvelope); + return { + ...rest, + 'response.chat.clientFrames': null, + 'response.http.headers': forClient, + 'response.chat.openaiResponses.rendered': move(failure.body), + 'response.http.status': failure.status, + }; + } + + try { + // The frames never reach a client, so the record is the only place they survive at all: + // the tee sits above the stateful half, where they are this protocol's own and carry + // the ids the resource below is assembled under. Reading is what records, and the fold + // on the next line is the read. + const frames = recordStream( + wrapOpenAIResponsesStatefulOutput(answer.frames as AsyncIterable>, use.gateway), + use.gateway.dump, + ); + const persisted = await collectClientFrames(frames, collectOpenAIResponsesProtocolEventsToResult); + return { + ...rest, + 'response.chat.clientFrames': move(frames), + 'response.http.headers': forClient, + 'response.chat.openaiResponses.rendered': move( + completeOpenAIResponsesCompaction(persisted, openaiResponsesCreatedAt(use.gateway)) as unknown as Record, + ), + 'response.http.status': back['response.http.status'], + 'response.chat.openaiResponses.streamedUsage': move( + withVerdict(rest['response.chat.openaiResponses.streamedUsage'], compactionFailed(persisted)), + ), + }; + } catch (error) { + // Nothing has gone out yet, so the fault is still a status. A compaction the gateway + // could not finish — an upstream that stated no counts, a turn that could not be + // stored — is not one it can answer, and the client is told what broke. + return { + ...rest, + 'response.chat.clientFrames': null, + 'response.http.headers': forClient, + 'response.chat.openaiResponses.rendered': move(internalErrorEnvelope(error)), + 'response.http.status': 502, + 'response.chat.openaiResponses.streamedUsage': move(withVerdict(rest['response.chat.openaiResponses.streamedUsage'], true)), + }; + } + }, +}); + +/** Whether the compaction resource says the turn behind it failed. `status` is not a key the + * compaction resource declares — it survives the spread from the turn the upstream actually + * ran, on either ending — and it is authoritative: a compaction that surfaced as failed + * belongs in the error column rather than masquerading as a success. */ +const compactionFailed = (persisted: { readonly status?: string }): boolean => persisted.status === 'failed'; + +/** The accounting the epilogue settles from, with the verdict only the completed resource + * could give it. The edge is where a compaction resource comes into existence, so it is the + * one reader that can say whether the turn behind it succeeded. + * + * What it composes is still this run's own unfinished work, so the result is declared too: + * the promise below it settles first, and the run must wait for the one it hands up. */ +const withVerdict = (outcome: Deferred | null, failed: boolean): Deferred | null => { + if (outcome === null || !failed) return outcome; + return defer(outcome.then(read => ({ ...read, failed: true }))); +}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact/facts.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact/facts.ts new file mode 100644 index 0000000000..5352495e0c --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact/facts.ts @@ -0,0 +1,18 @@ +import type { Failure } from '../../../pipeline/facts.ts'; +import type { Fields } from '../facts.ts'; + +/** What a compaction wire hands up. A compaction is a stream by the time it leaves either + * ending — the envelope is expanded into the events the stateful half reads — so the value + * arm this protocol's response key also admits is not one this chain ever produces. */ +export type Compacted = { + [P in K]: { readonly kind: 'stream'; readonly frames: AsyncIterable } | Failure; +}; + +export type OpenAIResponsesCompactEntry = Fields< + 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'request.chat.openaiResponses' | 'serve.model' +>; + +export type OpenAIResponsesCompactExit = Fields< + 'response.chat.openaiResponses.rendered' | 'response.chat.openaiResponses.streamedUsage' + | 'response.http.status' | 'response.http.jsonBody' | 'response.http.headers' | 'response.usage.billable' +>; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compact/wire.ts b/packages/gateway/src/data-plane/chat/openai-responses/compact/wire.ts new file mode 100644 index 0000000000..1f8eb91954 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/compact/wire.ts @@ -0,0 +1,29 @@ +import { composeChat as compose } from '../../compose.ts'; +import { dialChatWire } from '../../dial-wire.ts'; +import type { ChatWire } from '../../wire.ts'; +import { decryptNativeCompaction } from '../compact-decrypt.ts'; +import { OPENAI_RESPONSES_STREAMED_USAGE } from '../facts.ts'; +import { meterUsage } from '../meter.ts'; +import { summarizeForCompaction } from '../summarize-for-compaction.ts'; +import { openaiResponsesTarget } from '../target.ts'; +import { openaiResponsesWireRules, openaiResponsesWire } from '../wire.ts'; +import { openaiResponsesWireFor } from '../wires.ts'; +import { callOpenAIResponsesCompactUpstream } from './call-upstream.ts'; + +/** The generate fork, as the simulation reaches it. A summarization is an ordinary turn, so + * it is dialled on the wires an ordinary turn is dialled on. */ +const dialSummarizationWire = dialChatWire({ + source: 'request.chat.openaiResponses', + needs: ['request.chat.openaiResponses', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], + provides: ['response.chat.openaiResponses', OPENAI_RESPONSES_STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], + pick: endpoints => openaiResponsesTarget.pick(endpoints), + wire: openaiResponsesWireFor, +}); + +const nativeCompactionWire: ChatWire = compose('openaiResponsesCompactNative', [meterUsage(OPENAI_RESPONSES_STREAMED_USAGE), ...openaiResponsesWireRules, callOpenAIResponsesCompactUpstream]); + +export const compactionWire: ChatWire = compose('openaiResponsesCompactDecrypted', [decryptNativeCompaction({ native: nativeCompactionWire, replay: compose('openaiResponsesCompactReplay', openaiResponsesWire(OPENAI_RESPONSES_STREAMED_USAGE)), streamedUsage: OPENAI_RESPONSES_STREAMED_USAGE, compactEndpoint: true, asked: () => true })]); + +// The ending below picks this wire only for a candidate whose compactions are the shim's to +// simulate, so the ask is already answered: every turn that reaches it is one to summarize. +export const simulationWire: ChatWire = compose('openaiResponsesCompactSimulated', [summarizeForCompaction(() => true), dialSummarizationWire]); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/compaction-policy.ts b/packages/gateway/src/data-plane/chat/openai-responses/compaction-policy.ts new file mode 100644 index 0000000000..04e0030482 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/compaction-policy.ts @@ -0,0 +1,46 @@ +import { containsCompactionTrigger } from './compact-shim.ts'; +import type { CompactionAsk } from './summarize-for-compaction.ts'; +import { openaiResponsesTarget } from './target.ts'; +import type { AttemptSelector } from '../../pipeline/facts.ts'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; +import type { ModelCandidate } from '@floway-dev/provider'; + +// ── The compact shim ────────────────────────────────────────────────────────────────────── +// +// Two rules, composed by both of this protocol's chains, because a compaction the shim wrote +// is issued through one entry and read back through the other: Codex asks for one by ending +// a generate turn's input with a `compaction_trigger`, `/v1/responses/compact` asks for one +// by being called at all, and either way the envelope comes back to the client, who echoes it +// into the ordinary turns that follow. What the shim is, what it vendors and what it cannot +// reproduce is at `compact-shim.ts`. + +/** + * Whether this candidate's compactions are the shim's to simulate. + * + * Two ways to be, and an upstream only has to be one of them. The operator opted an OpenAI Responses + * upstream in with `openai-responses-compact-shim`, which is the say-so for one that would answer a + * compaction itself. Or the candidate has no OpenAI Responses endpoint at all: no translation + * carries a compaction, so an Anthropic Messages or OpenAI Chat Completions candidate has neither a compaction + * to dial nor a translator that models the item asking for one — the shim is structurally + * required there rather than chosen. + * + * One reading, taken wherever the question is asked, so a turn that asks for a compaction on + * its way through generation is answered with the compaction this protocol's own endpoint + * would have produced for it. + */ +export const simulatesCompaction = (candidate: ModelCandidate, attempt: AttemptSelector): boolean => + openaiResponsesTarget.pick(candidate.model.endpoints) !== 'openaiResponses' + || attempt.flags.includes('openai-responses-compact-shim'); + +/** + * What asking for a compaction looks like on this chain. + * + * Codex's RemoteCompactionV2 path asks for one inside an ordinary turn, by ending its input + * with the control item that requests it — so this chain reads the request rather than the + * operation, and reads it per turn. Where the shim is not this candidate's to run, the item + * travels on to an upstream whose own `/responses` endpoint answers it, which is what a + * native OpenAI Responses upstream does with it. + */ +export const asksForCompaction: CompactionAsk = (facts, use) => + containsCompactionTrigger((facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload).input) + && simulatesCompaction(use.resolveAttempt(facts['route.attempt']), facts['route.attempt']); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/disable-reasoning-on-forced-tool-choice.ts b/packages/gateway/src/data-plane/chat/openai-responses/disable-reasoning-on-forced-tool-choice.ts new file mode 100644 index 0000000000..0e3b676b78 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/disable-reasoning-on-forced-tool-choice.ts @@ -0,0 +1,46 @@ +import type { Chat } from '../facts.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; +import type { OpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; + +/** The reasoning sentinel a forced tool choice needs. `required` or a named tool is forced; + * `auto` and `none` leave the model free to reason. The `reasoning` object is replaced + * rather than merged — a summary has no meaning once there is nothing to summarize — and + * the sentinel is the gateway's canonical form, which is why this runs above the vendor + * normalizers that put it on the wire in a vendor's shape. */ +export const disableReasoningOnForcedToolChoiceForOpenAIResponses = defineStage< + Chat<'request.chat.openaiResponses' | 'route.attempt'>, + Chat<'request.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> +>({ + name: 'disableReasoningOnForcedToolChoice', + through: { + request: { + needs: ['request.chat.openaiResponses', 'route.attempt'], + consumes: [], + provides: ['request.chat.openaiResponses'], + }, + response: { needs: ['response.chat.openaiResponses'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.openaiResponses' | 'route.attempt'>, + Chat<'request.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> + >(() => ({ + request: facts => { + if (!facts['route.attempt'].flags.includes('disable-reasoning-on-forced-tool-choice')) { + return facts; + } + const payload = facts['request.chat.openaiResponses']; + if (!isForcedOpenAIResponsesToolChoice(payload.tool_choice)) return facts; + return { ...facts, 'request.chat.openaiResponses': move({ ...payload, reasoning: { effort: 'none' } }) }; + }, + })), +}); + +const isForcedOpenAIResponsesToolChoice = (choice: OpenAIResponsesPayload['tool_choice']): boolean => { + if (choice === undefined || choice === null) return false; + if (typeof choice === 'string') return choice === 'required'; + return true; +}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/emit.ts b/packages/gateway/src/data-plane/chat/openai-responses/emit.ts new file mode 100644 index 0000000000..be606c8246 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/emit.ts @@ -0,0 +1,223 @@ +import { wrapOpenAIResponsesClientEgress } from './client-output.ts'; +import { internalErrorEnvelope } from './errors.ts'; +import { OPENAI_RESPONSES_STREAMED_USAGE, type OpenAIResponsesStreamFraming, type Fields } from './facts.ts'; +import { isFailure, renderFailure, mintedErrorEnvelope } from '../../pipeline/facts.ts'; +import { isForwardableUpstreamHeader } from '../../shared/upstream-response.ts'; +import type { ChatServices } from '../services.ts'; +import { bindClientRelease, collectClientFrames, framedClientStream, withClientVerdict } from '../shared/client-stream.ts'; +import { recordStream } from '@floway-dev/dump'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; +import { collectOpenAIResponsesProtocolEventsToResult, openaiResponsesProtocolFrameToSSEFrame, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesStreamEvent, type ClientOpenAIResponsesStreamEvent, type ClientResponseResource } from '@floway-dev/protocols/openai-responses'; +import { toInternalDebugError } from '@floway-dev/provider'; + +/** + * The outermost edge. What reaches it is usually a stream — the upstream speaks SSE whatever + * the client asked for — so what this decides is whether the client sees the frames or the + * one response object they add up to. + * + * Collecting is therefore the edge's own work and not a second reading of the upstream: the + * same frames that would have gone out are folded here instead, by the protocol's own + * reassembly, which is what makes a stream that stopped short of its terminal event say so + * rather than answer with half a response. + * + * The stored-items membrane's other half runs here too, and it takes the client's own + * payload rather than the record's, because by the fork the record holds the prepared one — + * `previous_response_id` expanded away — and the resource echoes what the client asked with. + * + * A streamed answer is handed on in the framing the transport that opened the run writes. + * Only the last step differs: `renderSSE` is a wire format written over an HTTP body, and a + * transport that frames each event itself takes the events it would have been written from. + */ +export const emitOpenAIResponses = (framing: OpenAIResponsesStreamFraming) => defineStage< + Fields<'ingress.chat.openaiResponses.wantsStream' | 'request.chat.openaiResponses'>, + Fields<'ingress.chat.openaiResponses.wantsStream' | 'request.chat.openaiResponses'>, + Fields<'ingress.chat.openaiResponses.wantsStream' | 'response.chat.openaiResponses' | 'response.http.headers' | 'response.http.status' | 'response.chat.openaiResponses.streamedUsage'>, + Fields<'response.chat.openaiResponses.rendered' | 'response.http.status' | 'response.http.headers' | 'response.chat.clientFrames' | 'response.chat.openaiResponses.streamedUsage'>, + ChatServices +>({ + name: 'emitOpenAIResponses', + through: { + request: { needs: ['ingress.chat.openaiResponses.wantsStream', 'request.chat.openaiResponses'], consumes: [], provides: [] }, + response: { + needs: ['response.chat.openaiResponses', 'response.http.headers', 'response.http.status', OPENAI_RESPONSES_STREAMED_USAGE], + consumes: ['response.chat.openaiResponses', 'response.http.headers'], + provides: ['response.chat.clientFrames', 'response.chat.openaiResponses.rendered', 'response.http.status', 'response.http.headers', OPENAI_RESPONSES_STREAMED_USAGE], + }, + }, + execute: async (facts, next, use) => { + const client = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; + const back = await next(facts); + const { 'response.chat.openaiResponses': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); + + if (isFailure(answer)) { + const failure = renderFailure(answer, mintedErrorEnvelope); + return { + ...rest, + 'response.chat.clientFrames': null, + 'response.http.headers': forClient, + 'response.chat.openaiResponses.rendered': move(failure.body), + 'response.http.status': failure.status, + }; + } + // A turn the upstream answered with one body rather than a stream — the compaction + // envelope — is already the object the client is owed, so there is nothing to fold. + if (answer.kind === 'value') { + return { + ...rest, + 'response.chat.clientFrames': null, + 'response.http.headers': forClient, + 'response.chat.openaiResponses.rendered': move(answer.body as Record), + 'response.http.status': back['response.http.status'], + }; + } + + // Everything this protocol writes back into its own frames, in one place because every + // layer of it rewrites them and below the fold there would be nothing left to rewrite: + // the turn's own state sealed into the carrier a follow-up comes back on, each complete + // item stored under its exact id beneath one response id this gateway minted, and the resource + // completed to what the schema requires of it. It runs before the fold rather than beside it, + // so a client that did not ask to stream is answered with the object the persisted frames + // add up to. + const egress = wrapOpenAIResponsesClientEgress( + answer.frames as AsyncIterable>, + use.gateway, + client, + ); + // HTTP records the client-facing protocol before writing SSE or folding it into a + // resource. WebSocket records later, after its event-id/sequence/error projection. + if (!back['ingress.chat.openaiResponses.wantsStream']) { + const frames = recordStream(egress, use.gateway.dump); + try { + return { + ...rest, + 'response.chat.clientFrames': move(frames), + 'response.http.headers': forClient, + 'response.chat.openaiResponses.rendered': move( + await collectClientFrames(frames, collectOpenAIResponsesProtocolEventsToResult) as unknown as Record, + ), + 'response.http.status': back['response.http.status'], + }; + } catch (error) { + // Nothing has gone out yet, so the fault is still a status. A turn the gateway could + // not finish — the snapshot the next turn would read, most often — is not one it can + // answer, and the client is told what broke rather than handed the half that arrived. + use.gateway.dump?.failed(error); + return { + ...rest, + 'response.chat.clientFrames': move(frames), + 'response.http.headers': forClient, + 'response.chat.openaiResponses.rendered': move(internalErrorEnvelope(error)), + 'response.http.status': 502, + [OPENAI_RESPONSES_STREAMED_USAGE]: move(withClientVerdict(back[OPENAI_RESPONSES_STREAMED_USAGE], Promise.resolve(true))), + }; + } + } + if (framing === 'events') { + const clientStream = completeEventStream(egress, error => { use.gateway.dump?.failed(error); }); + const frames = clientStream.frames; + bindClientRelease(back, clientStream.release); + return { + ...rest, + 'response.chat.clientFrames': move(frames), + 'response.http.headers': forClient, + 'response.chat.openaiResponses.rendered': move(frames), + 'response.http.status': back['response.http.status'], + [OPENAI_RESPONSES_STREAMED_USAGE]: move(withClientVerdict(back[OPENAI_RESPONSES_STREAMED_USAGE], clientStream.failed)), + }; + } + let announced: ClientResponseResource | undefined; + const completed = framedClientStream( + completedSseFrames(egress), + frame => { + const wire = openaiResponsesProtocolFrameToSSEFrame(frame); + if (frame.type === 'event' && 'response' in frame.event) announced = frame.event.response; + return wire; + }, + error => [eventFrame(streamErrorEvent(error)), ...(announced === undefined ? [] : [eventFrame(streamFailedEvent(announced, error))])], + use.gateway.dump, + ); + bindClientRelease(back, completed.release); + return { + ...rest, + 'response.chat.clientFrames': move(completed.frames), + 'response.http.headers': forClient, + 'response.chat.openaiResponses.rendered': move(completed.rendered), + 'response.http.status': back['response.http.status'], + [OPENAI_RESPONSES_STREAMED_USAGE]: move(withClientVerdict(back[OPENAI_RESPONSES_STREAMED_USAGE], completed.failed)), + }; + }, +}); + +/** The spec nests the `error` event's payload under `error`, and both official SDKs key + * their mid-stream throw on exactly that key; the same fields at the top level are yielded + * to them as an ordinary event instead. + * https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L170-L177 + * https://github.com/openai/openai-node/blob/d77cf24d9f3885739c6cba76bc009abf0ab97428/src/core/streaming.ts#L69-L71 + * https://github.com/openai/openai-python/blob/3844843c277f42b0b18beaa58152cfda61df524a/src/openai/_streaming.py#L87-L98 */ +const streamErrorEvent = (error: unknown): ClientOpenAIResponsesStreamEvent => { + const debug = toInternalDebugError(error); + return { + type: 'error', + error: { + message: debug.message, + code: debug.type, + name: debug.name, + stack: debug.stack, + cause: debug.cause, + target_api: debug.target_api, + }, + } as unknown as ClientOpenAIResponsesStreamEvent; +}; + +/** "Any error incurred while streaming will be followed by a `response.failed` event." + * https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L430 */ +const streamFailedEvent = (announced: ClientResponseResource, error: unknown): ClientOpenAIResponsesStreamEvent => { + const debug = toInternalDebugError(error); + return { + type: 'response.failed', + response: { ...announced, status: 'failed', error: { code: debug.type, message: debug.message } }, + } as ClientOpenAIResponsesStreamEvent; +}; + +const completeEventStream = ( + frames: AsyncIterable>, + reportFailure: (error: unknown) => void, +): { frames: AsyncIterable>; failed: Promise; release: () => Promise } => { + let settle!: (failed: boolean) => void; + const failed = new Promise(resolve => { settle = resolve; }); + let started = false; + const stream = (async function* () { + started = true; + let failed = false; + let completed = false; + try { + for await (const frame of frames) if (frame.type !== 'done') yield frame; + completed = true; + } catch (error) { + failed = true; + reportFailure(error); + throw error; + } finally { + settle(failed || !completed); + } + })(); + return { + frames: { [Symbol.asyncIterator]: () => stream }, failed, release: async () => { + if (!started) settle(true); + await stream.return(); + }, + }; +}; + +const completedSseFrames = (frames: AsyncIterable>): AsyncIterable> => ({ + [Symbol.asyncIterator]: () => (async function* () { + for await (const frame of frames) if (frame.type !== 'done') yield frame; + yield doneFrame(); + })(), +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/errors.ts b/packages/gateway/src/data-plane/chat/openai-responses/errors.ts index e7e8886942..dbe3ee4b42 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/errors.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/errors.ts @@ -1,31 +1,22 @@ -import { appendFailedUpstreams } from '../../shared/failed-upstreams.ts'; -import { openAiErrorResult, type ChatServeFailure } from '../shared/errors.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import type { ApiErrorResult, ExecuteResult, PerformanceTelemetryContext } from '@floway-dev/provider'; +import type { ChatServeFailure } from '../shared/errors.ts'; +import { toInternalDebugError } from '@floway-dev/provider'; +/** The chat-wide refusals plus the one only this protocol can make: a turn that named an item + * by id which the store cannot resolve. It is thrown from inside the membrane's hydration + * rather than returned, because hydration walks the input and the item that is missing is + * found several frames below where the answer is written. */ export type OpenAIResponsesServeFailure = ChatServeFailure | { readonly kind: 'item-not-found'; readonly itemId: string }; -// Caller-input violations discovered by translation or the source affinity -// membrane share the OpenAI Responses 400 envelope. `performance` retains candidate -// attribution when validation fires after attempt dispatch. -export const openaiResponsesInputErrorResult = ( - error: { readonly message: string; readonly param?: string; readonly code?: string }, - performance?: PerformanceTelemetryContext, -): ApiErrorResult => - openAiErrorResult(400, error.message, { param: error.param ?? 'input', code: error.code ?? null }, performance); - -export const renderOpenAIResponsesFailure = ( - failure: OpenAIResponsesServeFailure, -): ExecuteResult> => { - switch (failure.kind) { - case 'item-not-found': - return openAiErrorResult(404, `Item with id '${failure.itemId}' not found.`, { param: 'input', code: null }); - case 'routing-unavailable': - return openAiErrorResult(400, failure.message, { param: 'input', code: 'responses_item_routing_unavailable' }); - case 'model-missing': - return openAiErrorResult(404, appendFailedUpstreams(`Model ${failure.model} is not available on any configured upstream.`, failure.failedUpstreams)); - case 'model-unsupported': - return openAiErrorResult(400, appendFailedUpstreams(`Model ${failure.model} does not support the /responses endpoint.`, failure.failedUpstreams)); - } +export const internalErrorEnvelope = (error: unknown) => { + const debug = toInternalDebugError(error); + return { + error: { + type: debug.type, + name: debug.name, + message: debug.message, + stack: debug.stack, + cause: debug.cause, + target_api: debug.target_api, + }, + }; }; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/expand-compactions.ts b/packages/gateway/src/data-plane/chat/openai-responses/expand-compactions.ts new file mode 100644 index 0000000000..7d852eca39 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/expand-compactions.ts @@ -0,0 +1,41 @@ +import type { Fields } from './facts.ts'; +import type { ChatServices } from '../services.ts'; +import { expandShimCompactionItems } from './compact-shim.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; + +/** + * Expands a compaction this gateway wrote back into the history it stood for. + * + * The shim's envelope carries the items it summarized, so a turn that echoes one back is sent + * that history rather than an opaque blob the upstream has no key for. A blob this gateway + * did not write — a native upstream's own encrypted compaction — fails the decode and rides + * through untouched, which is what lets an operator turn the flag off for an upstream that + * compacts natively. + * + */ +export const expandShimCompactions = defineStage< + Fields<'request.chat.openaiResponses'>, + Fields<'request.chat.openaiResponses'>, + Record, + Record, + ChatServices +>({ + name: 'expandShimCompactions', + through: { + request: { + needs: ['request.chat.openaiResponses'], + consumes: [], + provides: ['request.chat.openaiResponses'], + }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; + const expanded = expandShimCompactionItems(payload); + // A turn carrying none of ours hands the same payload on, so the record shows no change + // where none happened. + if (expanded === payload) return await next(facts); + return await next({ ...facts, 'request.chat.openaiResponses': move(expanded) }); + }, +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/facts.ts b/packages/gateway/src/data-plane/chat/openai-responses/facts.ts new file mode 100644 index 0000000000..ae7153247d --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/facts.ts @@ -0,0 +1,45 @@ +import type { StreamOutcome } from '../../pipeline/serve.ts'; +import type { ChatFacts } from '../facts.ts'; +import type { Deferred } from '@floway-dev/pipeline'; +import type { SseFrame, ProtocolFrame } from '@floway-dev/protocols/common'; +import type { ClientOpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; + +/** How the transport that opened the run frames a streamed answer. Both carry this + * protocol's own events and neither adds or drops one: SSE is the format an HTTP body is + * written in, terminator and all, and a WebSocket turn writes each event as a text frame of + * its own — so the transport that owns the socket takes the events and frames them itself. */ +export type OpenAIResponsesStreamFraming = 'sse' | 'events'; + +/** What this family adds to the chat space. */ +export interface OpenAIResponsesFacts extends ChatFacts { + /** Server-only state the gateway once attached to an item it emitted, keyed by that + * item's id, as the rows this turn hydrated carry it. It never reaches an upstream: it is + * what lets an item this turn re-emits be stored with the state it already had. + * + * Pairs rather than a `Map`, for the same reason the header keys are: a `Map` has no own + * properties, so it would be written into the dump as an empty object and the record would + * say the turn hydrated nothing. */ + 'request.chat.openaiResponses.privatePayloads': readonly (readonly [string, unknown])[]; + /** What the client is actually sent — an object when it asked for one, and the stream it + * asked to be streamed, in the framing its transport writes. The edge provides it, so a + * dump shows what the client received. */ + 'response.chat.openaiResponses.rendered': Record | AsyncIterable | AsyncIterable>; + /** What the upstream will have reported once the frames run out, and `null` when nothing + * streamed. Settling from this is the epilogue's job, after the drain. */ + 'response.chat.openaiResponses.streamedUsage': Deferred | null; +} + +export type Fields = { [P in K]: OpenAIResponsesFacts[P] }; + +/** This family's own reading, which every wire under it hands up. */ +export const OPENAI_RESPONSES_STREAMED_USAGE = 'response.chat.openaiResponses.streamedUsage'; + +export type OpenAIResponsesServeEntry = Fields< + 'ingress.http.headers' | 'ingress.chat.sourceProtocol' | 'ingress.chat.openaiResponses.wantsStream' + | 'request.chat.openaiResponses' | 'serve.model' +>; + +export type OpenAIResponsesServeExit = Fields< + 'response.chat.openaiResponses.rendered' | 'response.chat.openaiResponses.streamedUsage' + | 'response.http.status' | 'response.http.jsonBody' | 'response.http.headers' | 'response.usage.billable' +>; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools.ts new file mode 100644 index 0000000000..b2d50bdc46 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools.ts @@ -0,0 +1,448 @@ +// Hosted tools, as one stage. +// +// A hosted tool the upstream does not implement is emulated here: the tool is rewritten into a +// function tool on the way down, the model's call to it is executed by this gateway, its result +// is fed back, and the upstream is asked again — until the model stops calling it. What the +// client sees is one turn, with the hosted tool's own item and lifecycle events in it. +// +// That is a **fork the stage merges**, and it is why this is one stage rather than several. It +// descends more than once, and every descent after the first is asked with the previous turn's +// output folded back into the input; the frames of all of them are spliced into one stream +// under one synthesized response id. Nothing above sees the repetition — which is exactly what +// `writeSettlement` means when it says repetition passes through the stage that observes usage. +// +// Position: directly above the dial, below the fork. So a descent re-dials the *same* candidate, +// and an upstream that refuses mid-loop rides up to `failover`, which re-runs the whole loop +// against the next candidate rather than resuming a half-finished one. +// +// What every descent is billable for travels up separately and is concatenated, not summed: +// each turn is its own upstream call with its own pricing facts — a service tier and an input +// size that a later turn need not share — so the run settles one row per call rather than one +// row whose rate is a guess. + +import { buildErrorFromRefusal, consumeTurnStreaming, createMergeState, materializeAccumulatedOutput, materializeHostedToolItems, resolveHostedToolName, rewriteHostedToolChoice, rewriteHostedDeclarations, hostedToolChoiceToRestore, historicalClientCallableUsesName, isForcedHostedToolChoice, sumUsage, synthesizeTerminalEnvelope, transformHostedToolItems } from './hosted-tools/shared.ts'; +import { type ActiveHostedTool, type MergeState, type HostedToolDispatcher, type HostedToolRewrite, type HostedToolLoopState, type HostedToolRegistration, type HostedToolPrepareResult, type TurnSummary } from './hosted-tools/types.ts'; +import type { BillableEntity, Failure } from '../../pipeline/facts.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import { providerCalls } from '../../pipeline/provider-usage.ts'; +import type { StreamOutcome } from '../../pipeline/serve.ts'; +import type { ChatAnswer, ChatFacts } from '../facts.ts'; +import type { ChatServices } from '../services.ts'; +import { defer, defineStage, getFailureFacts, move, setRelease, type Owned, type Deferred } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import type { + CanonicalOpenAIResponsesPayload, + OpenAIResponsesInputItem, + OpenAIResponsesResult, + OpenAIResponsesStreamEvent, + OpenAIResponsesToolChoice, + OpenAIResponsesHostedTool, +} from '@floway-dev/protocols/openai-responses'; +import type { ChatTargetApi, ModelCandidate, OpenAIResponsesInvocation } from '@floway-dev/provider'; + +type R = { [P in K]: ChatFacts[P] }; + +/** What comes back through here. The streamed-usage key is the *source* family's rather than + * this protocol's — the same reason a wire is told it — so the slice is open at that one key + * and closed everywhere else. */ +type Answered = R<'response.chat.openaiResponses' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status'> + & { 'response.chat.openaiResponses.streamedUsage': Deferred | null } + & Record; + +/** One descent, as the loop reads it: the frames it is to splice, and what that call will turn + * out to have been billable for once they run out. */ +interface Turn { + readonly frames: AsyncIterable>; + readonly usage: Deferred | null; +} + +/** What the gateway answers a request its own tool declaration cannot accept. It is this + * gateway's refusal rather than an upstream's, so it carries the envelope a client reads and + * the 400 that goes with it. */ +const invalidRequest = ( + message: string, + param: string | null, + code: string | null | undefined, + errorType = 'invalid_request_error', +): Failure => { + const envelope = { + error: { message, type: errorType, param, code: code === undefined ? 'invalid_request_error' : code }, + }; + return { status: 400, message, envelope }; +}; + +/** + * Prepares every registered tool over the payload this attempt is about to send. + * + * The registrations are handed an invocation rather than facts, because what they read is the + * body and the candidate and nothing else — and the object they read it through is the stage's + * own, built here and never handed anywhere, so shaping it in place shapes nothing shared. + */ +const prepareHostedTools = async ( + registrations: readonly HostedToolRegistration[], + invocation: OpenAIResponsesInvocation, + gateway: ChatServices['gateway'], +): Promise<{ readonly active: readonly ActiveHostedTool[] } | { readonly refused: Failure }> => { + const active: ActiveHostedTool[] = []; + const preparedTools: Array> = []; + + // Validate the original request before any hosted rewrite changes the + // tool-array indexes used in error paths. + for (const prepareHostedTool of registrations) { + const prepared = await prepareHostedTool(invocation, gateway); + if (prepared.type === 'inactive') continue; + if (prepared.type === 'invalid-request') { + return { refused: invalidRequest(prepared.message, prepared.param, prepared.code, prepared.errorType) }; + } + preparedTools.push(prepared); + } + + for (const prepared of preparedTools) { + const currentTools = Array.isArray(invocation.payload.tools) ? invocation.payload.tools : []; + const toolName = resolveHostedToolName(prepared.baseToolName, currentTools, invocation.payload.input, invocation.payload.tool_choice); + const { hosted } = prepared; + const choice = invocation.payload.tool_choice; + let helperFunctionChoice: Exclude | undefined; + let rewrittenHelperChoice = choice; + if (hosted !== undefined && toolName !== prepared.baseToolName + && typeof choice === 'object' && choice !== null) { + // Forced and allowed-tools selectors share the same ownership rule. A + // bare helper spelling may be rebased only when no client owns it. + const clientOwnsName = currentTools.some(tool => + (tool.type === 'function' || tool.type === 'custom') && tool.name === prepared.baseToolName + && (!('namespace' in tool) || tool.namespace === undefined)) + || historicalClientCallableUsesName(prepared.baseToolName, invocation.payload.input); + if (!clientOwnsName) { + const rebaseSelector = (selector: T): T => { + if (typeof selector !== 'object' || selector === null || !('type' in selector) || selector.type !== 'function' + || !('name' in selector) || selector.name !== prepared.baseToolName + || ('namespace' in selector && selector.namespace !== undefined)) return selector; + return { ...selector, name: toolName }; + }; + if (choice.type === 'allowed_tools' && Array.isArray(choice.tools)) { + const tools = choice.tools.map(rebaseSelector); + if (tools.some((tool, index) => tool !== choice.tools[index])) rewrittenHelperChoice = { ...choice, tools }; + } else rewrittenHelperChoice = rebaseSelector(choice); + if (rewrittenHelperChoice !== choice) helperFunctionChoice = choice; + } + } + let canonicalHostedTool: OpenAIResponsesHostedTool | undefined = undefined; + if (hosted !== undefined) { + const rewrite = rewriteHostedDeclarations(invocation.payload, hosted, toolName); + canonicalHostedTool = rewrite.canonicalTopLevelHostedTool; + invocation.payload = rewrite.payload; + } + const originalToolChoice = helperFunctionChoice ?? hostedToolChoiceToRestore(choice, hosted, toolName); + if (helperFunctionChoice !== undefined) { + invocation.payload = { ...invocation.payload, tool_choice: rewrittenHelperChoice }; + } + active.push({ ...prepared, toolName, canonicalHostedTool, originalToolChoice }); + } + + return { active }; +}; + +/** What the stage is told rather than reaching for, because reaching for it would be this + * module importing the chain that composes it. `streamedUsage` is the family's own reading key; + * `targetOf` says which wire this candidate is reachable on, which is the one thing a + * registration reads about the dial it is preparing for. */ +export interface HostedToolWiring { + readonly streamedUsage: string; + readonly targetOf: (candidate: ModelCandidate) => ChatTargetApi; +} + +export const hostedTools = ( + registrations: readonly HostedToolRegistration[], + wiring: HostedToolWiring, +) => defineStage< + R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers' | 'serve.usage.prior'>, + R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers' | 'serve.usage.prior'>, + Answered, + Answered, + Answered, + ChatServices +>({ + name: 'hostedTools', + through: { + request: { + needs: ['request.chat.openaiResponses', 'route.attempt', 'ingress.http.headers', 'serve.usage.prior'], + consumes: [], + provides: ['request.chat.openaiResponses'], + }, + response: { + needs: ['response.chat.openaiResponses', wiring.streamedUsage, 'response.usage.billable', 'response.http.headers'], + consumes: ['response.http.body'], + provides: ['response.chat.openaiResponses', wiring.streamedUsage, 'response.usage.billable', 'response.http.body'], + }, + }, + // A tool declaration this gateway cannot accept is answered here rather than dialled: the + // upstream would have been asked for a tool it does not implement, on a body the dispatcher wrote. + return: { + provides: ['response.chat.openaiResponses', wiring.streamedUsage, 'response.usage.billable', 'response.http.headers', 'response.http.body', 'response.http.status'], + }, + execute: async (facts, next, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + // The stage's own view of the turn, which is what a registration reads and shapes. It is + // built here and handed nowhere, so the payload it ends up holding is what descends. + const invocation: OpenAIResponsesInvocation = { + payload: facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload, + candidate, + targetApi: wiring.targetOf(candidate), + headers: new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value])), + action: 'generate', + }; + + const prepared = await prepareHostedTools(registrations, invocation, use.gateway); + if ('refused' in prepared) { + await use.log.debug('refusing a hosted-tool declaration this gateway cannot accept'); + return move({ + ...facts, + 'response.chat.openaiResponses': prepared.refused, + [wiring.streamedUsage]: null, + 'response.usage.billable': [], + 'response.http.headers': [], + 'response.http.body': null, 'response.http.status': prepared.refused.status, + }) as never; + } + const { active } = prepared; + if (active.length === 0) return await next(facts); + + const rewrittenToolChoice = rewriteHostedToolChoice(invocation.payload.tool_choice, active); + if (rewrittenToolChoice !== invocation.payload.tool_choice) { + invocation.payload = { ...invocation.payload, tool_choice: rewrittenToolChoice }; + } + const canonicalInput = invocation.payload.input; + const nextInput = transformHostedToolItems(canonicalInput, active); + if (nextInput !== canonicalInput) invocation.payload = { ...invocation.payload, input: nextInput }; + + const hostedActive = active.filter( + (entry): entry is ActiveHostedTool & { hosted: HostedToolRewrite } => entry.hosted !== undefined, + ); + // A registration that only rewrote history has nothing to execute, so the turn is an + // ordinary one — dialled with the body it wrote and read by nobody here. + if (hostedActive.length === 0) return await descend(next, facts, invocation.payload); + + const dispatchers = new Map(); + for (const entry of hostedActive) dispatchers.set(entry.toolName, entry.hosted.dispatcher); + const loopState: HostedToolLoopState = { + iterationCount: 1, + remainingToolCalls: typeof invocation.payload.max_tool_calls === 'number' ? invocation.payload.max_tool_calls : undefined, + }; + const finalToolChoice = invocation.payload.tool_choice; + const demoteForcedHostedToolChoiceAfterFirstTurn = isForcedHostedToolChoice(finalToolChoice, dispatchers); + const back = await descend(next, facts, invocation.payload) as Answered & Record; + const first = turnOf(back, wiring.streamedUsage); + // An upstream that refused, or a compaction, is not a turn the loop can splice: it rides up + // as it came, and the tools this turn declared were simply never called. + if (first === null) return back as never; + + const merge = createMergeState(); + const billed: BillableEntity[] = []; + let settle!: (outcome: StreamOutcome) => void; + // Every descent's reading is awaited inside the loop, so this is complete by the time the + // spliced stream ends — which is the moment the run is answerable for what it billed. + const usage = defer(new Promise(resolve => { settle = resolve; })); + + let started = false; + let drainActive: () => Promise = async () => {}; + let closing: Promise | undefined; + const close = (): Promise => closing ??= (async () => { + await drainActive(); + if (!started) { + const outcome = first.usage === null ? { billable: back['response.usage.billable'], failed: false } : await first.usage; + settle({ ...outcome, failed: true }); + } + await iterator.return(); + })(); + const observeBody = (turn: Answered): void => { + if (!('response.http.body' in turn) || turn['response.http.body'] === null) return; + const body = turn['response.http.body'] as ReadableStream & Owned; + const drain = setRelease(body, async () => { + try { await drain(); } finally { await close(); } + }); + drainActive = drain; + }; + const iterator = (async function* () { + started = true; + yield* spliceTurns({ + next, facts, invocation, merge, loopState, demoteForcedHostedToolChoiceAfterFirstTurn, + first, dispatchers, store: use.gateway.store, canonicalInput, active, billed, settle, + streamedUsage: wiring.streamedUsage, candidate, + reportFailure: error => { use.gateway.dump?.failed(error); }, observeBody, + }); + })(); + observeBody(back); + return { + ...back, + 'response.chat.openaiResponses': move({ + kind: 'stream' as const, + frames: { [Symbol.asyncIterator]: () => ({ next: () => iterator.next() }) }, + }), + [wiring.streamedUsage]: move(usage), + // What was known when this stage handed up, which is what the first call reported. The + // figure every call turned out to cost arrives with the stream, through the reading above. + 'response.usage.billable': back['response.usage.billable'], + } as never; + }, +}); + +/** One descent, asked with the payload the dispatcher wrote. */ +const descend = async ( + next: (facts: R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers' | 'serve.usage.prior'>) => Promise, + facts: R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers' | 'serve.usage.prior'>, + payload: CanonicalOpenAIResponsesPayload, +): Promise => await next(move({ ...facts, 'request.chat.openaiResponses': move(payload) })); + +/** The descent's answer as a turn, or `null` where it was not one the loop can splice. The + * reading is read at the key the wire was told, which is the source family's own. */ +const turnOf = (back: Answered, streamedUsage: string): Turn | null => { + const answer = back['response.chat.openaiResponses'] as ChatAnswer; + if (isFailure(answer) || answer.kind !== 'stream') return null; + return { + frames: answer.frames as AsyncIterable>, + usage: back[streamedUsage] as Deferred | null, + }; +}; + +/** What a turn cost, once its own frames have run out. The loop reads them one at a time, so + * the running total is complete when the spliced stream reaches its terminal. */ +const accumulate = async (billed: BillableEntity[], turn: Turn): Promise => { + if (turn.usage === null) return false; + const outcome = await turn.usage; + billed.push(...outcome.billable); + return outcome.failed; +}; + +async function* consumeBilledTurn( + frames: AsyncGenerator, TurnSummary>, + turn: Turn, + billed: BillableEntity[], + failed: (value: boolean) => void, +): AsyncGenerator, TurnSummary> { + try { + return yield* frames; + } finally { + failed(await accumulate(billed, turn)); + } +} + +async function* spliceTurns(args: { + next: (facts: R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers' | 'serve.usage.prior'>) => Promise; + facts: R<'request.chat.openaiResponses' | 'route.attempt' | 'ingress.http.headers' | 'serve.usage.prior'>; + invocation: OpenAIResponsesInvocation; + merge: MergeState; + loopState: HostedToolLoopState; + demoteForcedHostedToolChoiceAfterFirstTurn: boolean; + first: Turn; + dispatchers: ReadonlyMap; + store: ChatServices['gateway']['store']; + canonicalInput: OpenAIResponsesInputItem[]; + active: readonly ActiveHostedTool[]; + billed: BillableEntity[]; + settle: (outcome: StreamOutcome) => void; + streamedUsage: string; + candidate: ModelCandidate; + reportFailure: (error: unknown) => void; + observeBody: (back: Answered) => void; +}): AsyncGenerator> { + const { invocation, merge, loopState, demoteForcedHostedToolChoiceAfterFirstTurn, dispatchers, store, active, billed, settle } = args; + const baseInput = args.canonicalInput; + let failed = false; + try { + let currentTurn: TurnSummary = yield* consumeBilledTurn(consumeTurnStreaming(args.first.frames, merge, true, dispatchers, loopState, active), args.first, billed, value => { failed ||= value; }); + merge.accumulatedUsage = sumUsage(merge.accumulatedUsage, currentTurn.turnUsage); + for (;;) { + const turn = currentTurn; + const executedHostedTools = turn.dispatched.length > 0; + + if (turn.terminalStatus.kind === 'failed') { + failed = true; + if (executedHostedTools) yield* materializeHostedToolItems(turn.dispatched, merge, store); + yield synthesizeTerminalEnvelope(merge, { kind: 'failed', error: turn.terminalStatus.response.error }, active); + return; + } + if (turn.terminalStatus.kind === 'incomplete') { + if (executedHostedTools) yield* materializeHostedToolItems(turn.dispatched, merge, store); + yield synthesizeTerminalEnvelope(merge, { kind: 'incomplete', incompleteDetails: turn.terminalStatus.response.incomplete_details }, active); + return; + } + if (turn.terminalStatus.kind === 'bare-error-pre-shell') { + failed = true; + yield synthesizeTerminalEnvelope(merge, { + kind: 'failed', + error: { code: turn.terminalStatus.error.code, message: turn.terminalStatus.error.message }, + }, active); + return; + } + if (!executedHostedTools && !turn.sawClientToolCall) { + yield synthesizeTerminalEnvelope(merge, { kind: 'completed' }, active); + return; + } + + yield* materializeHostedToolItems(turn.dispatched, merge, store); + if (turn.sawClientToolCall) { + yield synthesizeTerminalEnvelope(merge, { kind: 'completed' }, active); + return; + } + + // Accumulated output items are fed back as the next turn's input. An OpenAI Responses + // output item is a structural superset of the matching input item for every shape emitted + // here — messages, reasoning, function_call / function_call_output, and the hosted-tool + // items the dispatchers produce — so the reuse is sound; the cast only bridges the + // output/input naming. + const nextCanonicalInput = [ + ...baseInput, + ...materializeAccumulatedOutput(merge).map(item => item as OpenAIResponsesInputItem), + ]; + const { max_tool_calls: _spent, ...withoutCap } = invocation.payload; + invocation.payload = { + ...withoutCap, + input: transformHostedToolItems(nextCanonicalInput, active), + ...(loopState.remainingToolCalls === undefined ? {} : { max_tool_calls: Math.max(0, loopState.remainingToolCalls) }), + ...(demoteForcedHostedToolChoiceAfterFirstTurn ? { tool_choice: typeof withoutCap.tool_choice === 'object' && withoutCap.tool_choice?.type === 'allowed_tools' ? { ...withoutCap.tool_choice, mode: 'auto' as const } : 'auto' as const } : {}), + }; + loopState.iterationCount += 1; + + let back: Answered; + try { + back = await descend(args.next, { ...args.facts, 'serve.usage.prior': move([...args.facts['serve.usage.prior'], ...billed]) }, invocation.payload); + } catch (error) { + const atFailure = getFailureFacts(error); + if (atFailure !== undefined) billed.push(...providerCalls(args.candidate, atFailure)); + throw error; + } + args.observeBody(back); + const nextTurn = turnOf(back, args.streamedUsage); + if (nextTurn === null) { + failed = true; + billed.push(...back['response.usage.billable']); + yield synthesizeTerminalEnvelope(merge, { kind: 'failed', error: errorOf(back) }, active); + return; + } + currentTurn = yield* consumeBilledTurn(consumeTurnStreaming(nextTurn.frames, merge, false, dispatchers, loopState, active), nextTurn, billed, value => { failed ||= value; }); + merge.accumulatedUsage = sumUsage(merge.accumulatedUsage, currentTurn.turnUsage); + } + } catch (error) { + failed = true; + args.reportFailure(error); + if (merge.lastSeenModel === null) throw error; + yield synthesizeTerminalEnvelope(merge, { + kind: 'failed', + error: { + code: 'server_error', + message: `Upstream stream failed mid-response: ${error instanceof Error ? error.message : String(error)}`, + }, + }, active); + } finally { + settle({ billable: billed, failed }); + } +} + +/** A descent that did not answer with a stream, as the error the synthesized envelope carries. + * A compaction reaches here only if a client asked one to run tools, which no client does. */ +const errorOf = (back: Answered): NonNullable => { + const answer = back['response.chat.openaiResponses'] as ChatAnswer; + return isFailure(answer) + ? buildErrorFromRefusal(answer.status, answer.message) + : { code: 'server_error', message: 'the upstream answered a hosted-tool turn with a compaction' }; +}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tools/image-generation.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-generation.ts similarity index 58% rename from packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tools/image-generation.ts rename to packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-generation.ts index 6e4e0a083c..66b9bf0b96 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tools/image-generation.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-generation.ts @@ -1,14 +1,11 @@ -import { sleep } from '../../../../../shared/sleep.ts'; -import { enumerateModelCandidates } from '../../../../providers/resolution.ts'; -import { stampUpstreamCallStart } from '../../../../shared/attempt-timing.ts'; -import { appendFailedUpstreams } from '../../../../shared/failed-upstreams.ts'; -import type { AttemptState } from '../../../../shared/gateway-ctx.ts'; -import { recordPerformance, type PerformanceTelemetryContext } from '../../../../shared/telemetry/performance.ts'; -import { recordTokenUsage, tokenUsageFromOpenAIImagesBody } from '../../../../shared/telemetry/usage.ts'; -import { createExternalImageFetcher, type ExternalImageFetchResult } from '../../../shared/external-image-loader.ts'; -import type { ServerToolLifecycleEvent, ServerToolOutputItem, ServerToolRegistration, ServerToolTerminal } from '../server-tool-shim.ts'; -import { dimensionsFromBytes, getImageProcessor, type BackgroundScheduler } from '@floway-dev/platform'; -import { decodeForgivingBase64, encodeHex, isImageMediaType, mediaTypeEssence, parseSSEStream } from '@floway-dev/protocols/common'; +import { portableImageRequest, type ImageBackendConfig } from './image-sub-request/request.ts'; +import { imageTerminal, isRetryableImageError, serverError } from './image-sub-request/result.ts'; +import { runImageGenerationSubRequest } from './image-sub-request.ts'; +import type { HostedToolLifecycleEvent, HostedToolRegistration, HostedToolTerminal } from './types.ts'; +import type { GatewayCtx } from '../../../shared/gateway-ctx.ts'; +import { createExternalImageFetcher, type ExternalImageFetchResult } from '../../shared/external-image-loader.ts'; +import { dimensionsFromBytes } from '@floway-dev/platform'; +import { decodeForgivingBase64, isImageMediaType, mediaTypeEssence } from '@floway-dev/protocols/common'; import { collectOpenAIResponsesToolEntries, createRandomOpenAIResponsesItemId, @@ -19,12 +16,11 @@ import { type OpenAIResponsesInputImage, type OpenAIResponsesInputImageGenerationCall, type OpenAIResponsesInputItem, - type OpenAIResponsesOutputImageGenerationCall, type OpenAIResponsesTool, } from '@floway-dev/protocols/openai-responses'; -import { providerModelOf, type Fetcher, type OpenAIImagesEditsRequest, type Provider, type ModelCandidate, type ProviderModel } from '@floway-dev/provider'; +import { providerModelOf } from '@floway-dev/provider'; -export const SHIM_TOOL_NAME = 'image_generation'; +export const FUNCTION_TOOL_NAME = 'image_generation'; // Default image backend when the hosted tool omits `model`. gpt-image-2 is // the reference backend Azure's native OpenAI Responses `image_generation` routes @@ -32,7 +28,7 @@ export const SHIM_TOOL_NAME = 'image_generation'; export const DEFAULT_IMAGE_MODEL = 'gpt-image-2'; // Safety valve on the multi-turn ReAct loop: cap how many real image backend -// calls one response may dispatch (counted on `ShimState.imageDispatchCount`, +// calls one response may dispatch (counted on `HostedToolState.imageDispatchCount`, // not the shared ReAct turn count, so unrelated turns do not consume the // budget). Past the cap the dispatcher replays an exhausted-budget tool output // instead of hitting the backend, so a model that keeps retrying after failures @@ -41,7 +37,7 @@ const IMAGE_ITERATION_CAP = 10; // Public OpenAI Responses `image_generation` tool config enums (Azure-strict // surface). `webp` and arbitrary `WxH` sizes are rejected because the -// native Azure path rejects them; the shim mirrors that vocabulary rather +// native Azure path rejects them; the dispatcher mirrors that vocabulary rather // than passing them to a backend that would 400 with a different shape. const ALLOWED_SIZES = new Set(['1024x1024', '1024x1536', '1536x1024', 'auto']); const ALLOWED_QUALITIES = new Set(['low', 'medium', 'high', 'auto']); @@ -51,32 +47,8 @@ const ALLOWED_MODERATIONS = new Set(['auto', 'low']); const ALLOWED_ACTIONS = new Set(['generate', 'edit', 'auto']); const ALLOWED_INPUT_FIDELITY = new Set(['high', 'low']); -// gpt-image-* `/images/edits` accepts only these input image mimetypes; a live -// Azure probe confirmed png/jpeg/webp succeed while gif is rejected with -// `unsupported_file_mimetype`. Native OpenAI Responses accepts the same GIF and -// re-encodes it before editing, so the shim mirrors that behavior through the -// platform image processor. Common aliases are folded onto the backend form. -type EditMime = 'image/png' | 'image/jpeg' | 'image/webp'; - -const EDIT_MIME_ALIASES: Record = { - 'image/jpg': 'image/jpeg', - 'image/pjpeg': 'image/jpeg', - 'image/x-png': 'image/png', -}; -// The canonical edit-supported mimetype for a source, or null when the -// standalone endpoint requires local WebP transcoding first. -const editSupportedMime = (mime: string): EditMime | null => { - const canonical = EDIT_MIME_ALIASES[mime] ?? mime; - return canonical === 'image/png' || canonical === 'image/jpeg' || canonical === 'image/webp' - ? canonical - : null; -}; - -const editFileExt = (mime: EditMime): string => - mime === 'image/jpeg' ? 'jpg' : mime === 'image/webp' ? 'webp' : 'png'; - // The public `image_generation` tool-config surface. Azure rejects any other -// field with `unknown_parameter`, so the shim mirrors that strictness rather +// field with `unknown_parameter`, so the dispatcher mirrors that strictness rather // than silently forwarding unknown fields (which would diverge from the // emulated surface and hide client bugs). `n` is deliberately absent: Azure // echoes `n:1` internally but rejects a client-supplied `tools[].n`. @@ -89,7 +61,7 @@ const KNOWN_TOOL_FIELDS = new Set([ export const isHostedImageGenerationTool = (tool: OpenAIResponsesTool): tool is OpenAIResponsesHostedTool => tool.type === 'image_generation'; -// Identity canonicalization for image_generation: the shim doesn't +// Identity canonicalization for image_generation: the dispatcher doesn't // depend on filled defaults to run, and the OpenAI spec defaults for // `background` / `quality` / `size` / etc. observed via Azure echo // (all `'auto'`) signal "backend decides" rather than concrete values @@ -98,17 +70,11 @@ export const isHostedImageGenerationTool = (tool: OpenAIResponsesTool): tool is export const canonicalizeImageGenerationTool = (raw: OpenAIResponsesTool): OpenAIResponsesHostedTool | undefined => isHostedImageGenerationTool(raw) ? raw : undefined; -// A base64-data-URL or bare-base64 image source bound for an edit call. -// Bytes are held in a concrete ArrayBuffer so they can be wrapped in a Blob. interface ImageSource { bytes: ArrayBuffer; mimeType: string; } -interface PreparedImageSource extends ImageSource { - mimeType: EditMime; -} - interface RemoteImageSource { wireUrl: string; invalidUrlParam: string; @@ -120,39 +86,6 @@ type ImageSourceReference = ImageSource | RemoteImageSource; const isRemoteImageSource = (source: ImageSourceReference): source is RemoteImageSource => 'wireUrl' in source; -const prepareEditSources = async (sources: readonly ImageSource[]): Promise => { - const keyBySource = new Map>(); - const preparedByContent = new Map>(); - return await Promise.all(sources.map(async source => { - const mimeType = editSupportedMime(source.mimeType); - if (mimeType !== null) return { bytes: source.bytes, mimeType }; - - let keyPromise = keyBySource.get(source); - if (keyPromise === undefined) { - keyPromise = crypto.subtle.digest('SHA-256', source.bytes).then(buffer => { - const digest = encodeHex(new Uint8Array(buffer)); - return `${source.mimeType}\u0000${digest}`; - }); - keyBySource.set(source, keyPromise); - } - const key = await keyPromise; - - let prepared = preparedByContent.get(key); - if (prepared === undefined) { - // Native OpenAI Responses accepts formats such as GIF through its multimodal - // preprocessing, while the standalone edits endpoint accepts only - // PNG/JPEG/WebP. Re-encode locally to preserve the hosted-tool behavior. - // https://github.com/openai/openai-node/blob/ec2f57fd0d66e94782656b986d7b3eb03225369c/src/resources/images.ts#L560-L572 - prepared = getImageProcessor().compressToWebp(new Uint8Array(source.bytes), null).then(encoded => { - const bytes = encoded.buffer.slice(encoded.byteOffset, encoded.byteOffset + encoded.byteLength) as ArrayBuffer; - return { bytes, mimeType: 'image/webp' } satisfies PreparedImageSource; - }); - preparedByContent.set(key, prepared); - } - return await prepared; - })); -}; - const base64ToArrayBuffer = (b64: string): ArrayBuffer => { const bytes = decodeForgivingBase64(b64); return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer; @@ -180,7 +113,7 @@ const decodeInlineImage = ( } // A generated result is bare base64 on its first appearance and a data URL - // after the server-tool replay transform. Keying by decoded wire identity + // after the hosted-tool replay transform. Keying by decoded wire identity // lets both representations reuse the same bytes on later ReAct turns. const cacheKey = `${mimeType}\u0000${payload}`; if (cache?.has(cacheKey)) return cache.get(cacheKey) ?? null; @@ -239,56 +172,15 @@ const decodeInputImageDataUrl = ( } }; -// The orchestrator-visible tool config the shim layers onto the backend +// The orchestrator-visible tool config the dispatcher layers onto the backend // call. Mirrors Azure: the orchestrator only chooses `prompt`; everything -// here is read from the client's hosted-tool entry and applied by the shim. -export interface ImageGenerationConfig { - model: string; - size?: string; - quality?: string; - output_format?: 'png' | 'jpeg'; - background?: 'transparent' | 'opaque' | 'auto'; - moderation?: 'auto' | 'low'; - output_compression?: number; - // When > 0, the backend call is issued with `stream:true` and each - // progressively-rendered preview the backend emits is relayed as a native - // `image_generation_call.partial_image` frame. When 0/absent the backend - // is called non-streaming and no preview frames are produced. - partial_images?: number; - input_fidelity?: 'high' | 'low'; - // Inpainting mask materialized once at validation, forwarded to - // /images/edits as the standalone `mask` part. `file_id` masks are not - // supported (rejected at validation) — resolving them needs the Files API. +// here is read from the client's hosted-tool entry and applied by the dispatcher. +export interface ImageGenerationConfig extends ImageBackendConfig { mask?: ImageSourceReference; - action: 'generate' | 'edit' | 'auto'; } type MaterializedImageGenerationConfig = Omit & { mask?: ImageSource }; -const prepareEditRequest = async ( - sources: readonly ImageSource[], - config: MaterializedImageGenerationConfig, -): Promise<{ sources: readonly PreparedImageSource[]; mask?: PreparedImageSource }> => { - const originals = [...sources]; - if (config.mask !== undefined && !originals.includes(config.mask)) originals.push(config.mask); - const prepared = await prepareEditSources(originals); - const bySource = new Map(); - for (const [index, source] of originals.entries()) { - const wireSource = prepared[index]; - if (wireSource === undefined) throw new Error('Missing prepared image edit source'); - bySource.set(source, wireSource); - } - const wireSources = sources.map(source => { - const wireSource = bySource.get(source); - if (wireSource === undefined) throw new Error('Missing prepared image edit source'); - return wireSource; - }); - if (config.mask === undefined) return { sources: wireSources }; - const mask = bySource.get(config.mask); - if (mask === undefined) throw new Error('Missing prepared image edit mask'); - return { sources: wireSources, mask }; -}; - interface PrepareConfigError { message: string; param: string; @@ -319,7 +211,7 @@ const integerInRange = (value: unknown, param: string, min: number, max: number) }; // Validate one hosted `image_generation` declaration against the fields and -// values supported by this shim. The caller validates every +// values supported by this dispatcher. The caller validates every // declaration before selecting the last config, so an earlier invalid entry // fails. toolPath identifies its source carrier in the error. // https://developers.openai.com/api/docs/guides/tools-image-generation @@ -460,7 +352,7 @@ export const prepareImageGenerationConfig = ( }; // Single optional `prompt` parameter — matches the native `image_gen.imagegen` -// tool's surface (size/quality/etc. are NOT model-chosen; the shim layers them +// tool's surface (size/quality/etc. are NOT model-chosen; the dispatcher layers them // on from the client config, exactly like Azure). A minimal description // elicits native-quality refined prompts while costing ~50 input tokens vs // the native hosted tool's ~2300. @@ -837,489 +729,12 @@ export const resolveImageOperation = ( // continue. The full upstream error shape (type/code/message) is preserved so // the orchestrator can distinguish transient overload from a terminal // content-policy block. -type ImageError = { type: string; code: string; message: string; retryable: boolean }; - -// Server-resolved tool config echoed by the backend on both the partial_image -// frames and the final result (`background:"auto"` becomes the concrete value -// the server picked, etc.). Read straight off the backend rather than inferred -// from the request, so what we surface matches what was actually rendered. -interface EchoFields { - output_format?: 'png' | 'jpeg'; - quality?: 'low' | 'medium' | 'high'; - background?: 'transparent' | 'opaque'; - size?: string; -} - -export type ImageOutcome = - | { ok: true; b64: string; echo: EchoFields } - | { ok: false; error: ImageError }; - -// Project the server-resolved echo fields out of a backend payload (a response -// JSON body or an SSE event). Each field is validated against the public enum -// so a surprising backend value is dropped rather than echoed verbatim. -const extractEcho = (source: unknown): EchoFields => { - if (source === null || typeof source !== 'object') return {}; - const s = source as Record; - const echo: EchoFields = {}; - if (s.output_format === 'png' || s.output_format === 'jpeg') echo.output_format = s.output_format; - if (s.quality === 'low' || s.quality === 'medium' || s.quality === 'high') echo.quality = s.quality; - if (s.background === 'transparent' || s.background === 'opaque') echo.background = s.background; - if (typeof s.size === 'string') echo.size = s.size; - return echo; -}; - -const RETRYABLE_IMAGE_ERROR_CODES = new Set([ - 'EngineOverloaded', 'server_error', 'image_generation_server_error', 'image_generation_failed', -]); - -const isRetryableImageError = (code: string, type?: string): boolean => - RETRYABLE_IMAGE_ERROR_CODES.has(code) || (type !== undefined && RETRYABLE_IMAGE_ERROR_CODES.has(type)); - -const errorFromBody = (body: string, status: number): { type?: string; code: string; message: string } => { - try { - const parsed = JSON.parse(body) as { error?: { message?: unknown; code?: unknown; type?: unknown } }; - const err = parsed.error; - if (err !== undefined && err !== null) { - return { - ...(typeof err.type === 'string' ? { type: err.type } : {}), - message: typeof err.message === 'string' ? err.message : `Image backend returned HTTP ${status}`, - code: typeof err.code === 'string' ? err.code : `upstream_${status}`, - }; - } - } catch (e) { - if (!(e instanceof SyntaxError)) throw e; - } - return { message: `Image backend returned HTTP ${status}`, code: `upstream_${status}` }; -}; - -// Per-request inputs the dispatcher's backend call needs. Captured in the -// registration closure from `ctx`/`request` so the dispatcher stays free of the -// interceptor signature. Edit sources are NOT captured here — they are -// re-collected from the live `ctx.payload.input` at dispatch time so an image -// generated in an earlier turn (fed back as an `input_image`) becomes editable -// in a later turn. `imageDispatchCount` bounds how many real backend image -// calls one response may issue. -interface ShimState { +interface HostedToolState { config: MaterializedImageGenerationConfig; - apiKeyId: string; - upstreamIds: readonly string[] | null; - backgroundScheduler: BackgroundScheduler; - runtimeLocation: string; - downstreamAbortSignal: AbortSignal | undefined; + gateway: GatewayCtx; imageDispatchCount: number; } -const recordImageUsage = (state: ShimState, provider: Provider, model: ProviderModel, modelKey: string, responseBody: unknown): void => { - const usage = tokenUsageFromOpenAIImagesBody(responseBody); - if (usage === null) return; - const promise = recordTokenUsage(state.apiKeyId, { - model: model.id, - upstream: provider.upstreamId, - modelKey, - pricing: model.pricing ?? null, - }, usage).catch((error: unknown) => { - console.error('Failed to record image generation usage:', error); - }); - state.backgroundScheduler(promise); -}; - -const buildGenerationsBody = (prompt: string, config: ImageGenerationConfig, stream: boolean): Record => ({ - prompt, - // Public OpenAI Responses tool config forbids `n`, but the private standalone - // backend call always requests a single image, mirroring Azure's - // single-image OpenAI Responses behavior. - n: 1, - // `response_format` is intentionally not sent: gpt-image-* always returns - // base64 (`data[0].b64_json`) and rejects `response_format`, so the inline - // extraction below reads `b64_json` directly. - ...(config.size !== undefined ? { size: config.size } : {}), - ...(config.quality !== undefined ? { quality: config.quality } : {}), - ...(config.output_format !== undefined ? { output_format: config.output_format } : {}), - ...(config.background !== undefined ? { background: config.background } : {}), - ...(config.moderation !== undefined ? { moderation: config.moderation } : {}), - ...(config.output_compression !== undefined ? { output_compression: config.output_compression } : {}), - ...(stream ? { stream: true, partial_images: config.partial_images } : {}), -}); - -const buildEditsRequest = ( - prompt: string, - config: ImageGenerationConfig, - sources: readonly PreparedImageSource[], - mask: PreparedImageSource | undefined, - stream: boolean, -): OpenAIImagesEditsRequest => { - const parameters: Record = { - prompt, - n: 1, - ...(config.size === undefined ? {} : { size: config.size }), - ...(config.quality === undefined ? {} : { quality: config.quality }), - ...(config.output_format === undefined ? {} : { output_format: config.output_format }), - ...(config.background === undefined ? {} : { background: config.background }), - ...(config.moderation === undefined ? {} : { moderation: config.moderation }), - ...(config.output_compression === undefined ? {} : { output_compression: config.output_compression }), - ...(config.input_fidelity === undefined ? {} : { input_fidelity: config.input_fidelity }), - ...(stream ? { stream: true, partial_images: config.partial_images } : {}), - }; - const images = sources.map((source, index) => ({ - type: 'upload' as const, - file: { bytes: new Uint8Array(source.bytes), name: `image_${index}.${editFileExt(source.mimeType)}`, type: source.mimeType }, - })); - const maskFile = mask === undefined - ? undefined - : { bytes: new Uint8Array(mask.bytes), name: `mask.${editFileExt(mask.mimeType)}`, type: mask.mimeType }; - return { - images, - ...(maskFile === undefined ? {} : { mask: { type: 'upload' as const, file: maskFile } }), - parameters, - }; -}; - -const serverError = (e: unknown): ImageError => ({ - type: 'image_generation_error', - message: e instanceof Error ? e.message : String(e), - code: 'server_error', - retryable: true, -}); - -// Resolve the candidate that serves the configured image model for the -// target endpoint. A resolution/availability failure is normalized into -// an `ImageError` so the caller always produces a terminal image item. -const resolveImageCandidate = async ( - isEdit: boolean, - state: ShimState, -): Promise<{ ok: true; candidate: ModelCandidate } | { ok: false; error: ImageError }> => { - const endpointKey = isEdit ? 'openaiImagesEdits' : 'openaiImagesGenerations'; - const endpointPath = isEdit ? '/images/edits' : '/images/generations'; - let resolution; - try { - resolution = await enumerateModelCandidates({ - upstreamIds: state.upstreamIds, - model: state.config.model, - kind: 'image', - scheduler: state.backgroundScheduler, - runtimeLocation: state.runtimeLocation, - }); - } catch (e) { - return { ok: false, error: serverError(e) }; - } - const match = resolution.candidates.find(c => c.model.endpoints[endpointKey] !== undefined); - if (match !== undefined) { - return { ok: true, candidate: match }; - } - // Split on the resolver's `sawModel` signal the same way serve-prep.ts - // does for chat: an unknown model id ("model_not_found", 404-shaped) vs - // a model that exists under some catalog but cannot serve this op - // ("model_not_supported"). The latter splits further on whether the - // resolver's kind filter rejected the id (sawModel=true, candidates=[]: - // id exists but not as an image model) or the per-endpoint key did - // (candidates non-empty: image-kind upstreams exist but none expose the - // requested edits/generations endpoint). - if (!resolution.sawModel) { - return { - ok: false, - error: { - type: 'image_generation_error', - message: appendFailedUpstreams(`No upstream provides model '${state.config.model}'.`, resolution.failedUpstreams), - code: 'model_not_found', - retryable: false, - }, - }; - } - const message = resolution.candidates.length === 0 - ? `Model '${state.config.model}' is not an image model.` - : `No upstream supporting the ${endpointPath} endpoint provides model '${state.config.model}'.`; - return { - ok: false, - error: { - type: 'image_generation_error', - message: appendFailedUpstreams(message, resolution.failedUpstreams), - code: 'model_not_supported', - retryable: false, - }, - }; -}; - -// 60s cap matches the per-minute refill window of Azure TPM/RPM and -// openai.com tier image quotas — same clamp openai-python applies in -// [`_calculate_retry_timeout`](https://github.com/openai/openai-python/blob/d76d8c11c1da9f97aa8a0aaee8ccd44d2bc8f5e7/src/openai/_base_client.py#L789). -const RETRY_CAP_MS = 60_000; -const MAX_RATE_LIMIT_RETRIES = 2; - -// Header priority matches openai-python's `_parse_retry_after_header` with -// Azure's `x-ms-retry-after-ms` alias added. Treats <= 0 as "no hint" so the -// gpt-image-1 `retry-after: 0.0` quirk falls back to backoff instead of -// pretending the quota is free. -export const parseRetryAfterMs = (headers: Headers): number | null => { - for (const name of ['retry-after-ms', 'x-ms-retry-after-ms']) { - const raw = headers.get(name); - if (raw === null) continue; - const ms = Number(raw); - if (Number.isFinite(ms) && ms > 0) return ms; - } - const ra = headers.get('retry-after'); - if (ra !== null) { - const seconds = Number(ra); - if (Number.isFinite(seconds) && seconds > 0) return seconds * 1000; - const httpDateMs = Date.parse(ra); - if (!Number.isNaN(httpDateMs)) { - const delta = httpDateMs - Date.now(); - if (delta > 0) return delta; - } - } - return null; -}; - -// On 429, sleep for the upstream's retry hint (or jittered exponential -// backoff when absent) and replay the same backend call up to -// MAX_RATE_LIMIT_RETRIES times. The returned `response` always has a fresh, -// unread body — intermediate failed responses are drained inside the loop so -// the underlying socket can be reused while we sleep. -const issueImageCall = async ( - provider: Provider, - model: ProviderModel, - fetcher: Fetcher, - prompt: string, - editRequest: OpenAIImagesEditsRequest | null, - config: ImageGenerationConfig, - state: ShimState, - stream: boolean, - attempt: AttemptState, -): Promise<{ response: Response; modelKey: string }> => { - for (let retry = 0; ; retry++) { - const opts = { - fetcher, - waitUntil: state.backgroundScheduler, - headers: new Headers(), - // Stamp this image sub-call's OWN perf slot — never ctx.attempt — - // so the outer OpenAI Responses turn's upstream-call stamp is preserved. - // Perf recording lives at the sub-call's terminal boundary in - // streamImageGeneration; the retry loop overwrites this slot each - // retry so it reflects the dispatch that actually returned. - wrapUpstreamCall: stampUpstreamCallStart(attempt.timing), - }; - const { response, modelKey } = await (editRequest === null - ? provider.instance.callOpenAIImagesGenerations(model, buildGenerationsBody(prompt, config, stream), state.downstreamAbortSignal, opts) - : provider.instance.callOpenAIImagesEdits(model, editRequest, state.downstreamAbortSignal, opts)); - if (response.status !== 429 || retry >= MAX_RATE_LIMIT_RETRIES) return { response, modelKey }; - - // 25% jitter desynchronizes parallel callers so a burst of orchestrator - // turns doesn't all re-issue at the same instant. - const base = 1000 * 2 ** retry; - const backoffMs = base + Math.random() * base * 0.25; - const delayMs = Math.min(parseRetryAfterMs(response.headers) ?? backoffMs, RETRY_CAP_MS); - await response.text().catch(() => undefined); - await sleep(delayMs, state.downstreamAbortSignal); - } -}; - -// Consume a non-streaming backend response (partial_images = 0) into an -// outcome. Transport/backend failures become `{ok:false}` rather than -// throwing, so the caller always produces a terminal image item. -const consumeImageResponse = async ( - provider: Provider, - model: ProviderModel, - modelKey: string, - response: Response, - state: ShimState, -): Promise => { - const text = await response.text(); - if (!response.ok) { - const { type, code, message } = errorFromBody(text, response.status); - return { ok: false, error: { type: type ?? 'image_generation_error', code, message, retryable: isRetryableImageError(code, type) } }; - } - let parsed: unknown; - try { - parsed = JSON.parse(text); - } catch { - return { ok: false, error: { type: 'image_generation_error', message: 'Image backend returned a non-JSON success body.', code: 'server_error', retryable: true } }; - } - const b64 = (() => { - if (parsed === null || typeof parsed !== 'object') return null; - const data = (parsed as { data?: unknown }).data; - if (!Array.isArray(data) || data.length === 0) return null; - const first = data[0] as { b64_json?: unknown }; - return typeof first.b64_json === 'string' ? first.b64_json : null; - })(); - if (b64 === null) { - return { ok: false, error: { type: 'image_generation_error', message: 'Image backend response did not contain image bytes.', code: 'server_error', retryable: true } }; - } - recordImageUsage(state, provider, model, modelKey, parsed); - return { ok: true, b64, echo: extractEcho(parsed) }; -}; - -// Build the completed/failed `image_generation_call` output item plus its -// closing events. On success the final bytes ride `output_item.done.item.result` -// and a `response.image_generation_call.completed` closes the item; on failure -// neither `.completed` nor any `.partial_image` is emitted — only the failed -// `output_item.done`. -// -// `revised_prompt` is set to the orchestrator's prompt: the standalone images -// backend does no prompt rewriting and returns no `revised_prompt`, and the -// orchestrator's emitted prompt IS already its refined rewrite (it plays the -// role Azure's native flow gives the orchestrator), so it is the faithful -// source for this field. -export const imageTerminal = ( - prompt: string, - action: 'generate' | 'edit', - outcome: ImageOutcome, -): ServerToolTerminal => { - if (!outcome.ok) { - const item: ServerToolOutputItem & Omit = { - type: 'image_generation_call', - status: 'failed', - revised_prompt: prompt, - error: { message: outcome.error.message, code: outcome.error.code, type: outcome.error.type }, - }; - return { item, endEvents: [] }; - } - - const item: ServerToolOutputItem & Omit = { - type: 'image_generation_call', - status: 'completed', - action, - result: outcome.b64, - revised_prompt: prompt, - ...outcome.echo, - }; - return { item, endEvents: [{ type: 'response.image_generation_call.completed' }] }; -}; - -// One standalone-images SSE data line, folded into a backend-agnostic signal. -// The generations and edits endpoints use distinct event prefixes -// (`image_generation.*` vs `image_edit.*`); only the suffix is matched here. -type ImageStreamSignal = - | { kind: 'partial'; index: number; b64: string; echo: EchoFields } - | { kind: 'completed'; b64: string | undefined; usage: unknown; echo: EchoFields } - | { kind: 'error'; error: ImageError } - | null; - -export const parseImageStreamEvent = (data: string): ImageStreamSignal => { - let evt: { type?: unknown; partial_image_index?: unknown; b64_json?: unknown; usage?: unknown; error?: unknown }; - try { - evt = JSON.parse(data); - } catch { - return null; - } - const type = typeof evt.type === 'string' ? evt.type : ''; - if (type.endsWith('.partial_image')) { - return { - kind: 'partial', - index: typeof evt.partial_image_index === 'number' ? evt.partial_image_index : 0, - b64: typeof evt.b64_json === 'string' ? evt.b64_json : '', - echo: extractEcho(evt), - }; - } - if (type.endsWith('.completed')) { - return { kind: 'completed', b64: typeof evt.b64_json === 'string' ? evt.b64_json : undefined, usage: evt.usage, echo: extractEcho(evt) }; - } - if (type === 'error') { - const err = evt.error as { message?: unknown; code?: unknown; type?: unknown } | undefined; - const code = typeof err?.code === 'string' ? err.code : 'server_error'; - const errType = typeof err?.type === 'string' ? err.type : 'image_generation_error'; - return { - kind: 'error', - error: { type: errType, code, message: typeof err?.message === 'string' ? err.message : 'Image backend stream reported an error.', retryable: isRetryableImageError(code, errType) }, - }; - } - return null; -}; - -// Drive the backend and produce the deferred slot lifecycle: relay each -// progressively-rendered preview as a native `partial_image` frame, then -// return the terminal `image_generation_call` item. partial_images = 0 (or -// absent) takes a single non-streaming round-trip and yields no preview frames. -// -// Every sub-call records its OWN perf row under operation='image_generation' -// or 'image_edit' via a local AttemptState distinct from ctx.attempt (which -// belongs to the outer OpenAI Responses turn). firstOutputTokenAt stays null — image -// backends are single-body from the perf model's point of view, so the row -// lands in the neutral bucket with an honest requests + errors count and no -// synthesized TTFT. Resolution failures record no row: no upstream was ever -// dispatched. -const streamImageGeneration = ( - prompt: string, - action: 'generate' | 'edit', - isEdit: boolean, - sources: readonly ImageSource[], - state: ShimState, -) => async function* (): AsyncGenerator { - const resolved = await resolveImageCandidate(isEdit, state); - if (!resolved.ok) return imageTerminal(prompt, action, { ok: false, error: resolved.error }); - const { provider, fetcher } = resolved.candidate; - const model = providerModelOf(resolved.candidate); - const wantsPartials = (state.config.partial_images ?? 0) > 0; - - const attempt: AttemptState = { timing: { upstreamCallStartedAt: null, firstOutputTokenAt: null }, telemetry: undefined }; - const perfContext: PerformanceTelemetryContext = { - keyId: state.apiKeyId, - model: model.id, - upstream: provider.upstreamId, - operation: isEdit ? 'image_edit' : 'image_generation', - runtimeLocation: state.runtimeLocation, - }; - const finish = (outcome: ImageOutcome): ServerToolTerminal => { - recordPerformance({ attempt, backgroundScheduler: state.backgroundScheduler }, perfContext, !outcome.ok, 0, performance.now()); - return imageTerminal(prompt, action, outcome); - }; - - let response: Response; - let modelKey: string; - try { - let editRequest: OpenAIImagesEditsRequest | null = null; - if (isEdit) { - const prepared = await prepareEditRequest(sources, state.config); - editRequest = buildEditsRequest(prompt, state.config, prepared.sources, prepared.mask, wantsPartials); - } - ({ response, modelKey } = await issueImageCall( - provider, - model, - fetcher, - prompt, - editRequest, - state.config, - state, - wantsPartials, - attempt, - )); - } catch (e) { - return finish({ ok: false, error: serverError(e) }); - } - - if (!wantsPartials) { - return finish(await consumeImageResponse(provider, model, modelKey, response, state)); - } - - if (!response.ok) { - const { type, code, message } = errorFromBody(await response.text(), response.status); - return finish({ ok: false, error: { type: type ?? 'image_generation_error', code, message, retryable: isRetryableImageError(code, type) } }); - } - if (response.body === null) { - return finish({ ok: false, error: { type: 'image_generation_error', message: 'Image backend returned a streaming response with no body.', code: 'server_error', retryable: true } }); - } - - let finalB64: string | undefined; - let finalEcho: EchoFields = {}; - let usage: unknown; - for await (const frame of parseSSEStream(response.body, { signal: state.downstreamAbortSignal })) { - const signal = parseImageStreamEvent(frame.data); - if (signal === null) continue; - if (signal.kind === 'partial') { - yield { type: 'response.image_generation_call.partial_image', partial_image_index: signal.index, partial_image_b64: signal.b64, ...signal.echo }; - } else if (signal.kind === 'completed') { - finalB64 = signal.b64; - finalEcho = signal.echo; - usage = signal.usage; - } else { - return finish({ ok: false, error: signal.error }); - } - } - if (finalB64 === undefined) { - return finish({ ok: false, error: { type: 'image_generation_error', message: 'Image backend stream ended without a completed image.', code: 'server_error', retryable: true } }); - } - recordImageUsage(state, provider, model, modelKey, { usage }); - return finish({ ok: true, b64: finalB64, echo: finalEcho }); -}; - // Output-as-input round-trip: the multi-turn loop feeds accumulated // `image_generation_call` items back as the next turn's input, and client // histories may echo prior ones. Non-OpenAI-Responses upstreams can't read the item @@ -1395,7 +810,7 @@ export const transformInputItemsForImageGeneration = ( return out; }; -export const imageGenerationServerTool: ServerToolRegistration = async (invocation, gatewayCtx) => { +export const imageGenerationHostedTool: HostedToolRegistration = async (invocation, gatewayCtx) => { if (invocation.targetApi === 'openaiResponses' && !providerModelOf(invocation.candidate).enabledFlags.has('openai-responses-image-generation-shim')) { return { type: 'inactive' }; } @@ -1411,7 +826,7 @@ export const imageGenerationServerTool: ServerToolRegistration = async (invocati // the upstream can read them, but there is no hosted tool to dispatch. return { type: 'active', - baseToolName: SHIM_TOOL_NAME, + baseToolName: FUNCTION_TOOL_NAME, transformItems: transformInputItemsForImageGeneration, }; } @@ -1479,19 +894,15 @@ export const imageGenerationServerTool: ServerToolRegistration = async (invocati ...(mask === undefined ? {} : { mask }), }; - const state: ShimState = { + const state: HostedToolState = { config: materializedConfig, - apiKeyId: gatewayCtx.apiKeyId, - upstreamIds: gatewayCtx.upstreamIds, - backgroundScheduler: gatewayCtx.backgroundScheduler, - runtimeLocation: gatewayCtx.runtimeLocation, - downstreamAbortSignal: gatewayCtx.abortSignal, + gateway: gatewayCtx, imageDispatchCount: 0, }; return { type: 'active', - baseToolName: SHIM_TOOL_NAME, + baseToolName: FUNCTION_TOOL_NAME, transformItems: transformInputItemsForImageGeneration, hosted: { hostedTypes: ['image_generation'], @@ -1502,7 +913,7 @@ export const imageGenerationServerTool: ServerToolRegistration = async (invocati ? intercepted.arguments.prompt : ''; const id = synthesizeImageGenerationCallId(); - // Later ReAct turns include prior server-tool output in the live input. + // Later ReAct turns include prior hosted-tool output in the live input. // Resolve and validate again so action:auto can pivot from generation // to editing without bypassing the same source policy used at ingress. // The per-request inspector cache reuses bytes already decoded during @@ -1543,13 +954,24 @@ export const imageGenerationServerTool: ServerToolRegistration = async (invocati { type: 'response.image_generation_call.in_progress' }, { type: 'response.image_generation_call.generating' }, ], - run: streamImageGeneration( - promptArg, - operation.action, - operation.action === 'edit', - sources, - state, - ), + // Its own run: its own prologue, its own settlement, its own record. What this + // yields is that run's lifecycle, spliced into the turn that asked for it. + run: () => (async function* (): AsyncGenerator { + let call; + try { + call = await runImageGenerationSubRequest( + state.gateway, + portableImageRequest(promptArg, operation.action, state.config, sources), + ); + } catch (error) { + return imageTerminal(promptArg, operation.action, { ok: false, error: serverError(error) }); + } + try { + return yield* call.lifecycle; + } finally { + await call.drain(); + } + })(), }]; }, }, diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts new file mode 100644 index 0000000000..1617468b4d --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request.ts @@ -0,0 +1,64 @@ +import { imageGenerationSubRequestPipeline } from './image-sub-request/pipeline.ts'; +import type { ImageGenerationRequest } from './image-sub-request/request.ts'; +import type { HostedToolLifecycleEvent, HostedToolTerminal } from './types.ts'; +import { prologueFor } from '../../../pipeline/serve.ts'; +import type { GatewayCtx, AttemptState } from '../../../shared/gateway-ctx.ts'; +import { run, move } from '@floway-dev/pipeline'; + +/** + * The run a dispatcher call is. + * + * Its context is the parent's, with the three things a separate run owes itself: a start time of + * its own, an attempt slot of its own — so the outer turn's upstream stamp survives — and a + * record of its own. + */ +export const runImageGenerationSubRequest = async ( + parent: GatewayCtx, + request: ImageGenerationRequest, +): Promise<{ + readonly lifecycle: AsyncGenerator; + readonly drain: () => Promise; +}> => { + const path = request.action === 'edit' ? '/images/edits' : '/images/generations'; + const attempt: AttemptState = { timing: { firstOutputTokenAt: null, upstreamCallStartedAt: null }, telemetry: undefined }; + const wantsStream = (request.config.partial_images ?? 0) > 0; + const dump = parent.dump?.openSubRequest({ method: 'POST', path }, wantsStream, attempt.timing) ?? null; + const gateway: GatewayCtx = { + ...parent, + wantsStream, + requestStartedAt: Date.now(), + attempt, + dump, + }; + const prologue = prologueFor(gateway, { body: { bytes: new Uint8Array(), streamError: null }, headers: [] }, dump); + + try { + const { facts, drain } = await run( + imageGenerationSubRequestPipeline(request), + move({ 'request.imageGeneration.canonical': request, 'ingress.http.headers': [] }), + { ...prologue.services }, + ); + prologue.runDump?.afterRun(drain); + return { + lifecycle: facts['response.imageGeneration.lifecycle'], + drain: async () => { + let status = facts['response.http.status']; + try { + // Releasing the native body also drives an unread image to its usage terminal. + // Waiting for that reading first would prevent the owner from ever starting it. + await drain(); + } catch (error) { + dump?.failed(error); + if ((await facts['response.imageGeneration.lifecycleOutcome']).kind === 'exception') status = 500; + throw error; + } finally { + dump?.finalize(status, 0); + } + }, + }; + } catch (error) { + dump?.failed(error); + dump?.finalize(500, 0); + throw error; + } +}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/emit.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/emit.ts new file mode 100644 index 0000000000..1c6d4fb2da --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/emit.ts @@ -0,0 +1,133 @@ +import type { Fields, ImageLifecycleOutcome } from './facts.ts'; +import { errorFromBody, extractEcho, imageTerminal, isRetryableImageError, projectImageStreamEvent, serverError, type ImageOutcome } from './result.ts'; +import type { RunDump } from '../../../../../dump/run-sink.ts'; +import { isFailure } from '../../../../pipeline/facts.ts'; +import type { GatewayServices } from '../../../../pipeline/services.ts'; +import type { HostedToolLifecycleEvent, HostedToolTerminal } from '../types.ts'; +import { streamReferenceOf } from '@floway-dev/dump'; +import { defineStage, move, own, defer } from '@floway-dev/pipeline'; +import { eventFrame } from '@floway-dev/protocols/common'; +import type { CanonicalOpenAIImagesResponse } from '@floway-dev/protocols/openai-images'; + +export const emitHostedImageGeneration = defineStage< + Fields<'request.imageGeneration.canonical'>, + Fields<'request.imageGeneration.canonical'>, + Fields<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.usage.billable'>, + Fields<'response.imageGeneration.lifecycle' | 'response.imageGeneration.lifecycleOutcome' | 'response.openaiImages.streamedUsage' | 'response.http.status'>, + GatewayServices +>({ + name: 'emitHostedImageGeneration', + through: { + request: { needs: ['request.imageGeneration.canonical'], consumes: [], provides: [] }, + response: { + needs: ['response.openaiImages.canonical', 'response.openaiImages.streamedUsage', 'response.http.status', 'response.usage.billable'], + consumes: ['response.openaiImages.canonical'], provides: ['response.imageGeneration.lifecycle', 'response.imageGeneration.lifecycleOutcome', 'response.openaiImages.streamedUsage'], + }, + }, + execute: async (facts, next, use) => { + const back = await next(facts); + const { 'response.openaiImages.canonical': answer, ...rest } = back; + const request = facts['request.imageGeneration.canonical']; + const lifecycle = (async function* (): AsyncGenerator { + let outcome: ImageOutcome; + if (isFailure(answer)) { + const error = answer.body === undefined + ? { type: 'image_generation_error', code: answer.status === 404 ? 'model_not_found' : answer.status === 400 ? 'model_not_supported' : 'server_error', message: answer.message } + : errorFromBody(JSON.stringify(answer.body), answer.status); + outcome = { ok: false, error: { ...error, type: error.type ?? 'image_generation_error', retryable: isRetryableImageError(error.code, error.type) } }; + } else if (Symbol.asyncIterator in answer) { + const reader = answer[Symbol.asyncIterator](); + try { + for (;;) { + let step; + try { step = await reader.next(); } catch (error) { + use.gateway.dump?.failed(error); + outcome = { ok: false, error: serverError(error) }; + break; + } + if (step.done) { + outcome = { ok: false, error: serverError(new Error('Image backend stream ended without a completed image.')) }; + break; + } + const signal = projectImageStreamEvent(step.value); + if (signal === null) continue; + if (signal.kind === 'partial') { + yield { type: 'response.image_generation_call.partial_image', partial_image_index: signal.index, partial_image_b64: signal.b64, ...signal.echo }; + } else if (signal.kind === 'completed') { + return imageTerminal(request.prompt, request.action, signal.b64 === undefined + ? { ok: false, error: serverError(new Error('Image backend stream ended without a completed image.')) } + : { ok: true, b64: signal.b64, echo: signal.echo }); + } else { + return imageTerminal(request.prompt, request.action, { ok: false, error: signal.error }); + } + } + } finally { + await reader.return?.(); + } + } else { + outcome = imageOutcome(answer); + } + return imageTerminal(request.prompt, request.action, outcome); + })(); + // Formatting continues after the stage hands up. Releasing undemanded output stays + // neutral, while an observed emission exception must reach both settlement and drain. + const emission = Promise.withResolvers(); + let exception: Extract | null = null; + const recorded = recordLifecycle(lifecycle, use.gateway.dump); + const observed = (async function* () { + try { return yield* recorded; } catch (error) { + exception = move({ kind: 'exception' as const, error }); + use.gateway.dump?.failed(error); + await use.log.error('image lifecycle emission failed', { error }); + throw error; + } finally { + emission.resolve(exception ?? move({ kind: 'released' as const })); + } + })(); + const owned = own(Object.assign(observed, streamReferenceOf(recorded)), async () => { + try { await observed.return(undefined as never); } finally { + emission.resolve(exception ?? move({ kind: 'released' as const })); + } + if (exception !== null) throw exception.error; + }); + const reading = back['response.openaiImages.streamedUsage'] ?? Promise.resolve({ billable: back['response.usage.billable'], failed: false }); + return move({ + ...rest, + 'response.imageGeneration.lifecycle': owned, + 'response.imageGeneration.lifecycleOutcome': defer(emission.promise), + 'response.openaiImages.streamedUsage': defer(Promise.all([reading, emission.promise]).then(([outcome, completed]) => ({ + ...outcome, failed: outcome.failed || completed.kind === 'exception', + }))), + }); + }, +}); + +const imageOutcome = (answer: CanonicalOpenAIImagesResponse): ImageOutcome => { + const b64 = answer.images[0]?.base64; + return b64 === undefined + ? { ok: false, error: serverError(new Error('Image backend response did not contain image bytes.')) } + : { ok: true, b64, echo: extractEcho(answer.raw) }; +}; + +const recordLifecycle = (source: AsyncGenerator, dump: RunDump | null): AsyncGenerator => { + if (dump === null) return source; + const recording = dump.openStream(); + const generator = (async function* () { + let completed = false; + try { + for (;;) { + const step = await source.next(); + await recording.frame(eventFrame(step.value)); + if (step.done) { + completed = true; + await recording.end(); + return step.value; + } + yield step.value; + } + } finally { + if (!completed) await source.return(undefined as never); + } + })(); + return Object.assign(generator, recording.fact); +}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/facts.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/facts.ts new file mode 100644 index 0000000000..1c97eaa4e4 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/facts.ts @@ -0,0 +1,16 @@ +import type { ImageGenerationRequest } from './request.ts'; +import type { OpenAIImagesFacts } from '../../../../openai-images/facts.ts'; +import type { HostedToolLifecycleEvent, HostedToolTerminal } from '../types.ts'; +import type { Deferred } from '@floway-dev/pipeline'; + +export type ImageLifecycleOutcome = { readonly kind: 'released' } | { readonly kind: 'exception'; readonly error: unknown }; + +export interface ImageSubRequestFacts extends OpenAIImagesFacts { + 'request.imageGeneration.canonical': ImageGenerationRequest; + 'response.imageGeneration.lifecycleOutcome': Deferred; + 'response.imageGeneration.lifecycle': AsyncGenerator; +} + +export type Fields = { [P in K]: ImageSubRequestFacts[P] }; +export type ImageSubRequestEntry = Fields<'request.imageGeneration.canonical' | 'ingress.http.headers'>; +export type ImageSubRequestExit = Fields<'response.imageGeneration.lifecycle' | 'response.imageGeneration.lifecycleOutcome' | 'response.openaiImages.streamedUsage' | 'response.http.status'>; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/pipeline.ts new file mode 100644 index 0000000000..031cb1495d --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/pipeline.ts @@ -0,0 +1,17 @@ +import { emitHostedImageGeneration } from './emit.ts'; +import type { ImageSubRequestEntry, ImageSubRequestExit } from './facts.ts'; +import { prepareHostedImageGeneration } from './prepare.ts'; +import type { ImageGenerationRequest } from './request.ts'; +import { retryRateLimitedImages } from './retry.ts'; +import { callOpenAIImagesUpstream } from '../../../../openai-images/call-upstream.ts'; +import { openaiImagesModelStages } from '../../../../openai-images/pipeline.ts'; +import { writeSettlement } from '../../../../pipeline/settlement.ts'; +import { compose, type Pipeline } from '@floway-dev/pipeline'; + +export const imageGenerationSubRequestPipeline = (request: ImageGenerationRequest): Pipeline => + compose('imageGenerationSubRequest', [ + writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, 'response.openaiImages.streamedUsage'), + emitHostedImageGeneration, + prepareHostedImageGeneration, + ...openaiImagesModelStages({ operation: request.action === 'edit' ? 'edits' : 'generations' }, [retryRateLimitedImages, callOpenAIImagesUpstream]), + ]); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/prepare.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/prepare.ts new file mode 100644 index 0000000000..dfda8f9a4f --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/prepare.ts @@ -0,0 +1,27 @@ +import type { Fields } from './facts.ts'; +import { prepareImageRequest } from './request.ts'; +import type { GatewayServices } from '../../../../pipeline/services.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { openaiImagesRequestWantsStream } from '@floway-dev/protocols/openai-images'; + +export const prepareHostedImageGeneration = defineStage< + Fields<'request.imageGeneration.canonical'>, + Fields<'request.imageGeneration.canonical' | 'request.openaiImages.canonical' | 'serve.model' | 'ingress.openaiImages.wantsStream'>, + Fields<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.http.body' | 'response.usage.billable'>, + Fields<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.http.body' | 'response.usage.billable'>, + GatewayServices +>({ + name: 'prepareHostedImageGeneration', + through: { + request: { needs: ['request.imageGeneration.canonical'], consumes: [], provides: ['request.openaiImages.canonical', 'serve.model', 'ingress.openaiImages.wantsStream'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const request = await prepareImageRequest(facts['request.imageGeneration.canonical']); + return await next({ + ...facts, 'request.openaiImages.canonical': move(request), + 'serve.model': facts['request.imageGeneration.canonical'].config.model, + 'ingress.openaiImages.wantsStream': openaiImagesRequestWantsStream(request), + }); + }, +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/request.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/request.ts new file mode 100644 index 0000000000..6c24f7c00f --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/request.ts @@ -0,0 +1,186 @@ +import { getImageProcessor } from '@floway-dev/platform'; +import { encodeHex } from '@floway-dev/protocols/common'; +import type { CanonicalOpenAIImagesEditsRequest, CanonicalOpenAIImagesRequest } from '@floway-dev/protocols/openai-images'; + +// gpt-image-* `/images/edits` accepts only these input image mimetypes; a live +// Azure probe confirmed png/jpeg/webp succeed while gif is rejected with +// `unsupported_file_mimetype`. Native OpenAI Responses accepts the same GIF and +// re-encodes it before editing, so the dispatcher mirrors that behavior through the +// platform image processor. Common aliases are folded onto the backend form. +type EditMime = 'image/png' | 'image/jpeg' | 'image/webp'; + +const EDIT_MIME_ALIASES: Record = { + 'image/jpg': 'image/jpeg', + 'image/pjpeg': 'image/jpeg', + 'image/x-png': 'image/png', +}; +// The canonical edit-supported mimetype for a source, or null when the +// standalone endpoint requires local WebP transcoding first. +const editSupportedMime = (mime: string): EditMime | null => { + const canonical = EDIT_MIME_ALIASES[mime] ?? mime; + return canonical === 'image/png' || canonical === 'image/jpeg' || canonical === 'image/webp' + ? canonical + : null; +}; + +const editFileExt = (mime: EditMime): string => + mime === 'image/jpeg' ? 'jpg' : mime === 'image/webp' ? 'webp' : 'png'; + +export interface ImageBackendConfig { + model: string; + size?: string; + quality?: string; + output_format?: 'png' | 'jpeg'; + background?: 'transparent' | 'opaque' | 'auto'; + moderation?: 'auto' | 'low'; + output_compression?: number; + // When > 0, the backend call is issued with `stream:true` and each + // progressively-rendered preview the backend emits is relayed as a native + // `image_generation_call.partial_image` frame. When 0/absent the backend + // is called non-streaming and no preview frames are produced. + partial_images?: number; + input_fidelity?: 'high' | 'low'; + action: 'generate' | 'edit' | 'auto'; +} + +export interface ImageSource { + bytes: Uint8Array; + mimeType: string; +} +interface PreparedImageSource extends ImageSource { mimeType: EditMime } + +export interface ImageGenerationRequest { + prompt: string; + action: 'generate' | 'edit'; + config: ImageBackendConfig & { mask?: ImageSource }; + sources: readonly ImageSource[]; +} + +const prepareEditSources = async (sources: readonly ImageSource[]): Promise => { + const keyBySource = new Map>(); + const preparedByContent = new Map>(); + return await Promise.all(sources.map(async source => { + const mimeType = editSupportedMime(source.mimeType); + if (mimeType !== null) return { bytes: source.bytes, mimeType }; + + let keyPromise = keyBySource.get(source); + if (keyPromise === undefined) { + keyPromise = crypto.subtle.digest('SHA-256', source.bytes).then(buffer => { + const digest = encodeHex(new Uint8Array(buffer)); + return `${source.mimeType}\u0000${digest}`; + }); + keyBySource.set(source, keyPromise); + } + const key = await keyPromise; + + let prepared = preparedByContent.get(key); + if (prepared === undefined) { + // Native OpenAI Responses accepts formats such as GIF through its multimodal + // preprocessing, while the standalone edits endpoint accepts only + // PNG/JPEG/WebP. Re-encode locally to preserve the hosted-tool behavior. + // https://github.com/openai/openai-node/blob/ec2f57fd0d66e94782656b986d7b3eb03225369c/src/resources/images.ts#L560-L572 + prepared = getImageProcessor().compressToWebp(source.bytes, null).then(encoded => { + const bytes = new Uint8Array(encoded); + return { bytes, mimeType: 'image/webp' } satisfies PreparedImageSource; + }); + preparedByContent.set(key, prepared); + } + return await prepared; + })); +}; + +const prepareEditRequest = async ( + sources: readonly ImageSource[], + config: ImageBackendConfig & { mask?: ImageSource }, +): Promise<{ sources: readonly PreparedImageSource[]; mask?: PreparedImageSource }> => { + const originals = [...sources]; + if (config.mask !== undefined && !originals.includes(config.mask)) originals.push(config.mask); + const prepared = await prepareEditSources(originals); + return { + sources: prepared.slice(0, sources.length), + ...(config.mask === undefined ? {} : { mask: prepared[originals.indexOf(config.mask)]! }), + }; +}; + +const buildGenerationsBody = (prompt: string, config: ImageBackendConfig, stream: boolean): Record => ({ + prompt, + // Public OpenAI Responses tool config forbids `n`, but the private standalone + // backend call always requests a single image, mirroring Azure's + // single-image OpenAI Responses behavior. + n: 1, + // `response_format` is intentionally not sent: gpt-image-* always returns + // base64 (`data[0].b64_json`) and rejects `response_format`, so the inline + // extraction below reads `b64_json` directly. + ...(config.size !== undefined ? { size: config.size } : {}), + ...(config.quality !== undefined ? { quality: config.quality } : {}), + ...(config.output_format !== undefined ? { output_format: config.output_format } : {}), + ...(config.background !== undefined ? { background: config.background } : {}), + ...(config.moderation !== undefined ? { moderation: config.moderation } : {}), + ...(config.output_compression !== undefined ? { output_compression: config.output_compression } : {}), + ...(stream ? { stream: true, partial_images: config.partial_images } : {}), +}); + +const buildEditsRequest = ( + prompt: string, + config: ImageBackendConfig, + sources: readonly PreparedImageSource[], + mask: PreparedImageSource | undefined, + stream: boolean, +): CanonicalOpenAIImagesEditsRequest => { + const parameters: Record = { + prompt, + n: 1, + ...(config.size === undefined ? {} : { size: config.size }), + ...(config.quality === undefined ? {} : { quality: config.quality }), + ...(config.output_format === undefined ? {} : { output_format: config.output_format }), + ...(config.background === undefined ? {} : { background: config.background }), + ...(config.moderation === undefined ? {} : { moderation: config.moderation }), + ...(config.output_compression === undefined ? {} : { output_compression: config.output_compression }), + ...(config.input_fidelity === undefined ? {} : { input_fidelity: config.input_fidelity }), + ...(stream ? { stream: true, partial_images: config.partial_images } : {}), + }; + const images = sources.map((source, index) => ({ + kind: 'file' as const, + file: { bytes: source.bytes, fileName: `image_${index}.${editFileExt(source.mimeType)}`, mediaType: source.mimeType }, + })); + const maskFile = mask === undefined + ? undefined + : { bytes: mask.bytes, fileName: `mask.${editFileExt(mask.mimeType)}`, mediaType: mask.mimeType }; + return { + operation: 'edits', + images, + ...(maskFile === undefined ? {} : { mask: { kind: 'file' as const, file: maskFile } }), + parameters, + }; +}; + +export const prepareImageRequest = async (request: ImageGenerationRequest): Promise => { + const { prompt, config } = request; + const stream = (config.partial_images ?? 0) > 0; + if (request.action === 'generate') return { operation: 'generations', parameters: buildGenerationsBody(prompt, config, stream) }; + const prepared = await prepareEditRequest(request.sources, config); + return buildEditsRequest(prompt, config, prepared.sources, prepared.mask, stream); +}; + +export const portableImageRequest = ( + prompt: string, + action: 'generate' | 'edit', + config: ImageBackendConfig & { mask?: { bytes: ArrayBuffer; mimeType: string } }, + sources: readonly { bytes: ArrayBuffer; mimeType: string }[], +): ImageGenerationRequest => { + const converted = new Map(); + const convert = (source: { bytes: ArrayBuffer; mimeType: string }): ImageSource => { + let portable = converted.get(source); + if (portable === undefined) { + portable = { bytes: new Uint8Array(source.bytes), mimeType: source.mimeType }; + converted.set(source, portable); + } + return portable; + }; + const { mask, ...parameters } = config; + return { + prompt, action, + config: { ...parameters, ...(mask === undefined ? {} : { mask: convert(mask) }) }, + sources: sources.map(convert), + }; +}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/result.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/result.ts new file mode 100644 index 0000000000..cb6041416d --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/result.ts @@ -0,0 +1,133 @@ +import type { HostedToolOutputItem, HostedToolTerminal } from '../types.ts'; +import type { OpenAIImagesStreamEvent } from '@floway-dev/protocols/openai-images'; +import type { OpenAIResponsesOutputImageGenerationCall } from '@floway-dev/protocols/openai-responses'; + +export type ImageError = { type: string; code: string; message: string; retryable: boolean }; + +// Server-resolved tool config echoed by the backend on both the partial_image +// frames and the final result (`background:"auto"` becomes the concrete value +// the server picked, etc.). Read straight off the backend rather than inferred +// from the request, so what we surface matches what was actually rendered. +interface EchoFields { + output_format?: 'png' | 'jpeg'; + quality?: 'low' | 'medium' | 'high'; + background?: 'transparent' | 'opaque'; + size?: string; +} + +export type ImageOutcome = + | { ok: true; b64: string; echo: EchoFields } + | { ok: false; error: ImageError }; + +// Project the server-resolved echo fields out of a backend payload (a response +// JSON body or an SSE event). Each field is validated against the public enum +// so a surprising backend value is dropped rather than echoed verbatim. +export const extractEcho = (source: unknown): EchoFields => { + if (source === null || typeof source !== 'object') return {}; + const s = source as Record; + const echo: EchoFields = {}; + if (s.output_format === 'png' || s.output_format === 'jpeg') echo.output_format = s.output_format; + if (s.quality === 'low' || s.quality === 'medium' || s.quality === 'high') echo.quality = s.quality; + if (s.background === 'transparent' || s.background === 'opaque') echo.background = s.background; + if (typeof s.size === 'string') echo.size = s.size; + return echo; +}; + +const RETRYABLE_IMAGE_ERROR_CODES = new Set([ + 'EngineOverloaded', 'server_error', 'image_generation_server_error', 'image_generation_failed', +]); + +export const isRetryableImageError = (code: string, type?: string): boolean => + RETRYABLE_IMAGE_ERROR_CODES.has(code) || (type !== undefined && RETRYABLE_IMAGE_ERROR_CODES.has(type)); + +export const errorFromBody = (body: string, status: number): { type?: string; code: string; message: string } => { + try { + const parsed = JSON.parse(body) as { error?: { message?: unknown; code?: unknown; type?: unknown } }; + const err = parsed.error; + if (err !== undefined && err !== null) { + return { + ...(typeof err.type === 'string' ? { type: err.type } : {}), + message: typeof err.message === 'string' ? err.message : `Image backend returned HTTP ${status}`, + code: typeof err.code === 'string' ? err.code : `upstream_${status}`, + }; + } + } catch (e) { + if (!(e instanceof SyntaxError)) throw e; + } + return { message: `Image backend returned HTTP ${status}`, code: `upstream_${status}` }; +}; + +// Build the completed/failed `image_generation_call` output item plus its +// closing events. On success the final bytes ride `output_item.done.item.result` +// and a `response.image_generation_call.completed` closes the item; on failure +// neither `.completed` nor any `.partial_image` is emitted — only the failed +// `output_item.done`. +// +// `revised_prompt` is set to the orchestrator's prompt: the standalone images +// backend does no prompt rewriting and returns no `revised_prompt`, and the +// orchestrator's emitted prompt IS already its refined rewrite (it plays the +// role Azure's native flow gives the orchestrator), so it is the faithful +// source for this field. +export const imageTerminal = ( + prompt: string, + action: 'generate' | 'edit', + outcome: ImageOutcome, +): HostedToolTerminal => { + if (!outcome.ok) { + const item: HostedToolOutputItem & Omit = { + type: 'image_generation_call', + status: 'failed', + revised_prompt: prompt, + error: { message: outcome.error.message, code: outcome.error.code, type: outcome.error.type }, + }; + return { item, endEvents: [] }; + } + + const item: HostedToolOutputItem & Omit = { + type: 'image_generation_call', + status: 'completed', + action, + result: outcome.b64, + revised_prompt: prompt, + ...outcome.echo, + }; + return { item, endEvents: [{ type: 'response.image_generation_call.completed' }] }; +}; + +// Native image events become the hosted lifecycle without transport reframing. +// The generations and edits endpoints use distinct event prefixes +// (`image_generation.*` vs `image_edit.*`); only the suffix is matched here. +type ImageStreamSignal = + | { kind: 'partial'; index: number; b64: string; echo: EchoFields } + | { kind: 'completed'; b64: string | undefined; usage: unknown; echo: EchoFields } + | { kind: 'error'; error: ImageError } + | null; + +export const projectImageStreamEvent = (evt: OpenAIImagesStreamEvent): ImageStreamSignal => { + const type = typeof evt.type === 'string' ? evt.type : ''; + if (type.endsWith('.partial_image')) { + return { + kind: 'partial', + index: typeof evt.partial_image_index === 'number' ? evt.partial_image_index : 0, + b64: typeof evt.b64_json === 'string' ? evt.b64_json : '', + echo: extractEcho(evt), + }; + } + if (type.endsWith('.completed')) { + return { kind: 'completed', b64: typeof evt.b64_json === 'string' ? evt.b64_json : undefined, usage: evt.usage, echo: extractEcho(evt) }; + } + if (type === 'error') { + const err = evt.error as { message?: unknown; code?: unknown; type?: unknown } | undefined; + const code = typeof err?.code === 'string' ? err.code : 'server_error'; + const errType = typeof err?.type === 'string' ? err.type : 'image_generation_error'; + return { + kind: 'error', + error: { type: errType, code, message: typeof err?.message === 'string' ? err.message : 'Image backend stream reported an error.', retryable: isRetryableImageError(code, errType) }, + }; + } + return null; +}; + +export const serverError = (error: unknown): ImageError => ({ + type: 'image_generation_error', message: error instanceof Error ? error.message : String(error), code: 'server_error', retryable: true, +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/retry.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/retry.ts new file mode 100644 index 0000000000..f81a1970a7 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/image-sub-request/retry.ts @@ -0,0 +1,79 @@ +import { sleep } from '../../../../../shared/sleep.ts'; +import type { Fields } from '../../../../openai-images/facts.ts'; +import type { BillableEntity } from '../../../../pipeline/facts.ts'; +import type { StreamOutcome } from '../../../../pipeline/serve.ts'; +import type { GatewayServices } from '../../../../pipeline/services.ts'; +import { defineStage, move, defer, type Deferred } from '@floway-dev/pipeline'; + +// 60s cap matches the per-minute refill window of Azure TPM/RPM and +// openai.com tier image quotas — same clamp openai-python applies in +// [`_calculate_retry_timeout`](https://github.com/openai/openai-python/blob/d76d8c11c1da9f97aa8a0aaee8ccd44d2bc8f5e7/src/openai/_base_client.py#L789). +const RETRY_CAP_MS = 60_000; +const MAX_RATE_LIMIT_RETRIES = 2; + +// Header priority matches openai-python's `_parse_retry_after_header` with +// Azure's `x-ms-retry-after-ms` alias added. Treats <= 0 as "no hint" so the +// gpt-image-1 `retry-after: 0.0` quirk falls back to backoff instead of +// pretending the quota is free. +export const parseRetryAfterMs = (headers: Headers): number | null => { + for (const name of ['retry-after-ms', 'x-ms-retry-after-ms']) { + const raw = headers.get(name); + if (raw === null) continue; + const ms = Number(raw); + if (Number.isFinite(ms) && ms > 0) return ms; + } + const ra = headers.get('retry-after'); + if (ra !== null) { + const seconds = Number(ra); + if (Number.isFinite(seconds) && seconds > 0) return seconds * 1000; + const httpDateMs = Date.parse(ra); + if (!Number.isNaN(httpDateMs)) { + const delta = httpDateMs - Date.now(); + if (delta > 0) return delta; + } + } + return null; +}; + +export const retryRateLimitedImages = defineStage< + Fields<'request.openaiImages.canonical' | 'route.attempt' | 'serve.usage.prior'>, + Fields<'request.openaiImages.canonical' | 'route.attempt' | 'serve.usage.prior'>, + Fields<'response.http.status' | 'response.http.headers' | 'response.http.body' | 'response.usage.billable' | 'response.openaiImages.streamedUsage'>, + Fields<'response.http.status' | 'response.http.headers' | 'response.http.body' | 'response.usage.billable' | 'response.openaiImages.streamedUsage'>, + GatewayServices +>({ + name: 'retryRateLimitedImages', + through: { + request: { needs: ['request.openaiImages.canonical', 'route.attempt', 'serve.usage.prior'], consumes: [], provides: ['serve.usage.prior'] }, + response: { + needs: ['response.http.status', 'response.http.headers', 'response.http.body', 'response.usage.billable', 'response.openaiImages.streamedUsage'], + consumes: ['response.http.body'], + provides: ['response.http.body', 'response.usage.billable', 'response.openaiImages.streamedUsage'], + }, + }, + execute: async (facts, next, use) => { + const prior: BillableEntity[] = []; + for (let retry = 0; ; retry++) { + const back = await next({ ...facts, 'serve.usage.prior': move([...facts['serve.usage.prior'], ...prior]) }); + if (back['response.http.status'] !== 429 || retry >= MAX_RATE_LIMIT_RETRIES) { + if (prior.length === 0) return { ...back, 'serve.usage.prior': facts['serve.usage.prior'] }; + const pending = back['response.openaiImages.streamedUsage'] as Deferred | null; + const calls = move([...prior]); + return { + ...back, + 'serve.usage.prior': facts['serve.usage.prior'], + 'response.usage.billable': move([...calls, ...back['response.usage.billable']]), + 'response.openaiImages.streamedUsage': pending === null ? null + : move(defer(pending.then(outcome => ({ ...outcome, billable: move([...calls, ...outcome.billable]) })))), + }; + } + prior.push(...back['response.usage.billable']); + const base = 1000 * 2 ** retry; + const backoffMs = base + Math.random() * base * 0.25; + const headers = new Headers(back['response.http.headers'].map(([name, value]): [string, string] => [name, value])); + await sleep(Math.min(parseRetryAfterMs(headers) ?? backoffMs, RETRY_CAP_MS), use.gateway.abortSignal); + use.gateway.attempt.timing.upstreamCallStartedAt = null; + use.gateway.attempt.timing.firstOutputTokenAt = null; + } + }, +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request.ts new file mode 100644 index 0000000000..71c173aac4 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request.ts @@ -0,0 +1,46 @@ +import type { WebSearchRequest } from './search-sub-request/facts.ts'; +import { webSearchSubRequestPipeline } from './search-sub-request/pipeline.ts'; +import type { WebSearchRuntime } from './search-sub-request/services.ts'; +import { prologueFor } from '../../../pipeline/serve.ts'; +import type { GatewayCtx, AttemptState } from '../../../shared/gateway-ctx.ts'; +import type { WebSearchCallIR } from '../../../tools/web-search/operations.ts'; +import { run, move } from '@floway-dev/pipeline'; + +/** + * The run a dispatcher's search is. + * + * Its context is the parent's, with the three things a separate run owes itself: a start time of + * its own, an attempt slot of its own — so the outer turn's upstream stamp survives — and a + * record of its own. + */ +export const runWebSearchSubRequest = async ( + parent: GatewayCtx, + request: WebSearchRequest, + webSearch: WebSearchRuntime, +): Promise => { + const attempt: AttemptState = { timing: { firstOutputTokenAt: null, upstreamCallStartedAt: null }, telemetry: undefined }; + const dump = parent.dump?.openSubRequest({ method: 'POST', path: '/alpha/search' }, false, attempt.timing) ?? null; + const gateway: GatewayCtx = { + ...parent, + requestStartedAt: Date.now(), + attempt, + dump, + }; + const prologue = prologueFor(gateway, { body: { bytes: new Uint8Array(), streamError: null }, headers: [] }, dump); + + try { + const { facts, drain } = await run( + webSearchSubRequestPipeline, + move({ 'request.webSearch.canonical': request }), + { ...prologue.services, webSearch }, + ); + prologue.runDump?.afterRun(drain); + await drain(); + dump?.finalize(200, 0); + return facts['response.webSearch.ir']; + } catch (error) { + dump?.failed(error); + dump?.finalize(500, 0); + throw error; + } +}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/execute.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/execute.ts new file mode 100644 index 0000000000..0e14fb97aa --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/execute.ts @@ -0,0 +1,114 @@ +import type { Fields, WebSearchRequest } from './facts.ts'; +import type { SearchServices, WebSearchRuntime } from './services.ts'; +import { providerEntry } from '../../../../pipeline/provider-entry.ts'; +import { providerCalls } from '../../../../pipeline/provider-usage.ts'; +import { spentBody } from '../../../../pipeline/upstream-body.ts'; +import { upstreamPerformanceContext } from '../../../../shared/telemetry/attribution.ts'; +import { executeAlphaSearch } from '../../../../tools/web-search/alpha-search/execution.ts'; +import type { AlphaSearchDispatcher } from '../../../../tools/web-search/alpha-search/upstream.ts'; +import { assertLocalWebSearchSupport, executeOperationToIr, parseWebSearchOperations, runBackendSearchMulti, schemaErrorIr, startBatchFetch, UnsupportedLocalWebSearchFeatureError, unsupportedLocalWebSearchFeatureIr, type WebSearchCallIR, type WebSearchOperation } from '../../../../tools/web-search/operations.ts'; +import { exchangeResponse } from '@floway-dev/http/pipeline'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { OpenAIResponsesWebSearchAction } from '@floway-dev/protocols/openai-responses'; +import type { ProviderOperationRequest, ProviderResponse } from '@floway-dev/provider'; + +const ITERATION_CAP = 30; + +export const runWebSearchCall = defineStage< + Fields<'request.webSearch.canonical'>, + ProviderOperationRequest<'alphaSearch'>, + ProviderResponse, + Fields<'response.webSearch.ir' | 'response.usage.billable'>, + Fields<'response.webSearch.ir' | 'response.usage.billable'>, + SearchServices +>({ + name: 'runWebSearchCall', + into: { + request: { needs: ['request.webSearch.canonical'], consumes: [], provides: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'] }, + response: { needs: ['response.http.exchange', 'response.http.body', 'response.provider.called', 'response.provider.modelKey', 'response.provider.previousCalls'], consumes: ['response.http.exchange', 'response.http.body', 'response.provider.called', 'response.provider.modelKey', 'response.provider.previousCalls'], provides: ['response.webSearch.ir', 'response.usage.billable'] }, + }, + return: { provides: ['response.webSearch.ir', 'response.usage.billable'] }, + execute: async (facts, next, use) => { + let billable: Fields<'response.usage.billable'>['response.usage.billable'] = []; + const dispatch: AlphaSearchDispatcher = async (body, _signal, headers) => { + const alpha = use.webSearch.alpha; + if (alpha === undefined) throw new Error('Alpha Search was dispatched without a selected provider'); + const candidate = await alpha.resolveCandidate(); + const pipeline = candidate.provider.pipelines.alphaSearch; + if (pipeline === undefined) throw new Error(`Provider ${candidate.provider.kind} has no Alpha Search pipeline`); + const selector = use.rememberCandidates([candidate])[0]; + use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'chat'); + const { model: _callerModel, ...payload } = body; + const back = await next(providerEntry({ ...facts, 'route.attempt': selector, 'ingress.http.headers': [...headers] }, candidate, payload), pipeline); + billable = providerCalls(candidate, back as unknown as Record); + const exchange = back['response.http.exchange']; + if (exchange.type === 'transportFailure') throw exchange.error; + const response = exchangeResponse(exchange); + // The finite reader owns completion, including rejection; canceling its locked body would mask the read error. + spentBody(exchange.body); + const bytes = await response.arrayBuffer(); + return new Response(exchange.body === null ? null : bytes, { status: exchange.status, statusText: exchange.statusText, headers: exchange.headers.map(([name, value]): [string, string] => [name, value]) }); + }; + const ir = await execute(facts['request.webSearch.canonical'], use.webSearch, dispatch); + return move({ ...facts, 'response.webSearch.ir': ir, 'response.usage.billable': billable }); + }, +}); + +const execute = async (request: WebSearchRequest, runtime: WebSearchRuntime, dispatch: AlphaSearchDispatcher): Promise => { + const { commands, toolName, iterationCount } = request; + const parsed = parseWebSearchOperations(commands); + if (iterationCount > ITERATION_CAP) { + return schemaErrorIr( + 'tool budget exhausted', + 'Tool call budget exhausted', + `Web search iteration limit (${ITERATION_CAP}) reached. Further web_search calls in this response will return this same error. Summarize what you have already learned, and continue the task using other available tools (shell, file inspection, prior knowledge) or directly answer based on what you've gathered.`, + ); + } + if (parsed.kind === 'malformed' || parsed.ops.length === 0) { + return schemaErrorIr( + 'malformed dispatcher call arguments', + 'Malformed arguments', + 'Error: arguments must be a JSON object with sub-property arrays (search_query[], open[], find[]).', + ); + } + if (runtime.alpha !== undefined) { + const first = parsed.ops[0]; + let action: OpenAIResponsesWebSearchAction; + if (first.kind === 'search') { + const queries = parsed.ops.filter((op): op is Extract => op.kind === 'search').map(op => op.query); + action = queries.length === 1 + ? { type: 'search', query: queries[0], queries } + : { type: 'search', query: queries.join(' | '), queries }; + } else if (first.kind === 'open') { + action = { type: 'open_page', url: first.url }; + } else if (first.kind === 'find') { + action = { type: 'find_in_page', url: first.url, pattern: first.pattern }; + } else { + action = { type: 'search', query: Object.keys(commands).join(', ') }; + } + return await executeAlphaSearch({ dispatcher: dispatch, sessionId: runtime.alpha.sessionId, commands: request.commands, settings: request.settings, input: request.input, action, signal: runtime.session.signal }); + } + try { + assertLocalWebSearchSupport(commands); + } catch (error) { + if (error instanceof UnsupportedLocalWebSearchFeatureError) { + return unsupportedLocalWebSearchFeatureIr({ type: 'search', query: Object.keys(commands).join(', ') }, error.message); + } + throw error; + } + const session = { ...runtime.session, filters: request.filters, includeSearchActionSources: request.includeSearchActionSources }; + if (parsed.ops.length > 1 && parsed.ops.every(op => op.kind === 'search' && op.error === undefined)) { + return await runBackendSearchMulti(parsed.ops as Extract[], session); + } + if (parsed.ops.length > 1) { + return schemaErrorIr( + 'ambiguous dispatcher call', + 'Ambiguous tool call', + `Error: ambiguous \`${toolName}\` tool call — each function_call maps to one web_search_call. ` + + 'Multiple `search_query` entries are fine (they collapse into one search). ' + + 'For `open`/`find`, or any mix of kinds, split into one call per `open[]` entry, `find[]` entry, or `search_query[]` batch.', + ); + } + const batch = await startBatchFetch(parsed, session); + return await executeOperationToIr(parsed.ops[0], session, batch); +}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/facts.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/facts.ts new file mode 100644 index 0000000000..d2a1b6d3cc --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/facts.ts @@ -0,0 +1,20 @@ +import type { GatewayFacts } from '../../../../pipeline/facts.ts'; +import type { WebSearchCallIR, WebSearchFilters } from '../../../../tools/web-search/operations.ts'; +import type { OpenAIResponsesInputItem } from '@floway-dev/protocols/openai-responses'; + +export interface WebSearchRequest { + commands: Record; + toolName: string; + iterationCount: number; + filters: WebSearchFilters; + includeSearchActionSources: boolean; + settings: Record; + input: OpenAIResponsesInputItem[]; +} + +export interface WebSearchSubRequestFacts extends GatewayFacts { + 'request.webSearch.canonical': WebSearchRequest; + 'response.webSearch.ir': WebSearchCallIR; +} + +export type Fields = { [P in K]: WebSearchSubRequestFacts[P] }; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/pipeline.ts new file mode 100644 index 0000000000..3d52cf07b4 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/pipeline.ts @@ -0,0 +1,10 @@ +import { runWebSearchCall } from './execute.ts'; +import type { Fields } from './facts.ts'; +import { writeSettlement } from '../../../../pipeline/settlement.ts'; +import { compose, type Pipeline } from '@floway-dev/pipeline'; + +export const webSearchSubRequestPipeline: Pipeline, Fields<'response.webSearch.ir'>> = + compose('webSearchSubRequest', [ + writeSettlement(() => false), + runWebSearchCall, + ]); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/services.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/services.ts new file mode 100644 index 0000000000..e25d169ded --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/search-sub-request/services.ts @@ -0,0 +1,10 @@ +import type { GatewayServices } from '../../../../pipeline/services.ts'; +import type { WebSearchExecutionSession } from '../../../../tools/web-search/operations.ts'; +import type { ModelCandidate } from '@floway-dev/provider'; + +export interface WebSearchRuntime { + readonly session: Omit; + readonly alpha?: { readonly resolveCandidate: () => Promise; readonly sessionId: string }; +} + +export interface SearchServices extends GatewayServices { readonly webSearch: WebSearchRuntime } diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tool-shim.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/shared.ts similarity index 57% rename from packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tool-shim.ts rename to packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/shared.ts index b4bd058c67..405759e382 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tool-shim.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/shared.ts @@ -1,163 +1,10 @@ import { jsonrepair } from 'jsonrepair'; -import type { OpenAIResponsesInterceptor, OpenAIResponsesInvocation } from './types.ts'; +import type { MergeState, MergeUsage, ActiveHostedTool, HostedToolRewrite, HostedToolDispatcher, UpstreamTerminal, HostedToolLifecycleEvent, HostedToolOutputItem, HostedToolResultSlot, HostedToolTerminal, HostedToolLoopState, TurnSummary, InterceptedFunctionCall, DispatchedHostedToolSlot, SynthesizedTerminal } from './types.ts'; import { truncatePreservingCodePoints } from '../../../shared/text.ts'; -import type { ChatGatewayCtx } from '../../shared/gateway-ctx.ts'; import type { OpenAIResponsesStatefulStore } from '../items/store.ts'; -import type { InterceptorRun } from '@floway-dev/interceptor'; -import { eventFrame, sumBillableUsage, type ProtocolFrame } from '@floway-dev/protocols/common'; -import { - createRandomOpenAIResponsesItemId, - type CanonicalOpenAIResponsesPayload, - type OpenAIResponsesFunctionTool, - type OpenAIResponsesHostedTool, - type OpenAIResponsesInputItem, - type OpenAIResponsesOutputFunctionCall, - type OpenAIResponsesOutputItem, - type OpenAIResponsesResult, - type OpenAIResponsesStreamEvent, - type OpenAIResponsesTool, - type OpenAIResponsesToolChoice, -} from '@floway-dev/protocols/openai-responses'; -import type { EventResultMetadata, ExecuteResult } from '@floway-dev/provider'; - -export interface MergeUsage { - input_tokens?: number; - output_tokens?: number; - total_tokens?: number; - input_tokens_details?: { cached_tokens: number }; - output_tokens_details?: { reasoning_tokens: number }; -} - -export interface MergeState { - sequenceNumber: number; - outputIndex: number; - accumulatedOutput: Map; - accumulatedUsage: MergeUsage; - lastSeenModel: string | null; - synthesizedResponseId: string; - upstreamResponseSnapshot: OpenAIResponsesResult | undefined; -} - -export interface InterceptedFunctionCall { - callId: string; - name: string; - /** - * Parsed and jsonrepair-cleaned `arguments` object. `null` when the - * raw upstream string is not a JSON object even after jsonrepair — - * dispatchers handle that case via their own error path. Dispatchers - * that persist function-call inputs across turns should serialize - * this rather than the raw upstream string; replaying the raw form - * would re-break the next turn the same way it broke this one. - */ - arguments: Record | null; -} - -export interface ServerToolTerminal { - item: ServerToolOutputItem; - endEvents: ServerToolLifecycleEvent[]; - /** - * Optional server-only blob registered on the request's item store under - * `slot.id` before the slot's wire item leaves materialize. The persistence - * layer stores it in `payload.private`; the replay-side `transformItems` - * reads it back to reconstruct the full IR. - */ - privatePayload?: unknown; -} - -export interface ServerToolResultSlot { - id: string; - startItem: ServerToolOutputItem; - startEvents: readonly ServerToolLifecycleEvent[]; - // The deferred portion of a slot's lifecycle, driven at materialization - // time. It yields any intermediate lifecycle events as they arrive — e.g. - // progressively-rendered `image_generation_call.partial_image` frames a - // streamed backend delivers over the course of the call — and returns the - // terminal item plus its closing events (and an optional server-only - // `privatePayload`). A tool with no progressive output simply yields nothing - // and returns immediately. - run: () => AsyncGenerator; -} - -export type ServerToolOutputItem = { type: string; id?: string; [key: string]: unknown }; - -export type ServerToolLifecycleEvent = { type: string; [key: string]: unknown }; - -export interface ServerToolLoopState { - iterationCount: number; - remainingToolCalls: number | undefined; -} - -export interface DispatchedServerToolSlot { - intercepted: InterceptedFunctionCall; - slot: ServerToolResultSlot; - outputIndex: number; -} - -export type ServerToolDispatcher = (args: { - intercepted: InterceptedFunctionCall; - loopState: ServerToolLoopState; -}) => ServerToolResultSlot[]; - -// Keep hosted matching, function injection, and dispatch atomic so a -// registration cannot silently omit part of a server-tool family. -export interface ServerToolHostedDispatch { - hostedTypes: readonly string[]; - canonicalize: (raw: OpenAIResponsesTool) => OpenAIResponsesHostedTool | undefined; - buildFunctionTool: (canonical: OpenAIResponsesHostedTool, toolName: string) => OpenAIResponsesFunctionTool; - dispatcher: ServerToolDispatcher; -} - -export type ServerToolPrepareResult = - | { type: 'inactive' } - // `errorType` / `code` override the envelope for tools that emulate an - // upstream's rejection vocabulary. An omitted code falls back to the - // generic `invalid_request_error`; explicit null is preserved verbatim. - | { type: 'invalid-request'; message: string; param: string | null; errorType?: string; code?: string | null } - | { - type: 'active'; - baseToolName: string; - // History rewrite, applied whether or not the tool is hosted this - // turn so items echoed from a previous turn's output become - // upstream-readable even on a request that no longer declares the - // hosted tool. - transformItems?: (items: OpenAIResponsesInputItem[], toolName: string) => OpenAIResponsesInputItem[]; - // Present only when the request declares this hosted tool; absent for - // replay-only activation. - hosted?: ServerToolHostedDispatch; - }; - -export type ServerToolRegistration = (invocation: OpenAIResponsesInvocation, gatewayCtx: ChatGatewayCtx) => ServerToolPrepareResult | Promise; - -type ActiveServerTool = Extract & { - toolName: string; - // Hosted entries are rewritten at their input-item positions; only - // top-level replacements participate in response.tools restoration. - canonicalHostedTool: OpenAIResponsesHostedTool | undefined; - // Captures the exact forced choice shape before request rewriting. - originalToolChoice: Exclude | undefined; -}; - -// How a single upstream turn ended, as observed while consuming its -// stream. Carries the raw upstream `response` for failed/incomplete so -// the loop can lift the upstream `error` / `incomplete_details`, plus a -// `bare-error-pre-shell` variant for an `error` event that arrived -// before any `response.created` (no model known yet). Distinct from -// `SynthesizedTerminal`, which is the shim's own outgoing terminal. -export type UpstreamTerminal = - | { kind: 'completed' } - | { kind: 'failed'; response: OpenAIResponsesResult } - | { kind: 'incomplete'; response: OpenAIResponsesResult } - | { kind: 'bare-error-pre-shell'; error: { message: string; code: string } }; - -export interface TurnSummary { - dispatched: Array<{ intercepted: InterceptedFunctionCall; slots: DispatchedServerToolSlot[] }>; - sawClientToolCall: boolean; - turnUsage: MergeUsage; - terminalStatus: UpstreamTerminal; -} - -type LatestUpstreamMetadata = Pick; +import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; +import { createRandomOpenAIResponsesItemId, type OpenAIResponsesOutputItem, type OpenAIResponsesResult, type OpenAIResponsesToolChoice, type OpenAIResponsesTool, type OpenAIResponsesInputItem, type OpenAIResponsesHostedTool, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesStreamEvent, type OpenAIResponsesOutputFunctionCall } from '@floway-dev/protocols/openai-responses'; export const createMergeState = (): MergeState => ({ sequenceNumber: 0, @@ -165,7 +12,7 @@ export const createMergeState = (): MergeState => ({ accumulatedOutput: new Map(), accumulatedUsage: {}, lastSeenModel: null, - synthesizedResponseId: `resp_shim_${crypto.randomUUID().replace(/-/g, '')}`, + synthesizedResponseId: `resp_hosted_${crypto.randomUUID().replace(/-/g, '')}`, upstreamResponseSnapshot: undefined, }); @@ -226,9 +73,9 @@ const usageOf = (usage: OpenAIResponsesResult['usage']): MergeUsage => { return out; }; -const rewriteHostedToolChoice = ( +export const rewriteHostedToolChoice = ( toolChoice: OpenAIResponsesToolChoice | null | undefined, - active: readonly ActiveServerTool[], + active: readonly ActiveHostedTool[], ): OpenAIResponsesToolChoice | null | undefined => { if (toolChoice == null || typeof toolChoice === 'string') return toolChoice; if (toolChoice.type === 'allowed_tools' && Array.isArray(toolChoice.tools)) { @@ -247,9 +94,9 @@ const rewriteHostedToolChoice = ( return toolChoice; }; -const hostedToolChoiceToRestore = ( +export const hostedToolChoiceToRestore = ( choice: OpenAIResponsesToolChoice | null | undefined, - hosted: ServerToolHostedDispatch | undefined, + hosted: HostedToolRewrite | undefined, toolName: string, ): Exclude | undefined => { if (hosted === undefined || typeof choice !== 'object' || choice === null) return undefined; @@ -264,9 +111,9 @@ const hostedToolChoiceToRestore = ( return selectsHostedTool ? choice : undefined; }; -const isForcedServerToolChoice = ( +export const isForcedHostedToolChoice = ( choice: OpenAIResponsesToolChoice | null | undefined, - dispatchers: ReadonlyMap, + dispatchers: ReadonlyMap, ): boolean => { if (choice === 'required') return true; if (typeof choice !== 'object' || choice === null) return false; @@ -276,11 +123,11 @@ const isForcedServerToolChoice = ( && typeof selector.name === 'string' && dispatchers.has(selector.name)); }; -// The shim demotes forced choice to `auto` after the first turn, so synthesized +// The dispatcher demotes forced choice to `auto` after the first turn, so synthesized // echoes restore the captured client shape rather than the final upstream echo. const restoreEchoedToolChoice = ( toolChoice: OpenAIResponsesToolChoice | null | undefined, - active: readonly ActiveServerTool[], + active: readonly ActiveHostedTool[], ): OpenAIResponsesToolChoice | null | undefined => { for (const entry of active) { if (entry.originalToolChoice !== undefined) return entry.originalToolChoice; @@ -293,7 +140,7 @@ const restoreEchoedToolChoice = ( // by upstream for ordinary client tools. const restoreEchoedTools = ( tools: readonly OpenAIResponsesTool[] | undefined, - active: readonly ActiveServerTool[], + active: readonly ActiveHostedTool[], ): OpenAIResponsesTool[] | undefined => { if (tools === undefined) return undefined; return tools.map(tool => { @@ -307,7 +154,7 @@ const restoreEchoedTools = ( }); }; -export const resolveServerToolName = ( +export const resolveHostedToolName = ( baseName: string, tools: readonly OpenAIResponsesTool[], input: readonly OpenAIResponsesInputItem[] = [], @@ -351,10 +198,10 @@ export const resolveServerToolName = ( const candidate = `${baseName}_${i}`; if (!taken.has(candidate)) return candidate; } - throw new Error(`Unable to resolve a free server tool function name for ${baseName} within ${MAX_NAME_RESOLUTION_ATTEMPTS} attempts`); + throw new Error(`Unable to resolve a free hosted tool function name for ${baseName} within ${MAX_NAME_RESOLUTION_ATTEMPTS} attempts`); }; -const historicalClientCallableUsesName = (name: string, input: readonly OpenAIResponsesInputItem[]): boolean => +export const historicalClientCallableUsesName = (name: string, input: readonly OpenAIResponsesInputItem[]): boolean => input.some(item => { if (item.type === 'additional_tools' || item.type === 'tool_search_output') { return Array.isArray(item.tools) && item.tools.some(tool => @@ -368,9 +215,9 @@ const historicalClientCallableUsesName = (name: string, input: readonly OpenAIRe // Collapse matching hosted declarations within each tools array. Keep the // last declaration's configuration at the first matching slot so unrelated // tools retain their relative order. -const rewriteToolsForHostedShim = ( +export const rewriteHostedTools = ( tools: readonly OpenAIResponsesTool[], - hosted: ServerToolHostedDispatch, + hosted: HostedToolRewrite, toolName: string, ): { rewritten: OpenAIResponsesTool[]; canonicalHostedTool: OpenAIResponsesHostedTool } => { const rewritten: OpenAIResponsesTool[] = []; @@ -389,21 +236,21 @@ const rewriteToolsForHostedShim = ( canonicalHostedTool = canonical; } if (canonicalHostedTool === undefined) { - throw new Error('Hosted server-tool registration did not match any request tool'); + throw new Error('Hosted hosted-tool registration did not match any request tool'); } rewritten[replacementIndex] = hosted.buildFunctionTool(canonicalHostedTool, toolName); return { rewritten, canonicalHostedTool }; }; -const rewriteHostedDeclarations = ( +export const rewriteHostedDeclarations = ( payload: CanonicalOpenAIResponsesPayload, - hosted: ServerToolHostedDispatch, + hosted: HostedToolRewrite, toolName: string, ): { payload: CanonicalOpenAIResponsesPayload; canonicalTopLevelHostedTool: OpenAIResponsesHostedTool | undefined } => { let canonicalHostedTool: OpenAIResponsesHostedTool | undefined; const rewrite = (tools: OpenAIResponsesTool[]): OpenAIResponsesTool[] => { if (!tools.some(tool => hosted.canonicalize(tool) !== undefined)) return tools; - const result = rewriteToolsForHostedShim(tools, hosted, toolName); + const result = rewriteHostedTools(tools, hosted, toolName); canonicalHostedTool = result.canonicalHostedTool; return result.rewritten; }; @@ -415,7 +262,7 @@ const rewriteHostedDeclarations = ( return rewritten === item.tools ? item : { ...item, tools: rewritten }; }); if (canonicalHostedTool === undefined) { - throw new Error('Hosted server-tool registration did not match any request tool'); + throw new Error('Hosted hosted-tool registration did not match any request tool'); } return { payload: { ...payload, ...(tools === undefined ? {} : { tools }), input }, @@ -423,7 +270,7 @@ const rewriteHostedDeclarations = ( }; }; -export const parseServerToolArguments = (argumentsJson: string): Record | null => { +export const parseHostedToolArguments = (argumentsJson: string): Record | null => { if (argumentsJson === '') return {}; let parsed: unknown; try { @@ -441,11 +288,11 @@ const syntheticPrologueResponse = ( state: MergeState, id: string, model: string, - active: readonly ActiveServerTool[], + active: readonly ActiveHostedTool[], status: 'queued' | 'in_progress', ): OpenAIResponsesResult => { if (state.upstreamResponseSnapshot === undefined) { - throw new Error('Server-tool shim cannot synthesize an OpenAI Responses prologue envelope before an upstream response snapshot is captured.'); + throw new Error('Hosted-tool execution cannot synthesize an OpenAI Responses prologue envelope before an upstream response snapshot is captured.'); } const snapshot = state.upstreamResponseSnapshot; const restoredTools = restoreEchoedTools(snapshot.tools, active); @@ -506,11 +353,11 @@ const captureTerminalEvent = ( return null; }; -const stampServerToolEvent = ( +const stampHostedToolEvent = ( merge: MergeState, outputIndex: number, itemId: string, - event: ServerToolLifecycleEvent, + event: HostedToolLifecycleEvent, ): ProtocolFrame => eventFrame({ ...event, @@ -519,44 +366,44 @@ const stampServerToolEvent = ( sequence_number: merge.sequenceNumber++, } as OpenAIResponsesStreamEvent); -const attachServerToolItemId = (item: ServerToolOutputItem, id: string): OpenAIResponsesOutputItem => ({ ...item, id } as OpenAIResponsesOutputItem); +const attachHostedToolItemId = (item: HostedToolOutputItem, id: string): OpenAIResponsesOutputItem => ({ ...item, id } as OpenAIResponsesOutputItem); -const serverToolStartFrames = ( +const hostedToolStartFrames = ( merge: MergeState, outputIndex: number, - slot: ServerToolResultSlot, + slot: HostedToolResultSlot, ): ProtocolFrame[] => [ eventFrame({ type: 'response.output_item.added', output_index: outputIndex, - item: attachServerToolItemId(slot.startItem, slot.id), + item: attachHostedToolItemId(slot.startItem, slot.id), sequence_number: merge.sequenceNumber++, } as OpenAIResponsesStreamEvent), - ...slot.startEvents.map(event => stampServerToolEvent(merge, outputIndex, slot.id, event)), + ...slot.startEvents.map(event => stampHostedToolEvent(merge, outputIndex, slot.id, event)), ]; -const serverToolEndFrames = ( +const hostedToolEndFrames = ( merge: MergeState, outputIndex: number, - slot: ServerToolResultSlot, - result: ServerToolTerminal, + slot: HostedToolResultSlot, + result: HostedToolTerminal, ): ProtocolFrame[] => { const frames = [ - ...result.endEvents.map(event => stampServerToolEvent(merge, outputIndex, slot.id, event)), + ...result.endEvents.map(event => stampHostedToolEvent(merge, outputIndex, slot.id, event)), eventFrame({ type: 'response.output_item.done', output_index: outputIndex, - item: attachServerToolItemId(result.item, slot.id), + item: attachHostedToolItemId(result.item, slot.id), sequence_number: merge.sequenceNumber++, } as OpenAIResponsesStreamEvent), ]; - merge.accumulatedOutput.set(outputIndex, attachServerToolItemId(result.item, slot.id)); + merge.accumulatedOutput.set(outputIndex, attachHostedToolItemId(result.item, slot.id)); return frames; }; -const transformServerToolItems = ( +export const transformHostedToolItems = ( items: OpenAIResponsesInputItem[], - active: readonly ActiveServerTool[], + active: readonly ActiveHostedTool[], ): OpenAIResponsesInputItem[] => { let next = items; for (const entry of active) { @@ -569,11 +416,11 @@ export const consumeTurnStreaming = async function* ( frames: AsyncIterable>, merge: MergeState, isFirstTurn: boolean, - dispatchers: ReadonlyMap, - loopState: ServerToolLoopState, - active: readonly ActiveServerTool[], + dispatchers: ReadonlyMap, + loopState: HostedToolLoopState, + active: readonly ActiveHostedTool[], ): AsyncGenerator, TurnSummary> { - const dispatched: Array<{ intercepted: InterceptedFunctionCall; slots: DispatchedServerToolSlot[] }> = []; + const dispatched: Array<{ intercepted: InterceptedFunctionCall; slots: DispatchedHostedToolSlot[] }> = []; let sawClientToolCall = false; let turnUsage: MergeUsage = {}; let terminalStatus: UpstreamTerminal | undefined = undefined; @@ -594,7 +441,7 @@ export const consumeTurnStreaming = async function* ( const ensureModel = (): string => { if (merge.lastSeenModel === null) { - throw new Error('Server-tool shim cannot synthesize an OpenAI Responses envelope because upstream `response.created` did not report a `model` field.'); + throw new Error('Hosted-tool execution cannot synthesize an OpenAI Responses envelope because upstream `response.created` did not report a `model` field.'); } return merge.lastSeenModel; }; @@ -672,7 +519,7 @@ export const consumeTurnStreaming = async function* ( const item = event.item; if (item.type === 'function_call') { if (dispatchers.has(item.name)) { - // Reserve the downstream index the shim call occupies now, at + // Reserve the downstream index the dispatcher call occupies now, at // `.added`; the actual slot count is only known at `.done`, // where slot 0 takes this reserved index and any further slots // take fresh ones. Those stay contiguous because OpenAI Responses @@ -721,7 +568,7 @@ export const consumeTurnStreaming = async function* ( if (intercepted !== undefined) { if (event.item.type !== 'function_call' || event.item.name !== intercepted.intercepted.name || event.item.call_id !== intercepted.intercepted.callId || event.item.id !== intercepted.addedItem.id) { - throw new Error('Upstream changed a server-tool function identity before closing its call.'); + throw new Error('Upstream changed a hosted-tool function identity before closing its call.'); } const finalDispatcher = event.item.namespace === undefined ? dispatchers.get(event.item.name) : undefined; if (finalDispatcher === undefined) { @@ -746,14 +593,14 @@ export const consumeTurnStreaming = async function* ( } intercepted.intercepted.name = event.item.name; intercepted.argumentsJson = event.item.arguments; - intercepted.intercepted.arguments = parseServerToolArguments(intercepted.argumentsJson); + intercepted.intercepted.arguments = parseHostedToolArguments(intercepted.argumentsJson); const slots = finalDispatcher({ intercepted: intercepted.intercepted, loopState }); if (loopState.remainingToolCalls !== undefined) loopState.remainingToolCalls -= 1; - const dispatchedSlots: DispatchedServerToolSlot[] = []; + const dispatchedSlots: DispatchedHostedToolSlot[] = []; for (const [slotIndex, slot] of slots.entries()) { const outputIndex = slotIndex === 0 ? intercepted.reservedOutputIndex : merge.outputIndex++; dispatchedSlots.push({ intercepted: intercepted.intercepted, slot, outputIndex }); - yield* serverToolStartFrames(merge, outputIndex, slot); + yield* hostedToolStartFrames(merge, outputIndex, slot); } dispatched.push({ intercepted: intercepted.intercepted, slots: dispatchedSlots }); continue; @@ -864,7 +711,7 @@ export const consumeTurnStreaming = async function* ( output: [], status: 'failed', error: { - message: `Upstream emitted ${priorLabel} without closing shim call items at upstream output_index ${unmatched.join(', ')}.`, + message: `Upstream emitted ${priorLabel} without closing dispatcher call items at upstream output_index ${unmatched.join(', ')}.`, code: 'server_error', }, incomplete_details: null, @@ -877,11 +724,15 @@ export const consumeTurnStreaming = async function* ( const MAX_BODY_EXCERPT_CHARS = 512; -const buildErrorFromResult = ( - result: Exclude, { type: 'events' }>, +/** What a refusal mid-loop becomes on the synthesized terminal. An upstream that wrote an + * OpenAI-shaped envelope is quoted verbatim — its own code, type and sentence — because that is + * what a client reads; anything else is reported as the status plus an excerpt of what came. */ +export const buildErrorFromRefusal = ( + status: number, + body: string, ): NonNullable => { - if (result.type === 'internal-error') return { message: result.error.message, code: 'server_error' }; - const decoded = new TextDecoder('utf-8', { fatal: false }).decode(result.body); + const result = { status, body }; + const decoded = body; let parsed: unknown = undefined; try { parsed = JSON.parse(decoded); @@ -906,55 +757,22 @@ const buildErrorFromResult = ( }; }; -const invalidRequestEnvelope = ( - message: string, - param: string | null, - code: string | null | undefined, - errorType = 'invalid_request_error', -): ExecuteResult> => { - const body = JSON.stringify({ - error: { - message, - type: errorType, - param, - code: code === undefined ? 'invalid_request_error' : code, - }, - }); - return { - type: 'api-error', - source: 'gateway', - status: 400, - headers: new Headers({ 'content-type': 'application/json' }), - body: new TextEncoder().encode(body), - }; -}; - -// The terminal the shim emits downstream. Unlike `UpstreamTerminal` -// (what we observed), this carries only the already-extracted `error` / -// `incompleteDetails` the synthesized envelope needs; the output and -// usage come from accumulated shim state. There is no pre-shell variant -// here — synthesis always runs after a model is known. -type SynthesizedTerminal = - | { kind: 'completed' } - | { kind: 'failed'; error: OpenAIResponsesResult['error'] } - | { kind: 'incomplete'; incompleteDetails: OpenAIResponsesResult['incomplete_details'] }; - const SYNTHESIZED_TERMINAL_FRAME: Record = { completed: { type: 'response.completed', status: 'completed' }, failed: { type: 'response.failed', status: 'failed' }, incomplete: { type: 'response.incomplete', status: 'incomplete' }, }; -const synthesizeTerminalEnvelope = ( +export const synthesizeTerminalEnvelope = ( state: MergeState, kind: SynthesizedTerminal, - active: readonly ActiveServerTool[], + active: readonly ActiveHostedTool[], ): ProtocolFrame => { if (state.lastSeenModel === null) { - throw new Error('Server-tool shim cannot synthesize an OpenAI Responses terminal envelope before upstream `response.created` reports a model.'); + throw new Error('Hosted-tool execution cannot synthesize an OpenAI Responses terminal envelope before upstream `response.created` reports a model.'); } if (state.upstreamResponseSnapshot === undefined) { - throw new Error('Server-tool shim cannot synthesize an OpenAI Responses terminal envelope before upstream `response.created` is captured.'); + throw new Error('Hosted-tool execution cannot synthesize an OpenAI Responses terminal envelope before upstream `response.created` is captured.'); } const output = materializeAccumulatedOutput(state); const usage = usageForWire(state); @@ -989,281 +807,26 @@ const synthesizeTerminalEnvelope = ( } as OpenAIResponsesStreamEvent); }; -async function* materializeServerToolItems( - dispatched: ReadonlyArray<{ slots: DispatchedServerToolSlot[] }>, +export async function* materializeHostedToolItems( + dispatched: ReadonlyArray<{ slots: DispatchedHostedToolSlot[] }>, merge: MergeState, store: OpenAIResponsesStatefulStore, ): AsyncGenerator, void> { for (const d of dispatched) { for (const { slot, outputIndex } of d.slots) { const lifecycle = slot.run(); - let step = await lifecycle.next(); - while (!step.done) { - yield stampServerToolEvent(merge, outputIndex, slot.id, step.value); - step = await lifecycle.next(); - } - // Register private dispatcher state under the emitted item id so output - // persistence captures it and replay-side `transformItems` can restore - // it on the next loop turn. - store.registerPrivatePayload(slot.id, step.value.privatePayload); - yield* serverToolEndFrames(merge, outputIndex, slot, step.value); - } - } -} - -const accumulateBillableUsage = async ( - metadata: LatestUpstreamMetadata, - result: Extract>, { type: 'events' }>, -): Promise => { - const turn = result.finalMetadata === undefined ? undefined : (await result.finalMetadata).billableUsage; - metadata.billableUsage = sumBillableUsage(metadata.billableUsage, turn); -}; - -// Stream-backed billing metadata can depend on draining the events. Await it -// in finally to retain costs observed before a stream failure. -async function* consumeBilledTurn( - events: AsyncGenerator, TurnSummary>, - result: Extract>, { type: 'events' }>, - metadata: LatestUpstreamMetadata, -): AsyncGenerator, TurnSummary> { - try { - return yield* events; - } finally { - await accumulateBillableUsage(metadata, result); - } -} - -async function* runMultiTurnLoop(args: { - ctx: OpenAIResponsesInvocation; - run: InterceptorRun>>; - merge: MergeState; - loopState: ServerToolLoopState; - demoteForcedServerToolChoiceAfterFirstTurn: boolean; - turn1Iter: AsyncGenerator, TurnSummary>; - firstResult: Extract>, { type: 'events' }>; - dispatchers: ReadonlyMap; - store: OpenAIResponsesStatefulStore; - canonicalInput: OpenAIResponsesInputItem[]; - active: readonly ActiveServerTool[]; - metadata: LatestUpstreamMetadata; - resolveFinalMetadata: (m: EventResultMetadata) => void; -}): AsyncGenerator> { - const { ctx, run, merge, loopState, demoteForcedServerToolChoiceAfterFirstTurn, turn1Iter, dispatchers, store, active, metadata, resolveFinalMetadata } = args; - const baseInput = args.canonicalInput; - let midStreamError: unknown = undefined; - try { - let currentTurn: TurnSummary = yield* consumeBilledTurn(turn1Iter, args.firstResult, metadata); - merge.accumulatedUsage = sumUsage(merge.accumulatedUsage, currentTurn.turnUsage); - while (true) { - const turn = currentTurn; - const executedShim = turn.dispatched.length > 0; - - if (turn.terminalStatus.kind === 'failed') { - if (executedShim) yield* materializeServerToolItems(turn.dispatched, merge, store); - yield synthesizeTerminalEnvelope(merge, { kind: 'failed', error: turn.terminalStatus.response.error }, active); - return; - } - if (turn.terminalStatus.kind === 'incomplete') { - if (executedShim) yield* materializeServerToolItems(turn.dispatched, merge, store); - yield synthesizeTerminalEnvelope(merge, { kind: 'incomplete', incompleteDetails: turn.terminalStatus.response.incomplete_details }, active); - return; - } - if (turn.terminalStatus.kind === 'bare-error-pre-shell') { - yield synthesizeTerminalEnvelope(merge, { - kind: 'failed', - error: { code: turn.terminalStatus.error.code, message: turn.terminalStatus.error.message }, - }, active); - return; - } - if (!executedShim && !turn.sawClientToolCall) { - yield synthesizeTerminalEnvelope(merge, { kind: 'completed' }, active); - return; - } - - yield* materializeServerToolItems(turn.dispatched, merge, store); - if (turn.sawClientToolCall) { - yield synthesizeTerminalEnvelope(merge, { kind: 'completed' }, active); - return; - } - - // Accumulated output items are fed back as the next turn's input. - // A OpenAI Responses output item is a structural superset of the matching - // input item for every shape we emit here (messages, reasoning, - // function_call / function_call_output, and the server-tool items - // the dispatchers produce), so the reuse is sound; the cast only - // bridges the output/input naming. - const nextCanonicalInput = [ - ...baseInput, - ...materializeAccumulatedOutput(merge).map(item => item as OpenAIResponsesInputItem), - ]; - const nextPayload: CanonicalOpenAIResponsesPayload = { ...ctx.payload, input: transformServerToolItems(nextCanonicalInput, active) }; - if (loopState.remainingToolCalls !== undefined) { - nextPayload.max_tool_calls = Math.max(0, loopState.remainingToolCalls); - } else { - delete nextPayload.max_tool_calls; - } - ctx.payload = nextPayload; - - // OpenAI's allowed_tools mode "required" requires at least one listed tool; - // "auto" also permits a message. - // https://developers.openai.com/api/reference/resources/responses/methods/create - // After a server tool runs, Floway relaxes the forced choice for this internal - // continuation so the model can answer, keeping any allowed_tools subset. - if (demoteForcedServerToolChoiceAfterFirstTurn) { - const choice = ctx.payload.tool_choice; - ctx.payload = { - ...ctx.payload, - tool_choice: typeof choice === 'object' && choice?.type === 'allowed_tools' ? { ...choice, mode: 'auto' } : 'auto', - }; - } - loopState.iterationCount += 1; - - const nextResult = await run(); - if (nextResult.type !== 'events') { - yield synthesizeTerminalEnvelope(merge, { kind: 'failed', error: buildErrorFromResult(nextResult) }, active); - return; + try { + let step = await lifecycle.next(); + while (!step.done) { + yield stampHostedToolEvent(merge, outputIndex, slot.id, step.value); + step = await lifecycle.next(); + } + // Register dispatcher state before item.done makes its stored row reusable. + store.registerPrivatePayload(slot.id, step.value.privatePayload); + yield* hostedToolEndFrames(merge, outputIndex, slot, step.value); + } finally { + await lifecycle.return(undefined as never); } - metadata.modelIdentity = nextResult.modelIdentity; - metadata.performance = nextResult.performance; - currentTurn = yield* consumeBilledTurn( - consumeTurnStreaming(nextResult.events, merge, false, dispatchers, loopState, active), - nextResult, - metadata, - ); - merge.accumulatedUsage = sumUsage(merge.accumulatedUsage, currentTurn.turnUsage); - } - } catch (error) { - if (merge.lastSeenModel === null) { - midStreamError = error; - throw error; } - yield synthesizeTerminalEnvelope(merge, { - kind: 'failed', - error: { - code: 'server_error', - message: `Upstream stream failed mid-response: ${error instanceof Error ? error.message : String(error)}`, - }, - }, active); - } finally { - if (midStreamError === undefined) resolveFinalMetadata(metadata); } } - -export const withOpenAIResponsesServerToolShim = ( - registrations: readonly ServerToolRegistration[], -): OpenAIResponsesInterceptor => async (ctx, gatewayCtx, run) => { - const active: ActiveServerTool[] = []; - const preparedTools: Array> = []; - - // Validate the original request before any hosted rewrite changes the - // tool-array indexes used in error paths. - for (const prepareServerTool of registrations) { - const prepared = await prepareServerTool(ctx, gatewayCtx); - if (prepared.type === 'inactive') continue; - if (prepared.type === 'invalid-request') { - return invalidRequestEnvelope(prepared.message, prepared.param, prepared.code, prepared.errorType); - } - preparedTools.push(prepared); - } - - for (const prepared of preparedTools) { - const currentTools = Array.isArray(ctx.payload.tools) ? ctx.payload.tools : []; - const toolName = resolveServerToolName(prepared.baseToolName, currentTools, ctx.payload.input, ctx.payload.tool_choice); - const { hosted } = prepared; - const choice = ctx.payload.tool_choice; - let helperFunctionChoice: Exclude | undefined; - let rewrittenHelperChoice = choice; - if (hosted !== undefined && toolName !== prepared.baseToolName - && typeof choice === 'object' && choice !== null) { - // Forced and allowed-tools selectors share the same ownership rule. A - // bare helper spelling may be rebased only when no client owns it. - const clientOwnsName = currentTools.some(tool => - (tool.type === 'function' || tool.type === 'custom') && tool.name === prepared.baseToolName - && (!('namespace' in tool) || tool.namespace === undefined)) - || historicalClientCallableUsesName(prepared.baseToolName, ctx.payload.input); - if (!clientOwnsName) { - const rebaseSelector = (selector: T): T => { - if (typeof selector !== 'object' || selector === null || !('type' in selector) || selector.type !== 'function' - || !('name' in selector) || selector.name !== prepared.baseToolName - || ('namespace' in selector && selector.namespace !== undefined)) return selector; - return { ...selector, name: toolName }; - }; - if (choice.type === 'allowed_tools' && Array.isArray(choice.tools)) { - const tools = choice.tools.map(rebaseSelector); - if (tools.some((tool, index) => tool !== choice.tools[index])) rewrittenHelperChoice = { ...choice, tools }; - } else rewrittenHelperChoice = rebaseSelector(choice); - if (rewrittenHelperChoice !== choice) helperFunctionChoice = choice; - } - } - let canonicalHostedTool: OpenAIResponsesHostedTool | undefined = undefined; - if (hosted !== undefined) { - const rewrite = rewriteHostedDeclarations(ctx.payload, hosted, toolName); - canonicalHostedTool = rewrite.canonicalTopLevelHostedTool; - ctx.payload = rewrite.payload; - } - const originalToolChoice = helperFunctionChoice ?? hostedToolChoiceToRestore(choice, hosted, toolName); - if (helperFunctionChoice !== undefined) { - ctx.payload = { ...ctx.payload, tool_choice: rewrittenHelperChoice }; - } - active.push({ ...prepared, toolName, canonicalHostedTool, originalToolChoice }); - } - - if (active.length === 0) return await run(); - - const rewrittenToolChoice = rewriteHostedToolChoice(ctx.payload.tool_choice, active); - if (rewrittenToolChoice !== ctx.payload.tool_choice) { - ctx.payload = { ...ctx.payload, tool_choice: rewrittenToolChoice }; - } - - const canonicalInput = ctx.payload.input; - const nextInput = transformServerToolItems(canonicalInput, active); - if (nextInput !== canonicalInput) ctx.payload = { ...ctx.payload, input: nextInput }; - - const hostedActive = active.filter( - (entry): entry is ActiveServerTool & { hosted: ServerToolHostedDispatch } => - entry.hosted !== undefined, - ); - if (hostedActive.length === 0) return await run(); - - const dispatchers = new Map(); - for (const entry of hostedActive) dispatchers.set(entry.toolName, entry.hosted.dispatcher); - const loopState: ServerToolLoopState = { - iterationCount: 1, - remainingToolCalls: typeof ctx.payload.max_tool_calls === 'number' ? ctx.payload.max_tool_calls : undefined, - }; - const demoteForcedServerToolChoiceAfterFirstTurn = isForcedServerToolChoice(ctx.payload.tool_choice, dispatchers); - - const merge = createMergeState(); - const firstResult = await run(); - if (firstResult.type !== 'events') return firstResult; - const turn1Iter = consumeTurnStreaming(firstResult.events, merge, true, dispatchers, loopState, active); - - let resolveFinalMetadata!: (m: EventResultMetadata) => void; - const shimFinalMetadata = new Promise(resolve => { - resolveFinalMetadata = resolve; - }); - const metadata: LatestUpstreamMetadata = { - modelIdentity: firstResult.modelIdentity, - performance: firstResult.performance, - }; - - return { - ...firstResult, - events: runMultiTurnLoop({ - ctx, - run, - merge, - loopState, - demoteForcedServerToolChoiceAfterFirstTurn, - turn1Iter, - firstResult, - dispatchers, - store: gatewayCtx.store, - canonicalInput, - active, - metadata, - resolveFinalMetadata, - }), - finalMetadata: shimFinalMetadata, - }; -}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/types.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/types.ts new file mode 100644 index 0000000000..0b570a3997 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/types.ts @@ -0,0 +1,149 @@ +import type { ChatGatewayCtx } from '../../shared/gateway-ctx.ts'; +import type { OpenAIResponsesOutputItem, OpenAIResponsesResult, OpenAIResponsesTool, OpenAIResponsesHostedTool, OpenAIResponsesFunctionTool, OpenAIResponsesInputItem, OpenAIResponsesToolChoice } from '@floway-dev/protocols/openai-responses'; +import type { OpenAIResponsesInvocation } from '@floway-dev/provider'; + +export interface MergeUsage { + input_tokens?: number; + output_tokens?: number; + total_tokens?: number; + input_tokens_details?: { cached_tokens: number }; + output_tokens_details?: { reasoning_tokens: number }; +} + +export interface MergeState { + sequenceNumber: number; + outputIndex: number; + accumulatedOutput: Map; + accumulatedUsage: MergeUsage; + lastSeenModel: string | null; + synthesizedResponseId: string; + upstreamResponseSnapshot: OpenAIResponsesResult | undefined; +} + +export interface InterceptedFunctionCall { + callId: string; + name: string; + /** + * Parsed and jsonrepair-cleaned `arguments` object. `null` when the + * raw upstream string is not a JSON object even after jsonrepair — + * dispatchers handle that case via their own error path. Dispatchers + * that persist function-call inputs across turns should serialize + * this rather than the raw upstream string; replaying the raw form + * would re-break the next turn the same way it broke this one. + */ + arguments: Record | null; +} + +export interface HostedToolTerminal { + item: HostedToolOutputItem; + endEvents: HostedToolLifecycleEvent[]; + /** + * Optional server-only blob registered on the request's item store under + * `slot.id` before the slot's wire item leaves materialize. The persistence + * layer stores it in `payload.private`; the replay-side `transformItems` + * reads it back to reconstruct the full IR. + */ + privatePayload?: unknown; +} + +export interface HostedToolResultSlot { + id: string; + startItem: HostedToolOutputItem; + startEvents: readonly HostedToolLifecycleEvent[]; + // The deferred portion of a slot's lifecycle, driven at materialization + // time. It yields any intermediate lifecycle events as they arrive — e.g. + // progressively-rendered `image_generation_call.partial_image` frames a + // streamed backend delivers over the course of the call — and returns the + // terminal item plus its closing events (and an optional server-only + // `privatePayload`). A tool with no progressive output simply yields nothing + // and returns immediately. + run: () => AsyncGenerator; +} + +export type HostedToolOutputItem = { type: string; id?: string; [key: string]: unknown }; + +export type HostedToolLifecycleEvent = { type: string; [key: string]: unknown }; + +export interface HostedToolLoopState { + iterationCount: number; + remainingToolCalls: number | undefined; +} + +export interface DispatchedHostedToolSlot { + intercepted: InterceptedFunctionCall; + slot: HostedToolResultSlot; + outputIndex: number; +} + +export type HostedToolDispatcher = (args: { + intercepted: InterceptedFunctionCall; + loopState: HostedToolLoopState; +}) => HostedToolResultSlot[]; + +// Keep hosted matching, function injection, and dispatch atomic so a +// registration cannot silently omit part of a hosted-tool family. +export interface HostedToolRewrite { + hostedTypes: readonly string[]; + canonicalize: (raw: OpenAIResponsesTool) => OpenAIResponsesHostedTool | undefined; + buildFunctionTool: (canonical: OpenAIResponsesHostedTool, toolName: string) => OpenAIResponsesFunctionTool; + dispatcher: HostedToolDispatcher; +} + +export type HostedToolPrepareResult = + | { type: 'inactive' } + // `errorType` / `code` override the envelope for tools that emulate an + // upstream's rejection vocabulary. An omitted code falls back to the + // generic `invalid_request_error`; explicit null is preserved verbatim. + | { type: 'invalid-request'; message: string; param: string | null; errorType?: string; code?: string | null } + | { + type: 'active'; + baseToolName: string; + // History rewrite, applied whether or not the tool is hosted this + // turn so items echoed from a previous turn's output become + // upstream-readable even on a request that no longer declares the + // hosted tool. + transformItems?: (items: OpenAIResponsesInputItem[], toolName: string) => OpenAIResponsesInputItem[]; + // Present only when the request declares this hosted tool; absent for + // replay-only activation. + hosted?: HostedToolRewrite; + }; + +export type HostedToolRegistration = (invocation: OpenAIResponsesInvocation, gatewayCtx: ChatGatewayCtx) => HostedToolPrepareResult | Promise; + +export type ActiveHostedTool = Extract & { + toolName: string; + // Hosted entries are rewritten at their input-item positions; only + // top-level replacements participate in response.tools restoration. + canonicalHostedTool: OpenAIResponsesHostedTool | undefined; + // Captures the exact forced choice shape before request rewriting. + originalToolChoice: Exclude | undefined; +}; + +// How a single upstream turn ended, as observed while consuming its +// stream. Carries the raw upstream `response` for failed/incomplete so +// the loop can lift the upstream `error` / `incomplete_details`, plus a +// `bare-error-pre-shell` variant for an `error` event that arrived +// before any `response.created` (no model known yet). Distinct from +// `SynthesizedTerminal`, which is the dispatcher's own outgoing terminal. +export type UpstreamTerminal = + | { kind: 'completed' } + | { kind: 'failed'; response: OpenAIResponsesResult } + | { kind: 'incomplete'; response: OpenAIResponsesResult } + | { kind: 'bare-error-pre-shell'; error: { message: string; code: string } }; + +export interface TurnSummary { + dispatched: Array<{ intercepted: InterceptedFunctionCall; slots: DispatchedHostedToolSlot[] }>; + sawClientToolCall: boolean; + turnUsage: MergeUsage; + terminalStatus: UpstreamTerminal; +} + +// The terminal the dispatcher emits downstream. Unlike `UpstreamTerminal` +// (what we observed), this carries only the already-extracted `error` / +// `incompleteDetails` the synthesized envelope needs; the output and +// usage come from accumulated dispatcher state. There is no pre-shell variant +// here — synthesis always runs after a model is known. +export type SynthesizedTerminal = + | { kind: 'completed' } + | { kind: 'failed'; error: OpenAIResponsesResult['error'] } + | { kind: 'incomplete'; incompleteDetails: OpenAIResponsesResult['incomplete_details'] }; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tools/web-search.ts b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/web-search.ts similarity index 75% rename from packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tools/web-search.ts rename to packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/web-search.ts index 911e546e0a..f6bc849fe7 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tools/web-search.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/hosted-tools/web-search.ts @@ -1,35 +1,26 @@ -import { shortId } from '../../../../../shared/short-id.ts'; -import { truncatePreservingCodePoints } from '../../../../shared/text.ts'; -import { executeAlphaSearch } from '../../../../tools/web-search/alpha-search/execution.ts'; -import { resolveAlphaSearchDispatcher } from '../../../../tools/web-search/alpha-search/upstream.ts'; -import { loadWebSearchConfig } from '../../../../tools/web-search/config.ts'; -import { normalizeDomainEntry } from '../../../../tools/web-search/domain-normalize.ts'; +import type { WebSearchRuntime } from './search-sub-request/services.ts'; +import { runWebSearchSubRequest } from './search-sub-request.ts'; +import { type HostedToolOutputItem, type HostedToolRegistration } from './types.ts'; +import { shortId } from '../../../../shared/short-id.ts'; +import { truncatePreservingCodePoints } from '../../../shared/text.ts'; +import { resolveAlphaSearchCandidate } from '../../../tools/web-search/alpha-search/upstream.ts'; +import { loadWebSearchConfig } from '../../../tools/web-search/config.ts'; +import { normalizeDomainEntry } from '../../../tools/web-search/domain-normalize.ts'; import { actionSearchQueries, - assertLocalWebSearchSupport, CONTEXT_SIZE_TO_MAX_RESULTS, DEFAULT_SEARCH_CONTEXT_SIZE, - executeOperationToIr, isSearchContextSize, maxResultsForContextSize, - parseWebSearchOperations, renderWebSearchCallOutput, - runBackendSearchMulti, - schemaErrorIr, - startBatchFetch, - UnsupportedLocalWebSearchFeatureError, - unsupportedLocalWebSearchFeatureIr, - type ParsedWebSearchOperations, type WebSearchCallIR, - type WebSearchExecutionSession, type WebSearchFilters, - type WebSearchOperation, -} from '../../../../tools/web-search/operations.ts'; -import { resolveConfiguredWebSearchProvider } from '../../../../tools/web-search/provider.ts'; -import type { ConfiguredWebSearchProvider } from '../../../../tools/web-search/types.ts'; -import { type ServerToolLoopState, type ServerToolOutputItem, type ServerToolRegistration } from '../server-tool-shim.ts'; +} from '../../../tools/web-search/operations.ts'; +import { resolveConfiguredWebSearchProvider } from '../../../tools/web-search/provider.ts'; +import type { ConfiguredWebSearchProvider } from '../../../tools/web-search/types.ts'; import type { OpenAIResponsesFunctionTool, OpenAIResponsesFunctionToolCallItem, OpenAIResponsesHostedTool, OpenAIResponsesInputItem, OpenAIResponsesOutputWebSearchCall, OpenAIResponsesTool, OpenAIResponsesWebSearchAction } from '@floway-dev/protocols/openai-responses'; import { collectOpenAIResponsesToolEntries, createRandomOpenAIResponsesItemId, WEB_SEARCH_HOSTED_TYPE_NAMES } from '@floway-dev/protocols/openai-responses'; +import type { ModelCandidate } from '@floway-dev/provider'; import { providerModelOf } from '@floway-dev/provider'; // Runtime set derived from the canonical tuple declared next to @@ -38,9 +29,9 @@ import { providerModelOf } from '@floway-dev/provider'; // https://github.com/openai/openai-python/blob/e75766769547601a25ed83b666c4d0fd046881f0/src/openai/types/responses/web_search_preview_tool.py export const WEB_SEARCH_HOSTED_TYPES: ReadonlySet = new Set(WEB_SEARCH_HOSTED_TYPE_NAMES); -// Function-name regex `^[a-zA-Z0-9_-]+$` forbids dots, so the shim call +// Function-name regex `^[a-zA-Z0-9_-]+$` forbids dots, so the dispatcher call // uses the underscored form of the model's training-time `web.run`. -export const SHIM_TOOL_NAME = 'web_search'; +export const FUNCTION_TOOL_NAME = 'web_search'; // Put the selected hosted declaration's `user_location` in the replacement // function description so the model can use it as the default for local searches. @@ -59,7 +50,7 @@ const formatUserLocation = (loc: NonNullable): // local Tavily, Jina, and Microsoft Web IQ execution rejects the fields it // cannot implement before dispatch. // https://github.com/openai/codex/blob/2f19a57704fb7b1db032bc38cf995034254eaebb/codex-rs/codex-api/src/search.rs#L31-L213 -export const buildShimFunctionTool = ( +export const buildFunctionTool = ( canonical: OpenAIResponsesHostedTool, name: string, ): OpenAIResponsesFunctionTool => { @@ -331,8 +322,8 @@ const validateDomainListEntry = ( return { ok: true }; }; -// Validate the hosted web-search fields the shim interprets. The model-bound -// replacement uses the shim command schema; backend settings such as +// Validate the hosted web-search fields the dispatcher interprets. The model-bound +// replacement uses the dispatcher command schema; backend settings such as // `external_web_access` are handled separately, while unhandled fields such // as `return_token_budget` are omitted from that request. const validateHostedEntry = (tool: OpenAIResponsesHostedTool): PrepareToolsError | null => { @@ -380,7 +371,7 @@ const validateHostedEntry = (tool: OpenAIResponsesHostedTool): PrepareToolsError // Validate every hosted declaration before selecting the last one's runtime // filters, so an earlier invalid declaration cannot be masked. -export const prepareToolsForShim = ( +export const prepareTools = ( tools: OpenAIResponsesTool[], paths?: readonly string[], ): PrepareToolsResult => { @@ -410,9 +401,9 @@ export const prepareToolsForShim = ( const MAX_MALFORMED_WIRE_DUMP_CHARS = 1024; /** - * Persistent `payload.private` shape for one `web_search_call`. One shim call + * Persistent `payload.private` shape for one `web_search_call`. One dispatcher call * function_call corresponds to exactly one wsc and one op — multi-op - * shim calls (multi-kind mix or multi-instance same-kind) are rejected at + * dispatcher calls (multi-kind mix or multi-instance same-kind) are rejected at * dispatch with an `ambiguous` error, so there is never an array to * denormalize. The persisted-payload key IS the wsc id, so we don't repeat * it inside. @@ -424,13 +415,13 @@ const MAX_MALFORMED_WIRE_DUMP_CHARS = 1024; * so the upstream model's prior assistant turn looks bit-exact. * * - `ir` stores the action, structured results, and optional upstream - * model-facing output straight from `planShimSlots`. Replay uses + * model-facing output straight from `planDispatchSlots`. Replay uses * `renderWebSearchCallOutput`, which preserves that output when present * and otherwise renders the action and results. * * Version-tagged: an unknown `v` falls through the no-payload branch in * `transformInputItemsForWebSearch` (action re-serialized into the - * shim call shape, output replaced with the not-preserved notice). Starts + * dispatcher call shape, output replaced with the not-preserved notice). Starts * at 1; bump only on a wire-incompatible change after release. */ export interface WebSearchCallPrivatePayload { @@ -459,12 +450,12 @@ export const synthesizeWebSearchCallId = (): string => createRandomOpenAIRespons // replay call_id never reads as a web-search item id in logs. const synthesizeReplayCallId = (): string => shortId('cc_replay'); -// Re-serializes a wire `action` back into the shim's JSON arguments +// Re-serializes a wire `action` back into the dispatcher's JSON arguments // shape (`{search_query:[{q}]}` / `{open:[{ref_id}]}` / // `{find:[{ref_id,pattern}]}`). Used only on the replay-fallback path to // fill the paired function_call's `arguments` when no private payload // exists; the happy path replays the upstream's original args verbatim. -const actionToShimCallArgsJson = (action: OpenAIResponsesWebSearchAction): string => { +const actionToFunctionCallArgsJson = (action: OpenAIResponsesWebSearchAction): string => { switch (action.type) { case 'search': return JSON.stringify({ @@ -496,9 +487,9 @@ const actionToShimCallArgsJson = (action: OpenAIResponsesWebSearchAction): strin // 2. No payload (`store: false`, expired, foreign id, cross-account, or // schema-version mismatch): degrade to a synthesized pair whose // `function_call.arguments` is the wire action re-serialized into -// the shim call shape (so the model still sees what it asked for) and +// the dispatcher call shape (so the model still sees what it asked for) and // whose `function_call_output` text is the not-preserved placeholder. -// The shim deliberately does not read `item.results` from the +// The dispatcher deliberately does not read `item.results` from the // wire — turn 1's wire results may or may not exist depending on the // client's `include` opt-in, and trusting them across the wire would // couple state correctness to client storage discipline. @@ -562,7 +553,7 @@ export const transformInputItemsForWebSearch = ( type: 'function_call', call_id: callId, name: toolName, - arguments: actionToShimCallArgsJson(item.action), + arguments: actionToFunctionCallArgsJson(item.action), status: 'completed', }, { @@ -577,135 +568,7 @@ export const transformInputItemsForWebSearch = ( return out; }; -// The shim's execution session plus the one wire-shaping flag that lives -// only on the OpenAI Responses side. -interface ShimState extends WebSearchExecutionSession { - // Set when the client passed `include: ["web_search_call.results"]` on - // the request. Native OpenAI Responses gates the `results` field on this - // include token; the shim follows suit on the wire item — but the IR - // (and therefore `payload.private`) always carries the real results - // so a subsequent turn echoing the item id can be hydrated regardless. - includeSearchResults: boolean; - executeAlpha?: (commands: Record, action: OpenAIResponsesWebSearchAction) => Promise; -} - -const ITERATION_CAP = 30; - -const eagerResolver = (promise: Promise): (() => Promise) => { - const settled = promise.then( - value => ({ ok: true as const, value }), - error => ({ ok: false as const, error }), - ); - return async () => { - const result = await settled; - if (result.ok) return result.value; - throw result.error; - }; -}; - -const planShimSlots = ( - parsed: ParsedWebSearchOperations, - commands: Record, - toolName: string, - state: ShimState, - loopState: ServerToolLoopState, -): { id: string; resolve: () => Promise } => { - if (loopState.iterationCount > ITERATION_CAP) { - return { - id: synthesizeWebSearchCallId(), - resolve: async () => schemaErrorIr( - 'tool budget exhausted', - 'Tool call budget exhausted', - `Web search iteration limit (${ITERATION_CAP}) reached. Further web_search calls in this response will return this same error. Summarize what you have already learned, and continue the task using other available tools (shell, file inspection, prior knowledge) or directly answer based on what you've gathered.`, - ), - }; - } - - if (parsed.kind === 'malformed' || parsed.ops.length === 0) { - return { - id: synthesizeWebSearchCallId(), - resolve: async () => schemaErrorIr( - 'malformed shim call arguments', - 'Malformed arguments', - 'Error: arguments must be a JSON object with sub-property arrays (search_query[], open[], find[]).', - ), - }; - } - - const executeAlpha = state.executeAlpha; - if (executeAlpha !== undefined) { - const first = parsed.ops[0]; - let action: OpenAIResponsesWebSearchAction; - if (first.kind === 'search') { - const queries = parsed.ops.filter((op): op is Extract => op.kind === 'search').map(op => op.query); - action = queries.length === 1 - ? { type: 'search', query: queries[0], queries } - : { type: 'search', query: queries.join(' | '), queries }; - } else if (first.kind === 'open') { - action = { type: 'open_page', url: first.url }; - } else if (first.kind === 'find') { - action = { type: 'find_in_page', url: first.url, pattern: first.pattern }; - } else { - action = { type: 'search', query: Object.keys(commands).join(', ') }; - } - return { id: synthesizeWebSearchCallId(), resolve: eagerResolver(executeAlpha(commands, action)) }; - } - - try { - assertLocalWebSearchSupport(commands); - } catch (error) { - if (error instanceof UnsupportedLocalWebSearchFeatureError) { - return { - id: synthesizeWebSearchCallId(), - resolve: async () => unsupportedLocalWebSearchFeatureIr( - { type: 'search', query: Object.keys(commands).join(', ') }, - error.message, - ), - }; - } - return { - id: synthesizeWebSearchCallId(), - resolve: async () => { throw error; }, - }; - } - - // Multi-`search_query` entries collapse into one wsc with a multi-query - // action (`{type:'search', queries:[...]}`) — protocol-native and the - // only same-kind shape that fits in one wsc. Require every entry to - // parse cleanly; one malformed entry forces the model to fix all of - // them rather than silently dropping a search. - if (parsed.ops.length > 1 && parsed.ops.every(op => op.kind === 'search' && op.error === undefined)) { - const searchOps = parsed.ops as Array>; - return { - id: synthesizeWebSearchCallId(), - resolve: eagerResolver(runBackendSearchMulti(searchOps, state)), - }; - } - - // Any other multi-op shape cannot reduce to a single wsc action: `open` - // and `find` actions each carry one url/pattern, and mixed kinds have - // incompatible action types. Surface as ambiguous and let the model - // split into independent calls. - if (parsed.ops.length > 1) { - return { - id: synthesizeWebSearchCallId(), - resolve: async () => schemaErrorIr( - 'ambiguous shim call', - 'Ambiguous tool call', - `Error: ambiguous \`${toolName}\` tool call — each function_call maps to one web_search_call. ` - + 'Multiple `search_query` entries are fine (they collapse into one search). ' - + 'For `open`/`find`, or any mix of kinds, split into one call per `open[]` entry, `find[]` entry, or `search_query[]` batch.', - ), - }; - } - - return { - id: synthesizeWebSearchCallId(), - resolve: eagerResolver(startBatchFetch(parsed, state).then(async batch => await executeOperationToIr(parsed.ops[0], state, batch))), - }; -}; - -export const webSearchServerTool: ServerToolRegistration = async (invocation, gatewayCtx) => { +export const webSearchHostedTool: HostedToolRegistration = async (invocation, gatewayCtx) => { if (invocation.targetApi === 'openaiResponses' && !providerModelOf(invocation.candidate).enabledFlags.has('openai-responses-web-search-shim')) { return { type: 'inactive' }; } @@ -716,7 +579,7 @@ export const webSearchServerTool: ServerToolRegistration = async (invocation, ga const hasReplayInput = invocation.payload.input.some(i => i.type === 'web_search_call'); if (!hasHostedWebSearch && !hasReplayInput) return { type: 'inactive' }; - const prepared = prepareToolsForShim(tools, declarations.map(entry => entry.path)); + const prepared = prepareTools(tools, declarations.map(entry => entry.path)); if (!prepared.ok) { return { type: 'invalid-request', @@ -731,50 +594,52 @@ export const webSearchServerTool: ServerToolRegistration = async (invocation, ga let configuredProvider: Promise | undefined; const hosted = tools.filter(isHostedWebSearchTool).at(-1); const settings = alphaSearchSettingsFromHosted(hosted); - const state: ShimState = { - filters, + const session: WebSearchRuntime['session'] = { pageCache: new Map(), getProvider: () => { configuredProvider ??= Promise.resolve(resolveConfiguredWebSearchProvider(webSearchConfig)); return configuredProvider; }, apiKeyId: gatewayCtx.apiKeyId, - includeSearchResults: includeArray.includes('web_search_call.results'), - includeSearchActionSources: includeArray.includes('web_search_call.action.sources'), ...(gatewayCtx.abortSignal !== undefined ? { signal: gatewayCtx.abortSignal } : {}), }; - if (webSearchConfig.passthroughOpenAiSearch.enabled) { - const dispatcher = resolveAlphaSearchDispatcher({ - config: webSearchConfig.passthroughOpenAiSearch, - upstreamIds: gatewayCtx.upstreamIds, - scheduler: gatewayCtx.backgroundScheduler, - runtimeLocation: gatewayCtx.runtimeLocation, - }); - const sessionId = crypto.randomUUID(); - state.executeAlpha = async (commands, action) => await executeAlphaSearch({ - dispatcher: await dispatcher, - sessionId, - commands, - settings, - input: invocation.payload.input, - action, - signal: gatewayCtx.abortSignal, - }); - } + let selectedAlphaCandidate: Promise | undefined; + const webSearch: WebSearchRuntime = { + session, + ...(webSearchConfig.passthroughOpenAiSearch.enabled ? { + alpha: { + resolveCandidate: () => selectedAlphaCandidate ??= resolveAlphaSearchCandidate({ + config: webSearchConfig.passthroughOpenAiSearch, + upstreamIds: gatewayCtx.upstreamIds, + scheduler: gatewayCtx.backgroundScheduler, + runtimeLocation: gatewayCtx.runtimeLocation, + }), + sessionId: crypto.randomUUID(), + }, + } : {}), + }; return { type: 'active', - baseToolName: SHIM_TOOL_NAME, + baseToolName: FUNCTION_TOOL_NAME, transformItems: (items, toolName) => transformInputItemsForWebSearch(items, toolName, id => gatewayCtx.store.getPrivatePayload(id)), ...(hasHostedWebSearch ? { hosted: { hostedTypes: WEB_SEARCH_HOSTED_TYPE_NAMES, canonicalize: canonicalizeWebSearchTool, - buildFunctionTool: buildShimFunctionTool, + buildFunctionTool, dispatcher: ({ intercepted, loopState }) => { const commands = intercepted.arguments ?? {}; - const slot = planShimSlots(parseWebSearchOperations(intercepted.arguments), commands, intercepted.name, state, loopState); + const id = synthesizeWebSearchCallId(); + const search = runWebSearchSubRequest(gatewayCtx, { + commands, toolName: intercepted.name, iterationCount: loopState.iterationCount, filters, + includeSearchActionSources: includeArray.includes('web_search_call.action.sources'), + settings, input: invocation.payload.input, + }, webSearch).then( + value => ({ ok: true as const, value }), + error => ({ ok: false as const, error }), + ); const functionCallItem: OpenAIResponsesFunctionToolCallItem = { type: 'function_call', call_id: intercepted.callId, @@ -787,19 +652,24 @@ export const webSearchServerTool: ServerToolRegistration = async (invocation, ga status: 'completed', }; return [{ - id: slot.id, + id, startItem: { type: 'web_search_call', status: 'in_progress' }, startEvents: [ { type: 'response.web_search_call.in_progress' }, { type: 'response.web_search_call.searching' }, ], + // Its own run: its own prologue, its own settlement, its own record. The + // commands, filters, and conversation enter the child record before its backend + // executes; live caches and provider handles remain in its services. run: async function* run() { - const ir = await slot.resolve(); + const result = await search; + if (!result.ok) throw result.error; + const ir = result.value; // `results` is gated on the client's `include` // opt-in to match native OpenAI Responses' default wire // shape; the IR keeps them either way for the // private-payload round-trip. - const item: ServerToolOutputItem & Omit = state.includeSearchResults + const item: HostedToolOutputItem & Omit = includeArray.includes('web_search_call.results') ? { type: 'web_search_call', status: 'completed', action: ir.action, results: ir.results } : { type: 'web_search_call', status: 'completed', action: ir.action }; const privatePayload: WebSearchCallPrivatePayload = { diff --git a/packages/gateway/src/data-plane/chat/openai-responses/http.ts b/packages/gateway/src/data-plane/chat/openai-responses/http.ts index 67fb8bf981..9e40c306bf 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/http.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/http.ts @@ -1,127 +1,130 @@ -import { openaiResponsesInputErrorResult } from './errors.ts'; +// POST /v1/responses and POST /v1/responses/compact, both served through their own pipeline. +// Generation is one chain; compaction is a second operation over the same protocol and is a +// chain of its own rather than another wire under the first. +// +// Each entry is a prologue and an epilogue around that chain: read what the client sent, hand +// it over, and turn what the run answered with into a response. Everything between is stages, +// the stored-items membrane included — which is why a continuation that does not resolve is +// answered by the chain on both routes rather than caught as a throw on either. + +import { openaiResponsesCompactPipeline } from './compact.ts'; import { createOpenAIResponsesHttpStore } from './items/store.ts'; -import { respondOpenAIResponses, respondOpenAIResponsesFailure } from './respond.ts'; -import { PreviousResponseNotFoundError } from './serve-prep.ts'; -import { openaiResponsesServe } from './serve.ts'; +import { openaiResponsesServePipeline } from './pipeline.ts'; import type { AuthedContext } from '../../../middleware/auth.ts'; -import { backgroundSchedulerFromContext } from '../../../runtime/background.ts'; -import { createGatewayCtxFromHono, finalizeGatewayResponse, type GatewayCtx } from '../../shared/gateway-ctx.ts'; -import { inboundHeaders } from '../../shared/inbound-headers.ts'; -import { readRequestBody, takeRequestBody, type RequestBody } from '../../shared/request-body.ts'; -import { settle } from '../../shared/telemetry/settle.ts'; -import { createChatGatewayCtxFromHono, type ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import { providerModelsUnavailableResponse } from '../shared/upstream-models-error.ts'; +import { isFrames, openPrologue, readIngress, serveThrough, type Ingress } from '../../pipeline/serve.ts'; +import { finalizeGatewayResponse } from '../../shared/gateway-ctx.ts'; +import { openChatPrologue } from '../prologue.ts'; +import { move } from '@floway-dev/pipeline'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesRequestPayload } from '@floway-dev/protocols/openai-responses'; -import { internalErrorResult, toInternalDebugError } from '@floway-dev/provider'; import { canonicalizeOpenAIResponsesPayload, TranslatorInputError } from '@floway-dev/translate'; -// OpenAI's verbatim previous_response_not_found envelope. Codex compares this -// body byte-for-byte against upstream — see the cross-references on -// `PreviousResponseNotFoundError` in serve-prep.ts. -const previousResponseNotFoundResponse = (id: string): Response => +/** The read, as a value rather than a throw, because a pipelined entry decides what to do + * about a body it could not read before it opens a run rather than after one unwound. + * + * What the reader rejected travels with the rejection. This protocol's envelope names the + * field at fault and the code for the condition, and those are statements only the reader + * can make: `input` and `input[0]` are different sentences, and a body with no `model` is + * refused in words OpenAI's own clients already parse. */ +const readRequest = (bytes: Uint8Array): + | { type: 'ok'; payload: CanonicalOpenAIResponsesPayload } + | { type: 'invalid'; message: string; param?: string; code?: string } => { + try { + return { type: 'ok', payload: canonicalizeOpenAIResponsesPayload(JSON.parse(new TextDecoder().decode(bytes)) as OpenAIResponsesRequestPayload) }; + } catch (error) { + if (!(error instanceof TranslatorInputError)) { + return { type: 'invalid', message: error instanceof Error ? error.message : String(error) }; + } + return { + type: 'invalid', + message: error.message, + ...(error.param === undefined ? {} : { param: error.param }), + ...(error.code === undefined ? {} : { code: error.code }), + }; + } +}; + +/** The rejection, as this protocol's clients read one. A reader that named neither field + * still fills both slots, because the envelope declares them and `input` is where a body + * this endpoint could not read went wrong. */ +const invalidRequestResponse = (invalid: { message: string; param?: string; code?: string }): Response => Response.json( { error: { - message: `Previous response with id '${id}' not found.`, + message: invalid.message, type: 'invalid_request_error', - param: 'previous_response_id', - code: 'previous_response_not_found', + param: invalid.param ?? 'input', + code: invalid.code ?? null, }, }, { status: 400 }, ); -// Surfaces a pre-stream throw (malformed JSON body, an interceptor crash, -// etc.) as an OpenAI-Responses-shaped 502 with the same internal-error envelope the -// in-flow `internal-error` ExecuteResult produces. A -// `ProviderModelsUnavailableError` carrying an upstream HTTP body relays -// that body verbatim — the upstream's `/models` 401 IS the diagnostic. The -// caller passes its outer `ctx` when one was already constructed (so the -// dump row preserves the model attribution the request-time -// `requestedModel` stamped, and the throwing-candidate telemetry stamped -// in serve.ts survives onto the error row); a fresh ctx is minted only -// for pre-parse failures where no payload was available to read model from. -const respondWithInternalError = async (c: AuthedContext, error: unknown, requestBody: RequestBody, ctx?: GatewayCtx): Promise => { - const verbatim = providerModelsUnavailableResponse(error); - if (verbatim !== null) return verbatim; - const effectiveCtx = ctx ?? createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - const result = internalErrorResult(502, toInternalDebugError(error), effectiveCtx.attempt.telemetry); - const response = respondOpenAIResponsesFailure(result, effectiveCtx); - return finalizeGatewayResponse(effectiveCtx, response); -}; - -// Pre-stream throw dispatcher. `PreviousResponseNotFoundError` and the -// translator-input case render protocol-shaped 400s; anything else falls -// through to the internal-error 502 path. -const respondToThrow = async (c: AuthedContext, error: unknown, requestBody: RequestBody, ctx?: GatewayCtx): Promise => { - if (error instanceof PreviousResponseNotFoundError) { - const response = previousResponseNotFoundResponse(error.previousResponseId); - ctx?.dump?.error('gateway'); - return ctx ? finalizeGatewayResponse(ctx, response) : response; - } - if (error instanceof TranslatorInputError) { - const effectiveCtx = ctx ?? createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - const response = respondOpenAIResponsesFailure(openaiResponsesInputErrorResult(error, effectiveCtx.attempt.telemetry), effectiveCtx); - return finalizeGatewayResponse(effectiveCtx, response); - } - return await respondWithInternalError(c, error, requestBody, ctx); +/** A request the gateway refused before it reached a pipeline: there is no model to resolve + * and no attempt to make, so there is nothing for a run to record beyond the refusal. */ +const refuse = (c: AuthedContext, ingress: Ingress, invalid: { message: string; param?: string; code?: string }): Response => { + const refused = openPrologue(c, ingress, { wantsStream: false }); + refused.gateway.dump?.error('gateway'); + return finalizeGatewayResponse(refused.gateway, invalidRequestResponse(invalid)); }; -const parsePayload = (requestBody: RequestBody): CanonicalOpenAIResponsesPayload => - canonicalizeOpenAIResponsesPayload(JSON.parse(new TextDecoder().decode(requestBody.bytes)) as OpenAIResponsesRequestPayload); - export const openaiResponsesHttp = { generate: async (c: AuthedContext): Promise => { - const requestBody = await readRequestBody(c); - let ctx: ChatGatewayCtx | undefined; - try { - const payload = parsePayload(requestBody); - const wantsStream = payload.stream === true; - ctx = createChatGatewayCtxFromHono(c, { wantsStream, requestBody: takeRequestBody(requestBody), model: payload.model, backgroundScheduler: backgroundSchedulerFromContext(c) }, (apiKey, requestStartedAt) => createOpenAIResponsesHttpStore(apiKey, requestStartedAt, payload.store ?? undefined)); - const result = await openaiResponsesServe.generate({ payload, ctx, headers: inboundHeaders(c) }); - const response = await respondOpenAIResponses(c, result, wantsStream, ctx, payload); - return finalizeGatewayResponse(ctx, response); - } catch (error) { - return await respondToThrow(c, error, requestBody, ctx); - } + const ingress = await readIngress(c); + const request = readRequest(ingress.body.bytes); + if (request.type === 'invalid') return refuse(c, ingress, request); + + const { payload } = request; + const wantsStream = payload.stream === true; + const prologue = openChatPrologue(c, ingress, { + wantsStream, + model: payload.model, + storeFactory: (apiKey, requestStartedAt) => createOpenAIResponsesHttpStore(apiKey, requestStartedAt, payload.store ?? undefined), + }); + + return await serveThrough( + c, + prologue, + openaiResponsesServePipeline(), + move({ + 'ingress.http.headers': prologue.headers, + 'ingress.chat.sourceProtocol': 'openaiResponses', + 'ingress.chat.openaiResponses.wantsStream': wantsStream, + 'request.chat.openaiResponses': payload, + 'serve.model': payload.model, + }) as never, + facts => { + const rendered = facts['response.chat.openaiResponses.rendered']; + if (isFrames(rendered)) return { frames: rendered }; + return { body: facts['response.http.jsonBody']!, contentType: 'application/json' }; + }, + ); }, compact: async (c: AuthedContext): Promise => { - const requestBody = await readRequestBody(c); - let ctx: ChatGatewayCtx | undefined; - try { - const payload = parsePayload(requestBody); - ctx = createChatGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), model: payload.model, backgroundScheduler: backgroundSchedulerFromContext(c) }, (apiKey, requestStartedAt) => createOpenAIResponsesHttpStore(apiKey, requestStartedAt, payload.store ?? undefined)); - const result = await openaiResponsesServe.compact({ payload, ctx, headers: inboundHeaders(c) }); - if (result.type === 'result') { - // Compact drains the upstream stream into a single compaction - // resource with no per-token stamps; recordPerformance therefore - // lands in the neutral bucket (request counted, no TTFT/TPOT sample). - // `status` is not a `CompactResource` key — it survives the spread - // from the upstream turn — and it is authoritative for failure: a - // compact that surfaced as `response.failed` must be recorded as such - // so it shows up in the error column instead of masquerading as a - // success. - const failed = result.result.status === 'failed'; - if (failed) { - ctx.dump?.failed('compact envelope status=failed'); - } else { - ctx.dump?.success(result.modelIdentity, result.usage); - } - settle( - ctx, - result.performance, - result.modelIdentity, - result.usage, - failed, - ); - const compactResponse = Response.json(result.result); - return finalizeGatewayResponse(ctx, compactResponse); - } - const response = await respondOpenAIResponses(c, result, false, ctx, payload); - return finalizeGatewayResponse(ctx, response); - } catch (error) { - return await respondToThrow(c, error, requestBody, ctx); - } + const ingress = await readIngress(c); + const request = readRequest(ingress.body.bytes); + if (request.type === 'invalid') return refuse(c, ingress, request); + + const { payload } = request; + const prologue = openChatPrologue(c, ingress, { + wantsStream: false, + model: payload.model, + storeFactory: (apiKey, requestStartedAt) => createOpenAIResponsesHttpStore(apiKey, requestStartedAt, payload.store ?? undefined), + }); + + return await serveThrough( + c, + prologue, + openaiResponsesCompactPipeline(), + move({ + 'ingress.http.headers': prologue.headers, + 'ingress.chat.sourceProtocol': 'openaiResponses', + 'request.chat.openaiResponses': payload, + 'serve.model': payload.model, + }) as never, + // A compaction is one resource however the turn went, so there is never a stream to + // write: the frames it came as were read into that resource before the run answered. + facts => ({ body: facts['response.http.jsonBody']!, contentType: 'application/json' }), + ); }, }; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/hydrate-stored-items.ts b/packages/gateway/src/data-plane/chat/openai-responses/hydrate-stored-items.ts new file mode 100644 index 0000000000..54f0d690c6 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/hydrate-stored-items.ts @@ -0,0 +1,116 @@ +import type { Fields } from './facts.ts'; +import type { ChatServices } from '../services.ts'; +import type { OpenAIResponsesServeFailure } from './errors.ts'; +import { hydrateOpenAIResponsesPayload } from './items/hydrate.ts'; +import { expandPreviousResponseId, PreviousResponseNotFoundError } from './serve-prep.ts'; +import { tryCatchChatServeFailure } from '../shared/errors.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; + +/** A refusal the membrane makes on its own, in the shape every other refusal in this chain + * takes: an empty billed set and empty headers are what "no upstream was called" looks like + * on those two keys, and the envelope is what an OpenAI client reads. */ +const refuseStoredItems = ( + status: number, + message: string, + extra: { readonly param: string; readonly code: string | null }, +): Fields<'response.chat.openaiResponses' | 'response.chat.openaiResponses.streamedUsage' | 'response.usage.billable' | 'response.http.headers' | 'response.http.body' | 'response.http.status'> => ({ + 'response.usage.billable': [], + 'response.http.headers': [], + 'response.http.body': null, 'response.http.status': status, + 'response.chat.openaiResponses.streamedUsage': null, + 'response.chat.openaiResponses': { + status, + message, + envelope: { error: { message, type: 'invalid_request_error', ...extra } }, + }, +}); + +/** + * The stored-items membrane, on the way in. + * + * Three things, in an order the store fixes. A `previous_response_id` becomes the snapshot's + * items in front of this turn's input, and the id itself never reaches a wire — it names + * something only this gateway holds. Every item the turn now names is read back out of the + * store, so an item the client echoed by id is sent as the row we stored rather than as the + * client's copy of it, and whatever server-only state that row carried comes back with it. + * Then the items this turn *adds* are staged, so the snapshot the next turn continues from + * holds them alongside the history it inherited. + * + * It sits above the resolver because everything after it routes on the result: the payload + * affinity reads is the hydrated one, and a turn whose continuation does not resolve has + * nowhere to be routed to. Both of its refusals are answers it already holds, which is why + * it carries the `return` trait alongside `through`. + */ +export const hydrateStoredItems = defineStage< + Fields<'request.chat.openaiResponses'>, + Fields<'request.chat.openaiResponses' | 'request.chat.openaiResponses.privatePayloads'>, + Record, + Record, + Fields<'response.chat.openaiResponses' | 'response.chat.openaiResponses.streamedUsage' | 'response.usage.billable' | 'response.http.headers' | 'response.http.body' | 'response.http.status'>, + ChatServices +>({ + name: 'hydrateStoredItems', + through: { + request: { + needs: ['request.chat.openaiResponses'], + consumes: [], + provides: ['request.chat.openaiResponses', 'request.chat.openaiResponses.privatePayloads'], + }, + response: { needs: [], consumes: [], provides: [] }, + }, + return: { + provides: [ + 'response.chat.openaiResponses', + 'response.chat.openaiResponses.streamedUsage', + 'response.usage.billable', + 'response.http.headers', 'response.http.body', 'response.http.status', + ], + }, + execute: async (facts, next, use) => { + const store = use.gateway.store; + // The key holds what a client may send, whose `input` is a string or a list; this chain + // runs on the canonical form the entry normalized it to. + const asked = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; + + let expanded: CanonicalOpenAIResponsesPayload; + try { + expanded = await expandPreviousResponseId(asked, store); + } catch (error) { + if (!(error instanceof PreviousResponseNotFoundError)) throw error; + // OpenAI's own words for a continuation that does not resolve, byte for byte: Codex + // compares this body against upstream's. See the cross-references on + // `PreviousResponseNotFoundError`. + return move({ + ...facts, + ...refuseStoredItems(400, error.message, { param: 'previous_response_id', code: 'previous_response_not_found' }), + }); + } + + // Both lists: what the turn now names is read by id, and what the client itself sent is + // read by item hash as well, so a body repeated verbatim finds the row it already made. + await store.loadInputItems(expanded.input, asked.input); + + let hydrated: ReturnType; + try { + hydrated = hydrateOpenAIResponsesPayload(expanded, store); + } catch (error) { + const failure = tryCatchChatServeFailure(error); + if (failure?.kind !== 'item-not-found') throw error; + return move({ + ...facts, + ...refuseStoredItems(404, `Item with id '${failure.itemId}' not found.`, { param: 'input', code: null }), + }); + } + + // The client's own input, not the expansion's `item_reference` prefix: the prefix is + // already in the snapshot this turn inherited, and staging it again would repeat it. + await store.stageInputItems(asked.input); + + return await next({ + ...facts, + 'request.chat.openaiResponses': move(hydrated.payload), + 'request.chat.openaiResponses.privatePayloads': move([...hydrated.privatePayloads]), + }); + }, +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/answer-websocket-warmup.ts b/packages/gateway/src/data-plane/chat/openai-responses/interceptors/answer-websocket-warmup.ts deleted file mode 100644 index a043d5bf3a..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/answer-websocket-warmup.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { OpenAIResponsesInterceptor } from './types.ts'; -import { telemetryModelIdentity } from '../../../shared/telemetry/attribution.ts'; -import { syntheticEventsFromResult } from '../items/output.ts'; -import type { OpenAIResponsesResult } from '@floway-dev/protocols/openai-responses'; -import { eventResult, providerModelOf } from '@floway-dev/provider'; - -// Codex opens every session with a WebSocket prewarm: a `response.create` that -// carries the session's instructions and tools with `generate: false`. It is -// connection setup rather than inference, so Codex waits only for the terminal -// `response.completed` and then continues from that response's id, sending -// just the items the prewarm did not already carry. -// https://github.com/openai/codex/blob/6989c6548b3737f108e2bb5ae1171b1d2032e30c/codex-rs/core/src/client.rs#L17-L18 -// https://github.com/openai/codex/blob/6989c6548b3737f108e2bb5ae1171b1d2032e30c/codex-rs/core/src/client.rs#L2025 -// https://github.com/openai/codex/blob/6989c6548b3737f108e2bb5ae1171b1d2032e30c/codex-rs/core/src/client.rs#L2181-L2184 -// -// No upstream call can stand in for it. The Codex HTTP backend rejects the -// field with `{"detail":"Unsupported parameter: generate"}`, and a translated -// target drops it and runs a full, billed generation. The gateway answers the -// prewarm itself: by now serve preparation has resolved the model, expanded -// any `previous_response_id`, and staged this request's input, so the empty -// completed response commits a snapshot of exactly that input and the next -// turn's continuation replays it. An id no upstream serves still fails before -// this runs. -// -// No `performance` context on the result: a turn that never dialed the -// upstream has no latency to report. The usage row still lands, at zero, so the -// request stays visible in the dashboard. -export const answerWebSocketWarmup: OpenAIResponsesInterceptor = async (ctx, _gatewayCtx, run) => { - // TODO: In the pipeline architecture, answer generate:false locally only for WebSocket requests. - if (ctx.payload.generate !== false) return await run(); - const result: OpenAIResponsesResult = { - // Replaced by the client-output boundary's own response id. - id: '', - object: 'response', - model: ctx.payload.model, - status: 'completed', - output: [], - error: null, - incomplete_details: null, - usage: { input_tokens: 0, output_tokens: 0, total_tokens: 0 }, - }; - return eventResult(syntheticEventsFromResult(result), telemetryModelIdentity(ctx.candidate, providerModelOf(ctx.candidate).id)); -}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/apply-role-compatibility.ts b/packages/gateway/src/data-plane/chat/openai-responses/interceptors/apply-role-compatibility.ts deleted file mode 100644 index ce9664d570..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/apply-role-compatibility.ts +++ /dev/null @@ -1,35 +0,0 @@ -import type { OpenAIResponsesInterceptor } from './types.ts'; -import type { OpenAIResponsesInputItem } from '@floway-dev/protocols/openai-responses'; -import { providerModelOf } from '@floway-dev/provider'; - -export const withRoleCompatibilityApplied: OpenAIResponsesInterceptor = (ctx, _gatewayCtx, run) => { - if (ctx.targetApi !== 'openaiResponses') return run(); - - const flags = providerModelOf(ctx.candidate).enabledFlags; - const rewriteSystemToDeveloper = flags.has('rewrite-system-to-developer'); - const rewriteDeveloperToSystem = flags.has('rewrite-developer-to-system'); - const rewriteMidConvSystemToUser = flags.has('rewrite-mid-conv-system-to-user'); - if (!rewriteSystemToDeveloper && !rewriteDeveloperToSystem && !rewriteMidConvSystemToUser) return run(); - - let crossedLeadingSystemRun = false; - ctx.payload = { - ...ctx.payload, - input: ctx.payload.input.map(item => { - let mapped: OpenAIResponsesInputItem = item; - if (mapped.type === 'message' && rewriteSystemToDeveloper && mapped.role === 'system') { - mapped = { ...mapped, role: 'developer' }; - } - if (mapped.type === 'message' && rewriteDeveloperToSystem && mapped.role === 'developer') { - mapped = { ...mapped, role: 'system' }; - } - const isSystemMessage = mapped.type === 'message' && mapped.role === 'system'; - if (!crossedLeadingSystemRun && !isSystemMessage) crossedLeadingSystemRun = true; - if (rewriteMidConvSystemToUser && crossedLeadingSystemRun && mapped.type === 'message' && mapped.role === 'system') { - mapped = { ...mapped, role: 'user' }; - } - return mapped; - }), - }; - - return run(); -}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/disable-reasoning-on-forced-tool-choice.ts b/packages/gateway/src/data-plane/chat/openai-responses/interceptors/disable-reasoning-on-forced-tool-choice.ts deleted file mode 100644 index f6979ad429..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/disable-reasoning-on-forced-tool-choice.ts +++ /dev/null @@ -1,25 +0,0 @@ - -import type { OpenAIResponsesInterceptor } from './types.ts'; -import type { OpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; -import { providerModelOf } from '@floway-dev/provider'; - -// Opt-in workaround for upstreams where forced `tool_choice` and enabled -// reasoning do not compose. Sets the gateway's canonical "no reasoning" -// sentinel `reasoning: { effort: 'none' }` (also OpenAI Responses API's documented -// disable value). Any active Vendor: * flag's last-running normalizer then -// translates that into the vendor's wire form. Sibling fields on the -// `reasoning` object (e.g. `summary`) are dropped — they have no meaning -// when reasoning is disabled. -const hasForcedToolChoice = (payload: OpenAIResponsesPayload): boolean => { - const toolChoice = payload.tool_choice; - if (toolChoice === undefined || toolChoice === null) return false; - if (typeof toolChoice === 'string') return toolChoice === 'required'; - return true; -}; - -export const withReasoningDisabledOnForcedToolChoice: OpenAIResponsesInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('disable-reasoning-on-forced-tool-choice')) return await run(); - if (!hasForcedToolChoice(ctx.payload)) return await run(); - ctx.payload = { ...ctx.payload, reasoning: { effort: 'none' } }; - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/index.ts b/packages/gateway/src/data-plane/chat/openai-responses/interceptors/index.ts deleted file mode 100644 index ebadcb0af7..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/index.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { answerWebSocketWarmup } from './answer-websocket-warmup.ts'; -import { withRoleCompatibilityApplied } from './apply-role-compatibility.ts'; -import { withOpenAIResponsesCollaborationShim } from './collaboration-shim.ts'; -import { withOpenAIResponsesCompactShim } from './compact-shim.ts'; -import { withReasoningDisabledOnForcedToolChoice } from './disable-reasoning-on-forced-tool-choice.ts'; -import { withEmptyToolsToolChoiceNormalized } from './normalize-empty-tools-tool-choice.ts'; -import { withExclusiveCachedTokensNormalized } from './normalize-exclusive-cached-tokens.ts'; -import { withOpenAIResponsesServerToolShim } from './server-tool-shim.ts'; -import { imageGenerationServerTool } from './server-tools/image-generation.ts'; -import { webSearchServerTool } from './server-tools/web-search.ts'; -import { withPromptCacheKeyStripped } from './strip-prompt-cache-key.ts'; -import type { OpenAIResponsesInterceptor } from './types.ts'; -import { withVendorDeepSeekOpenAIResponsesNormalize } from './vendor-deepseek-normalize.ts'; -import { withVendorQwenOpenAIResponsesNormalize } from './vendor-qwen-normalize.ts'; - -// Unified OpenAI Responses interceptor list. All entries are attached to every -// candidate; each interceptor's body decides whether to act (flag-gated entries -// early-return on `providerModelOf(ctx.candidate).enabledFlags.has(flagId)`). -// -// Translated requests re-enter the selected target protocol's chain. The role -// compatibility entry therefore acts only when OpenAI Responses is the final target, -// after pairwise translation has finished. -// -// Order matters: earlier entries wrap later ones. -// - answerWebSocketWarmup: runs outermost so a WebSocket `generate: false` prewarm is -// answered before any shim or upstream call can turn it into a generation. -// - withOpenAIResponsesCompactShim: wraps the remaining shims so the action pivot -// ('compact' → 'generate' for the inner summarization turn) is visible -// to every downstream interceptor + the provider terminal. Also -// responsible for inbound expansion of prior shim-encoded compaction -// items so the upstream sees the summarized history. -// - withOpenAIResponsesCollaborationShim: keeps one plaintext namespace around -// the complete server-tool loop and restores client identities on egress. -// - withOpenAIResponsesServerToolShim: wraps the multi-turn ReAct loop around -// the rest of the chain. -// - withReasoningDisabledOnForcedToolChoice: gated by -// `disable-reasoning-on-forced-tool-choice`. -// - withEmptyToolsToolChoiceNormalized: gated by -// `empty-tools-tool-choice-none`. Runs inside the server-tool shim so a -// tool injected by that shim prevents the empty-list rewrite. -// - withRoleCompatibilityApplied: applies role flags in the fixed order -// `system → developer → system → user`; later rewrites are authoritative -// when flags overlap, and the final step affects only mid-conversation system. -// - withPromptCacheKeyStripped: gated by `strip-prompt-cache-key`. Drops -// the top-level `prompt_cache_key` field for upstreams that reject it -// as an unknown argument (e.g. Azure DeepSeek). Runs before vendor -// normalizers so vendor-specific translation sees the already-stripped -// canonical payload. -// - withExclusiveCachedTokensNormalized: unconditional on an OpenAI Responses -// target. Folds the cache buckets back into `input_tokens` whenever -// `total_tokens` witnesses that the upstream reports them alongside it, -// and consults `usage-exclusive-cached-tokens` for the responses whose -// totals witness nothing — so the flag is a declaration input rather than -// a gate. -// - withVendor*OpenAIResponsesNormalize: gated by `vendor-`. Registered after -// the role-compatibility entry so each gets the final say on the outbound wire -// body. -export const openaiResponsesInterceptors: readonly OpenAIResponsesInterceptor[] = [ - answerWebSocketWarmup, - withOpenAIResponsesCompactShim, - withOpenAIResponsesCollaborationShim, - withOpenAIResponsesServerToolShim([ - webSearchServerTool, - imageGenerationServerTool, - ]), - withReasoningDisabledOnForcedToolChoice, - withEmptyToolsToolChoiceNormalized, - withRoleCompatibilityApplied, - withPromptCacheKeyStripped, - withExclusiveCachedTokensNormalized, - withVendorDeepSeekOpenAIResponsesNormalize, - withVendorQwenOpenAIResponsesNormalize, -]; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/normalize-empty-tools-tool-choice.ts b/packages/gateway/src/data-plane/chat/openai-responses/interceptors/normalize-empty-tools-tool-choice.ts deleted file mode 100644 index 6c1eb3b482..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/normalize-empty-tools-tool-choice.ts +++ /dev/null @@ -1,14 +0,0 @@ -import type { OpenAIResponsesInterceptor } from './types.ts'; -import { collectOpenAIResponsesTools } from '@floway-dev/protocols/openai-responses'; -import { providerModelOf } from '@floway-dev/provider'; - -// Opt-in compatibility rewrite for upstreams that reject an active -// `tool_choice` when the caller explicitly supplies no tools. This runs after -// server-tool shims have had the opportunity to inject a usable tool. -export const withEmptyToolsToolChoiceNormalized: OpenAIResponsesInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('empty-tools-tool-choice-none')) return await run(); - if (!Array.isArray(ctx.payload.tools) || ctx.payload.tools.length !== 0) return await run(); - if (collectOpenAIResponsesTools(ctx.payload).length !== 0) return await run(); - ctx.payload = { ...ctx.payload, tool_choice: 'none' }; - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/normalize-exclusive-cached-tokens.ts b/packages/gateway/src/data-plane/chat/openai-responses/interceptors/normalize-exclusive-cached-tokens.ts deleted file mode 100644 index 2fb5c82527..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/normalize-exclusive-cached-tokens.ts +++ /dev/null @@ -1,76 +0,0 @@ -// OpenAI-Responses-shape counterpart of the OpenAI Chat Completions normalizer of the same -// name. The evidence for the two conventions, the decision rule, and the two -// contradictions that raise are documented once, at -// `../../openai-chat-completions/interceptors/normalize-exclusive-cached-tokens.ts` -// and at `foldsExclusiveCacheTokens`. -// -// OpenAI Responses carries usage on `event.response.usage` and names the buckets -// `input_tokens` / `input_tokens_details.{cached_tokens, cache_write_tokens}`. -// Every event that carries a response resource repeats the whole resource, so -// the rewrite applies to each of them rather than to a single terminal frame. - -import type { OpenAIResponsesInterceptor } from './types.ts'; -import { asJsonObject, type JsonObject, readJsonNumber } from '../../../../shared/json-helpers.ts'; -import { foldsExclusiveCacheTokens } from '../../../shared/telemetry/usage.ts'; -import { eventFrame } from '@floway-dev/protocols/common'; -import type { OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { providerModelOf } from '@floway-dev/provider'; - -const rewriteInboundUsage = ( - event: OpenAIResponsesStreamEvent, - declaredExclusive: boolean, - identity: string, -): OpenAIResponsesStreamEvent => { - if (!('response' in event)) return event; - const response = asJsonObject(event.response); - const usage = asJsonObject(response?.usage); - if (!response || !usage) return event; - const inputTokens = readJsonNumber(usage.input_tokens); - const outputTokens = readJsonNumber(usage.output_tokens); - if (inputTokens == null || outputTokens == null) return event; - - const details = asJsonObject(usage.input_tokens_details); - const cacheRead = readJsonNumber(details?.cached_tokens) ?? 0; - const cacheWrite = readJsonNumber(details?.cache_write_tokens) ?? 0; - if (cacheRead === 0 && cacheWrite === 0) return event; - - const fold = foldsExclusiveCacheTokens(declaredExclusive, { - inputTokens, - outputTokens, - totalTokens: readJsonNumber(usage.total_tokens) ?? undefined, - cacheRead, - cacheWrite, - }, identity); - if (!fold) return event; - - const nextUsage: JsonObject = { ...usage, input_tokens: inputTokens + cacheRead + cacheWrite }; - const nextResponse: JsonObject = { ...response, usage: nextUsage }; - return { ...event, response: nextResponse } as unknown as OpenAIResponsesStreamEvent; -}; - -export const withExclusiveCachedTokensNormalized: OpenAIResponsesInterceptor = async (ctx, _gatewayCtx, run) => { - // Runs only where the wire it speaks about is; see the OpenAI Chat Completions - // counterpart. - if (ctx.targetApi !== 'openaiResponses') return await run(); - - const model = providerModelOf(ctx.candidate); - const declaredExclusive = model.enabledFlags.has('usage-exclusive-cached-tokens'); - const identity = `${ctx.candidate.provider.upstreamId}/${model.id}`; - - const result = await run(); - if (result.type !== 'events') return result; - - return { - ...result, - events: (async function* () { - for await (const frame of result.events) { - if (frame.type !== 'event') { - yield frame; - continue; - } - const event = rewriteInboundUsage(frame.event, declaredExclusive, identity); - yield event === frame.event ? frame : eventFrame(event); - } - })(), - }; -}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/strip-prompt-cache-key.ts b/packages/gateway/src/data-plane/chat/openai-responses/interceptors/strip-prompt-cache-key.ts deleted file mode 100644 index c2045ccdfb..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/strip-prompt-cache-key.ts +++ /dev/null @@ -1,15 +0,0 @@ -import type { OpenAIResponsesInterceptor } from './types.ts'; -import { providerModelOf } from '@floway-dev/provider'; - -// Opt-in workaround for upstreams that reject `prompt_cache_key` as an unknown -// request argument (e.g. Azure DeepSeek). Drop the top-level field before the -// request reaches the terminal. OpenAI-native and truly OpenAI-compatible -// OpenAI Responses upstreams accept it for prefix-cache attribution, so removal only -// happens under the flag. -export const withPromptCacheKeyStripped: OpenAIResponsesInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('strip-prompt-cache-key')) return await run(); - if (ctx.payload.prompt_cache_key === undefined) return await run(); - const { prompt_cache_key: _stripped, ...rest } = ctx.payload; - ctx.payload = rest; - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/types.ts b/packages/gateway/src/data-plane/chat/openai-responses/interceptors/types.ts deleted file mode 100644 index 9922fc9c16..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/types.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { TokenUsage } from '../../../../repo/types.ts'; -import type { ChatGatewayCtx } from '../../shared/gateway-ctx.ts'; -import type { Interceptor } from '@floway-dev/interceptor'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import type { EventResultMetadata, ExecuteResult, OpenAIResponsesInvocation, TelemetryModelIdentity } from '@floway-dev/provider'; - -export type { OpenAIResponsesInvocation }; - -// The chain runner produces an event stream for both actions — the attempt -// drains it into a single non-streaming result when the caller's intent action -// was 'compact', and serve completes that result into the compaction resource. -// `modelIdentity`, `usage`, and `performance` carry the per-turn attribution -// forward so the http layer records the success path identically to streaming -// generate. -// The non-streaming result is parameterized because `/responses/compact` -// answers with `CompactResource` rather than the response resource, and its -// own completion narrows the type further. -export type OpenAIResponsesAttemptResult = - | ExecuteResult> - | { - readonly type: 'result'; - readonly result: Result; - readonly modelIdentity: TelemetryModelIdentity; - readonly usage: TokenUsage | null; - readonly performance: EventResultMetadata['performance']; - }; - -export type OpenAIResponsesInterceptor = Interceptor< - OpenAIResponsesInvocation, - ChatGatewayCtx, - ExecuteResult> ->; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/vendor-deepseek-normalize.ts b/packages/gateway/src/data-plane/chat/openai-responses/interceptors/vendor-deepseek-normalize.ts deleted file mode 100644 index 922e47a92d..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/vendor-deepseek-normalize.ts +++ /dev/null @@ -1,44 +0,0 @@ -// DeepSeek wire-dialect normalizer for the OpenAI Responses protocol. Always- -// attached; flag-gated by `vendor-deepseek`. Runs last among interceptors -// so it has the final say on the outbound wire body. -// -// Outbound (request → upstream): -// -// - `reasoning.effort: 'none'` is the gateway's canonical "no reasoning" -// sentinel (produced when an Anthropic Messages source had `thinking: { type: -// 'disabled' }`, etc.). DeepSeek uses a top-level -// `thinking: { type: 'disabled' }` field instead. We strip the entire -// `reasoning` object and emit the DeepSeek form. -// -// Inbound: nothing today — the OpenAI-Responses-target dialect quirks that exist -// on OpenAI Chat Completions (assistant `reasoning_content` field, `prompt_cache_*_tokens` -// usage) have no OpenAI-Responses-shape equivalent that has surfaced. Add hooks -// here if vendor-specific OpenAI Responses inbound rewrites become necessary. -// -// Reference: -// - https://api-docs.deepseek.com/zh-cn/guides/thinking_mode - -import type { OpenAIResponsesInterceptor } from './types.ts'; -import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; -import { providerModelOf } from '@floway-dev/provider'; - -interface DeepSeekDisableField { - thinking?: { type: 'disabled' }; -} - -type CanonicalOpenAIResponsesPayloadWithDeepSeekDisable = Omit & DeepSeekDisableField; - -const stripCanonicalReasoningSentinel = (payload: CanonicalOpenAIResponsesPayload): CanonicalOpenAIResponsesPayload => { - if (payload.reasoning?.effort !== 'none') return payload; - const { reasoning: _stripped, ...rest } = payload; - const out: CanonicalOpenAIResponsesPayloadWithDeepSeekDisable = { ...rest, thinking: { type: 'disabled' } }; - return out as CanonicalOpenAIResponsesPayload; -}; - -export const withVendorDeepSeekOpenAIResponsesNormalize: OpenAIResponsesInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('vendor-deepseek')) return await run(); - - ctx.payload = stripCanonicalReasoningSentinel(ctx.payload); - - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/vendor-qwen-normalize.ts b/packages/gateway/src/data-plane/chat/openai-responses/interceptors/vendor-qwen-normalize.ts deleted file mode 100644 index 0e6c1e9458..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/vendor-qwen-normalize.ts +++ /dev/null @@ -1,29 +0,0 @@ -// Qwen wire-dialect normalizer for the OpenAI Responses protocol. Always-attached; -// flag-gated by `vendor-qwen`. Runs last among interceptors so it has the -// final say on the outbound wire body. -// -// Outbound (request → upstream): -// -// - `reasoning.effort: 'none'` is the gateway's canonical "no reasoning" -// sentinel. Qwen uses a top-level `enable_thinking: false` field instead. -// We strip the entire `reasoning` object and emit the Qwen form. -// -// Inbound: nothing today. -// -// Reference: -// - https://www.alibabacloud.com/help/en/model-studio/deep-thinking - -import type { OpenAIResponsesInterceptor } from './types.ts'; -import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; -import { providerModelOf } from '@floway-dev/provider'; - -export const withVendorQwenOpenAIResponsesNormalize: OpenAIResponsesInterceptor = async (ctx, _gatewayCtx, run) => { - if (!providerModelOf(ctx.candidate).enabledFlags.has('vendor-qwen')) return await run(); - - if (ctx.payload.reasoning?.effort === 'none') { - const { reasoning: _stripped, ...rest } = ctx.payload; - ctx.payload = { ...rest, enable_thinking: false } as CanonicalOpenAIResponsesPayload; - } - - return await run(); -}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/items/output.ts b/packages/gateway/src/data-plane/chat/openai-responses/items/output.ts index 7b091407e2..330000fda7 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/items/output.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/items/output.ts @@ -10,7 +10,7 @@ import { isOpenAIResponsesCompactionItem, openaiResponsesResultToEvents, type Op // separately before a successful terminal frame. Failed/error terminals keep // completed item rows but never a snapshot. // -// One client response can span several upstream calls behind the server-tool +// One client response can span several upstream calls behind the hosted-tool // runtime. The caller mints its envelope id once, and this wrapper applies it // to every queued/created/in-progress and terminal response envelope without // changing any output item. diff --git a/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts b/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts index 73e1305789..8cd26591ef 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/items/store.ts @@ -37,7 +37,7 @@ export interface OpenAIResponsesStatefulStore { persistOutputItem(row: StoredOpenAIResponsesItem): Promise; commitSnapshot(responseId: string, mode: OpenAIResponsesSnapshotMode, outputItemIds: readonly string[]): Promise; // Per-attempt transient state. `beginAttempt` reseeds the private-payload - // scratchpad from hydrated items; interceptors can add server-only state + // scratchpad from hydrated items; a hosted tool can add server-only state // during the turn and output capture persists it with the exact wire item. beginAttempt(privatePayloads: ReadonlyMap): void; registerPrivatePayload(id: string, privatePayload: unknown): void; @@ -107,6 +107,12 @@ export class LayeredOpenAIResponsesStatefulStore implements OpenAIResponsesState await this.loadItems({ ids: [...ids], itemHashes: [...itemHashes] }); } + // Copied on the way out, which is the one copy in this file the pipeline's own rule asks for + // rather than forbids. What hydration builds from this row is handed to `move()`, so it is + // deep-frozen the moment it becomes a fact — and the row is a live cache entry that outlives + // the turn. Handing the cache's own object over would freeze the cache, which is the same + // hazard `GatewayFacts` states about putting a live handle in the record: the writes the + // store relies on would then fail where nothing is watching. getItemById(id: string): StoredOpenAIResponsesItem | undefined { const row = this.loadedItems.get(id); return row === undefined ? undefined : cloneStoredOpenAIResponsesItem(row); @@ -157,17 +163,22 @@ export class LayeredOpenAIResponsesStatefulStore implements OpenAIResponsesState await Promise.all(this.options.writes.map(write => write.insertSnapshot(snapshot))); } + // The scratchpad holds what it was handed. These three used to copy on the way in and again + // on the way out, so that a caller writing into a payload could not reach the store's own — + // but the payloads arriving here came out of the record and are frozen, and the one reader + // splices a sub-object of them into an array rather than writing to it. Copying bought + // isolation from a write nothing performs, at a deep copy per stored item per attempt. beginAttempt(privatePayloads: ReadonlyMap): void { this.privatePayloads.clear(); - for (const [id, payload] of privatePayloads) this.privatePayloads.set(id, structuredClone(payload)); + for (const [id, payload] of privatePayloads) this.privatePayloads.set(id, payload); } registerPrivatePayload(id: string, privatePayload: unknown): void { - if (privatePayload !== undefined) this.privatePayloads.set(id, structuredClone(privatePayload)); + if (privatePayload !== undefined) this.privatePayloads.set(id, privatePayload); } getPrivatePayload(id: string): unknown { - return structuredClone(this.privatePayloads.get(id)); + return this.privatePayloads.get(id); } private async loadItems(query: { ids: readonly string[]; itemHashes: readonly string[] }): Promise { @@ -389,7 +400,7 @@ export const createOpenAIResponsesHttpStore = (apiKey: OpenAIResponsesStatePolic // Non-OpenAI-Responses sources (Anthropic Messages / Gemini generateContent / OpenAI Chat Completions) never persist // OpenAI Responses items, even when translation enters an OpenAI Responses attempt — but the -// server-tool shim still runs there, and its request-private payload +// hosted-tool stage still runs there, and its request-private payload // scratchpad lives on the store. So they get a store with no backing: it holds // per-attempt state in memory and reads/writes nothing durable, keeping the // store present on every chat ctx. diff --git a/packages/gateway/src/data-plane/chat/openai-responses/meter.ts b/packages/gateway/src/data-plane/chat/openai-responses/meter.ts new file mode 100644 index 0000000000..05e18bc993 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/meter.ts @@ -0,0 +1,99 @@ +import type { StreamOutcome } from '../../pipeline/serve.ts'; +import { meterChatWire } from '../meter.ts'; +import { billableUsageFromOpenAIResponsesEvent } from './usage.ts'; +import type { BillableEntity } from '../../pipeline/facts.ts'; +import { isFirstOutputTokenFrame } from '../shared/first-output-token.ts'; +import { chatUsageMeasurement, type ChatBillableUsage } from '../shared/usage.ts'; +import { defer, type Deferred } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import { isOpenAIResponsesTerminalEvent, OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import type { TelemetryModelIdentity } from '@floway-dev/provider'; + +export const meterUsage = (streamedUsage: string) => meterChatWire({ + wire: 'openaiResponses', + answer: 'response.chat.openaiResponses', + streamedUsage, + read: meterOpenAIResponses, +}); + +/** Reads the upstream's own usage off its own events as they pass, so the reading costs one + * pass and the client's stream is what drives it. OpenAI Responses states its counts on the + * lifecycle envelopes, and only one carrying real counts replaces the running figure, so an + * envelope that states none cannot wipe a good reading. */ +const meterOpenAIResponses = ( + source: AsyncIterable>, + identity: TelemetryModelIdentity, + attempt: { firstOutputTokenAt: number | null }, +): { readonly frames: AsyncIterable>; readonly outcome: Deferred } => { + const completion = Promise.withResolvers(); + const outcome = defer(completion.promise); + let settled = false; + // Running out without the terminal frame is what "it did not finish" means, and it is known + // at the same moment the usage is. + let sawTerminal = false; + let failed = false; + const generator = (async function* () { + let reported: ChatBillableUsage | undefined; + const finish = (): void => { + if (settled) return; + settled = true; + try { completion.resolve({ billable: [billedOpenAIResponsesEntity(identity, reported)], failed: failed || !sawTerminal }); } catch (error) { + completion.reject(error); + throw error; + } + }; + try { + for await (const frame of source) { + // Time to first token is measured where the token is, which is the only place that + // knows a frame carries generated content rather than the envelope around it. + if (attempt.firstOutputTokenAt === null && isFirstOutputTokenFrame(frame, 'openaiResponses')) { + attempt.firstOutputTokenAt = performance.now(); + } + if (frame.type !== 'event') { + yield frame; + continue; + } + const usage = billableUsageFromOpenAIResponsesEvent(frame.event); + if (usage !== null) reported = usage; + // Read off the frame rather than off having been resumed past it. The stage that + // stores the turn's items stops reading at the terminal event, so the resumption + // never comes: what this loop would learn from it is already true when the frame + // arrives, and the turn ended the way the upstream said it did whether or not + // anything downstream asked for another. + if (isOpenAIResponsesTerminalEvent(frame.event)) { + sawTerminal = true; + failed = frame.event.type === 'response.failed' || frame.event.type === 'error'; + finish(); + } + yield frame; + // The turn is over, so there is nothing further to read. An upstream that holds the + // connection open past its terminal event would otherwise hold the client's stream + // open with it; returning here closes the read, which cancels the upstream. + if (sawTerminal) return; + } + // Frames ran out with no terminal event, which is a turn nobody can answer from: the + // response was never stated complete, incomplete or failed. + throw new Error(OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE); + } finally { + // Reached however the frames ended — the terminal event, a client that stopped + // reading, or a broken upstream — because tokens the upstream already metered are + // billable whatever happened to the downstream half. + finish(); + } + })(); + return { frames: { [Symbol.asyncIterator]: () => generator }, outcome }; +}; + +/** An upstream that reported nothing leaves no quantities at all, which is a different + * statement from reporting zero. + * + * A reading that did arrive is converted rather than cast: a billed entity is keyed by + * billing metric, which is not the shape a protocol reports in. The tier rides along + * because on this protocol it is not a quantity but a rate selector — `service_tier` states + * the tier the turn was actually served at, and that is the pricing entry it is billed + * under. */ +export const billedOpenAIResponsesEntity = (identity: TelemetryModelIdentity, usage: ChatBillableUsage | undefined): BillableEntity => { + if (usage === undefined) return { identity, quantities: {} }; + const measurement = chatUsageMeasurement(usage); + return { identity, quantities: measurement.quantities, pricingFacts: measurement.pricingFacts }; +}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/normalize-assistant-content.ts b/packages/gateway/src/data-plane/chat/openai-responses/normalize-assistant-content.ts new file mode 100644 index 0000000000..88da83bcea --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/normalize-assistant-content.ts @@ -0,0 +1,39 @@ +import type { Fields } from './facts.ts'; +import type { ChatServices } from '../services.ts'; +import { normalizeAssistantInputText } from './items/normalize-assistant-content.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; + +/** + * What an upstream's OpenAI Responses endpoint accepts on an assistant item it produced itself. + * Copilot's compaction translation and Azure-native compaction both emit assistant messages + * whose content blocks carry `type: 'input_text'`, and both then refuse those same items + * echoed back as input on the next turn. Every way prior upstream-produced history reaches a + * wire arrives here — a direct client echo, the snapshot the membrane expanded, a compaction + * tail — so this is the one place the canonical assistant content type is put back. + * + * Only this wire needs it. Both translators read `input_text` and `output_text` the same way + * on assistant content, so a turn that leaves for Anthropic Messages or OpenAI Chat Completions never carried + * the disagreement in the first place. + */ +export const normalizeAssistantContentForOpenAIResponses = defineStage< + Fields<'request.chat.openaiResponses'>, + Fields<'request.chat.openaiResponses'>, + Record, + Record, + ChatServices +>({ + name: 'normalizeAssistantContentForOpenAIResponses', + through: { + request: { needs: ['request.chat.openaiResponses'], consumes: [], provides: ['request.chat.openaiResponses'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; + const input = normalizeAssistantInputText(payload.input); + // A rewrite that changed nothing hands the same payload on, so the record shows no + // change where none happened. + if (input === payload.input) return await next(facts); + return await next({ ...facts, 'request.chat.openaiResponses': move({ ...payload, input }) }); + }, +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/normalize-empty-tools-tool-choice.ts b/packages/gateway/src/data-plane/chat/openai-responses/normalize-empty-tools-tool-choice.ts new file mode 100644 index 0000000000..781b29d7a0 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/normalize-empty-tools-tool-choice.ts @@ -0,0 +1,5 @@ +import { normalizeEmptyTools } from '../shared/normalize-empty-tools.ts'; +import { collectOpenAIResponsesTools, type CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; + +export const normalizeEmptyToolsForOpenAIResponses = normalizeEmptyTools('request.chat.openaiResponses', payload => + payload.tool_choice === 'none' || collectOpenAIResponsesTools(payload as CanonicalOpenAIResponsesPayload).length > 0 ? payload : { ...payload, tool_choice: 'none' }); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/normalize-exclusive-cached-tokens.ts b/packages/gateway/src/data-plane/chat/openai-responses/normalize-exclusive-cached-tokens.ts new file mode 100644 index 0000000000..906779f451 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/normalize-exclusive-cached-tokens.ts @@ -0,0 +1,80 @@ +import { asJsonObject, type JsonObject } from '../../../shared/json-helpers.ts'; +import type { AttemptSelector } from '../../pipeline/facts.ts'; +import type { Chat } from '../facts.ts'; +import { withCacheBucketsFolded, type CacheBucketNames } from '../shared/cache-buckets.ts'; +import { answerWithFrames, rewritingEvents } from '../shared/frames.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; +import type { OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; + +/** + * Restores OpenAI's inclusive input-token contract on an upstream that reports the cache + * buckets alongside `input_tokens` instead of inside it. + * + * Unconditional on this chain rather than flag-gated: `foldsExclusiveCacheTokens` owns the + * decision and reads `total_tokens` as the witness, and the `usage-exclusive-cached-tokens` + * flag is a declaration input for the responses whose totals witness nothing. Every event + * that carries a response resource repeats the whole resource, so the rewrite applies to each + * of them rather than to a single terminal frame. The evidence for the two conventions and + * the contradictions that raise are documented at `foldsExclusiveCacheTokens` and at + * `withCacheBucketsFolded`, which is the fold itself. + */ +export const normalizeExclusiveCachedTokensForOpenAIResponses = defineStage< + Chat<'route.attempt'>, + Chat<'route.attempt'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> +>({ + name: 'normalizeExclusiveCachedTokens', + through: { + request: { needs: ['route.attempt'], consumes: [], provides: [] }, + response: { needs: ['response.chat.openaiResponses'], consumes: [], provides: ['response.chat.openaiResponses'] }, + }, + execute: transform< + Chat<'route.attempt'>, + Chat<'route.attempt'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> + >(() => { + // Which attempt this is gets read on the way down and spoken about on the way back, which + // is the order `transform` runs the two halves in. + let attempt!: AttemptSelector; + return { + request: facts => { + attempt = facts['route.attempt']; + return facts; + }, + response: facts => { + const declaredExclusive = attempt.flags.includes('usage-exclusive-cached-tokens'); + const identity = `${attempt.upstreamId}/${attempt.modelId}`; + const answer = answerWithFrames( + facts['response.chat.openaiResponses'], + frames => rewritingEvents(frames, event => foldOpenAIResponsesUsage(event, declaredExclusive, identity)), + ); + return answer === null ? facts : { ...facts, 'response.chat.openaiResponses': move(answer) }; + }, + }; + }), +}); + +/** OpenAI Responses carries usage on `event.response.usage` and names the buckets `input_tokens` and + * `input_tokens_details.{cached_tokens, cache_write_tokens}`. */ +const OPENAI_RESPONSES_CACHE_BUCKETS: CacheBucketNames = { + input: 'input_tokens', + output: 'output_tokens', + details: 'input_tokens_details', + cacheWrite: ['cache_write_tokens'], +}; + +const foldOpenAIResponsesUsage = ( + event: OpenAIResponsesStreamEvent, + declaredExclusive: boolean, + identity: string, +): OpenAIResponsesStreamEvent => { + if (!('response' in event)) return event; + const response = asJsonObject(event.response); + const usage = asJsonObject(response?.usage); + if (response === null || usage === null) return event; + const folded = withCacheBucketsFolded(usage, OPENAI_RESPONSES_CACHE_BUCKETS, declaredExclusive, identity); + if (folded === usage) return event; + return { ...event, response: { ...response, usage: folded } as JsonObject } as unknown as OpenAIResponsesStreamEvent; +}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts new file mode 100644 index 0000000000..15de8359be --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/pipeline.ts @@ -0,0 +1,72 @@ +import { hydrateStoredItems } from './hydrate-stored-items.ts'; +import { failover } from '../../pipeline/failover.ts'; +import { materializeAttempt } from '../materialize-attempt.ts'; +import { resolveChatCandidates } from '../resolve-candidates.ts'; +import { answerOpenAIResponsesWebSocketWarmup } from './answer-websocket-warmup.ts'; +import { beginStoredAttempt } from './begin-stored-attempt.ts'; +import { projectOpenAIResponsesCollaboration } from './collaboration-shim.ts'; +import { asksForCompaction } from './compaction-policy.ts'; +import { emitOpenAIResponses } from './emit.ts'; +import { expandShimCompactions } from './expand-compactions.ts'; +import { OPENAI_RESPONSES_STREAMED_USAGE, type OpenAIResponsesServeEntry, type OpenAIResponsesServeExit } from './facts.ts'; +import { projectOpenAIResponsesWebSocket, type OpenAIResponsesWebSocketEntry, type OpenAIResponsesWebSocketExit } from './project-websocket.ts'; +import { serializeClientJson } from '../../pipeline/serialize-client-json.ts'; +import { composeChat as compose } from '../compose.ts'; +import { dialChatWire } from '../dial-wire.ts'; +import { imageGenerationHostedTool } from './hosted-tools/image-generation.ts'; +import { webSearchHostedTool } from './hosted-tools/web-search.ts'; +import { hostedTools } from './hosted-tools.ts'; +import { normalizeEmptyToolsForOpenAIResponses } from './normalize-empty-tools-tool-choice.ts'; +import { summarizeForCompaction } from './summarize-for-compaction.ts'; +import { openaiResponsesNarrowing, openaiResponsesTarget } from './target.ts'; +import { openaiResponsesWireFor } from './wires.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import { writeSettlement } from '../../pipeline/settlement.ts'; +import type { Pipeline, Stage } from '@floway-dev/pipeline'; + +export const openaiResponsesServePipeline = (): Pipeline => + compose('openaiResponsesServe', [ + writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, OPENAI_RESPONSES_STREAMED_USAGE), + serializeClientJson('response.chat.openaiResponses.rendered'), + emitOpenAIResponses('sse'), + ...openaiResponsesModelStages(false), + ]); + +export const openaiResponsesWebSocketPipeline = (): Pipeline => + compose('openaiResponsesWebSocket', [ + writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, OPENAI_RESPONSES_STREAMED_USAGE), + projectOpenAIResponsesWebSocket, + emitOpenAIResponses('events'), + ...openaiResponsesModelStages(true), + ]); + +const openaiResponsesModelStages = (warmup: boolean): readonly Stage[] => [ + hydrateStoredItems, + resolveChatCandidates(openaiResponsesNarrowing), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.chat.openaiResponses'?: unknown })['response.chat.openaiResponses']), + owns: ['response.http.body'], + pendingUsage: OPENAI_RESPONSES_STREAMED_USAGE, + }), + materializeAttempt('request.chat.openaiResponses'), + beginStoredAttempt, + ...warmup ? [answerOpenAIResponsesWebSocketWarmup] : [], + expandShimCompactions, + projectOpenAIResponsesCollaboration, + summarizeForCompaction(asksForCompaction), + // Directly above the dial, because every descent it makes is another dial of this same + // candidate: a hosted tool the upstream does not implement is emulated by asking again with + // the tool's result folded back in, and the frames of every ask are spliced into one turn. + hostedTools([webSearchHostedTool, imageGenerationHostedTool], { + streamedUsage: OPENAI_RESPONSES_STREAMED_USAGE, + targetOf: candidate => openaiResponsesTarget.pick(candidate.model.endpoints), + }), + normalizeEmptyToolsForOpenAIResponses, + dialChatWire({ + source: 'request.chat.openaiResponses', + needs: ['request.chat.openaiResponses', 'ingress.http.headers', 'ingress.chat.sourceProtocol'], + provides: ['response.chat.openaiResponses', OPENAI_RESPONSES_STREAMED_USAGE, 'response.usage.billable', 'response.http.headers'], + pick: endpoints => openaiResponsesTarget.pick(endpoints), + wire: openaiResponsesWireFor, + }), +]; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/project-websocket.ts b/packages/gateway/src/data-plane/chat/openai-responses/project-websocket.ts new file mode 100644 index 0000000000..d3c0ff8918 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/project-websocket.ts @@ -0,0 +1,277 @@ +import type { Fields, OpenAIResponsesServeEntry, OpenAIResponsesServeExit } from './facts.ts'; +import { DOWNSTREAM_KEEP_ALIVE_INTERVAL_MS } from '../../shared/sse.ts'; +import type { ChatServices } from '../services.ts'; +import { bindClientRelease, withClientVerdict } from '../shared/client-stream.ts'; +import { recordStream, streamReferenceOf } from '@floway-dev/dump'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; +import { OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE, isOpenAIResponsesTerminalEvent, type ClientOpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import { toInternalDebugError } from '@floway-dev/provider'; + +// Our implementor slug prefixes the keep-alive's wire type; the spec reserves +// every unprefixed type for itself, gives `acme:trace_event` as the form, and +// makes `type` and `sequence_number` the only mandatory fields — which is all +// this frame carries. +// https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L758-L765 +// A slug type is inert in openai-node's WebSocket reader: the frame is emitted +// under its own literal type name, nothing is listening on that name, and only +// a frame typed `error` is routed into the socket's error path. +// https://github.com/openai/openai-node/blob/d77cf24d9f3885739c6cba76bc009abf0ab97428/src/resources/responses/ws-base.ts#L346-L371 +export const KEEP_ALIVE_EVENT_TYPE = 'floway:keep_alive'; + +// Extended reasoning turns go completely silent: upstream sends SSE +// `ping` events, `parseOpenAIResponsesStream` drops them, and no frame at +// all reaches this socket for minutes. A silent Workers WebSocket +// does not reliably survive that. Cloudflare states the +// teardown without naming a duration — "when no data is transmitted +// in either direction for a period of time" — and probing a +// `workers.dev` endpoint built on this handler's own `WebSocketPair` +// shape found no constant to design against: from one vantage point +// an idle socket lived a full hour, 4/4, while from another 13 of 16 +// idle sockets died between 215.8 s and 1788.2 s, median around +// 660 s. Teardown is path-dependent and stochastic, and it is always +// a silent EOF — across roughly 40 observed teardowns, not one CLOSE +// frame and not one RST, so the failure carries no protocol-level +// signal. +// https://developers.cloudflare.com/network/websockets/#idle-timeout +// +// Cloudflare's stated remedy, a client-side ping/pong heartbeat, +// does not cover it: on the path that drops, 6 of 9 sockets pinging +// every 30 s died anyway, one of them after 61.5 s. A +// server-originated text frame did cover it — 12/12 survived on that +// same path, with ≤400 s intervals holding and ≥500 s failing. Why a +// data frame outlives a protocol ping there was not established. +// Sending a ping is not open to us regardless: workerd's `WebSocket` +// exposes only accept/send/close/(de)serializeAttachment, and the kj +// layer beneath states the omission as a design decision ("Ping/Pong +// … are not exposed through this interface"). RFC 6455 §5.5.2 is the +// right mechanism, it is unreachable here, and it would not help the +// client that needs it most either, since Codex's frame pump +// swallows control frames and only a text frame rearms its 300 s +// idle timeout. +// https://github.com/cloudflare/workerd/blob/26b5461b7dcc640bb16072f1ba6f2c6df82572ba/src/workerd/api/web-socket.h#L346-L394 +// https://github.com/capnproto/capnproto/blob/e9fa5c7dc98192fc0dc0098ec770db68f997a938/c%2B%2B/src/kj/compat/http.h#L622-L631 +// https://github.com/openai/codex/blob/e6cfd40c3f444aadd6017c9eeab01db70f48961a/codex-rs/codex-api/src/endpoint/responses_websocket.rs#L91-L101 +// https://github.com/openai/codex/blob/e6cfd40c3f444aadd6017c9eeab01db70f48961a/codex-rs/codex-api/src/endpoint/responses_websocket.rs#L695-L699 +// https://github.com/openai/codex/blob/e6cfd40c3f444aadd6017c9eeab01db70f48961a/codex-rs/model-provider-info/src/lib.rs#L26 +// +// So the keep-alive is a text frame whose `type` no client +// recognizes. The spec's extension section governs its shape: an +// implementor slug prefix plus a `sequence_number`. A keep-alive is +// neither a delta nor a state-machine event, so it cannot be spelled +// as a `response.*` event; the slug form is what makes it ignorable +// without loss. openai-node's SSE `responses.stream()` helper is the +// one client that treats a prefixed type as fatal — its accumulator +// closes its `switch` on `assertNever` — and it is out of reach here: +// Floway's SSE keep-alive is a comment line, and this frame exists +// only on the WebSocket transport. +// https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L758 +// https://github.com/openai/openai-node/blob/d77cf24d9f3885739c6cba76bc009abf0ab97428/src/lib/responses/ResponseAccumulator.ts#L387-L389 +// +// Held back until the turn's first event has gone out. That gate is +// parity with the stricter transport, not a demand of this one: both +// SDKs' SSE stream helpers refuse anything before `response.created` +// — openai-node rejects even its own `keepalive` type there — while +// every WebSocket reader we can inspect tolerates an unknown type at +// any position, openai-node emitting it under a name nothing listens +// on, openai-python constructing it unchecked, and Codex tracing and +// discarding it. The window before the first event therefore stays +// unprotected, and closing it is a behavior question rather than a +// client-compatibility one. +// https://github.com/openai/openai-node/blob/d77cf24d9f3885739c6cba76bc009abf0ab97428/src/lib/responses/ResponseAccumulator.ts#L25-L31 +// https://github.com/openai/openai-python/blob/3844843c277f42b0b18beaa58152cfda61df524a/src/openai/lib/streaming/responses/_responses.py#L369-L370 +// https://github.com/openai/openai-python/blob/3844843c277f42b0b18beaa58152cfda61df524a/src/openai/resources/responses/responses.py#L4493-L4502 +// https://github.com/openai/codex/blob/e6cfd40c3f444aadd6017c9eeab01db70f48961a/codex-rs/codex-api/src/sse/responses.rs#L466-L472 + +export interface OpenAIResponsesWebSocketPacket { + readonly text: string; + readonly frame: ProtocolFrame> | null; +} + +export type OpenAIResponsesWebSocketEntry = OpenAIResponsesServeEntry & { + readonly 'ingress.chat.openaiResponses.eventId': string | undefined; +}; +export type OpenAIResponsesWebSocketExit = Omit & { + readonly 'response.chat.openaiResponses.websocket': AsyncIterable; +}; + +export const projectOpenAIResponsesWebSocket = defineStage< + Pick, + Pick, + Fields<'response.chat.openaiResponses.rendered' | 'response.chat.openaiResponses.streamedUsage' | 'response.http.status' | 'response.chat.clientFrames'> & Pick, + Pick & Fields<'response.chat.openaiResponses.streamedUsage' | 'response.chat.clientFrames'>, + ChatServices +>({ + name: 'projectOpenAIResponsesWebSocket', + through: { + request: { needs: ['ingress.chat.openaiResponses.eventId'], consumes: [], provides: [] }, + response: { + needs: ['response.chat.openaiResponses.rendered', 'response.chat.openaiResponses.streamedUsage', 'response.http.status', 'response.chat.clientFrames'], + consumes: ['response.chat.openaiResponses.rendered', 'response.chat.clientFrames', 'response.chat.openaiResponses.streamedUsage'], + provides: ['response.chat.openaiResponses.websocket', 'response.chat.clientFrames', 'response.chat.openaiResponses.streamedUsage'], + }, + }, + execute: async (facts, next, use) => { + const back = await next(facts); + const { 'response.chat.openaiResponses.rendered': rendered, ...rest } = back; + const eventId = back['ingress.chat.openaiResponses.eventId']; + let started = false; + const verdict = Promise.withResolvers(); + const packet = (event: Record, recorded = true): OpenAIResponsesWebSocketPacket => { + const payload = eventId === undefined ? event : { ...event, event_id: eventId }; + const text = JSON.stringify(payload); + return { text, frame: recorded ? eventFrame(payload) : null }; + }; + const generator = (async function* () { + started = true; + let completed = false; + let failed = false; + try { + if (!(Symbol.asyncIterator in rendered)) { + failed = true; + yield packet({ type: 'error', status: back['response.http.status'], error: normalizeErrorBody(rendered, back['response.http.status']) }); + } else { + const iterator = (rendered as AsyncIterable>)[Symbol.asyncIterator](); + const sequence = createDownstreamSequence(); + let pendingNext = pendingWsFrameResult(iterator.next()); + let terminalEvent: ClientOpenAIResponsesStreamEvent | undefined; + let streamed = false; + let exhausted = false; + try { + while (true) { + if (use.gateway.abortSignal?.aborted) return; + const next = await nextFrameOrKeepAlive(pendingNext); + if (next.type === 'keep-alive') { + if (!streamed) continue; + yield packet({ type: KEEP_ALIVE_EVENT_TYPE, sequence_number: sequence.take() }, false); + continue; + } + if (next.type === 'next-error') throw next.error; + if (next.result.done) { exhausted = true; break; } + const frame = next.result.value; + pendingNext = pendingWsFrameResult(iterator.next()); + if (frame.type !== 'event' || terminalEvent !== undefined) continue; + const event = frame.event; + if (event.type === 'error' || event.type === 'response.failed') failed = true; + // A terminal follows committed item/snapshot writes. Hold it until source + // exhaustion, so a follow-up starts only after all work behind it completed. + if (isOpenAIResponsesTerminalEvent(event)) { terminalEvent = event; continue; } + yield packet(sequence.renumber(event) as unknown as Record); + streamed = true; + } + } finally { + if (!exhausted) await iterator.return?.(); + } + if (terminalEvent === undefined) throw new Error(OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE); + yield packet(sequence.renumber(terminalEvent) as unknown as Record); + } + completed = true; + } catch (error) { + failed = true; + use.gateway.dump?.failed(error); + if (!use.gateway.abortSignal?.aborted) yield packet({ type: 'error', status: 500, error: { ...toInternalDebugError(error), code: 'internal_error' } }); + completed = true; + } finally { + verdict.resolve(failed || !completed); + } + })(); + const packets = recordStream({ [Symbol.asyncIterator]: () => generator }, use.gateway.dump, value => value.frame); + const reference = streamReferenceOf(packets); + bindClientRelease(back, async () => { + if (!started) verdict.resolve(true); + await generator.return(); + }); + return { + ...rest, + 'response.chat.openaiResponses.websocket': move(packets), + 'response.chat.clientFrames': move({ ...reference, async *[Symbol.asyncIterator]() { for await (const value of packets) if (value.frame !== null) yield value.frame; } }), + 'response.chat.openaiResponses.streamedUsage': move(withClientVerdict(back['response.chat.openaiResponses.streamedUsage'], verdict.promise)), + }; + }, +}); + +type WsFrameRaceResult = + | { type: 'frame'; result: IteratorResult> } + | { type: 'next-error'; error: unknown } + | { type: 'keep-alive' }; + +const pendingWsFrameResult = (pendingNext: Promise>>): Promise => + pendingNext.then( + (result): WsFrameRaceResult => ({ type: 'frame', result }), + (error): WsFrameRaceResult => ({ type: 'next-error', error }), + ); + +// The interval is the one already shared with SSE rather than a WebSocket +// constant of its own. Widening the gap between server data frames on a +// dropping path put the boundary between 400 s, which still held the socket +// open, and 500 s, which did not, so 15 s is far more frequent than the +// mechanism needs. It is kept because widening it buys nothing: a keep-alive +// is ~70 bytes, and the earliest unprotected idle teardown seen on that same +// path was 215.8 s, so an interval chosen for economy would spend a real +// margin against a stochastic teardown to save nothing. +const nextFrameOrKeepAlive = async (pendingFrame: Promise): Promise => { + let timeoutId: ReturnType | undefined; + const keepAlive = new Promise(resolve => { + timeoutId = setTimeout(() => resolve({ type: 'keep-alive' }), DOWNSTREAM_KEEP_ALIVE_INTERVAL_MS); + }); + try { + return await Promise.race([pendingFrame, keepAlive]); + } finally { + if (timeoutId !== undefined) clearTimeout(timeoutId); + } +}; + +interface DownstreamSequence { + renumber(event: ClientOpenAIResponsesStreamEvent): ClientOpenAIResponsesStreamEvent; + take(): number; +} + +// A WebSocket turn shares one sequence space with the streaming-HTTP events it +// carries, so a keep-alive cannot sit outside that numbering: it takes a real +// slot and every later event is shifted past it. The number is always present +// and always numeric, because a resuming openai-python client compares it with +// `>` against its `starting_after` cursor and `None` there raises a TypeError. +// https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L117 +// https://github.com/openai/openai-python/blob/3844843c277f42b0b18beaa58152cfda61df524a/src/openai/lib/streaming/responses/_responses.py#L59 +// +// Upstream numbering is left untouched until the first keep-alive, and a +// keep-alive can only follow an event that already went out, so the slot it +// takes is always known. +const createDownstreamSequence = (): DownstreamSequence => { + let shift = 0; + let next = 0; + return { + renumber: event => { + if (event.sequence_number === undefined) return event; + const sequenceNumber = event.sequence_number + shift; + next = sequenceNumber + 1; + return { ...event, sequence_number: sequenceNumber }; + }, + take: () => { + shift += 1; + const taken = next; + next += 1; + return taken; + }, + }; +}; + +const normalizeErrorBody = (body: unknown, status: number): Record => { + const source = body && typeof body === 'object' && 'error' in body && typeof (body as { error?: unknown }).error === 'object' + ? (body as { error: Record }).error + : body && typeof body === 'object' + ? body as Record + : {}; + const type = typeof source.type === 'string' + ? source.type + : status >= 500 ? 'server_error' : 'invalid_request_error'; + const message = typeof source.message === 'string' + ? source.message + : `OpenAI Responses request failed with status ${status}.`; + return { + ...source, + type, + code: typeof source.code === 'string' ? source.code : type, + message, + }; +}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/respond.ts b/packages/gateway/src/data-plane/chat/openai-responses/respond.ts deleted file mode 100644 index b693bcf274..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-responses/respond.ts +++ /dev/null @@ -1,180 +0,0 @@ -import type { Context } from 'hono'; -import { streamSSE } from 'hono/streaming'; - -import { wrapOpenAIResponsesClientEgress } from './client-output.ts'; -import type { GatewayCtx } from '../../shared/gateway-ctx.ts'; -import { type StreamCompletion, writeSSEFrames } from '../../shared/sse.ts'; -import { recordFailedRequest } from '../../shared/telemetry/performance.ts'; -import { settle } from '../../shared/telemetry/settle.ts'; -import { tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; -import { forwardUpstreamHeaders, mergeForwardedUpstreamHeaders } from '../../shared/upstream-response.ts'; -import { SourceStreamState, eventResultMetadata, plainResultToResponse } from '../shared/respond.ts'; -import { doneFrame, eventFrame, type ProtocolFrame, sseCommentFrame, sseFrame } from '@floway-dev/protocols/common'; -import { openaiResponsesProtocolFrameToSSEFrame, OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE, collectOpenAIResponsesProtocolEventsToResult } from '@floway-dev/protocols/openai-responses'; -import { isOpenAIResponsesTerminalEvent, type CanonicalOpenAIResponsesPayload, type ClientResponseResource, type ClientOpenAIResponsesStreamEvent, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { type ExecuteResult, type PlainResult, type InternalDebugError, toInternalDebugError } from '@floway-dev/provider'; -import { apiErrorToResponse } from '@floway-dev/provider'; - -// Renders an OpenAI Responses failure that never opened a stream. Separate entry -// because a request that fails before its payload parses has no payload to -// answer with, and the events path below requires one. -export const respondOpenAIResponsesFailure = ( - result: Exclude>, { type: 'events' }> | PlainResult, - ctx: GatewayCtx, -): Response => { - if (result.type === 'api-error') { - recordFailedRequest(ctx, result.performance); - ctx.dump?.error(result.source, result.upstreamId); - return apiErrorToResponse(result); - } - - if (result.type === 'internal-error') { - recordFailedRequest(ctx, result.performance); - ctx.dump?.failed(result.error.message); - return internalOpenAIResponsesErrorResponse(result.status, result.error); - } - - if (result.status >= 400) { - ctx.dump?.error(result.upstreamId !== undefined ? 'upstream' : 'gateway', result.upstreamId); - } - return plainResultToResponse(result); -}; - -// Renders an upstream OpenAI Responses result into the client HTTP/SSE response. An -// events result drains to one JSON body (non-streaming) or is proxied frame by -// frame (streaming); anything else is a pre-stream failure. -export const respondOpenAIResponses = async ( - c: Context, - result: ExecuteResult> | PlainResult, - wantsStream: boolean, - ctx: GatewayCtx, - request: CanonicalOpenAIResponsesPayload, -): Promise => { - if (result.type !== 'events') return respondOpenAIResponsesFailure(result, ctx); - - const state = new SourceStreamState(); - const observed = observeOpenAIResponsesFrames(result.events, state, ctx); - const frames = wrapOpenAIResponsesClientEgress(observed, ctx, request); - - if (!wantsStream) { - try { - const response = await collectOpenAIResponsesProtocolEventsToResult(frames); - const metadata = await eventResultMetadata(result); - const usage = tokenUsageFromBillableUsage(metadata.billableUsage); - ctx.dump?.success(metadata.modelIdentity, usage); - settle(ctx, metadata.performance, metadata.modelIdentity, usage, state.failed || response.status === 'failed'); - return Response.json(response, { headers: mergeForwardedUpstreamHeaders(undefined, result.headers) }); - } catch (error) { - recordFailedRequest(ctx, result.performance); - ctx.dump?.failed(error); - return internalOpenAIResponsesErrorResponse(502, toInternalDebugError(error)); - } - } - - forwardUpstreamHeaders(c, result.headers); - const response = streamSSE(c, async stream => { - let completion: StreamCompletion = 'error'; - try { - completion = await writeSSEFrames(stream, openaiResponsesSseFrames(frames, state, ctx), { - keepAlive: { frame: sseCommentFrame('keepalive') }, - ...(ctx.downstreamAbortController !== undefined ? { downstreamAbortController: ctx.downstreamAbortController } : {}), - }); - } finally { - const metadata = await eventResultMetadata(result); - const failed = state.failedAfter(completion); - if (failed) { - ctx.dump?.failed(`responses stream failed (completion=${completion}, source-failed=${state.failed})`); - } else { - ctx.dump?.success(metadata.modelIdentity, tokenUsageFromBillableUsage(metadata.billableUsage)); - } - settle(ctx, metadata.performance, metadata.modelIdentity, tokenUsageFromBillableUsage(metadata.billableUsage), failed); - } - }); - - return response; -}; - -// --- error rendering --- - -const internalOpenAIResponsesErrorResponse = (status: number, error: InternalDebugError): Response => - Response.json({ - error: { - type: error.type, - name: error.name, - message: error.message, - stack: error.stack, - cause: error.cause, - target_api: error.target_api, - }, - }, { status }); - -// The spec nests the `error` event's payload under `error`, and both official -// SDKs key their mid-stream throw on exactly that key; the same fields at the -// top level are yielded to them as an ordinary event instead. -// https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L170-L177 -// https://github.com/openai/openai-node/blob/d77cf24d9f3885739c6cba76bc009abf0ab97428/src/core/streaming.ts#L69-L71 -// https://github.com/openai/openai-python/blob/3844843c277f42b0b18beaa58152cfda61df524a/src/openai/_streaming.py#L87-L98 -const internalOpenAIResponsesStreamErrorEvent = (error: unknown): ClientOpenAIResponsesStreamEvent => { - const debug = toInternalDebugError(error); - return { - type: 'error', - error: { - message: debug.message, - code: debug.type, - name: debug.name, - stack: debug.stack, - cause: debug.cause, - target_api: debug.target_api, - }, - } as unknown as ClientOpenAIResponsesStreamEvent; -}; - -// --- frame observation --- - -const isOpenAIResponsesTerminalFrame = (frame: ProtocolFrame) => frame.type === 'event' && isOpenAIResponsesTerminalEvent(frame.event); - -const observeOpenAIResponsesFrames = async function* (frames: AsyncIterable>, state: SourceStreamState, ctx: GatewayCtx) { - for await (const frame of frames) { - await ctx.dump?.frame(frame); - const failed = frame.type === 'event' && (frame.event.type === 'error' || frame.event.type === 'response.failed'); - if (failed) state.failed = true; - if (isOpenAIResponsesTerminalFrame(frame) && !failed) state.completed = true; - yield frame; - if (isOpenAIResponsesTerminalFrame(frame)) return; - } - throw new Error(OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE); -}; - -// "Any error incurred while streaming will be followed by a `response.failed` -// event." -// https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L430 -const openaiResponsesFailedEvent = (resource: ClientResponseResource, error: unknown): ClientOpenAIResponsesStreamEvent => { - const debug = toInternalDebugError(error); - return { - type: 'response.failed', - response: { ...resource, status: 'failed', error: { code: debug.type, message: debug.message } }, - } as ClientOpenAIResponsesStreamEvent; -}; - -const openaiResponsesSseFrames = async function* (frames: AsyncIterable>, state: SourceStreamState, ctx: GatewayCtx) { - let announced: ClientResponseResource | undefined; - try { - for await (const frame of frames) { - if (frame.type === 'event' && 'response' in frame.event) announced = frame.event.response; - yield openaiResponsesProtocolFrameToSSEFrame(frame); - } - // The SSE transport terminates on the literal `[DONE]` payload: - // https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx?plain=1#L84 - yield openaiResponsesProtocolFrameToSSEFrame(doneFrame()); - } catch (error) { - state.failed = true; - const errorEvent = internalOpenAIResponsesStreamErrorEvent(error); - await ctx.dump?.frame(eventFrame(errorEvent)); - yield sseFrame(JSON.stringify(errorEvent), 'error'); - if (announced !== undefined) { - const failedFrame = eventFrame(openaiResponsesFailedEvent(announced, error)); - await ctx.dump?.frame(failedFrame); - yield openaiResponsesProtocolFrameToSSEFrame(failedFrame); - } - } -}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/response-resource.ts b/packages/gateway/src/data-plane/chat/openai-responses/response-resource.ts index ee26f896e0..c84878fbbe 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/response-resource.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/response-resource.ts @@ -18,7 +18,7 @@ import type { // the only place Floway states one, and only where the wire schema forbids both // omission and `null`. // -// The other half of that rule lives in the server-tool shim, which shipped +// The other half of that rule lives in the hosted-tool shim, which shipped // "absent-echo (no tools synthesized when upstream omits it)" in 44322150d // (#125) and whose synthesis paths throw on an absent `upstreamResponseSnapshot` // rather than fabricate a resource never captured. The two rules do not conflict @@ -33,12 +33,10 @@ import type { // rows carried by `response.output_item.done` and a snapshot record of their // ordered ids, and awaits that before the terminal frame leaves it, so no // response resource is ever stored. Affinity egress runs underneath too, and -// billing reads `billableUsage` off the `ExecuteResult` rather than the -// resource. One reader sits above this stage, the non-streaming `settle` -// call, and it reads only `status`, which rides through from the upstream and -// is never stated here. The WebSocket transport buffers the terminal event -// and flushes it last, branching only on the event type; it reads nothing off -// the resource at all. +// billing uses its own measured usage facts. Client collection runs above +// this projection, so a mandatory wire default cannot become an upstream +// usage observation. The WebSocket transport buffers the terminal event +// and flushes it last, branching on its type. // // #125's "no tools synthesized when upstream omits it" therefore still holds: // the interior resource carries no `tools`, and egress states `[]` on the way @@ -169,7 +167,7 @@ export const completeResponseResource = ( // Gateway state. `created_at` and `store` have no candidate chain because no // other source can know them. `completed_at` overrides any upstream value // because one client turn can span several upstream calls behind the - // server-tool runtime, so no single upstream's completion instant describes + // hosted-tool runtime, so no single upstream's completion instant describes // it. created_at: sources.createdAt, completed_at: terminal ? Math.floor(Date.now() / 1000) : null, diff --git a/packages/gateway/src/data-plane/chat/openai-responses/serve-prep.ts b/packages/gateway/src/data-plane/chat/openai-responses/serve-prep.ts index fa86dfe0f2..8942f36340 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/serve-prep.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/serve-prep.ts @@ -1,23 +1,11 @@ -import { analyzeOpenAIResponsesAffinity } from './affinity/ingress.ts'; -import { openaiResponsesTarget } from './attempt.ts'; -import { renderOpenAIResponsesFailure, type OpenAIResponsesServeFailure } from './errors.ts'; -import { hydrateOpenAIResponsesPayload } from './items/hydrate.ts'; import type { OpenAIResponsesStatefulStore } from './items/store.ts'; -import { enumerateModelCandidates } from '../../providers/resolution.ts'; -import type { AffinityCandidateSelection } from '../shared/affinity/index.ts'; -import { selectAffinityCandidates } from '../shared/affinity/index.ts'; -import { noViableCandidateFailure, tryCatchChatServeFailure } from '../shared/errors.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import type { ModelCandidate, ExecuteResult } from '@floway-dev/provider'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; // Thrown when a request names a `previous_response_id` that the store cannot -// resolve. The HTTP/WS entry layer catches this and renders the OpenAI-shaped -// 400 body verbatim — clients (codex) compare it byte-for-byte against -// upstream OpenAI's `previous_response_not_found` envelope, so the rendering -// stays at the entry boundary instead of being folded into the generic -// ChatServeFailure renderer. +// resolve. The stage that hydrates catches this and answers with the +// OpenAI-shaped 400 body verbatim — clients (codex) compare it byte-for-byte +// against upstream OpenAI's `previous_response_not_found` envelope, so what +// they read is the upstream's own wording rather than this gateway's. // // Verbatim payload cross-verified from real upstream captures: // - https://github.com/cline/cline/issues/9399 @@ -38,7 +26,7 @@ export class PreviousResponseNotFoundError extends Error { // then drops `previous_response_id` from the payload (the snapshot id is a // gateway concept and never reaches the upstream wire). Native-entry only: // translated payloads coming in from another protocol's attempt never carry -// `previous_response_id`, so this prep runs in serve and not in attempt. +// `previous_response_id`, so this runs above the fork and never on a wire. export const expandPreviousResponseId = async ( payload: CanonicalOpenAIResponsesPayload, store: OpenAIResponsesStatefulStore, @@ -58,63 +46,3 @@ export const expandPreviousResponseId = async ( ], }; }; - -export type OpenAIResponsesServePlan = - | { readonly kind: 'failure'; readonly result: ExecuteResult> } - | { - readonly kind: 'ready'; - readonly affinitySelection: AffinityCandidateSelection; - readonly privatePayloads: ReadonlyMap; - readonly candidates: readonly ModelCandidate[]; - }; - -// Runs the native source preparation both `openaiResponsesServe.generate` and -// `openaiResponsesServe.compact` need before dispatching to `openaiResponsesAttempt`: -// expand any `previous_response_id`, load and hydrate stored items, prepare -// affinity, stage the user input, and return the selected candidate list. -// Returns a rendered failure result when no candidate is viable so the -// caller can surface it directly without re-deriving the model-error -// branch. The caller iterates the candidates — a successful attempt is the -// final answer, a per-candidate failure falls through to the next entry. -export const prepareOpenAIResponsesServePlan = async (args: { - readonly payload: CanonicalOpenAIResponsesPayload; - readonly ctx: ChatGatewayCtx; -}): Promise => { - const { payload, ctx } = args; - const store = ctx.store; - const prepared = await expandPreviousResponseId(payload, store); - const { candidates, sawModel, failedUpstreams } = await enumerateModelCandidates({ - upstreamIds: ctx.upstreamIds, - model: prepared.model, - kind: 'chat', - scheduler: ctx.backgroundScheduler, - runtimeLocation: ctx.runtimeLocation, - }); - const viable = candidates.filter(c => openaiResponsesTarget.canServe(c.model.endpoints)); - await store.loadInputItems(prepared.input, payload.input); - let hydrated: ReturnType; - try { - hydrated = hydrateOpenAIResponsesPayload(prepared, store); - } catch (error) { - const failure = tryCatchChatServeFailure(error); - if (failure === null) throw error; - return { kind: 'failure', result: renderOpenAIResponsesFailure(failure) }; - } - const affinity = await analyzeOpenAIResponsesAffinity(hydrated.payload, ctx.affinity.codec); - const selection = selectAffinityCandidates(viable, affinity); - if ('kind' in selection) return { kind: 'failure', result: renderOpenAIResponsesFailure(selection) }; - // Stage the user-supplied input from the original payload — not the - // expansion's `item_reference` prefix — so the next-turn snapshot picks - // up the new user items in addition to the prior snapshot history. - // Runs after the affinity walk so any `item_reference` in user-supplied - // input has its target row loaded. - await store.stageInputItems(payload.input); - - if (selection.candidates.length === 0) { - return { - kind: 'failure', - result: renderOpenAIResponsesFailure(noViableCandidateFailure(sawModel, prepared.model, failedUpstreams)), - }; - } - return { kind: 'ready', affinitySelection: selection, privatePayloads: hydrated.privatePayloads, candidates: selection.candidates }; -}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/serve.ts b/packages/gateway/src/data-plane/chat/openai-responses/serve.ts deleted file mode 100644 index 50bc773d5c..0000000000 --- a/packages/gateway/src/data-plane/chat/openai-responses/serve.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { openaiResponsesAttempt } from './attempt.ts'; -import { openaiResponsesCreatedAt, wrapOpenAIResponsesStatefulOutput } from './client-output.ts'; -import { completeOpenAIResponsesCompaction } from './compaction-resource.ts'; -import type { OpenAIResponsesAttemptResult } from './interceptors/types.ts'; -import { syntheticEventsFromCompaction } from './items/output.ts'; -import { prepareOpenAIResponsesServePlan } from './serve-prep.ts'; -import { iterateCandidates } from '../../shared/iterate-candidates.ts'; -import type { ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { collectOpenAIResponsesProtocolEventsToResult, type CanonicalOpenAIResponsesPayload, type ClientOpenAIResponsesCompaction, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import type { ExecuteResult } from '@floway-dev/provider'; - -interface OpenAIResponsesServeArgs { - readonly payload: CanonicalOpenAIResponsesPayload; - readonly ctx: ChatGatewayCtx; - readonly headers: Headers; -} - -export const openaiResponsesServe = { - generate: async (args: OpenAIResponsesServeArgs): Promise>> => { - const { payload, ctx, headers } = args; - const plan = await prepareOpenAIResponsesServePlan({ payload, ctx }); - if (plan.kind === 'failure') return plan.result; - // Iterate the affinity-selected candidates: success (SSE stream opened) is the - // final answer; per-candidate failures fall through so a transient - // 5xx/429/network does not become the request's verdict when another - // candidate can serve. The last failure surfaces verbatim on exhaustion. - // Each attempt stamps its private prepared-payload clone with the - // candidate's canonical model id. - const result = await iterateCandidates( - plan.candidates, - 'openaiResponsesServe.generate', - ctx, - 'chat', - async candidate => { - const result = await openaiResponsesAttempt.generate({ - payload: plan.affinitySelection.payloadFor(candidate), - sourceState: { - privatePayloads: plan.privatePayloads, - }, - ctx, - candidate, - headers, - }); - if (result.type === 'events') ctx.affinity.select(candidate); - return result; - }, - ); - return result; - }, - - compact: async (args: OpenAIResponsesServeArgs): Promise> => { - const { payload, ctx, headers } = args; - // Compact accepts `previous_response_id` (the official endpoint documents - // it). When present serve-prep expands it the same way generate does so - // stored history is hydrated before candidate dispatch. - // - // For non-openai-responses targets the openai-responses-compact-shim picks up the - // request inside the interceptor chain, flips action='compact' to - // 'generate', runs a SUMMARIZATION_PROMPT turn through translation, and - // re-tags the result as compact on the way out. - const plan = await prepareOpenAIResponsesServePlan({ payload, ctx }); - if (plan.kind === 'failure') return plan.result; - const result = await iterateCandidates( - plan.candidates, - 'openaiResponsesServe.compact', - ctx, - 'chat', - async candidate => { - const result = await openaiResponsesAttempt.invoke({ - payload: plan.affinitySelection.payloadFor(candidate), - sourceState: { - privatePayloads: plan.privatePayloads, - }, - action: 'compact', - ctx, - candidate, - headers, - }); - if (result.type === 'result') ctx.affinity.select(candidate); - return result; - }, - ); - if (result.type !== 'result') return result; - - const stored = wrapOpenAIResponsesStatefulOutput(syntheticEventsFromCompaction(result.result), ctx); - const persisted = await collectOpenAIResponsesProtocolEventsToResult(stored); - return { - ...result, - result: completeOpenAIResponsesCompaction(persisted, openaiResponsesCreatedAt(ctx)), - }; - }, -}; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/strip-prompt-cache-key.ts b/packages/gateway/src/data-plane/chat/openai-responses/strip-prompt-cache-key.ts new file mode 100644 index 0000000000..b71c9dba95 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/strip-prompt-cache-key.ts @@ -0,0 +1,35 @@ +import type { Chat } from '../facts.ts'; +import { withoutKeys } from '../shared/structural.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; + +/** Drops a field the upstream would reject as an unknown argument. It runs above the vendor + * normalizers so each of them sees the already-stripped canonical payload. */ +export const stripPromptCacheKeyForOpenAIResponses = defineStage< + Chat<'request.chat.openaiResponses' | 'route.attempt'>, + Chat<'request.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> +>({ + name: 'stripPromptCacheKey', + through: { + request: { + needs: ['request.chat.openaiResponses', 'route.attempt'], + consumes: [], + provides: ['request.chat.openaiResponses'], + }, + response: { needs: ['response.chat.openaiResponses'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.openaiResponses' | 'route.attempt'>, + Chat<'request.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> + >(() => ({ + request: facts => { + if (!facts['route.attempt'].flags.includes('strip-prompt-cache-key')) return facts; + const payload = facts['request.chat.openaiResponses']; + const stripped = withoutKeys(payload, ['prompt_cache_key']); + return stripped === payload ? facts : { ...facts, 'request.chat.openaiResponses': move(stripped) }; + }, + })), +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/summarize-for-compaction.ts b/packages/gateway/src/data-plane/chat/openai-responses/summarize-for-compaction.ts new file mode 100644 index 0000000000..bd21f48d35 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/summarize-for-compaction.ts @@ -0,0 +1,79 @@ +import { OPENAI_RESPONSES_STREAMED_USAGE, type Fields } from './facts.ts'; +import type { ChatServices } from '../services.ts'; +import { summarizationTurnFor, summaryTextFrom, EMPTY_SUMMARY_MESSAGE, buildCompactionEnvelope } from './compact-shim.ts'; +import { syntheticEventsFromResult } from './items/output.ts'; +import { isFailure } from '../../pipeline/facts.ts'; +import type { StreamOutcome } from '../../pipeline/serve.ts'; +import { defineStage, move, type Use, type Deferred } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import { collectOpenAIResponsesProtocolEventsToResult, createRandomOpenAIResponsesItemId, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; + +/** How the chain that composed the stage below says a turn reaching it asked for a compaction. + * It is the whole gate, so each chain states its own: what asks for a compaction differs by + * operation, and no action travels in the record for the stage to read instead. */ +export type CompactionAsk = ( + facts: Fields<'request.chat.openaiResponses' | 'route.attempt'>, + use: Use, +) => boolean; + +/** + * Answers a turn that asked for a compaction with one, over an upstream that has no + * compaction wire. + * + * It rewrites the turn on the way down — the compactor's prompt at the head of the history, + * the trigger stripped, a terminal nudge appended, nothing persisted upstream — and folds the + * generated summary into an envelope of this gateway's own on the way back. Below it is the + * ordinary generate fork, which is what makes every wire generation can take a wire a + * compaction can be simulated over. + * + * What the simulation cannot reproduce is stated where it happens, at `summarizationTurnFor`. + */ +export const summarizeForCompaction = (asked: CompactionAsk) => defineStage< + Fields<'request.chat.openaiResponses' | 'route.attempt'>, + Fields<'request.chat.openaiResponses'>, + Fields<'response.chat.openaiResponses' | 'response.chat.openaiResponses.streamedUsage' | 'response.usage.billable'>, + Fields<'response.chat.openaiResponses' | 'response.chat.openaiResponses.streamedUsage' | 'response.usage.billable'>, + ChatServices +>({ + name: 'summarizeForCompaction', + through: { + request: { + needs: ['request.chat.openaiResponses', 'route.attempt'], + consumes: [], + provides: ['request.chat.openaiResponses'], + }, + response: { + needs: ['response.chat.openaiResponses', OPENAI_RESPONSES_STREAMED_USAGE, 'response.usage.billable'], + consumes: [], + provides: ['response.chat.openaiResponses', OPENAI_RESPONSES_STREAMED_USAGE, 'response.usage.billable'], + }, + }, + execute: async (facts, next, use) => { + if (!asked(facts, use)) return await next(facts); + const payload = facts['request.chat.openaiResponses'] as CanonicalOpenAIResponsesPayload; + const back = await next({ ...facts, 'request.chat.openaiResponses': move(summarizationTurnFor(payload)) }); + + const answer = back['response.chat.openaiResponses']; + // An upstream that refused is handed on as it came: the client learns the compaction + // failed rather than being given a silent empty envelope. One that answered with a single + // envelope compacted on its own, and an envelope is already the answer — there are no + // frames to fold and nothing this layer could add to it. + if (isFailure(answer) || answer.kind !== 'stream') return back; + + const frames = answer.frames as AsyncIterable>; + const collected = await collectOpenAIResponsesProtocolEventsToResult(frames); + + const summaryText = summaryTextFrom(collected.output); + if (summaryText.length === 0) { + // A summarization that closed no text produced no summary, and the blob is the whole of + // what the next turn inherits — so this is a candidate that did not do the job rather + // than a fault that ends the request, and the fork can try another. + const reading = back[OPENAI_RESPONSES_STREAMED_USAGE] as Deferred | null; + const billable = reading === null ? back['response.usage.billable'] : (await reading).billable; + return { ...back, 'response.chat.openaiResponses': move({ status: 502, message: EMPTY_SUMMARY_MESSAGE }), 'response.usage.billable': move(billable), [OPENAI_RESPONSES_STREAMED_USAGE]: null }; + } + + const synthesized = buildCompactionEnvelope(createRandomOpenAIResponsesItemId('compaction'), summaryText, collected); + return { ...back, 'response.chat.openaiResponses': move({ kind: 'stream' as const, frames: syntheticEventsFromResult(synthesized) }) }; + }, +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/target.ts b/packages/gateway/src/data-plane/chat/openai-responses/target.ts new file mode 100644 index 0000000000..4dd2a3a8a1 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/target.ts @@ -0,0 +1,33 @@ +import type { ChatNarrowing } from '../resolve-candidates.ts'; +import type { Fields } from './facts.ts'; +import { chatTargetPicker } from '../shared/target-picker.ts'; +import { analyzeOpenAIResponsesAffinity } from './affinity/ingress.ts'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; + +/** `/v1/responses` prefers its own wire, then the translated Anthropic Messages path, then + * the translated OpenAI Chat Completions path. */ +export const openaiResponsesTarget = chatTargetPicker(['openaiResponses', 'anthropicMessages', 'openaiChatCompletions']); + +export const openaiResponsesNarrowing: ChatNarrowing, 'request.chat.openaiResponses'> = ({ + requestKey: 'request.chat.openaiResponses', + canServe: candidate => openaiResponsesTarget.canServe(candidate.model.endpoints), + affinity: async (payload, gateway) => await analyzeOpenAIResponsesAffinity(payload as CanonicalOpenAIResponsesPayload, gateway.affinity.codec), + unsupported: model => `Model ${model} does not support the /responses endpoint.`, + refuse: (status, message, reason) => ({ + 'response.chat.openaiResponses.streamedUsage': null, + 'response.chat.openaiResponses': { + status, + message, + // What an OpenAI client reads: the condition's own type, and for a turn whose carried + // state cannot be routed, the field at fault and the code that names it. + envelope: { + error: { + message, + type: 'invalid_request_error', + ...(reason === 'routing-unavailable' ? { param: 'input', code: 'responses_item_routing_unavailable' } : {}), + }, + }, + }, + }), + refuses: ['response.chat.openaiResponses', 'response.chat.openaiResponses.streamedUsage'], +}); diff --git a/packages/gateway/src/data-plane/chat/openai-responses/usage.ts b/packages/gateway/src/data-plane/chat/openai-responses/usage.ts index 0610b04f17..3280831519 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/usage.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/usage.ts @@ -1,4 +1,5 @@ -import { billableServiceTier, splitInclusiveInputTokens, type BillableUsage } from '@floway-dev/protocols/common'; +import type { ChatBillableUsage } from '../shared/usage.ts'; +import { billableServiceTier, splitInclusiveInputTokens } from '@floway-dev/protocols/common'; import { openaiResponsesResultFromStreamEvent, type OpenAIResponsesResult, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; // service_tier reports the tier actually served and therefore selects the @@ -8,7 +9,7 @@ import { openaiResponsesResultFromStreamEvent, type OpenAIResponsesResult, type // body can be priced through the same helper. export const billableUsageFromOpenAIResponsesResult = ( response: { readonly usage?: OpenAIResponsesResult['usage']; readonly service_tier?: OpenAIResponsesResult['service_tier'] }, -): BillableUsage | null => { +): ChatBillableUsage | null => { const usage = response.usage; if (!usage) return null; const cacheWrite = usage.input_tokens_details?.cache_write_tokens ?? 0; @@ -19,17 +20,17 @@ export const billableUsageFromOpenAIResponsesResult = ( ); const tier = billableServiceTier(response.service_tier); return { - input, + ...(usage.input_tokens === undefined ? {} : { input }), cacheRead, cacheWrite, // OpenAI Responses has no cache-write TTL split; every write bills at one rate. cacheWrite1h: 0, - output: usage.output_tokens, + ...(usage.output_tokens === undefined ? {} : { output: usage.output_tokens }), ...(tier !== null ? { tier } : {}), }; }; -export const billableUsageFromOpenAIResponsesEvent = (event: OpenAIResponsesStreamEvent): BillableUsage | null => { +export const billableUsageFromOpenAIResponsesEvent = (event: OpenAIResponsesStreamEvent): ChatBillableUsage | null => { const response = openaiResponsesResultFromStreamEvent(event); return response === null ? null : billableUsageFromOpenAIResponsesResult(response); }; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/vendor-deep-seek-normalize.ts b/packages/gateway/src/data-plane/chat/openai-responses/vendor-deep-seek-normalize.ts new file mode 100644 index 0000000000..def8113019 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/vendor-deep-seek-normalize.ts @@ -0,0 +1,43 @@ +import type { Chat } from '../facts.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; +import type { OpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; + +/** DeepSeek says "no reasoning" with a top-level `thinking: { type: 'disabled' }` rather than + * with the canonical sentinel, so the sentinel is put into its wire form here — last among + * this chain's rewrites, because a vendor normalizer has the final say on the outbound body. + * https://api-docs.deepseek.com/zh-cn/guides/thinking_mode */ +export const vendorDeepSeekNormalizeForOpenAIResponses = defineStage< + Chat<'request.chat.openaiResponses' | 'route.attempt'>, + Chat<'request.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> +>({ + name: 'vendorDeepSeekNormalize', + through: { + request: { + needs: ['request.chat.openaiResponses', 'route.attempt'], + consumes: [], + provides: ['request.chat.openaiResponses'], + }, + response: { needs: ['response.chat.openaiResponses'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.openaiResponses' | 'route.attempt'>, + Chat<'request.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> + >(() => ({ + request: facts => { + if (!facts['route.attempt'].flags.includes('vendor-deepseek')) return facts; + const payload = facts['request.chat.openaiResponses']; + if (payload.reasoning?.effort !== 'none') return facts; + const { reasoning, ...rest } = payload; + const normalized: OpenAIResponsesPayloadWithDeepSeekThinking = { ...rest, thinking: { type: 'disabled' } }; + return { ...facts, 'request.chat.openaiResponses': move(normalized) }; + }, + })), +}); + +/** Neither field is an OpenAI Responses field, so each is declared beside the vendor that reads it + * rather than widening the protocol's own request type. */ +type OpenAIResponsesPayloadWithDeepSeekThinking = Omit & { thinking: { type: 'disabled' } }; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/vendor-qwen-normalize.ts b/packages/gateway/src/data-plane/chat/openai-responses/vendor-qwen-normalize.ts new file mode 100644 index 0000000000..ba7797c519 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/vendor-qwen-normalize.ts @@ -0,0 +1,39 @@ +import type { Chat } from '../facts.ts'; +import { defineStage, transform, move } from '@floway-dev/pipeline'; +import type { OpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; + +/** Qwen says the same thing with a top-level `enable_thinking: false`. + * https://www.alibabacloud.com/help/en/model-studio/deep-thinking */ +export const vendorQwenNormalizeForOpenAIResponses = defineStage< + Chat<'request.chat.openaiResponses' | 'route.attempt'>, + Chat<'request.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> +>({ + name: 'vendorQwenNormalize', + through: { + request: { + needs: ['request.chat.openaiResponses', 'route.attempt'], + consumes: [], + provides: ['request.chat.openaiResponses'], + }, + response: { needs: ['response.chat.openaiResponses'], consumes: [], provides: [] }, + }, + execute: transform< + Chat<'request.chat.openaiResponses' | 'route.attempt'>, + Chat<'request.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'>, + Chat<'response.chat.openaiResponses'> + >(() => ({ + request: facts => { + if (!facts['route.attempt'].flags.includes('vendor-qwen')) return facts; + const payload = facts['request.chat.openaiResponses']; + if (payload.reasoning?.effort !== 'none') return facts; + const { reasoning, ...rest } = payload; + const normalized: OpenAIResponsesPayloadWithQwenThinking = { ...rest, enable_thinking: false }; + return { ...facts, 'request.chat.openaiResponses': move(normalized) }; + }, + })), +}); + +type OpenAIResponsesPayloadWithQwenThinking = Omit & { enable_thinking: false }; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts b/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts index 5632cbdac9..fa9a6d4271 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/websocket.ts @@ -1,25 +1,33 @@ +// The OpenAI Responses WebSocket transport: a second entry against the same chain `POST +// /v1/responses` runs. One `response.create` frame is one turn and one turn is one run, so +// nothing about a turn outlives the frame that opened it except what the connection itself +// holds — the session's item store and the ids a continuation may name. +// +// Entering the pipeline system takes no capability, so this handler opens a run exactly as a +// Hono route handler does. What it does not do is answer through `serveThrough`: that builds +// an HTTP `Response`, and a turn here writes its own frames to a socket, counts them, and +// closes its own recording. + import type { Context } from 'hono'; -import { wrapOpenAIResponsesClientEgress } from './client-output.ts'; -import { createOpenAIResponsesWsSession } from './items/store.ts'; -import { PreviousResponseNotFoundError } from './serve-prep.ts'; -import { openaiResponsesServe } from './serve.ts'; -import type { TurnDump } from '../../../dump/turn-dump.ts'; +import { createOpenAIResponsesWsSession, type OpenAIResponsesStatefulStore } from './items/store.ts'; +import { openaiResponsesWebSocketPipeline } from './pipeline.ts'; +import type { OpenAIResponsesWebSocketExit } from './project-websocket.ts'; +import { openRunDump } from '../../../dump/run-sink.ts'; +import type { RunDump } from '../../../dump/run-sink.ts'; import { apiKeyFromContext, authenticateApiKey, type AuthedContext } from '../../../middleware/auth.ts'; +import type { ApiKey } from '../../../repo/types.ts'; import { backgroundSchedulerFromContext } from '../../../runtime/background.ts'; -import { inboundHeaders } from '../../shared/inbound-headers.ts'; -import { takeRequestBody } from '../../shared/request-body.ts'; -import { DOWNSTREAM_KEEP_ALIVE_INTERVAL_MS, type StreamCompletion } from '../../shared/sse.ts'; -import { recordFailedRequest } from '../../shared/telemetry/performance.ts'; -import { settle } from '../../shared/telemetry/settle.ts'; -import { tokenUsageFromBillableUsage } from '../../shared/telemetry/usage.ts'; +import { prologueFor, type Ingress } from '../../pipeline/serve.ts'; +import type { AttemptState } from '../../shared/gateway-ctx.ts'; +import { takeRequestBody, type RequestBody } from '../../shared/request-body.ts'; +import { type StreamCompletion } from '../../shared/sse.ts'; +import type { ChatPrologue } from '../prologue.ts'; import { createChatGatewayCtxFromHono, type ChatGatewayCtx } from '../shared/gateway-ctx.ts'; -import { SourceStreamState, eventResultMetadata } from '../shared/respond.ts'; +import { SourceStreamState } from '../shared/source-stream-state.ts'; +import { move, run } from '@floway-dev/pipeline'; import type { BackgroundScheduler } from '@floway-dev/platform'; -import { isJsonMediaType, type ProtocolFrame } from '@floway-dev/protocols/common'; -import { OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE } from '@floway-dev/protocols/openai-responses'; -import { isOpenAIResponsesTerminalEvent, type CanonicalOpenAIResponsesPayload, type ClientOpenAIResponsesStreamEvent, type OpenAIResponsesRequestPayload, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import type { ExecuteResult } from '@floway-dev/provider'; +import { isOpenAIResponsesTerminalEvent, type CanonicalOpenAIResponsesPayload, type ClientOpenAIResponsesStreamEvent, type OpenAIResponsesRequestPayload } from '@floway-dev/protocols/openai-responses'; import { toInternalDebugError } from '@floway-dev/provider'; import { canonicalizeOpenAIResponsesPayload, TranslatorInputError } from '@floway-dev/translate'; @@ -34,17 +42,6 @@ interface OpenAIResponsesWebSocketSocket { const UTF8_ENCODER = new TextEncoder(); -// Our implementor slug prefixes the keep-alive's wire type; the spec reserves -// every unprefixed type for itself, gives `acme:trace_event` as the form, and -// makes `type` and `sequence_number` the only mandatory fields — which is all -// this frame carries. -// https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L758-L765 -// A slug type is inert in openai-node's WebSocket reader: the frame is emitted -// under its own literal type name, nothing is listening on that name, and only -// a frame typed `error` is routed into the socket's error path. -// https://github.com/openai/openai-node/blob/d77cf24d9f3885739c6cba76bc009abf0ab97428/src/resources/responses/ws-base.ts#L346-L371 -export const KEEP_ALIVE_EVENT_TYPE = 'floway:keep_alive'; - interface OpenAIResponsesWebSocketHandlers { onMessage(event: { readonly data: unknown }, socket: OpenAIResponsesWebSocketSocket): void; onClose(event: unknown, socket: OpenAIResponsesWebSocketSocket): void; @@ -56,12 +53,12 @@ type OpenAIResponsesWebSocketUpgradeResolver = ( events: OpenAIResponsesWebSocketHandlers, ) => Response | Promise; -let _openai_responsesWebSocketUpgradeResolver: OpenAIResponsesWebSocketUpgradeResolver | null = null; +let _responsesWebSocketUpgradeResolver: OpenAIResponsesWebSocketUpgradeResolver | null = null; export const initOpenAIResponsesWebSocketUpgradeResolver = ( resolver: OpenAIResponsesWebSocketUpgradeResolver, ): void => { - _openai_responsesWebSocketUpgradeResolver = resolver; + _responsesWebSocketUpgradeResolver = resolver; }; declare const WebSocketPair: { @@ -88,8 +85,8 @@ export const openaiResponsesWebSocket = async (c: AuthedContext): Promise 0)` loop rather than a single // `Promise.allSettled(pendingWork)` snapshot: the in-flight message - // handler running at close time may still enqueue a final - // dump.finalize / settle / recordFailedRequest from its finally/catch after - // `sessionClosed` resolves. The loop keeps going until the Set is - // genuinely empty, which is bounded because `closed = true` short- - // circuits future message handlers at the top of `handleClientMessage`. - // - // That loop only ever runs while the Set is non-empty, which is why the - // message chain itself is tracked as session work in `onMessage`. A turn - // schedules nothing until its terminal `finally` — dump.finalize and - // settle are the only background work it produces, and both come after an - // `await` there — so a client closing mid-turn finds the Set empty at the - // exact moment `sessionClosed` resolves, and the drain would fall through - // before the turn recorded anything. Holding the chain keeps the lifetime - // open until the handler has unwound and its records are in the Set. + // handler running at close time may still enqueue a final write from its + // finally/catch after `sessionClosed` resolves. The loop keeps going until + // the Set is genuinely empty, which is bounded because `closed = true` + // short-circuits future message handlers at the top of + // `handleClientMessage`. const pendingWork = new Set>(); let sessionClosedResolve: (() => void) | undefined; const sessionClosed = new Promise(resolve => { sessionClosedResolve = resolve; }); @@ -192,7 +181,6 @@ const createOpenAIResponsesWebSocketEvents = (c: AuthedContext): OpenAIResponses .catch(error => { if (!closed) sendError(socket, 500, serverErrorEnvelope(error)); }); - sessionScheduler(queue); }, }; }; @@ -202,6 +190,62 @@ interface OpenAIResponsesWsTurnFailure { fail(status: number, error: Record): void; } +/** + * A turn's own run. + * + * `openChatPrologue` reads the turn's shape off the Hono context, and a turn on this + * transport shares none of that shape with the request that opened the socket. It is a `WS` + * turn on a connection whose upgrade was a `GET`; its body is the frame that just arrived + * rather than the upgrade's, which had none; it owns the abort controller the session cancels + * it with; and its background work outlives the fetch that returned the 101, so the scheduler + * it settles and records through is the session's rather than that fetch's. What it does not + * differ in is the services a chat run is given, which is why it hands those back in the shape + * every chat entry hands them over in. + */ +const openOpenAIResponsesWebSocketTurn = ( + c: AuthedContext, + turn: { + readonly payload: CanonicalOpenAIResponsesPayload; + readonly body: RequestBody; + readonly headers: Ingress['headers']; + readonly downstreamAbortController: AbortController; + readonly backgroundScheduler: BackgroundScheduler; + readonly store: (apiKey: ApiKey, requestStartedAt: number) => OpenAIResponsesStatefulStore; + }, +): ChatPrologue => { + // The frame is this turn's request body, so an operator reading the dashboard sees the + // exact `response.create` that opened it, under its own `WS /v1/responses` row rather than + // under the upgrade that carried it. + const attempt: AttemptState = { timing: { firstOutputTokenAt: null, upstreamCallStartedAt: null }, telemetry: undefined }; + const dump = openRunDump( + apiKeyFromContext(c), + { method: 'WS', path: new URL(c.req.raw.url).pathname, body: turn.body }, + turn.backgroundScheduler, + true, + attempt.timing, + ); + const gateway = createChatGatewayCtxFromHono(c, { + wantsStream: true, + attempt, + model: turn.payload.model, + requestBody: takeRequestBody(turn.body), + downstreamAbortController: turn.downstreamAbortController, + backgroundScheduler: turn.backgroundScheduler, + dump, + }, turn.store); + const base = prologueFor(gateway, { body: turn.body, headers: turn.headers }, dump); + + return { + ...base, + gateway, + services: { + ...base.services, + gateway, + selectAffinity: candidate => { gateway.affinity.select(candidate); }, + }, + }; +}; + const handleClientMessage = async ( c: AuthedContext, socket: OpenAIResponsesWebSocketSocket, @@ -217,20 +261,17 @@ const handleClientMessage = async ( let ctx: ChatGatewayCtx | undefined; let previousResponseId: string | undefined; - // "If a continuation turn fails with a `4xx` or `5xx` error, the server MUST - // evict the referenced `previous_response_id` from the connection-local - // cache. A later attempt to continue from that evicted `store=false` - // response ID on the same connection MUST fail with - // `previous_response_not_found`." + // "If a continuation turn fails with a `4xx` or `5xx` error, the server MUST evict the + // referenced `previous_response_id` from the connection-local cache. A later attempt to + // continue from that evicted `store=false` response ID on the same connection MUST fail + // with `previous_response_not_found`." // https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L127 // - // Eviction is wired at two points because a failing turn can leave through - // two exits that do not share a path. `fail` covers the turn that answers - // the client with an error envelope, including the `api-error` and - // `internal-error` results, which return before the streaming loop is - // entered; the loop's `finally` covers the turn that ends failed without - // one. A throw inside the loop is the single path that takes both, and - // `Map.delete` makes the second call inert. + // Eviction is wired at two points because a failing turn can leave through two exits that + // do not share a path. `fail` covers the turn that answers the client with an error + // envelope, including the refusals the chain hands back as a body rather than a stream; the + // streaming loop's `finally` covers the turn that ends failed without one. A throw inside + // the loop is the single path that takes both, and `Map.delete` makes the second call inert. const turnFailure: OpenAIResponsesWsTurnFailure = { evict: () => { if (ctx === undefined || previousResponseId === undefined) return; @@ -243,11 +284,10 @@ const handleClientMessage = async ( }; try { - // Capture raw frame bytes up front so they're available as the dump's - // request body when `ctx` is constructed below. Payloads that fail to - // parse never reach ctx construction, so no dump record is emitted for - // them — there is no api-key-scoped turn to attribute them to. - const requestBody = { bytes: wsDataToBytes(data), streamError: null }; + // Capture raw frame bytes up front so they're available as the run's request body when + // the turn is opened below. Payloads that fail to parse never reach that point, so no + // dump record is emitted for them — there is no api-key-scoped turn to attribute them to. + const requestBody: RequestBody = { bytes: wsDataToBytes(data), streamError: null }; if (!(await authenticateApiKey(c, authenticatedRawKey))) { turnFailure.fail(401, { type: 'authentication_error', @@ -280,41 +320,36 @@ const handleClientMessage = async ( : Object.fromEntries(Object.entries(message).filter(([key]) => key !== 'type' && key !== 'event_id')); const payload = openaiResponsesPayloadFromClientSource(source); previousResponseId = payload.previous_response_id ?? undefined; - ctx = createChatGatewayCtxFromHono(c, { - wantsStream: true, + + const prologue = openOpenAIResponsesWebSocketTurn(c, { + payload, + body: requestBody, + // The upgrade's own headers are the connection's, and every turn on it is dialled with + // them: this transport has no per-turn header surface, so what a client wants a later + // turn to carry it carries in the frame body instead. + headers: [...c.req.raw.headers], downstreamAbortController, - // The WS upgrade has no HTTP body; the dump's request body is the - // per-turn JSON frame bytes so an operator reading the dashboard - // sees the exact `response.create` payload the client sent. - requestBody: takeRequestBody(requestBody), - method: 'WS', - model: payload.model, backgroundScheduler, - }, (apiKey, requestStartedAt) => session.createStore(apiKey, requestStartedAt, payload.store ?? undefined)); - - let result; - try { - result = await openaiResponsesServe.generate({ payload, ctx, headers: inboundHeaders(c) }); - } catch (error) { - if (signal.aborted || isClosed()) return; - // The HTTP entry renders this verbatim envelope as a 400; WS surfaces the - // same body nested under the spec's WebSocket error envelope so clients - // can still compare error.message byte-for-byte against upstream. - if (error instanceof PreviousResponseNotFoundError) { - turnFailure.fail(400, { - message: error.message, - type: 'invalid_request_error', - param: 'previous_response_id', - code: 'previous_response_not_found', - }); - ctx.dump?.failed(error); - ctx.dump?.finalize(400, []); - return; - } - throw error; - } - - await respondOpenAIResponsesWebSocket({ socket, eventId, signal, isClosed, result, ctx, payload, turnFailure }); + store: (apiKey, requestStartedAt) => session.createStore(apiKey, requestStartedAt, payload.store ?? undefined), + }); + ctx = prologue.gateway; + + const { facts, drain } = await run( + openaiResponsesWebSocketPipeline(), + move({ + 'ingress.http.headers': prologue.headers, + 'ingress.chat.sourceProtocol': 'openaiResponses', + // A turn on this transport always streams, whatever the client wrote. + 'ingress.chat.openaiResponses.wantsStream': true, + 'ingress.chat.openaiResponses.eventId': eventId, + 'request.chat.openaiResponses': payload, + 'serve.model': payload.model, + }) as never, + prologue.services as never, + ); + prologue.runDump?.afterRun(drain); + + await respondOpenAIResponsesWebSocket({ socket, signal, isClosed, prologue, facts, drain, turnFailure }); } catch (error) { if (signal.aborted || isClosed()) return; if (error instanceof TranslatorInputError) { @@ -336,14 +371,8 @@ const handleClientMessage = async ( } turnFailure.fail(500, serverErrorEnvelope(error)); if (ctx !== undefined) { - // Mid-attempt throws (interceptor bug, translation error, provider-layer JS - // exception not represented as a ChatServeFailure) never reach the - // respondOpenAIResponsesWebSocket result branches, so their `recordFailedRequest` - // call would be skipped. Attribute the failure to the last upstream stamped - // synchronously by `openaiResponsesServe.generate`, matching the HTTP transports. - recordFailedRequest(ctx, ctx.attempt.telemetry); ctx.dump?.failed(error); - ctx.dump?.finalize(500, []); + ctx.dump?.finalize(500, 0); } } }; @@ -370,341 +399,42 @@ const openaiResponsesPayloadFromClientSource = (source: object): CanonicalOpenAI const respondOpenAIResponsesWebSocket = async (input: { readonly socket: OpenAIResponsesWebSocketSocket; - readonly eventId: string | undefined; readonly signal: AbortSignal; readonly isClosed: () => boolean; - readonly result: ExecuteResult>; - readonly ctx: ChatGatewayCtx; - readonly payload: CanonicalOpenAIResponsesPayload; + readonly prologue: ChatPrologue; + readonly facts: OpenAIResponsesWebSocketExit; + readonly drain: () => Promise; readonly turnFailure: OpenAIResponsesWsTurnFailure; }): Promise => { - const { socket, eventId, signal, isClosed, result, ctx, payload, turnFailure } = input; - if (result.type === 'api-error') { - recordFailedRequest(ctx, result.performance); - ctx.dump?.error(result.source, result.upstreamId); - turnFailure.fail(result.status, normalizeErrorBody(parseMaybeJson(result.body, result.headers), result.status)); - ctx.dump?.finalize(result.status, []); - return; - } - - if (result.type === 'internal-error') { - recordFailedRequest(ctx, result.performance); - ctx.dump?.failed(result.error.message); - turnFailure.fail(result.status, internalErrorEnvelope(result.error)); - ctx.dump?.finalize(result.status, []); - return; - } - + const { socket, signal, isClosed, prologue, facts, drain, turnFailure } = input; + const ctx = prologue.gateway; const state = new SourceStreamState(); let completion: StreamCompletion = 'error'; try { - let terminalEvent: ClientOpenAIResponsesStreamEvent | undefined; - const observed = observeOpenAIResponsesWebSocketFrames(result.events, state, ctx); - const output = wrapOpenAIResponsesClientEgress(observed, ctx, payload); - const iterator = output[Symbol.asyncIterator](); - let pendingNext = pendingWsFrameResult(iterator.next()); - let completed = false; - let stoppedByDownstream = false; - let streamed = false; - const sequence = createDownstreamSequence(); - - const stopForDownstream = (): void => { - stoppedByDownstream = true; - completion = 'cancel'; - }; - - try { - while (true) { - if (signal.aborted || isClosed()) { - stopForDownstream(); - return; - } - - const next = await nextFrameOrKeepAlive(pendingNext); - - if (next.type === 'keep-alive') { - // Extended reasoning turns go completely silent: upstream sends SSE - // `ping` events, `parseOpenAIResponsesStream` drops them, and no frame at - // all reaches this socket for minutes. A silent Workers WebSocket - // does not reliably survive that. Cloudflare states the - // teardown without naming a duration — "when no data is transmitted - // in either direction for a period of time" — and probing a - // `workers.dev` endpoint built on this handler's own `WebSocketPair` - // shape found no constant to design against: from one vantage point - // an idle socket lived a full hour, 4/4, while from another 13 of 16 - // idle sockets died between 215.8 s and 1788.2 s, median around - // 660 s. Teardown is path-dependent and stochastic, and it is always - // a silent EOF — across roughly 40 observed teardowns, not one CLOSE - // frame and not one RST, so the failure carries no protocol-level - // signal. - // https://developers.cloudflare.com/network/websockets/#idle-timeout - // - // Cloudflare's stated remedy, a client-side ping/pong heartbeat, - // does not cover it: on the path that drops, 6 of 9 sockets pinging - // every 30 s died anyway, one of them after 61.5 s. A - // server-originated text frame did cover it — 12/12 survived on that - // same path, with ≤400 s intervals holding and ≥500 s failing. Why a - // data frame outlives a protocol ping there was not established. - // Sending a ping is not open to us regardless: workerd's `WebSocket` - // exposes only accept/send/close/(de)serializeAttachment, and the kj - // layer beneath states the omission as a design decision ("Ping/Pong - // … are not exposed through this interface"). RFC 6455 §5.5.2 is the - // right mechanism, it is unreachable here, and it would not help the - // client that needs it most either, since Codex's frame pump - // swallows control frames and only a text frame rearms its 300 s - // idle timeout. - // https://github.com/cloudflare/workerd/blob/26b5461b7dcc640bb16072f1ba6f2c6df82572ba/src/workerd/api/web-socket.h#L346-L394 - // https://github.com/capnproto/capnproto/blob/e9fa5c7dc98192fc0dc0098ec770db68f997a938/c%2B%2B/src/kj/compat/http.h#L622-L631 - // https://github.com/openai/codex/blob/e6cfd40c3f444aadd6017c9eeab01db70f48961a/codex-rs/codex-api/src/endpoint/responses_websocket.rs#L91-L101 - // https://github.com/openai/codex/blob/e6cfd40c3f444aadd6017c9eeab01db70f48961a/codex-rs/codex-api/src/endpoint/responses_websocket.rs#L695-L699 - // https://github.com/openai/codex/blob/e6cfd40c3f444aadd6017c9eeab01db70f48961a/codex-rs/model-provider-info/src/lib.rs#L26 - // - // So the keep-alive is a text frame whose `type` no client - // recognizes. The spec's extension section governs its shape: an - // implementor slug prefix plus a `sequence_number`. A keep-alive is - // neither a delta nor a state-machine event, so it cannot be spelled - // as a `response.*` event; the slug form is what makes it ignorable - // without loss. openai-node's SSE `responses.stream()` helper is the - // one client that treats a prefixed type as fatal — its accumulator - // closes its `switch` on `assertNever` — and it is out of reach here: - // Floway's SSE keep-alive is a comment line, and this frame exists - // only on the WebSocket transport. - // https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L758 - // https://github.com/openai/openai-node/blob/d77cf24d9f3885739c6cba76bc009abf0ab97428/src/lib/responses/ResponseAccumulator.ts#L387-L389 - // - // Held back until the turn's first event has gone out. That gate is - // parity with the stricter transport, not a demand of this one: both - // SDKs' SSE stream helpers refuse anything before `response.created` - // — openai-node rejects even its own `keepalive` type there — while - // every WebSocket reader we can inspect tolerates an unknown type at - // any position, openai-node emitting it under a name nothing listens - // on, openai-python constructing it unchecked, and Codex tracing and - // discarding it. The window before the first event therefore stays - // unprotected, and closing it is a behavior question rather than a - // client-compatibility one. - // https://github.com/openai/openai-node/blob/d77cf24d9f3885739c6cba76bc009abf0ab97428/src/lib/responses/ResponseAccumulator.ts#L25-L31 - // https://github.com/openai/openai-python/blob/3844843c277f42b0b18beaa58152cfda61df524a/src/openai/lib/streaming/responses/_responses.py#L369-L370 - // https://github.com/openai/openai-python/blob/3844843c277f42b0b18beaa58152cfda61df524a/src/openai/resources/responses/responses.py#L4493-L4502 - // https://github.com/openai/codex/blob/e6cfd40c3f444aadd6017c9eeab01db70f48961a/codex-rs/codex-api/src/sse/responses.rs#L466-L472 - if (!streamed) continue; - if (!sendJson(socket, { type: KEEP_ALIVE_EVENT_TYPE, sequence_number: sequence.take() }, eventId, ctx.dump)) { - stopForDownstream(); - return; - } - continue; - } - if (next.type === 'next-error') throw next.error; - if (next.result.done) { - completed = true; - break; - } - - const frame = next.result.value; - pendingNext = pendingWsFrameResult(iterator.next()); - if (frame.type !== 'event') continue; - - const event = frame.event; - - // The wrapped terminal event arrives only after its item and snapshot - // writes have committed, but the generator still has work to drain - // behind it. Buffer it here and flush it once the loop has run to - // completion, so the terminal event is the last frame of the turn and - // is itself the signal that a follow-up turn may reference this - // response. WebSocket carries the same streaming event objects as - // streaming HTTP, and the specification defines no frame after the - // terminal event: - // https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L117 - // Live captures agree. Azure Foundry OpenAI-Responses-over-WebSocket, and the - // ChatGPT backend the Codex CLI talks to, both end a turn on - // `response.completed` and send nothing further while the socket stays - // open and idle. Codex's own reader breaks its loop on that event - // rather than waiting for any trailing envelope: - // https://github.com/openai/codex/blob/acd540f1581bf30f963fccbcce43ac494102242c/codex-rs/codex-api/src/endpoint/responses_websocket.rs#L792-L799 - if (terminalEvent !== undefined) continue; - - if (isOpenAIResponsesTerminalEvent(event)) { - terminalEvent = event; - continue; - } - - if (!sendOpenAIResponsesEvent(socket, sequence.renumber(event), eventId, ctx.dump)) { - stopForDownstream(); - return; - } - streamed = true; - } - } finally { - if (!completed) { - const stopped = iterator.return?.(undefined); - if (stoppedByDownstream) stopped?.catch(() => {}); - else await stopped; - } - } - - if (terminalEvent === undefined) { - throw new Error(OPENAI_RESPONSES_MISSING_TERMINAL_MESSAGE); - } - // Renumbered here rather than where it was buffered: keep-alives can still - // fire while the generator drains behind the terminal event, and each of - // those takes a slot that has to land before the terminal event's own. - if (!sendOpenAIResponsesEvent(socket, sequence.renumber(terminalEvent), eventId, ctx.dump)) { - completion = 'cancel'; - return; + for await (const packet of facts['response.chat.openaiResponses.websocket']) { + if (signal.aborted || isClosed() || !sendText(socket, packet.text, ctx.dump)) { completion = 'cancel'; return; } + if (packet.frame?.type !== 'event') continue; + const event = packet.frame.event; + if (event.type === 'error' || event.type === 'response.failed') state.failed = true; + if (isOpenAIResponsesTerminalEvent(event as unknown as ClientOpenAIResponsesStreamEvent) && !state.failed) state.completed = true; } completion = 'eof'; - } catch (error) { - if (signal.aborted || isClosed()) { - completion = 'cancel'; - return; - } - state.failed = true; - turnFailure.fail(500, serverErrorEnvelope(error)); } finally { - const metadata = await eventResultMetadata(result); + await drain(); const failed = state.failedAfter(completion); if (failed) { - // `fail` cannot carry the eviction for every failed turn: one that - // streamed an `error` or `response.failed` terminal answered the client - // with an event rather than an error envelope, and one the client - // abandoned before the terminal event answered with nothing at all. - // Both settle here. For the abandoned turn the eviction is inert — the - // connection-local cache dies with the socket. turnFailure.evict(); - ctx.dump?.failed(`responses ws turn failed (completion=${completion}, source-failed=${state.failed})`); - } else ctx.dump?.success(metadata.modelIdentity, tokenUsageFromBillableUsage(metadata.billableUsage)); - ctx.dump?.finalize(failed ? 500 : 200, []); - settle(ctx, metadata.performance, metadata.modelIdentity, tokenUsageFromBillableUsage(metadata.billableUsage), failed); - } -}; - -const observeOpenAIResponsesWebSocketFrames = async function* ( - frames: AsyncIterable>, - state: SourceStreamState, - ctx: ChatGatewayCtx, -): AsyncGenerator> { - for await (const frame of frames) { - await ctx.dump?.frame(frame); - if (frame.type === 'event') { - const event = frame.event; - const failed = event.type === 'error' || event.type === 'response.failed'; - if (failed) state.failed = true; - if (isOpenAIResponsesTerminalEvent(event) && !failed) state.completed = true; + ctx.dump?.failed(`openai-responses ws turn failed (completion=${completion}, source-failed=${state.failed})`, { fallback: true }); } - yield frame; + ctx.dump?.finalize(failed ? 500 : facts['response.http.status'], 0); } }; -type WsFrameRaceResult = - | { type: 'frame'; result: IteratorResult> } - | { type: 'next-error'; error: unknown } - | { type: 'keep-alive' }; - -const pendingWsFrameResult = (pendingNext: Promise>>): Promise => - pendingNext.then( - (result): WsFrameRaceResult => ({ type: 'frame', result }), - (error): WsFrameRaceResult => ({ type: 'next-error', error }), - ); - -// The interval is the one already shared with SSE rather than a WebSocket -// constant of its own. Widening the gap between server data frames on a -// dropping path put the boundary between 400 s, which still held the socket -// open, and 500 s, which did not, so 15 s is far more frequent than the -// mechanism needs. It is kept because widening it buys nothing: a keep-alive -// is ~70 bytes, and the earliest unprotected idle teardown seen on that same -// path was 215.8 s, so an interval chosen for economy would spend a real -// margin against a stochastic teardown to save nothing. -const nextFrameOrKeepAlive = async (pendingFrame: Promise): Promise => { - let timeoutId: ReturnType | undefined; - const keepAlive = new Promise(resolve => { - timeoutId = setTimeout(() => resolve({ type: 'keep-alive' }), DOWNSTREAM_KEEP_ALIVE_INTERVAL_MS); - }); - try { - return await Promise.race([pendingFrame, keepAlive]); - } finally { - if (timeoutId !== undefined) clearTimeout(timeoutId); - } -}; - -interface DownstreamSequence { - renumber(event: ClientOpenAIResponsesStreamEvent): ClientOpenAIResponsesStreamEvent; - take(): number; -} - -// A WebSocket turn shares one sequence space with the streaming-HTTP events it -// carries, so a keep-alive cannot sit outside that numbering: it takes a real -// slot and every later event is shifted past it. The number is always present -// and always numeric, because a resuming openai-python client compares it with -// `>` against its `starting_after` cursor and `None` there raises a TypeError. -// https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L117 -// https://github.com/openai/openai-python/blob/3844843c277f42b0b18beaa58152cfda61df524a/src/openai/lib/streaming/responses/_responses.py#L59 -// -// Upstream numbering is left untouched until the first keep-alive, and a -// keep-alive can only follow an event that already went out, so the slot it -// takes is always known. -const createDownstreamSequence = (): DownstreamSequence => { - let shift = 0; - let next = 0; - return { - renumber: event => { - if (event.sequence_number === undefined) return event; - const sequenceNumber = event.sequence_number + shift; - next = sequenceNumber + 1; - return { ...event, sequence_number: sequenceNumber }; - }, - take: () => { - shift += 1; - const taken = next; - next += 1; - return taken; - }, - }; -}; - -const parseMaybeJson = (body: Uint8Array, headers: Headers): unknown => { - const text = new TextDecoder().decode(body); - if (!isJsonMediaType(headers.get('content-type'))) return { message: text }; - try { - return JSON.parse(text) as unknown; - } catch { - return { message: text }; - } -}; - -const internalErrorEnvelope = (error: Extract>, { type: 'internal-error' }>['error']): Record => ({ - type: error.type, - code: error.type, - name: error.name, - message: error.message, - stack: error.stack, - cause: error.cause, - target_api: error.target_api, -}); - const serverErrorEnvelope = (error: unknown): Record => ({ ...toInternalDebugError(error), code: 'internal_error', }); -const normalizeErrorBody = (body: unknown, status: number): Record => { - const source = body && typeof body === 'object' && 'error' in body && typeof (body as { error?: unknown }).error === 'object' - ? (body as { error: Record }).error - : body && typeof body === 'object' - ? body as Record - : {}; - const type = typeof source.type === 'string' - ? source.type - : status >= 500 ? 'server_error' : 'invalid_request_error'; - const message = typeof source.message === 'string' - ? source.message - : `OpenAI Responses request failed with status ${status}.`; - return { - ...source, - type, - code: typeof source.code === 'string' ? source.code : type, - message, - }; -}; - // "WebSocket failures MUST be sent as a JSON `error` envelope with a `status` // code and an `error.code`." // https://github.com/openresponses/openresponses/blob/92c12d96d7b61d6d15e2214daa5e9c6000ab6e1c/src/specifications/2026-04-24.mdx#L166 @@ -716,43 +446,29 @@ const sendError = ( status: number, error: Record, eventId?: string, - dump?: TurnDump | null, + dump?: RunDump | null, ): void => { sendJson(socket, { type: 'error', status, error }, eventId, dump); }; -// A turn's own frames go out through this entry, which accepts only a stream -// event whose response resource has already passed the client-facing egress -// stage. The two frames Floway synthesizes for the transport itself — the -// `error` envelope and the `floway:keep_alive` keep-alive — carry no response -// resource at all, and neither belongs to the stream-event union: the error -// envelope is not a streaming event, and the keep-alive is a slug-prefixed -// extension the union deliberately does not model. Both keep the untyped -// `sendJson`. -const sendOpenAIResponsesEvent = ( - socket: OpenAIResponsesWebSocketSocket, - event: ClientOpenAIResponsesStreamEvent, - eventId?: string, - dump?: TurnDump | null, -): boolean => sendJson(socket, event, eventId, dump); - const sendJson = ( socket: OpenAIResponsesWebSocketSocket, value: unknown, eventId?: string, - dump?: TurnDump | null, + dump?: RunDump | null, ): boolean => { if (socket.readyState !== 1) return false; const payload = eventId === undefined || !value || typeof value !== 'object' ? value : { ...value, event_id: eventId }; - let text: string; - try { - text = JSON.stringify(payload); - socket.send(text); - } catch { - return false; - } + return sendText(socket, JSON.stringify(payload), dump); +}; + +const sendText = (socket: OpenAIResponsesWebSocketSocket, text: string, dump?: RunDump | null): boolean => { + if (socket.readyState !== 1) return false; + try { socket.send(text); } catch { return false; } dump?.recordSentPayloadBytes(UTF8_ENCODER.encode(text).byteLength); return true; }; + +export { KEEP_ALIVE_EVENT_TYPE } from './project-websocket.ts'; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/wire.ts b/packages/gateway/src/data-plane/chat/openai-responses/wire.ts new file mode 100644 index 0000000000..2ef26d3a3d --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/wire.ts @@ -0,0 +1,43 @@ +import { applyRoleCompatibilityToOpenAIResponses } from './apply-role-compatibility.ts'; +import { callOpenAIResponsesUpstream } from './call-upstream.ts'; +import { disableReasoningOnForcedToolChoiceForOpenAIResponses } from './disable-reasoning-on-forced-tool-choice.ts'; +import { meterUsage } from './meter.ts'; +import { normalizeAssistantContentForOpenAIResponses } from './normalize-assistant-content.ts'; +import { normalizeEmptyToolsForOpenAIResponses } from './normalize-empty-tools-tool-choice.ts'; +import { normalizeExclusiveCachedTokensForOpenAIResponses } from './normalize-exclusive-cached-tokens.ts'; +import { stripPromptCacheKeyForOpenAIResponses } from './strip-prompt-cache-key.ts'; +import { vendorDeepSeekNormalizeForOpenAIResponses } from './vendor-deep-seek-normalize.ts'; +import { vendorQwenNormalizeForOpenAIResponses } from './vendor-qwen-normalize.ts'; +import type { Stage } from '@floway-dev/pipeline'; + +/** + * What every turn this wire sends is subject to, whichever operation asked for it. + * + * Every source protocol that reaches an upstream over this endpoint runs these, whether the + * client spoke OpenAI Responses or a handoff arrived here — which is what makes the three rules + * belong to the wire. The role rewrite and the assistant-content rewrite both state what an + * upstream's OpenAI Responses endpoint accepts; the cache-bucket fold speaks about the usage *this* + * wire reports and about the flag that describes it, and a translator emits the canonical + * form, which is the one case the fold has nothing to do with. + * + * They are named apart from the dial because a compaction is dialled differently and is + * subject to the same three. + */ +export const openaiResponsesWireRules: readonly Stage[] = [ + normalizeEmptyToolsForOpenAIResponses, + normalizeAssistantContentForOpenAIResponses, + disableReasoningOnForcedToolChoiceForOpenAIResponses, + applyRoleCompatibilityToOpenAIResponses, + stripPromptCacheKeyForOpenAIResponses, + normalizeExclusiveCachedTokensForOpenAIResponses, + vendorDeepSeekNormalizeForOpenAIResponses, + vendorQwenNormalizeForOpenAIResponses, +]; + +/** The OpenAI Responses wire, as the chain that dials it. The meter is above every rule, which + * is what makes the figure it bills the one the client is shown. */ +export const openaiResponsesWire = (streamedUsage: string): readonly Stage[] => [ + meterUsage(streamedUsage), + ...openaiResponsesWireRules, + callOpenAIResponsesUpstream, +]; diff --git a/packages/gateway/src/data-plane/chat/openai-responses/wires.ts b/packages/gateway/src/data-plane/chat/openai-responses/wires.ts new file mode 100644 index 0000000000..590c1c8288 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/openai-responses/wires.ts @@ -0,0 +1,48 @@ +import { composeChat as compose } from '../compose.ts'; +import type { ChatServices } from '../services.ts'; +import type { ChatWire } from '../wire.ts'; +import { decryptNativeCompaction } from './compact-decrypt.ts'; +import { containsCompactionTrigger } from './compact-shim.ts'; +import { OPENAI_RESPONSES_STREAMED_USAGE } from './facts.ts'; +import { openaiResponsesWire } from './wire.ts'; +import { anthropicMessagesWire } from '../anthropic-messages/wire.ts'; +import { handOff } from '../handoff.ts'; +import { openaiChatCompletionsWire } from '../openai-chat-completions/wire.ts'; +import { createExternalImageLoader } from '../shared/external-image-loader.ts'; +import type { Use } from '@floway-dev/pipeline'; +import type { ChatTargetApi, ModelCandidate } from '@floway-dev/provider'; +import { translateOpenAIResponsesViaAnthropicMessages, translateOpenAIResponsesViaOpenAIChatCompletions } from '@floway-dev/translate'; + +/** The three wires `/v1/responses` can be served on. Its own is the bare wire; each translated + * one is a handoff and then the target protocol's own wire. */ +export const openaiResponsesWireFor = (target: ChatTargetApi, candidate: ModelCandidate, use: Use): ChatWire => { + switch (target) { + case 'openaiResponses': + { + const native = compose, Record>('openaiResponsesNative', openaiResponsesWire(OPENAI_RESPONSES_STREAMED_USAGE)); + return compose('openaiResponsesNativeCompaction', [decryptNativeCompaction({ native, replay: native, streamedUsage: OPENAI_RESPONSES_STREAMED_USAGE, compactEndpoint: false, asked: payload => containsCompactionTrigger(payload.input) })]); + } + case 'anthropicMessages': + return compose('openaiResponsesViaAnthropicMessages', [ + handOff({ + from: { request: 'request.chat.openaiResponses', response: 'response.chat.openaiResponses' }, + to: { request: 'request.chat.anthropicMessages', response: 'response.chat.anthropicMessages' }, + trip: async payload => await translateOpenAIResponsesViaAnthropicMessages(payload, { + model: candidate.model.id, + fallbackMaxOutputTokens: candidate.model.limits.max_output_tokens, + loadRemoteImage: createExternalImageLoader(use.gateway.abortSignal), + }), + }), + ...anthropicMessagesWire(OPENAI_RESPONSES_STREAMED_USAGE), + ]); + case 'openaiChatCompletions': + return compose('openaiResponsesViaOpenAIChatCompletions', [ + handOff({ + from: { request: 'request.chat.openaiResponses', response: 'response.chat.openaiResponses' }, + to: { request: 'request.chat.openaiChatCompletions', response: 'response.chat.openaiChatCompletions' }, + trip: async payload => await translateOpenAIResponsesViaOpenAIChatCompletions(payload, { model: candidate.model.id }), + }), + ...openaiChatCompletionsWire(OPENAI_RESPONSES_STREAMED_USAGE), + ]); + } +}; diff --git a/packages/gateway/src/data-plane/chat/prologue.ts b/packages/gateway/src/data-plane/chat/prologue.ts new file mode 100644 index 0000000000..276ce462b0 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/prologue.ts @@ -0,0 +1,43 @@ +// Chat admission adds the request-owned store and affinity ports to shared run services. + +import type { ChatServices } from './services.ts'; +import { createChatGatewayCtxFromHono, type ChatGatewayCtx } from './shared/gateway-ctx.ts'; +import type { AuthedContext } from '../../middleware/auth.ts'; +import type { ApiKey } from '../../repo/types.ts'; +import { gatewayCtxOptions, prologueFor, runDumpOf, type Ingress, type Prologue } from '../pipeline/serve.ts'; +import type { OpenAIResponsesStatefulStore } from './openai-responses/items/store.ts'; + +export interface ChatPrologue extends Prologue { + readonly services: ChatServices; + readonly gateway: ChatGatewayCtx; +} + +export const openChatPrologue = ( + c: AuthedContext, + ingress: Ingress, + options: { + readonly wantsStream: boolean; + readonly model?: string; + /** Native OpenAI Responses entries persist their items; every other source gets a scratchpad, + * so the hosted-tool shim's request-private state always has a home. */ + readonly storeFactory: (apiKey: ApiKey, requestStartedAt: number) => OpenAIResponsesStatefulStore; + }, +): ChatPrologue => { + // One options object, read twice: the context is built from it, and the run recording it + // opened is what the runner's events are written to. Building it a second time would call + // `takeRequestBody` again and hand the dump an empty buffer — which is exactly how every + // chat turn once came to write a record holding no events. + const ctxOptions = gatewayCtxOptions(c, ingress, options); + const gateway = createChatGatewayCtxFromHono(c, ctxOptions, options.storeFactory); + const base = prologueFor(gateway, ingress, runDumpOf(ctxOptions)); + + return { + ...base, + gateway, + services: { + ...base.services, + gateway, + selectAffinity: candidate => { gateway.affinity.select(candidate); }, + }, + }; +}; diff --git a/packages/gateway/src/data-plane/chat/resolve-candidates.ts b/packages/gateway/src/data-plane/chat/resolve-candidates.ts new file mode 100644 index 0000000000..d0b5eaf37b --- /dev/null +++ b/packages/gateway/src/data-plane/chat/resolve-candidates.ts @@ -0,0 +1,97 @@ +import type { ChatFacts, ChatRequestKey } from './facts.ts'; +import type { ChatServices } from './services.ts'; +import { enumerateModelCandidates } from '../providers/resolution.ts'; +import { appendFailedUpstreams } from '../shared/failed-upstreams.ts'; +import { selectAffinityCandidates } from './shared/affinity/index.ts'; +import type { AffinityRequestAnalysis } from './shared/affinity/selection.ts'; +import type { ChatGatewayCtx } from './shared/gateway-ctx.ts'; +import type { Slice } from './stage-contracts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { ModelCandidate } from '@floway-dev/provider'; + +/** Why the gateway refused before it reached an upstream. A protocol's envelope names the + * condition in its own words, and this is what it names. */ +export type ChatRefusal = 'routing-unavailable' | 'model-unsupported' | 'model-missing'; + +/** How a source protocol narrows and orders the candidates that could serve it. */ +export interface ChatNarrowing { + readonly requestKey: RequestKey; + /** Which upstream wires this source prefers, in order. A candidate none of whose endpoints + * appear here cannot serve the request whatever else it offers. */ + readonly canServe: (candidate: ModelCandidate) => boolean; + /** What the client's own turn says about where it must go. Client-carried state — a + * an OpenAI Responses `previous_response_id`, an encrypted reasoning blob — pins the turn to the + * upstream that issued it, so this runs before any candidate is tried. */ + readonly affinity: (payload: Pick[RequestKey], gateway: ChatGatewayCtx) => Promise>; + readonly unsupported: (model: string) => string; + readonly refuse: (status: number, message: string, reason: ChatRefusal) => Refusal; + readonly refuses: readonly (keyof Refusal)[]; +} + +/** + * Provides `serve.candidates` in the order affinity asks for, or answers with the failure + * that says why there are none. + * + * Affinity can refuse outright — a turn whose state requires two incompatible upstreams has + * nowhere to go — and that is an answer this stage already holds, which is why it carries + * the `return` trait alongside `through`. + */ +export const resolveChatCandidates = (narrowing: ChatNarrowing) => defineStage< + Slice<'serve.model'> & Pick, + Slice<'serve.model' | 'serve.candidates'> & Pick & Pick, + Slice<'response.usage.billable' | 'response.http.headers' | 'response.http.body' | 'response.http.status'>, + Slice<'response.usage.billable' | 'response.http.headers' | 'response.http.body' | 'response.http.status'>, + Slice<'response.usage.billable' | 'response.http.headers' | 'response.http.body' | 'response.http.status'> & Refusal, + ChatServices +>({ + name: 'resolveChatCandidates', + through: { + request: { needs: ['serve.model', narrowing.requestKey], consumes: [], provides: ['serve.candidates', 'request.chat.candidatePayloads'] }, + response: { needs: ['response.usage.billable', 'response.http.headers'], consumes: [], provides: [] }, + }, + return: { provides: ['response.usage.billable', 'response.http.headers', 'response.http.body', 'response.http.status', ...narrowing.refuses] }, + execute: async (facts, next, use) => { + const model = facts['serve.model']; + const { candidates, sawModel, failedUpstreams } = await enumerateModelCandidates({ + upstreamIds: use.gateway.upstreamIds, + model, + kind: 'chat', + scheduler: use.gateway.backgroundScheduler, + runtimeLocation: use.gateway.runtimeLocation, + }); + + // An empty billed set is what "we did not call an upstream" looks like, and an empty + // header list is the same statement on the other key. + const refuse = (status: number, message: string, reason: ChatRefusal) => + move({ + ...facts, + 'response.usage.billable': [], + 'response.http.headers': [], + 'response.http.body': null, 'response.http.status': status, + ...narrowing.refuse(status, message, reason), + }); + + const affinity = await narrowing.affinity(facts[narrowing.requestKey] as ChatFacts[RequestKey], use.gateway); + const viable = candidates.filter(candidate => narrowing.canServe(candidate)); + const selection = selectAffinityCandidates(viable, affinity); + // A turn whose carried state needs two upstreams at once is a request the client can fix + // by not carrying it, which is what every family called this before: a 400, not a + // conflict with something the gateway holds. + if ('kind' in selection) return refuse(400, selection.message, 'routing-unavailable'); + if (selection.candidates.length === 0) { + const missing = sawModel + ? narrowing.unsupported(model) + : `Model ${model} is not available on any configured upstream.`; + return refuse(sawModel ? 400 : 404, appendFailedUpstreams(missing, failedUpstreams), sawModel ? 'model-unsupported' : 'model-missing'); + } + + const selectors = use.rememberCandidates(selection.candidates); + // Selection and payload projection are pure after carrier decoding. Snapshot only the + // accepted candidates' request values; live provider instances remain in the registry. + const payloads = Object.fromEntries(selectors.map((selector, index) => [ + selector.candidateId, selection.payloadFor(selection.candidates[index]), + ])); + await use.log.debug('resolved chat candidates', { model, viable: selection.candidates.length, resolved: candidates.length }); + return await next({ ...facts, 'serve.candidates': move(selectors), 'request.chat.candidatePayloads': move(payloads) }); + }, +}); diff --git a/packages/gateway/src/data-plane/chat/services.ts b/packages/gateway/src/data-plane/chat/services.ts new file mode 100644 index 0000000000..7523f99cb1 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/services.ts @@ -0,0 +1,8 @@ +import type { GatewayServices } from '../pipeline/services.ts'; +import type { ChatGatewayCtx } from './shared/gateway-ctx.ts'; +import type { ModelCandidate, ProviderChatServices } from '@floway-dev/provider'; + +export interface ChatServices extends GatewayServices, ProviderChatServices { + readonly gateway: ChatGatewayCtx; + readonly selectAffinity: (candidate: ModelCandidate) => void; +} diff --git a/packages/gateway/src/data-plane/chat/shared/affinity/selection.ts b/packages/gateway/src/data-plane/chat/shared/affinity/selection.ts index be657d077a..14d71dc4f8 100644 --- a/packages/gateway/src/data-plane/chat/shared/affinity/selection.ts +++ b/packages/gateway/src/data-plane/chat/shared/affinity/selection.ts @@ -19,8 +19,6 @@ export interface AffinityCandidateSelection { readonly payloadFor: (candidate: ModelCandidate) => T; } -export type AffinitySelectionFailure = Extract; - export type OptionalAffinityBlobProjection = | { readonly kind: 'preserve'; readonly value: string; readonly preferred: boolean } | { readonly kind: 'remove'; readonly degrades: boolean; readonly preferred: boolean }; @@ -135,7 +133,7 @@ export const defineAffinityRequest = ( export const selectAffinityCandidates = ( candidates: readonly ModelCandidate[], affinity: AffinityRequestAnalysis, -): AffinityCandidateSelection | AffinitySelectionFailure => { +): AffinityCandidateSelection | ChatServeFailure => { if (affinity.requiredTargets.length > 1) { return { kind: 'routing-unavailable', diff --git a/packages/gateway/src/data-plane/chat/shared/alias-rules.ts b/packages/gateway/src/data-plane/chat/shared/alias-rules.ts index c4c0300a23..b0738b2ff9 100644 --- a/packages/gateway/src/data-plane/chat/shared/alias-rules.ts +++ b/packages/gateway/src/data-plane/chat/shared/alias-rules.ts @@ -1,97 +1,104 @@ // Post-translate rule overlay. The alias resolver tags each alias-origin // candidate with `.rules`; each terminal wire call reads them off the -// dispatching candidate and writes onto the target IR's NATIVE slot before +// dispatching candidate and puts them on the target IR's NATIVE slot before // dispatching. Rules that a target protocol cannot express are silently // dropped — the wire has nowhere to put them. // // Structuring the overlay this way keeps every translate pair pure // native↔native and eliminates the fan-out of Floway-extension fields onto // each source IR. +// +// Each overlay returns the payload it produced rather than writing into the one it was handed. +// What it is handed descends from a record, and a record's values are frozen — an overlay that +// wrote in place would work only for as long as its caller happened to have rebuilt the level +// it writes to, which is the shape of failure that reached a client as a 502 once already. import type { AnthropicMessagesPayload, AnthropicMessagesThinkingDisplay } from '@floway-dev/protocols/anthropic-messages'; import { isFastServiceTier, type AliasRules } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; import type { OpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; -const hasReasoning = (rules: AliasRules): rules is AliasRules & { reasoning: NonNullable } => - rules.reasoning !== undefined; +type Reasoning = NonNullable; -export const applyRulesToUpstreamOpenAIChatCompletions = (body: OpenAIChatCompletionsPayload, rules: AliasRules): void => { - if (hasReasoning(rules)) { - const { effort } = rules.reasoning; - if (effort !== undefined) body.reasoning_effort = effort; - // `budget_tokens`, `adaptive`, and `summary` have no native OpenAI Chat Completions - // Completions slot; drop silently. - } - if (rules.verbosity !== undefined) body.verbosity = rules.verbosity; - if (rules.serviceTier !== undefined) body.service_tier = rules.serviceTier; +const reasoningOf = (rules: AliasRules): Reasoning => rules.reasoning ?? {}; + +export const applyRulesToUpstreamOpenAIChatCompletions = (body: T, rules: AliasRules): T => { + // `budget_tokens`, `adaptive`, and `summary` have no native OpenAI Chat Completions slot; + // drop silently. + const { effort } = reasoningOf(rules); + return { + ...body, + ...(effort === undefined ? {} : { reasoning_effort: effort }), + ...(rules.verbosity === undefined ? {} : { verbosity: rules.verbosity }), + ...(rules.serviceTier === undefined ? {} : { service_tier: rules.serviceTier }), + } as T; }; -export const applyRulesToUpstreamOpenAIResponses = (body: OpenAIResponsesPayload, rules: AliasRules): void => { - if (hasReasoning(rules)) { - const { effort, summary } = rules.reasoning; - if (effort !== undefined || summary !== undefined) { - const existing = body.reasoning ?? {}; - body.reasoning = { - ...existing, - ...(effort !== undefined ? { effort } : {}), - ...(summary !== undefined ? { summary } : {}), - }; - } - // `budget_tokens` and `adaptive` have no native OpenAI Responses slot; drop - // silently. - } - if (rules.verbosity !== undefined) { - body.text = { ...body.text, verbosity: rules.verbosity }; - } - if (rules.serviceTier !== undefined) body.service_tier = rules.serviceTier; +export const applyRulesToUpstreamOpenAIResponses = (body: T, rules: AliasRules): T => { + // `budget_tokens` and `adaptive` have no native OpenAI Responses slot; drop silently. + const { effort, summary } = reasoningOf(rules); + return { + ...body, + ...(effort === undefined && summary === undefined ? {} : { + reasoning: { + ...body.reasoning, + ...(effort === undefined ? {} : { effort }), + ...(summary === undefined ? {} : { summary }), + }, + }), + ...(rules.verbosity === undefined ? {} : { text: { ...body.text, verbosity: rules.verbosity } }), + ...(rules.serviceTier === undefined ? {} : { service_tier: rules.serviceTier }), + } as T; }; -export const applyRulesToUpstreamAnthropicMessages = (body: AnthropicMessagesPayload, rules: AliasRules): void => { - if (hasReasoning(rules)) { - const { effort, budget_tokens, adaptive, summary } = rules.reasoning; +export const applyRulesToUpstreamAnthropicMessages = (body: T, rules: AliasRules): T => { + // `verbosity` has no native Anthropic Messages slot; drop silently. + const reasoning = reasoningOf(rules); + const thinking = anthropicMessagesThinking(body.thinking, reasoning); + const reasoned = { + ...body, // Anthropic stores explicit effort in `output_config.effort`; budget / // adaptive ride on `thinking.*`. Splitting them so both can be set in // the same overlay (effort fixed + budget pinned, e.g.) without one // erasing the other. - if (effort !== undefined) { - body.output_config = { ...body.output_config, effort }; - } - const display = summary !== undefined ? mapSummaryToAnthropicMessagesDisplay(summary) : undefined; - const displayPart = display !== undefined ? { display } : {}; - if (adaptive === true) { - // Adaptive auto-determines the budget; strip any client-set - // `budget_tokens` so the alias rule's mode isn't accompanied by a - // sibling budget the operator didn't ask for. - const { budget_tokens: _drop, ...priorThinking } = body.thinking ?? {}; - body.thinking = { ...priorThinking, type: 'adaptive', ...displayPart }; - } else if (budget_tokens !== undefined) { - body.thinking = { ...body.thinking, type: 'enabled', budget_tokens, ...displayPart }; - } else if (display !== undefined) { - // Anthropic discards `thinking.display` unless a mode is set; default - // to the enabled variant so the summary hint reaches the wire. - body.thinking = { ...body.thinking, type: 'enabled', ...displayPart }; - } - } - // `verbosity` has no native Anthropic Messages slot; drop silently. - if (rules.serviceTier !== undefined) { - // The cross-protocol bridge in translate maps `speed: 'fast'` ↔ the - // OpenAI accelerated lane; on a native Anthropic Messages target an alias - // rule naming that lane lands on `speed` so the upstream sees Fast Mode - // through its native field. Both OpenAI spellings count, since OpenAI - // itself accepts `priority` and `fast` interchangeably. Other tier values - // pass through on `service_tier` since Anthropic Messages's native request - // enum (`auto`/`standard_only`) doesn't model them. Whichever branch we - // take, clear the sibling field so the upstream never sees two tiers in - // conflict. - if (isFastServiceTier(rules.serviceTier)) { - body.speed = 'fast'; - delete body.service_tier; - } else { - body.service_tier = rules.serviceTier; - delete body.speed; - } + ...(reasoning.effort === undefined ? {} : { output_config: { ...body.output_config, effort: reasoning.effort } }), + ...(thinking === undefined ? {} : { thinking }), + } as T; + if (rules.serviceTier === undefined) return reasoned; + + // The cross-protocol bridge in translate maps `speed: 'fast'` ↔ + // `service_tier: 'fast'`; on a native Anthropic Messages target the alias rule + // `serviceTier: 'fast'` lands on `speed` so the upstream sees Fast Mode + // through its native field. Other tier values pass through on + // `service_tier` since Anthropic Messages's native enum (`auto`/`standard_only`) + // doesn't model them. Whichever branch we take, the sibling field is gone + // so the upstream never sees two tiers in conflict. + const { speed: _speed, service_tier: _serviceTier, ...tierless } = reasoned; + return (isFastServiceTier(rules.serviceTier) + ? { ...tierless, speed: 'fast' } + : { ...tierless, service_tier: rules.serviceTier }) as T; +}; + +/** The `thinking` block an overlay asks for, or `undefined` where it asks for none. */ +const anthropicMessagesThinking = ( + prior: AnthropicMessagesPayload['thinking'], + reasoning: Reasoning, +): AnthropicMessagesPayload['thinking'] | undefined => { + const { budget_tokens, adaptive, summary } = reasoning; + const display = summary === undefined ? undefined : mapSummaryToAnthropicMessagesDisplay(summary); + const displayPart = display === undefined ? {} : { display }; + + if (adaptive === true) { + // Adaptive auto-determines the budget; a client-set `budget_tokens` goes, so the alias + // rule's mode is not accompanied by a sibling budget the operator didn't ask for. + const { budget_tokens: _drop, ...priorThinking } = prior ?? {}; + return { ...priorThinking, type: 'adaptive', ...displayPart }; } + if (budget_tokens !== undefined) return { ...prior, type: 'enabled', budget_tokens, ...displayPart }; + // Anthropic discards `thinking.display` unless a mode is set; default + // to the enabled variant so the summary hint reaches the wire. + if (display !== undefined) return { ...prior, type: 'enabled', ...displayPart }; + return undefined; }; // Collapse OpenAI-style summary presets onto Anthropic's structured diff --git a/packages/gateway/src/data-plane/chat/shared/cache-buckets.ts b/packages/gateway/src/data-plane/chat/shared/cache-buckets.ts new file mode 100644 index 0000000000..23285fceeb --- /dev/null +++ b/packages/gateway/src/data-plane/chat/shared/cache-buckets.ts @@ -0,0 +1,68 @@ +import { readJsonNumber, asJsonObject, type JsonObject } from '../../../shared/json-helpers.ts'; +import { foldsExclusiveCacheTokens } from '../../shared/telemetry/usage.ts'; + +/** Where a protocol keeps the numbers the cache-bucket fold reads. The names are the whole of + * what a protocol contributes: the decision, the two contradictions it raises and the + * arithmetic are one rule, and it is written once below. */ +export interface CacheBucketNames { + readonly input: string; + readonly output: string; + readonly details: string; + /** Read in the order given, because an upstream projecting Anthropic's cache-write bucket + * onto an OpenAI-shaped usage block may spell it either way. */ + readonly cacheWrite: readonly string[]; +} + +/** + * Restores OpenAI's inclusive input-token contract on a usage block that reports the cache + * buckets alongside the input total instead of inside it. + * + * OpenAI states the subset relationship outright — "Cached tokens here are counted as a subset + * of input tokens, meaning input tokens will include cached and uncached tokens" + * (https://github.com/openai/openai-openapi/blob/d4fb706e6e05d4cc9f1b33ca59b6e4f3e8edd439/openapi.yaml#L51043-L51049) + * — and every downstream consumer here subtracts on that basis. Anthropic takes the opposite + * convention: `input_tokens` counts only what was neither read from nor written to the cache, + * and the three buckets sum to the real input + * (https://platform.claude.com/docs/en/docs/build-with-claude/prompt-caching). + * + * A gateway that fronts an Anthropic-shaped upstream and projects it into an OpenAI shape can + * carry the exclusive convention through to a wire that declares the inclusive one. Portkey + * does exactly that — it assigns Anthropic's `input_tokens` straight to `prompt_tokens` while + * summing `total_tokens` from all four buckets + * (https://github.com/Portkey-AI/gateway/blob/669825cbe89ee51569918b8f78a9db486fd69dd4/src/providers/anthropic/chatComplete.ts#L612-L627). + * Charm Hyper produces the same shape by subtracting the cached prefix out of the input total + * it received: for one observed kimi-k3 turn it reported `prompt_tokens 479, cached_tokens + * 13312, completion_tokens 373, total_tokens 14164`, where 479 + 13312 + 373 = 14164. + * + * `foldsExclusiveCacheTokens` owns the decision and the two contradictions that must not pass + * silently; the `usage-exclusive-cached-tokens` flag is its declaration input. `total_tokens` + * itself is left alone: under the exclusive convention it already counts the real input, so + * the rewritten input plus output is what it was equal to all along. A block with nothing to + * fold comes back by identity. + */ +export const withCacheBucketsFolded = ( + usage: JsonObject, + names: CacheBucketNames, + declaredExclusive: boolean, + identity: string, +): JsonObject => { + const inputTokens = readJsonNumber(usage[names.input]); + const outputTokens = readJsonNumber(usage[names.output]); + if (inputTokens == null || outputTokens == null) return usage; + + const details = asJsonObject(usage[names.details]); + const cacheRead = readJsonNumber(details?.cached_tokens) ?? 0; + const cacheWrite = names.cacheWrite.map(name => readJsonNumber(details?.[name])).find(value => value != null) ?? 0; + if (cacheRead === 0 && cacheWrite === 0) return usage; + + const folds = foldsExclusiveCacheTokens(declaredExclusive, { + inputTokens, + outputTokens, + totalTokens: readJsonNumber(usage.total_tokens) ?? undefined, + cacheRead, + cacheWrite, + }, identity); + if (!folds) return usage; + + return { ...usage, [names.input]: inputTokens + cacheRead + cacheWrite }; +}; diff --git a/packages/gateway/src/data-plane/chat/shared/client-stream.ts b/packages/gateway/src/data-plane/chat/shared/client-stream.ts new file mode 100644 index 0000000000..cbbeae07df --- /dev/null +++ b/packages/gateway/src/data-plane/chat/shared/client-stream.ts @@ -0,0 +1,72 @@ +import type { RunDump } from '../../../dump/run-sink.ts'; +import type { StreamOutcome } from '../../pipeline/serve.ts'; +import { recordStream, streamReferenceOf } from '@floway-dev/dump'; +import { defer, setRelease, type Owned, type Deferred } from '@floway-dev/pipeline'; +import type { ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; + +// Collectors stop at the protocol terminal. Keep that stop local to the fold so the same +// recorded cursor can still prove source exhaustion and publish its stream.end marker. +export const collectClientFrames = async ( + frames: AsyncIterable, + collect: (frames: AsyncIterable) => Promise, +): Promise => { + const cursor = frames[Symbol.asyncIterator](); + const result = await collect({ [Symbol.asyncIterator]: () => ({ next: () => cursor.next() }) }); + while (!(await cursor.next()).done) continue; + return result; +}; + +export const framedClientStream = >( + source: AsyncIterable, + render: (frame: Frame) => SseFrame, + errorFrames: (error: unknown) => readonly Frame[], + dump: RunDump | null, +): { + readonly frames: AsyncIterable; + readonly rendered: AsyncIterable; + readonly failed: Promise; + readonly release: () => Promise; +} => { + let settle!: (failed: boolean) => void; + const failed = new Promise(resolve => { settle = resolve; }); + let started = false; + const generator = (async function* () { + started = true; + let failed = false; + let completed = false; + try { + for await (const frame of source) yield { frame, wire: render(frame) }; + completed = true; + } catch (error) { + failed = true; + dump?.failed(error); + for (const frame of errorFrames(error)) yield { frame, wire: render(frame) }; + } finally { + settle(failed || !completed); + } + })(); + // A protocol frame is published only after it has a valid transport representation. + // The pair keeps serialization errors inside completion without formatting a second time. + const packets = recordStream({ [Symbol.asyncIterator]: () => generator }, dump, packet => packet.frame); + const reference = streamReferenceOf(packets); + return { + frames: { ...reference, [Symbol.asyncIterator]: () => (async function* () { for await (const packet of packets) yield packet.frame; })() }, + rendered: { ...reference, [Symbol.asyncIterator]: () => (async function* () { for await (const packet of packets) yield packet.wire; })() }, + failed, + release: async () => { + if (!started) settle(true); + await generator.return(); + }, + }; +}; + +export const withClientVerdict = (reading: Deferred | null, failed: Promise): Deferred | null => + reading === null ? null : defer(Promise.all([reading, failed]).then(([outcome, clientFailed]) => ({ ...outcome, failed: outcome.failed || clientFailed }))); + +export const bindClientRelease = (facts: object, release: () => Promise): void => { + if (!('response.http.body' in facts) || facts['response.http.body'] === null) return; + const body = facts['response.http.body'] as ReadableStream & Owned; + const drain = setRelease(body, async () => { + try { await drain(); } finally { await release(); } + }); +}; diff --git a/packages/gateway/src/data-plane/chat/shared/errors.ts b/packages/gateway/src/data-plane/chat/shared/errors.ts index 51629a9f89..c73f78ac9e 100644 --- a/packages/gateway/src/data-plane/chat/shared/errors.ts +++ b/packages/gateway/src/data-plane/chat/shared/errors.ts @@ -1,14 +1,14 @@ -import type { ApiErrorResult, PerformanceTelemetryContext } from '@floway-dev/provider'; +// Failures a chat protocol makes before it reaches an upstream, as a throw the chain that can +// answer them catches. Unexpected throws bubble as-is. +// +// Almost every refusal is a value on the record — narrowing states its own, and the edge +// renders it. What is left here is the one shape that cannot be: a fault found several frames +// inside a walk, where the answer is written by the stage that started the walk. -// Failures a chat protocol can render before reaching an upstream; unexpected -// throws bubble as-is. `failedUpstreams` on model-{missing,unsupported} -// carries upstream names with a recorded catalog-refresh failure. The error -// may predate this resolution and may accompany a usable last-known-good -// catalog; empty means no consulted snapshot records such a failure. -export type ChatServeFailure = - | { readonly kind: 'model-missing'; readonly model: string; readonly failedUpstreams: readonly string[] } - | { readonly kind: 'model-unsupported'; readonly model: string; readonly failedUpstreams: readonly string[] } - | { readonly kind: 'routing-unavailable'; readonly message: string }; +/** `routing-unavailable` is what affinity says when a turn carries state for an upstream no + * candidate can be routed to. Its message is the caller's to read: only the selection knows + * which of the carried targets went missing. */ +export type ChatServeFailure = { readonly kind: 'routing-unavailable'; readonly message: string }; class ChatServeFailureError extends Error { readonly failure: TFailure; @@ -25,34 +25,3 @@ export const throwChatServeFailure = (error: unknown): TFailure | null => error instanceof ChatServeFailureError ? error.failure as TFailure : null; - -export const openAiErrorResult = ( - status: number, - message: string, - extra?: { readonly param: string; readonly code: string | null }, - performance?: PerformanceTelemetryContext, -): ApiErrorResult => ({ - type: 'api-error', - source: 'gateway', - status, - headers: new Headers({ 'content-type': 'application/json' }), - body: new TextEncoder().encode(JSON.stringify({ - error: { message, type: 'invalid_request_error', ...extra }, - })), - ...(performance ? { performance } : {}), -}) satisfies ApiErrorResult; - -// Builds the failure value every serve dispatches with after `canServe` has -// dropped every candidate: `sawModel=true` means the inbound id exists in -// some upstream's catalog but no upstream wire reaches it for this protocol, -// rendered as 400 model-unsupported; `sawModel=false` means no upstream knows -// the id at all, rendered as 404 model-missing. The per-protocol failure -// renderer turns the value into its own envelope shape. -export const noViableCandidateFailure = ( - sawModel: boolean, - model: string, - failedUpstreams: readonly string[], -): ChatServeFailure => - sawModel - ? { kind: 'model-unsupported', model, failedUpstreams } - : { kind: 'model-missing', model, failedUpstreams }; diff --git a/packages/gateway/src/data-plane/chat/shared/frames.ts b/packages/gateway/src/data-plane/chat/shared/frames.ts new file mode 100644 index 0000000000..e543e606ad --- /dev/null +++ b/packages/gateway/src/data-plane/chat/shared/frames.ts @@ -0,0 +1,33 @@ +import { isFailure } from '../../pipeline/facts.ts'; +import type { ChatAnswer } from '../facts.ts'; +import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; + +/** The response half of every rule here that rewrites a stream: a refusal and a collected + * body have no frames to rewrite, and telling the three apart is reading a value rather than + * dispatching on a declaration. `null` says there was nothing to do, so the caller hands the + * record it was given straight on. */ +export const answerWithFrames = ( + answer: ChatAnswer, + rewrite: (frames: AsyncIterable>) => AsyncGenerator>, +): ChatAnswer | null => { + if (isFailure(answer) || answer.kind !== 'stream') return null; + const frames = answer.frames as AsyncIterable>; + return { kind: 'stream' as const, frames: { [Symbol.asyncIterator]: () => rewrite(frames) } }; +}; + +/** A stream with each event rewritten one for one. The frame comes back by identity when the + * rewrite changed nothing, which is the same conditional-write convention the request-side + * rules follow — and a transport frame is not an event, so it rides through untouched. */ +export const rewritingEvents = async function* ( + frames: AsyncIterable>, + rewrite: (event: Event) => Event, +): AsyncGenerator> { + for await (const frame of frames) { + if (frame.type !== 'event') { + yield frame; + continue; + } + const event = rewrite(frame.event); + yield event === frame.event ? frame : eventFrame(event); + } +}; diff --git a/packages/gateway/src/data-plane/chat/shared/gateway-ctx.ts b/packages/gateway/src/data-plane/chat/shared/gateway-ctx.ts index 38313d6183..5bef9d8ffc 100644 --- a/packages/gateway/src/data-plane/chat/shared/gateway-ctx.ts +++ b/packages/gateway/src/data-plane/chat/shared/gateway-ctx.ts @@ -7,11 +7,10 @@ import type { OpenAIResponsesStatefulStore } from '../openai-responses/items/sto // Chat-protocol ctx adds the affinity membrane and the OpenAI Responses item store. // The store is present on every chat ctx: native OpenAI Responses entries supply a // persisting factory, non-OpenAI-Responses sources a no-backing scratchpad store, so -// the server-tool shim's request-private state always has a home. Every chat +// the hosted-tool shim's request-private state always has a home. Every chat // HTTP/WS entry constructs this via `createChatGatewayCtxFromHono` and threads -// it through serve → narrow → attempt. Passthrough endpoints (OpenAI -// Embeddings / OpenAI Images / OpenAI Audio Transcriptions / OpenAI -// Completions) have no stored-items concept and stay on plain `GatewayCtx`. +// it through candidate selection and protocol stages. Non-chat endpoints +// have no stored-items concept and use `GatewayCtx`. export interface ChatGatewayCtx extends GatewayCtx { readonly affinity: AffinityRequestContext; readonly store: OpenAIResponsesStatefulStore; diff --git a/packages/gateway/src/data-plane/chat/shared/normalize-empty-tools.ts b/packages/gateway/src/data-plane/chat/shared/normalize-empty-tools.ts new file mode 100644 index 0000000000..d85088abdb --- /dev/null +++ b/packages/gateway/src/data-plane/chat/shared/normalize-empty-tools.ts @@ -0,0 +1,21 @@ +import type { ChatFacts, Chat } from '../facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; + +type EmptyToolsProtocol = 'request.chat.openaiChatCompletions' | 'request.chat.openaiResponses' | 'request.chat.anthropicMessages'; + +export const normalizeEmptyTools = (key: Key, rewrite: (payload: ChatFacts[Key]) => ChatFacts[Key]) => defineStage< + Chat, Chat, Record, Record +>({ + name: 'normalizeEmptyToolsToolChoice', + through: { + request: { needs: [key, 'route.attempt'], consumes: [], provides: [key] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + if (!facts['route.attempt'].flags.includes('empty-tools-tool-choice-none')) return await next(facts); + const payload = facts[key]; + if (!Array.isArray(payload.tools) || payload.tools.length !== 0) return await next(facts); + const rewritten = rewrite(payload); + return await next(rewritten === payload ? facts : { ...facts, [key]: move(rewritten) } as never); + }, +}); diff --git a/packages/gateway/src/data-plane/chat/shared/provider-answer.ts b/packages/gateway/src/data-plane/chat/shared/provider-answer.ts new file mode 100644 index 0000000000..493fe16819 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/shared/provider-answer.ts @@ -0,0 +1,28 @@ +import { providerCalls } from '../../pipeline/provider-usage.ts'; +import { dialFailure, readUpstreamBody, spentBody } from '../../pipeline/upstream-body.ts'; +import type { ChatAnswer } from '../facts.ts'; +import { exchangeResponse } from '@floway-dev/http/pipeline'; +import type { Facts } from '@floway-dev/pipeline'; +import type { ChatProviderOperation, ModelCandidate, ProviderChatResponse } from '@floway-dev/provider'; + +export const providerChatAnswer = async (candidate: ModelCandidate, back: ProviderChatResponse) => { + const { 'response.http.exchange': exchange, 'response.provider.output': output } = back; + const billable = providerCalls(candidate, back as unknown as Facts); + if (exchange.type === 'transportFailure') return { + answer: dialFailure(exchange.error), status: 502, headers: [], billable, body: null, + }; + if (output !== null) return { + answer: output as ChatAnswer, + status: 'kind' in output ? exchange.status : output.status, + headers: exchange.headers, + billable, + body: 'kind' in output && output.kind === 'stream' ? exchange.body : spentBody(exchange.body), + }; + const response = exchangeResponse(exchange); + spentBody(exchange.body); + const body = await readUpstreamBody(response); + return { + answer: { status: exchange.status, message: body.text, ...('json' in body ? { body: body.json } : {}) }, + status: exchange.status, headers: exchange.headers, billable, body: exchange.body, + }; +}; diff --git a/packages/gateway/src/data-plane/chat/shared/provider-stream-result.ts b/packages/gateway/src/data-plane/chat/shared/provider-stream-result.ts deleted file mode 100644 index 859f208344..0000000000 --- a/packages/gateway/src/data-plane/chat/shared/provider-stream-result.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { isFirstOutputTokenFrame } from './first-output-token.ts'; -import type { GatewayCtx } from '../../shared/gateway-ctx.ts'; -import { telemetryModelIdentity, upstreamPerformanceContext } from '../../shared/telemetry/attribution.ts'; -import type { BillableUsage, ProtocolFrame } from '@floway-dev/protocols/common'; -import { eventResult, readUpstreamApiError, type ChatTargetApi, type EventResultMetadata, type ExecuteResult, type ModelCandidate, type ProviderStreamResult } from '@floway-dev/provider'; - -export const providerStreamResultToExecuteResult = async ( - providerResult: ProviderStreamResult, - candidate: ModelCandidate, - targetApi: ChatTargetApi, - ctx: GatewayCtx, - // Reads the upstream's own usage off one of its events, in the upstream's - // own protocol. This is the only place pricing figures are produced; nothing - // downstream re-derives them from the translated result the client receives. - readBillableUsage: (event: TEvent) => BillableUsage | null, -): Promise>> => { - const context = upstreamPerformanceContext(ctx, candidate, 'chat'); - if (!providerResult.ok) { - return { ...(await readUpstreamApiError(providerResult.response, candidate.provider.upstreamId)), performance: context }; - } - const identity = telemetryModelIdentity(candidate, providerResult.modelKey); - let resolveFinal!: (metadata: EventResultMetadata) => void; - const finalMetadata = new Promise(resolve => { resolveFinal = resolve; }); - // Only a report carrying real counts replaces the running figure, so a - // trailing empty usage frame cannot wipe a good one. Held outside the - // generator so an abandoned stream can still settle with what it saw. - let billableUsage: BillableUsage | undefined; - const settleMetadata = (): void => resolveFinal({ - modelIdentity: identity, - ...(context !== undefined ? { performance: context } : {}), - ...(billableUsage !== undefined ? { billableUsage } : {}), - }); - // Every streaming response now resolves its cost here, and the respond - // layer awaits it in a `finally`. A transport that walks away without - // closing the generator would otherwise hang that await forever, so the - // abort settles it too; whichever fires first wins, and the later call is a - // no-op. - ctx.abortSignal?.addEventListener('abort', settleMetadata, { once: true }); - const stampedEvents = (async function* () { - try { - for await (const frame of providerResult.events) { - if (ctx.attempt.timing.firstOutputTokenAt === null && isFirstOutputTokenFrame(frame, targetApi)) { - ctx.attempt.timing.firstOutputTokenAt = performance.now(); - } - if (frame.type === 'event') { - const reported = readBillableUsage(frame.event); - if (reported !== null) billableUsage = reported; - } - yield frame; - } - } finally { - settleMetadata(); - } - })(); - return { - ...eventResult(stampedEvents, identity, { performance: context, headers: providerResult.headers }), - finalMetadata, - }; -}; diff --git a/packages/gateway/src/data-plane/chat/shared/respond.ts b/packages/gateway/src/data-plane/chat/shared/respond.ts deleted file mode 100644 index 284010379b..0000000000 --- a/packages/gateway/src/data-plane/chat/shared/respond.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { StreamCompletion } from '../../shared/sse.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { plainResult } from '@floway-dev/provider'; -import type { EventResultMetadata, ExecuteResult, PlainResult } from '@floway-dev/provider'; - -// Emits a measurement endpoint's already-shaped body verbatim. The endpoint's -// `attempt` owns all shaping — the success body and any source-specific error -// envelope — so every source's `respond` renders a plain result identically. -export const plainResultToResponse = (result: PlainResult): Response => - new Response(result.body.slice().buffer, { status: result.status, headers: result.headers }); - -// Used by count_tokens endpoints that either pass through the upstream body -// or wrap an already-built error/success Response. -export const plainResultFromResponse = async (response: Response, upstream?: string): Promise => - plainResult( - response.status, - new Headers({ 'content-type': response.headers.get('content-type') ?? 'application/json' }), - new Uint8Array(await response.arrayBuffer()), - upstream, - ); - -// Per-stream observation accumulated by each source's frame observer and read -// back when the response settles: did the stream fail, did it reach its -// terminal frame, and the last frame-level usage worth billing. -export class SourceStreamState { - failed = false; - completed = false; - - failedAfter(completion: StreamCompletion): boolean { - return completion === 'error' || this.failed || (completion === 'cancel' && !this.completed); - } -} - -export const eventResultMetadata = async (result: Extract>, { type: 'events' }>): Promise => - await (result.finalMetadata ?? { - modelIdentity: result.modelIdentity, - ...(result.performance ? { performance: result.performance } : {}), - }); diff --git a/packages/gateway/src/data-plane/chat/shared/role-compatibility.ts b/packages/gateway/src/data-plane/chat/shared/role-compatibility.ts new file mode 100644 index 0000000000..d95eb422d0 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/shared/role-compatibility.ts @@ -0,0 +1,44 @@ + +export interface RoleRewrite { + readonly systemToDeveloper: boolean; + readonly developerToSystem: boolean; + readonly midConversationSystemToUser: boolean; +} + +/** A flag is data on an upstream model, and this is a stage reading it. There is no + * category of stage a flag switches on or off. */ +export const rolesFor = (flags: readonly string[]): RoleRewrite | null => { + const rewrite: RoleRewrite = { + systemToDeveloper: flags.includes('rewrite-system-to-developer'), + developerToSystem: flags.includes('rewrite-developer-to-system'), + midConversationSystemToUser: flags.includes('rewrite-mid-conv-system-to-user'), + }; + return rewrite.systemToDeveloper || rewrite.developerToSystem || rewrite.midConversationSystemToUser + ? rewrite + : null; +}; + +/** The three rewrites in their settled order, and the state that makes "mid-conversation" + * mean what it says: the last step affects only a system message that appears after the + * leading run, which is why the fold carries a flag rather than testing the index. + * + * Written once, over roles alone. A protocol walks its own items, hands in the role it read + * off one and takes back the role to write, so the ordering exists in one place however + * differently the two protocols that use it shape a turn. An item with no role at all — + * an OpenAI Responses reasoning item between two system messages — still crosses the leading run, + * which is what makes the system message after it mid-conversation. */ +export const roleRewriter = (rewrite: RoleRewrite) => { + let crossedLeadingSystemRun = false; + return (role: Role): Role | RewrittenRole => { + let result: Role | RewrittenRole = role; + if (rewrite.systemToDeveloper && result === 'system') result = 'developer'; + if (rewrite.developerToSystem && result === 'developer') result = 'system'; + if (!crossedLeadingSystemRun && result !== 'system') crossedLeadingSystemRun = true; + if (rewrite.midConversationSystemToUser && crossedLeadingSystemRun && result === 'system') result = 'user'; + return result; + }; +}; + +/** The only roles the fold ever writes. Every protocol it runs on admits all three, so a + * rewritten role needs no cast to go back onto the item it came from. */ +type RewrittenRole = 'system' | 'developer' | 'user'; diff --git a/packages/gateway/src/data-plane/chat/shared/source-stream-state.ts b/packages/gateway/src/data-plane/chat/shared/source-stream-state.ts new file mode 100644 index 0000000000..203035dec9 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/shared/source-stream-state.ts @@ -0,0 +1,16 @@ +// What a source's stream turned out to be, read back when the turn settles. +// +// One transport still needs it: the OpenAI Responses WebSocket writes each frame itself and has to +// decide, when the socket closes, whether what it wrote was a finished turn. Every HTTP +// family answers the same question from its own metered reading instead. + +import type { StreamCompletion } from '../../shared/sse.ts'; + +export class SourceStreamState { + failed = false; + completed = false; + + failedAfter(completion: StreamCompletion): boolean { + return completion === 'error' || this.failed || (completion === 'cancel' && !this.completed); + } +} diff --git a/packages/gateway/src/data-plane/chat/shared/structural.ts b/packages/gateway/src/data-plane/chat/shared/structural.ts new file mode 100644 index 0000000000..d05933ba94 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/shared/structural.ts @@ -0,0 +1,22 @@ + +/** Removing a field inside a value is the same move as removing a fact, and a value that + * carried none of them comes back by identity so the rule costs nothing where it does not + * apply. */ +export const withoutKeys = (value: T, keys: readonly (keyof T)[]): T => { + if (!keys.some(key => key in value)) return value; + const removed = new Set(keys as readonly string[]); + return Object.fromEntries(Object.entries(value).filter(([key]) => !removed.has(key))) as T; +}; + +/** `map`, but the array comes back by identity when every element did. Written once because + * every rule here walks a list of something — messages, input items, content parts — and + * the identity is the whole reason a layer costs only what it actually changed. */ +export const mapKeepingIdentity = (items: T[], rewrite: (item: T) => T): T[] => { + let changed = false; + const mapped = items.map(item => { + const next = rewrite(item); + if (next !== item) changed = true; + return next; + }); + return changed ? mapped : items; +}; diff --git a/packages/gateway/src/data-plane/chat/shared/translate-traverse.ts b/packages/gateway/src/data-plane/chat/shared/translate-traverse.ts deleted file mode 100644 index a8691ae279..0000000000 --- a/packages/gateway/src/data-plane/chat/shared/translate-traverse.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { DumpAccumulator } from '../../../dump/accumulator.ts'; -import type { TurnDump } from '../../../dump/turn-dump.ts'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ChatTargetApi, ExecuteResult } from '@floway-dev/provider'; -import type { TranslateTripResult } from '@floway-dev/translate'; - -// Protocol-agnostic capture hook for the pre-translation (target-protocol) -// upstream view. `traverseTranslation` invokes it as it relays the inner -// attempt's frames/bytes through `trip.events`; the call site constructs one -// from its `(ctx.dump, targetApi)` via `captureFromDump`. The target protocol -// is statically known at every call site (the enclosing `if`/`case` branch), -// so the dashboard can render the captured frames with the right per-protocol -// serializer without re-deriving it from the client path. -export interface TranslationUpstreamCapture { - upstreamFrame(frame: ProtocolFrame): void; - upstreamApiError(error: { status: number; headers: Headers; body: Uint8Array }): void; -} - -// `dump` is null when the api key opts out of dumps; returns `undefined` so -// `traverseTranslation` skips all upstream capture work (no tee, no api-error -// copy) — zero per-request cost on that path. `targetApi` is stamped eagerly so -// `meta.targetApi` is set even when the upstream stream produces zero frames. -export const captureFromDump = ( - dump: TurnDump | null, - targetApi: ChatTargetApi, -): TranslationUpstreamCapture | undefined => { - if (dump === null) return undefined; - if (!(dump instanceof DumpAccumulator)) throw new Error('Legacy chat translation requires an edge dump recording'); - dump.setUpstreamTargetApi(targetApi); - const upstreamFrame = (frame: ProtocolFrame) => dump.upstreamFrame(frame); - const upstreamApiError = (error: { status: number; headers: Headers; body: Uint8Array }) => dump.upstreamApiError(error); - return { upstreamFrame, upstreamApiError }; -}; - -// Threads a translate trip around an inner attempt. The trip itself is async -// (the real `@floway-dev/translate` pair functions resolve a `Promise`), so -// `translate` returns the trip object behind a promise. The pair functions -// take `(src, ctx)`; this helper's `translate` parameter stays unary so each -// caller closes over its own `ctx` (`p => translateXViaY(p, ctx)`). -// -// On an upstream api-error the trip's optional `apiError` hook is invoked so -// the pair can rewrite the body into the source protocol's envelope — the -// canonical case is `anthropic-messages-via-*` translating an upstream context-window -// error into the Anthropic `prompt is too long:` shape Claude Code recognizes -// for auto-compaction. Pairs that don't set `apiError` (or return `undefined`) -// pass the upstream body through verbatim. -// -// When `capture` is provided, the ORIGINAL target-protocol frames (events -// path) or the verbatim upstream api-error envelope (api-error path) are tee'd -// to the dump BEFORE translation/rewrite, so the dump records what the upstream -// actually sent independent of what Floway translated it into. -export const traverseTranslation = async ( - payload: SP, - translate: (p: SP) => Promise>, - innerAttempt: (translated: TP) => Promise>>, - capture?: TranslationUpstreamCapture, -): Promise>> => { - const trip = await translate(payload); - const inner = await innerAttempt(trip.target); - if (inner.type === 'events') { - const events = capture === undefined - ? inner.events - : teeUpstreamFrames(inner.events, capture.upstreamFrame); - return { ...inner, events: trip.events(events) }; - } - // Capture the ORIGINAL upstream api-error envelope before the optional - // `trip.apiError` rewrite. Fires for EVERY upstream api-error — even when no - // rewrite happens (trip.apiError undefined or returns undefined), because the - // verbatim target-protocol body is still the pre-translation view operators - // need to diagnose upstream behavior. - if (inner.type === 'api-error' && inner.source === 'upstream') { - capture?.upstreamApiError({ status: inner.status, headers: inner.headers, body: inner.body }); - if (trip.apiError !== undefined) { - const rewritten = trip.apiError({ status: inner.status, headers: inner.headers, body: inner.body }); - if (rewritten !== undefined) return { ...inner, status: rewritten.status, headers: rewritten.headers, body: rewritten.body }; - } - } - return inner; -}; - -// Tee that pushes each original target-protocol frame to the dump BEFORE -// handing it to `trip.events`. Lockstep with the downstream respond layer's -// iteration: as the respond layer pulls `result.events`, the tee pulls -// `inner.events`, captures, and yields; by the time `finalize`→`write` runs, -// the upstream frame log is complete. No buffering — one array push per frame. -const teeUpstreamFrames = async function* ( - events: AsyncIterable>, - capture: (frame: ProtocolFrame) => void, -): AsyncGenerator> { - for await (const frame of events) { - capture(frame); - yield frame; - } -}; diff --git a/packages/gateway/src/data-plane/chat/shared/upstream-models-error.ts b/packages/gateway/src/data-plane/chat/shared/upstream-models-error.ts deleted file mode 100644 index a4de64406e..0000000000 --- a/packages/gateway/src/data-plane/chat/shared/upstream-models-error.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { ProviderModelsUnavailableError } from '@floway-dev/provider'; - -// A custom upstream's `/models` endpoint failing is the user's first signal -// that their upstream is misconfigured — relay the verbatim status/body so -// the caller sees the upstream's diagnostic, not a generic gateway envelope. -// Network-level failures (no httpResponse) return null; callers fall through -// to their own internal-error rendering for those because there is no -// upstream body to relay. -export const providerModelsUnavailableResponse = (error: unknown): Response | null => { - if (!(error instanceof ProviderModelsUnavailableError) || !error.httpResponse) return null; - const { status, headers, body } = error.httpResponse; - return new Response(body, { status, headers }); -}; diff --git a/packages/gateway/src/data-plane/chat/shared/usage.ts b/packages/gateway/src/data-plane/chat/shared/usage.ts new file mode 100644 index 0000000000..003ded5773 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/shared/usage.ts @@ -0,0 +1,24 @@ +import type { TokenUsage } from '../../../repo/types.ts'; +import { tokenUsageQuantities } from '../../../repo/usage-metrics.ts'; +import { tokenUsage } from '../../shared/telemetry/usage.ts'; +import type { UsageMeasurement } from '../../shared/telemetry/usage.ts'; +import type { BillableUsage } from '@floway-dev/protocols/common'; + +export type ChatBillableUsage = Omit & { readonly input?: number; readonly output?: number }; + +export const chatUsageMeasurement = (usage: ChatBillableUsage): UsageMeasurement => { + const tokens: TokenUsage = { + ...tokenUsage({ input_cache_read: usage.cacheRead, input_cache_write: usage.cacheWrite, input_cache_write_1h: usage.cacheWrite1h }), + ...(usage.input === undefined ? {} : { input: usage.input }), + ...(usage.output === undefined ? {} : { output: usage.output }), + ...(usage.tier === undefined ? {} : { tier: usage.tier }), + }; + return { + quantities: tokenUsageQuantities(tokens), + pricingFacts: { + ...(usage.tier === undefined ? {} : { serviceTier: usage.tier }), + ...(usage.input === undefined ? {} : { inputTokens: usage.input + usage.cacheRead + usage.cacheWrite + usage.cacheWrite1h }), + }, + dumpTokenUsage: tokens, + }; +}; diff --git a/packages/gateway/src/data-plane/chat/stage-contracts.ts b/packages/gateway/src/data-plane/chat/stage-contracts.ts new file mode 100644 index 0000000000..1cd97e00d2 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/stage-contracts.ts @@ -0,0 +1,3 @@ +import type { GatewayFacts } from '../pipeline/facts.ts'; + +export type Slice = { [P in K]: GatewayFacts[P] }; diff --git a/packages/gateway/src/data-plane/chat/wire.ts b/packages/gateway/src/data-plane/chat/wire.ts new file mode 100644 index 0000000000..d7e0b125c8 --- /dev/null +++ b/packages/gateway/src/data-plane/chat/wire.ts @@ -0,0 +1,10 @@ +import type { ChatFacts } from './facts.ts'; +import type { Pipeline } from '@floway-dev/pipeline'; + +export type RequestKey = Extract; + +/** One wire, as the chain that dials it. The keys are erased at this seam for the same + * reason they are erased on `Stage`: what a wire reads and hands up is checked by assembly + * and by the runner against declarations, and a family's slice type is nobody's business + * in between. */ +export type ChatWire = Pipeline, Record>; diff --git a/packages/gateway/src/data-plane/models/gemini.ts b/packages/gateway/src/data-plane/models/gemini.ts index e991aae5ab..535b601863 100644 --- a/packages/gateway/src/data-plane/models/gemini.ts +++ b/packages/gateway/src/data-plane/models/gemini.ts @@ -6,7 +6,7 @@ import { getRepo } from '../../repo/index.ts'; import type { ModelAliasesRepo } from '../../repo/types.ts'; import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; -import { geminiGenerateContentStatusForHttpStatus } from '../chat/gemini-generate-content/errors.ts'; +import { geminiGenerateContentErrorResponse } from '../chat/gemini-generate-content/errors.ts'; import { enumerateAddressableModelIds, listedRealModels } from '../shared/listing/addressable.ts'; import { mergeAliasesIntoModels } from '../shared/listing/alias.ts'; import type { ModelPricing } from '@floway-dev/protocols/common'; @@ -49,14 +49,8 @@ const toGeminiModel = (model: InternalModel): GeminiModel => { }; }; -const geminiError = (status: number, message: string): Response => - Response.json( - { error: { code: status, message, status: geminiGenerateContentStatusForHttpStatus(status) } }, - { status: status as 400 | 404 | 500 | 502 }, - ); - const geminiModelLoadError = (error: unknown): Response => - geminiError(502, error instanceof Error ? error.message : String(error)); + geminiGenerateContentErrorResponse(502, error instanceof Error ? error.message : String(error)); // Real chat models plus chat-kind alias entries; collision and dedupe ride // on the shared `mergeAliasesIntoModels` helper so /v1beta/models stays in @@ -99,13 +93,13 @@ export const serveGeminiModels = async (c: Context): Promise => { export const serveGeminiModelInfo = async (c: Context): Promise => { const rawModelId = c.req.param('modelId'); - if (!rawModelId) return geminiError(404, 'Model not found: '); + if (!rawModelId) return geminiGenerateContentErrorResponse(404, 'Model not found: '); const modelId = rawModelId.replace(/^models\//, ''); try { const scheduleRefresh = createModelsRefreshScheduler(getRuntimeLocation(c.req.raw), backgroundSchedulerFromContext(c)); const model = (await loadGeminiModels(effectiveUpstreamIdsFromContext(c), scheduleRefresh, getRepo().modelAliases)).find(candidate => candidate.baseModelId === modelId || candidate.name === `models/${modelId}`); - if (!model) return geminiError(404, `Model not found: ${modelId}`); + if (!model) return geminiGenerateContentErrorResponse(404, `Model not found: ${modelId}`); return Response.json(model); } catch (error) { return geminiModelLoadError(error); diff --git a/packages/gateway/src/data-plane/openai-audio/emit.ts b/packages/gateway/src/data-plane/openai-audio/emit.ts index 6f7f3b1fe4..7990c9404d 100644 --- a/packages/gateway/src/data-plane/openai-audio/emit.ts +++ b/packages/gateway/src/data-plane/openai-audio/emit.ts @@ -1,8 +1,7 @@ import type { OpenAIAudioTranscriptionEvents, Fields } from './facts.ts'; -import { isFailure } from '../pipeline/facts.ts'; +import { isFailure, mintedErrorEnvelope, renderFailure } from '../pipeline/facts.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import { defineStage, move } from '@floway-dev/pipeline'; -import { renderErrorEnvelope } from '@floway-dev/protocols/common'; import { renderOpenAIAudioTranscription, type CanonicalOpenAIAudioTranscription } from '@floway-dev/protocols/openai-audio'; const isEvents = (answer: CanonicalOpenAIAudioTranscription | OpenAIAudioTranscriptionEvents): answer is OpenAIAudioTranscriptionEvents => @@ -54,7 +53,7 @@ export const emitOpenAIAudioTranscription = defineStage< 'response.http.headers': forClient, 'response.http.status': answer.status, 'response.openaiAudioTranscription.mediaType': 'application/json', - 'response.openaiAudioTranscription.rendered': move(renderErrorEnvelope(answer.message, answer.body)), + 'response.openaiAudioTranscription.rendered': move(renderFailure(answer, mintedErrorEnvelope).body), }; } return { diff --git a/packages/gateway/src/data-plane/openai-completions/emit.ts b/packages/gateway/src/data-plane/openai-completions/emit.ts index 224625d53e..ed8232eb85 100644 --- a/packages/gateway/src/data-plane/openai-completions/emit.ts +++ b/packages/gateway/src/data-plane/openai-completions/emit.ts @@ -1,11 +1,11 @@ import type { OpenAICompletionsFacts, OpenAICompletionsFrames, Fields } from './facts.ts'; import type { RunDump } from '../../dump/run-sink.ts'; -import { isFailure } from '../pipeline/facts.ts'; +import { isFailure, mintedErrorEnvelope, renderFailure } from '../pipeline/facts.ts'; import type { GatewayServices } from '../pipeline/services.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import { recordStream, streamReferenceOf } from '@floway-dev/dump'; import { defineStage, move } from '@floway-dev/pipeline'; -import { renderErrorEnvelope, isOpenAIUsageOnlyEventShape, type SseFrame } from '@floway-dev/protocols/common'; +import { isOpenAIUsageOnlyEventShape, type SseFrame } from '@floway-dev/protocols/common'; import { openaiCompletionsProtocolFrameToSSEFrame } from '@floway-dev/protocols/openai-completions'; const isFrames = (answer: OpenAICompletionsFacts['response.openaiCompletions.payload']): answer is OpenAICompletionsFrames => @@ -68,7 +68,7 @@ const rendered = ( wantsUsageChunk: boolean, dump: RunDump | null, ): OpenAICompletionsFacts['response.openaiCompletions.rendered'] => - isFailure(answer) ? renderErrorEnvelope(answer.message, answer.body) + isFailure(answer) ? renderFailure(answer, mintedErrorEnvelope).body : isFrames(answer) ? renderSSE(wantsUsageChunk ? answer : recordStream(withoutUsage(answer), dump)) : answer; diff --git a/packages/gateway/src/data-plane/openai-embeddings/emit.ts b/packages/gateway/src/data-plane/openai-embeddings/emit.ts index 1b0ce7a199..915f055612 100644 --- a/packages/gateway/src/data-plane/openai-embeddings/emit.ts +++ b/packages/gateway/src/data-plane/openai-embeddings/emit.ts @@ -1,8 +1,7 @@ import type { Fields } from './facts.ts'; -import { isFailure } from '../pipeline/facts.ts'; +import { isFailure, mintedErrorEnvelope, renderFailure } from '../pipeline/facts.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import { defineStage, move } from '@floway-dev/pipeline'; -import { renderErrorEnvelope } from '@floway-dev/protocols/common'; import { renderOpenAIEmbeddingsResponse } from '@floway-dev/protocols/openai-embeddings'; /** @@ -42,7 +41,7 @@ export const emitOpenAIEmbeddings = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.openaiEmbeddings.rendered': move(renderErrorEnvelope(answer.message, answer.body)), + 'response.openaiEmbeddings.rendered': move(renderFailure(answer, mintedErrorEnvelope).body), // The upstream's own status, or the gateway's own when it refused before dialling. // A client is not owed the upstream's exact bytes; it is owed the truth about what // happened, and a 429 arriving as a 200 is not that. diff --git a/packages/gateway/src/data-plane/openai-embeddings/usage.ts b/packages/gateway/src/data-plane/openai-embeddings/usage.ts deleted file mode 100644 index 78ec49ccdc..0000000000 --- a/packages/gateway/src/data-plane/openai-embeddings/usage.ts +++ /dev/null @@ -1,10 +0,0 @@ -import type { TokenUsage } from '../../repo/types.ts'; -import { tokenUsage } from '../shared/telemetry/usage.ts'; - -export const tokenUsageFromOpenAIEmbeddingsBody = (body: unknown): TokenUsage | null => { - if (!body || typeof body !== 'object') return null; - const { usage } = body as { usage?: unknown }; - if (!usage || typeof usage !== 'object') return null; - const promptTokens = (usage as { prompt_tokens?: unknown }).prompt_tokens; - return typeof promptTokens === 'number' ? tokenUsage({ input: promptTokens }) : null; -}; diff --git a/packages/gateway/src/data-plane/openai-images/emit.ts b/packages/gateway/src/data-plane/openai-images/emit.ts index cb910c3af0..32d7a4044f 100644 --- a/packages/gateway/src/data-plane/openai-images/emit.ts +++ b/packages/gateway/src/data-plane/openai-images/emit.ts @@ -1,9 +1,9 @@ import type { OpenAIImagesFrames, Fields, OpenAIImagesFacts } from './facts.ts'; -import { isFailure } from '../pipeline/facts.ts'; +import { isFailure, mintedErrorEnvelope, renderFailure } from '../pipeline/facts.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import { streamReferenceOf } from '@floway-dev/dump'; import { defineStage, move } from '@floway-dev/pipeline'; -import { renderErrorEnvelope, type SseFrame } from '@floway-dev/protocols/common'; +import { type SseFrame } from '@floway-dev/protocols/common'; import { renderOpenAIImagesResponse, openaiImagesStreamEventToSSEFrame, type CanonicalOpenAIImagesResponse } from '@floway-dev/protocols/openai-images'; const isFrames = (answer: CanonicalOpenAIImagesResponse | OpenAIImagesFrames): answer is OpenAIImagesFrames => @@ -51,7 +51,7 @@ export const emitOpenAIImages = defineStage< }); const rendered = (answer: OpenAIImagesFacts['response.openaiImages.canonical']): OpenAIImagesFacts['response.openaiImages.rendered'] => - isFailure(answer) ? renderErrorEnvelope(answer.message, answer.body) + isFailure(answer) ? renderFailure(answer, mintedErrorEnvelope).body : isFrames(answer) ? renderSSE(answer) : renderOpenAIImagesResponse(answer); diff --git a/packages/gateway/src/data-plane/openai-images/pipeline.ts b/packages/gateway/src/data-plane/openai-images/pipeline.ts index 8e5449412a..0b7610efd0 100644 --- a/packages/gateway/src/data-plane/openai-images/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-images/pipeline.ts @@ -7,7 +7,7 @@ import { failover } from '../pipeline/failover.ts'; import { resolveCandidates } from '../pipeline/resolve-candidates.ts'; import { serializeClientJson } from '../pipeline/serialize-client-json.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; -import { compose, type Pipeline } from '@floway-dev/pipeline'; +import { compose, type Pipeline, type Stage } from '@floway-dev/pipeline'; import type { CanonicalOpenAIImagesRequest } from '@floway-dev/protocols/openai-images'; export const openaiImagesServePipeline = (request: CanonicalOpenAIImagesRequest): Pipeline => @@ -15,11 +15,15 @@ export const openaiImagesServePipeline = (request: CanonicalOpenAIImagesRequest) writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, 'response.openaiImages.streamedUsage'), serializeClientJson('response.openaiImages.rendered'), emitOpenAIImages, - resolveCandidates(narrowing(request)), - failover({ - failed: handedUp => isFailure((handedUp as { 'response.openaiImages.canonical'?: unknown })['response.openaiImages.canonical']), - owns: ['response.http.body'], - pendingUsage: 'response.openaiImages.streamedUsage', - }), - callOpenAIImagesUpstream, + ...openaiImagesModelStages(request), ]); + +export const openaiImagesModelStages = (request: Pick, dispatchStages: readonly Stage[] = [callOpenAIImagesUpstream]): readonly Stage[] => [ + resolveCandidates(narrowing(request)), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.openaiImages.canonical'?: unknown })['response.openaiImages.canonical']), + owns: ['response.http.body'], + pendingUsage: 'response.openaiImages.streamedUsage', + }), + ...dispatchStages, +]; diff --git a/packages/gateway/src/data-plane/openai-images/target.ts b/packages/gateway/src/data-plane/openai-images/target.ts index bc83782a13..fe140e8d70 100644 --- a/packages/gateway/src/data-plane/openai-images/target.ts +++ b/packages/gateway/src/data-plane/openai-images/target.ts @@ -11,7 +11,7 @@ const ENDPOINT = { /** A candidate that cannot serve *this* request is not a candidate. One family covers two * endpoints and an upstream may expose either without the other, so which one is asked for is * what narrows the list. */ -export const narrowing = (request: CanonicalOpenAIImagesRequest) => ({ +export const narrowing = (request: Pick) => ({ kind: 'image' as const, reject: (candidate: ModelCandidate) => candidate.model.endpoints[ENDPOINT[request.operation]] === undefined ? `the upstream does not expose the OpenAI Images ${request.operation} endpoint` diff --git a/packages/gateway/src/data-plane/pipeline/attempt-body.ts b/packages/gateway/src/data-plane/pipeline/attempt-body.ts new file mode 100644 index 0000000000..34075be29a --- /dev/null +++ b/packages/gateway/src/data-plane/pipeline/attempt-body.ts @@ -0,0 +1,24 @@ +// Alias rules apply to the addressed candidate. Providers receive model-free +// request content and stamp their own wire model; unchanged descendants retain +// their identity through the immutable handoff. + +import type { ModelCandidate } from '@floway-dev/provider'; + +/** + * The body for this attempt: the record's request, addressed to the model the candidate + * resolved, with the alias' own rules applied over it. + * + * The id the client addressed does not travel. An alias is a gateway concept — the provider + * re-stamps whatever it resolved upstream — so the key is dropped rather than forwarded, and + * the alias' rules apply to the body that is actually sent. + */ +export const bodyForAttempt = ( + recorded: T, + candidate: ModelCandidate, + applyRules: (body: T, rules: NonNullable) => T, +): Omit => { + const addressed = { ...recorded, model: candidate.model.id } as T; + const payload = candidate.rules === undefined ? addressed : applyRules(addressed, candidate.rules); + const { model: _addressed, ...body } = payload; + return body; +}; diff --git a/packages/gateway/src/data-plane/pipeline/facts.ts b/packages/gateway/src/data-plane/pipeline/facts.ts index 401fdb9e5e..4110e864d6 100644 --- a/packages/gateway/src/data-plane/pipeline/facts.ts +++ b/packages/gateway/src/data-plane/pipeline/facts.ts @@ -2,7 +2,7 @@ import type { UsageQuantities } from '../../repo/types.ts'; import type { Secret, Owned } from '@floway-dev/pipeline'; -import type { PricingRuntimeFacts } from '@floway-dev/protocols/common'; +import { isJsonObject, renderErrorEnvelope, type PricingRuntimeFacts } from '@floway-dev/protocols/common'; import type { TelemetryModelIdentity } from '@floway-dev/provider'; /** Everything about an attempt that is data: which upstream, which model row on it, and the @@ -35,13 +35,56 @@ export interface BillableEntity { export interface Failure { readonly status: number; readonly message: string; - /** Complete parsed upstream error content when available. */ + /** The upstream's own body, parsed, when it sent one this protocol can carry. It is what the + * client is answered with — the refusing party's words rather than a restatement of them — + * and a body that was not JSON at all leaves this absent, because there is nothing to carry + * and an envelope minted from the status and the message is what stands in for it. */ readonly body?: unknown; + /** What the gateway itself would write, when the refusal is its own. A protocol's envelope + * carries more than a status and a sentence — which field was at fault, which code names + * the condition — and only whoever refused knows those, so it is written where the refusal + * is made rather than derived from the status where it is rendered. */ + readonly envelope?: Record; } export const isFailure = (value: unknown): value is Failure => typeof value === 'object' && value !== null && 'status' in value && 'message' in value; +/** What a failure is sent as. The two travel together because for some protocols they are one + * decision: a Google-RPC envelope states its code inside the body, so a status chosen apart + * from the envelope is a second answer to the same question. */ +export interface RenderedFailure { + readonly body: Record; + readonly status: number; +} + +/** + * What a client is sent for a failure, in three tiers, so no edge has to remember them. + * + * An upstream that sent a body is forwarded it — the answer to "why was I refused" is the + * refusing party's to give. A refusal the gateway made itself is written by whoever made it, + * because only there is it known which field was at fault and which code names the condition. + * The protocol's own rendering of a status and a sentence is what is left when neither holds. + * + * Only that third tier is the gateway's to word, and it is therefore the only one that may + * answer with a status other than the failure's: a forwarded body already states its own, and + * a refusal that wrote its own envelope wrote it against the status it refused with. So `mint` + * hands back both, and a protocol cannot state a code in the body that the response contradicts. + */ +export const renderFailure = (failure: Failure, mint: (failure: Failure) => RenderedFailure): RenderedFailure => { + if (isJsonObject(failure.body)) return { body: failure.body, status: failure.status }; + if (failure.envelope !== undefined) return { body: failure.envelope, status: failure.status }; + return mint(failure); +}; + +/** The third tier for every protocol whose envelope says nothing about the status — which is + * all of them but Gemini generateContent. */ +export const mintedAs = (envelope: (failure: Failure) => Record) => + (failure: Failure): RenderedFailure => ({ body: envelope(failure), status: failure.status }); + +/** The third tier for the OpenAI-shaped protocols, which is the same envelope in all three. */ +export const mintedErrorEnvelope = mintedAs(failure => renderErrorEnvelope(failure.message)); + export interface GatewayFacts { /** What the client sent, before anything read it. Every family hands these over, because * every ending forwards what a provider is allowed to forward of them. */ @@ -52,7 +95,8 @@ export interface GatewayFacts { 'serve.model': string; 'serve.candidates': readonly AttemptSelector[]; - /** A selector resolves a live candidate without freezing the provider's mutable catalog. */ + /** The selected candidate's immutable identity and flags. Provider instances, transport + * handles and catalog state stay in the candidate registry and are resolved by id. */ 'route.attempt': AttemptSelector; /** Admitted headers remain values until the provider shapes its authenticated HTTP request. */ diff --git a/packages/gateway/src/data-plane/pipeline/serve.ts b/packages/gateway/src/data-plane/pipeline/serve.ts index 002118c6e3..b3a0f5c8ff 100644 --- a/packages/gateway/src/data-plane/pipeline/serve.ts +++ b/packages/gateway/src/data-plane/pipeline/serve.ts @@ -12,19 +12,19 @@ import type { ContentfulStatusCode } from 'hono/utils/http-status'; import { createCandidateRegistry } from './candidates.ts'; import type { AttemptSelector, BillableEntity, GatewayFacts } from './facts.ts'; -import type { GatewayServices, RunGatewayCtx } from './services.ts'; +import type { GatewayServices } from './services.ts'; import { openRunDump, type RunDump } from '../../dump/run-sink.ts'; import { apiKeyFromContext, type AuthedContext } from '../../middleware/auth.ts'; import { internalErrorResponse } from '../../middleware/internal-error-response.ts'; import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; import { consoleLogSink } from '../../runtime/log.ts'; import { stampUpstreamCallStart } from '../shared/attempt-timing.ts'; -import { createGatewayCtxFromHono, finalizeGatewayResponse, type AttemptState } from '../shared/gateway-ctx.ts'; +import { createGatewayCtxFromHono, finalizeGatewayResponse, type CreateGatewayCtxOptions, type AttemptState, type GatewayCtx } from '../shared/gateway-ctx.ts'; import { readRequestBody, takeRequestBody, type RequestBody } from '../shared/request-body.ts'; import { writeSSEFrames } from '../shared/sse.ts'; import { recordStream } from '@floway-dev/dump'; import { run, getFailureFacts, type Pipeline } from '@floway-dev/pipeline'; -import { sseCommentFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { sseCommentFrame, type SseFrame, type SseWritableFrame } from '@floway-dev/protocols/common'; type Slice = { [P in K]: GatewayFacts[P] }; @@ -51,49 +51,59 @@ export const readIngress = async (c: Context): Promise => ({ export interface Prologue { readonly runDump: RunDump | null; readonly services: GatewayServices; - readonly gateway: RunGatewayCtx; + readonly gateway: GatewayCtx; readonly headers: readonly (readonly [string, string])[]; } -/** - * Opens a run: the request context the telemetry stages read, and the services the stages - * are given. It takes the bytes the handler has already read and hands them to the dump, - * which is what leaves the handler's own copy free to be released. - * - * The candidate store is the resolver the ruling names — "the resolver is the service and - * the selector is a fact". A `ModelCandidate` carries the provider's instance, its fetcher - * and its models cache; those never enter the record, because `move()` would freeze them - * and the provider's own cache refresh would break. So the stage that enumerates hands the - * live ones here, and the stage that dials asks for one back by selector. - */ +/** Opens the run with the request's timing state and live candidate registry. Only + * immutable selectors enter facts; provider instances and transport handles stay here. */ export const openPrologue = ( c: AuthedContext, ingress: Ingress, options: { readonly wantsStream: boolean; readonly model?: string }, ): Prologue => { + const options_ = gatewayCtxOptions(c, ingress, options); + return prologueFor(createGatewayCtxFromHono(c, options_), ingress, runDumpOf(options_)); +}; + +/** What a run's request context is built from. Exported because a family whose context is a + * richer one builds that instead, and both have to be built from the same read of the body: + * `takeRequestBody` empties what it is given, so calling this twice would hand the dump an + * empty buffer the second time. */ +export const gatewayCtxOptions = ( + c: AuthedContext, + ingress: Ingress, + options: { readonly wantsStream: boolean; readonly model?: string }, +): CreateGatewayCtxOptions => { const backgroundScheduler = backgroundSchedulerFromContext(c); const attempt: AttemptState = { timing: { firstOutputTokenAt: null, upstreamCallStartedAt: null }, telemetry: undefined }; // The shape follows the endpoint. A pipelined turn is recorded as its whole run — every // stage, both directions — so it opens that recording here instead of the edge one, and // no turn is ever written twice. - const runDump = openRunDump( + const dump = openRunDump( apiKeyFromContext(c), { method: c.req.method, path: new URL(c.req.raw.url).pathname, body: ingress.body }, backgroundScheduler, options.wantsStream, attempt.timing, ); - const gateway: RunGatewayCtx = { - ...createGatewayCtxFromHono(c, { - wantsStream: options.wantsStream, - attempt, - ...(options.model === undefined ? {} : { model: options.model }), - requestBody: takeRequestBody(ingress.body), - backgroundScheduler, - dump: runDump, - }), - dump: runDump, + return { + wantsStream: options.wantsStream, + attempt, + ...(options.model === undefined ? {} : { model: options.model }), + requestBody: takeRequestBody(ingress.body), + backgroundScheduler, + dump, }; +}; + +/** The run recording those options opened, for the caller that has to hand its sink to + * `run`. A context carries the recording; only the prologue needs the sink. */ +export const runDumpOf = (options: CreateGatewayCtxOptions): RunDump | null => + (options.dump ?? null) as RunDump | null; + +/** The services every run is given, over whichever context it was opened with. */ +export const prologueFor = (gateway: GatewayCtx, ingress: Ingress, runDump: RunDump | null = null): Prologue => { const candidates = createCandidateRegistry(); return { @@ -138,6 +148,10 @@ export type Rendered = /** An answer that *is* a stream. The frames go out as they arrive, which is why nothing * above waits for them and why what they billed is settled afterwards. */ readonly frames: AsyncIterable; + /** What is written on an idle connection to keep it open. A comment is invisible to any + * client, but a protocol that defines its own idle event is read by clients that expect + * one — Anthropic's `ping` is a frame Claude Code sees — so the family names it. */ + readonly keepAlive?: SseWritableFrame; }; /** Which of the two an answer turned out to be. A family's rendered fact carries whichever @@ -223,7 +237,7 @@ const serveRun = async < return finalizeGatewayResponse(prologue.gateway, streamSSE(c, async stream => { try { await writeSSEFrames(stream, answer.frames, { - keepAlive: { frame: sseCommentFrame('keepalive') }, + keepAlive: { frame: answer.keepAlive ?? sseCommentFrame('keepalive') }, ...(prologue.gateway.downstreamAbortController === undefined ? {} : { downstreamAbortController: prologue.gateway.downstreamAbortController }), diff --git a/packages/gateway/src/data-plane/pipeline/services.ts b/packages/gateway/src/data-plane/pipeline/services.ts index 55eab63a43..7c106a1e1e 100644 --- a/packages/gateway/src/data-plane/pipeline/services.ts +++ b/packages/gateway/src/data-plane/pipeline/services.ts @@ -7,15 +7,12 @@ // behavior and stay outside those values. import type { AttemptSelector } from './facts.ts'; -import type { RunDump } from '../../dump/run-sink.ts'; import type { GatewayCtx } from '../shared/gateway-ctx.ts'; import type { HttpServices } from '@floway-dev/http/pipeline'; import type { Event, Logger } from '@floway-dev/pipeline'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { ModelCandidate } from '@floway-dev/provider'; -export type RunGatewayCtx = Omit & { readonly dump: RunDump | null }; - export interface GatewayServices extends HttpServices { readonly recordProtocolFrames: >(frames: AsyncIterable) => AsyncIterable; /** The global sink. Every stage's lines reach it, tagged with the stage's name. */ @@ -26,7 +23,7 @@ export interface GatewayServices extends HttpServices { /** The request-scoped context the settlement and telemetry stages read. It is a service * and not a fact because it holds live handles — the scheduler, the abort signal. */ - readonly gateway: RunGatewayCtx; + readonly gateway: GatewayCtx; /** Turns a selector back into the thing that dials. The resolver is the service and the * selector is the fact: a per-upstream transport is not a fact and is not pinned at the * prologue either, so what is injected is the thing that resolves one and what travels diff --git a/packages/gateway/src/data-plane/rerank/emit.ts b/packages/gateway/src/data-plane/rerank/emit.ts index 0a1a60029b..ea7c41b2ef 100644 --- a/packages/gateway/src/data-plane/rerank/emit.ts +++ b/packages/gateway/src/data-plane/rerank/emit.ts @@ -1,5 +1,5 @@ import type { Fields } from './facts.ts'; -import { isFailure } from '../pipeline/facts.ts'; +import { isFailure, mintedErrorEnvelope, renderFailure } from '../pipeline/facts.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import { renderErrorEnvelope } from '@floway-dev/protocols/common'; @@ -41,7 +41,7 @@ export const emitRerank = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.rerank.rendered': move(renderErrorEnvelope(answer.message, answer.body)), + 'response.rerank.rendered': move(renderFailure(answer, mintedErrorEnvelope).body), // The upstream's own status, or the gateway's own when it refused before dialling. // A client is not owed the upstream's exact bytes; it is owed the truth about what // happened, and a 429 arriving as a 200 is not that. diff --git a/packages/gateway/src/data-plane/shared/gateway-ctx.ts b/packages/gateway/src/data-plane/shared/gateway-ctx.ts index 75016cbdad..dfb85371ba 100644 --- a/packages/gateway/src/data-plane/shared/gateway-ctx.ts +++ b/packages/gateway/src/data-plane/shared/gateway-ctx.ts @@ -1,7 +1,6 @@ import type { AttemptTiming } from './attempt-timing.ts'; import type { RequestBody } from './request-body.ts'; -import { openDumpAccumulator } from '../../dump/accumulator.ts'; -import type { TurnDump } from '../../dump/turn-dump.ts'; +import type { RunDump } from '../../dump/run-sink.ts'; import { apiKeyFromContext, type AuthedContext, effectiveUpstreamIdsFromContext } from '../../middleware/auth.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; @@ -32,23 +31,23 @@ export interface GatewayCtx { // Null when the api key has no dump retention configured, in which case // `finalizeGatewayResponse` short-circuits the dump tee and returns the // response untouched. - readonly dump: TurnDump | null; + readonly dump: RunDump | null; } export interface CreateGatewayCtxOptions { wantsStream: boolean; attempt?: AttemptState; - // What this turn is recorded as. The shape follows the endpoint: a pipelined one hands in - // its run recording, and everything else lets the factory open the edge accumulator. - // Absent and null differ — absent means "open the usual one", null means "record nothing". - dump?: TurnDump | null; + // What this turn is recorded as, opened by whoever opened the run — every entry does, through + // the prologue. `null` is a key with no retention configured, which is what makes recording + // conditional: there is nothing to hand the runner rather than a sink that discards. + dump: RunDump | null; // WebSocket-style call sites own the AbortController (so the upgrade // handler can cancel mid-stream); HTTP call sites let the factory mint one // when wantsStream is true. downstreamAbortController?: AbortController; // Already-buffered inbound request body bytes. HTTP handlers read them - // once via `readRequestBody` and pass them in so the dump accumulator's - // snapshot reflects the exact bytes the handler parsed. WebSocket + // once via `readRequestBody` and pass them in so the record holds the exact + // bytes the handler parsed. WebSocket // upgrades carry no HTTP body — the WS OpenAI Responses path passes the // per-turn JSON message bytes here so the dump captures the turn's // input verbatim. @@ -78,9 +77,7 @@ export const createGatewayCtxFromHono = (c: AuthedContext, opts: CreateGatewayCt const apiKey = apiKeyFromContext(c); const upstreamIds = effectiveUpstreamIdsFromContext(c); const attempt: AttemptState = opts.attempt ?? { timing: { firstOutputTokenAt: null, upstreamCallStartedAt: null }, telemetry: undefined }; - const dump = 'dump' in opts - ? opts.dump ?? null - : openDumpAccumulator(c, opts.method ?? c.req.method, apiKey, opts.requestBody, opts.backgroundScheduler, opts.wantsStream, attempt.timing); + const dump = opts.dump; if (opts.model !== undefined) dump?.requestedModel(opts.model); return { apiKeyId: apiKey.id, @@ -96,8 +93,8 @@ export const createGatewayCtxFromHono = (c: AuthedContext, opts: CreateGatewayCt }; }; -// Run the dump-accumulator's finalize tee on the outgoing Response. Every -// inbound HTTP wrapper returns its response through this seam so the dump -// pipeline applies uniformly across happy-path, error, and passthrough paths. +// Run the recording's finalize tee on the outgoing Response. Every inbound HTTP +// wrapper returns its response through this seam so the dump applies uniformly +// across successful and failed turns. export const finalizeGatewayResponse = (ctx: GatewayCtx, response: Response): Response => ctx.dump?.finalize(response) ?? response; diff --git a/packages/gateway/src/data-plane/shared/iterate-candidates.ts b/packages/gateway/src/data-plane/shared/iterate-candidates.ts deleted file mode 100644 index 4d9e6281d2..0000000000 --- a/packages/gateway/src/data-plane/shared/iterate-candidates.ts +++ /dev/null @@ -1,78 +0,0 @@ -import type { GatewayCtx } from './gateway-ctx.ts'; -import { upstreamPerformanceContext } from './telemetry/attribution.ts'; -import type { ModelCandidate, PerformanceOperation } from '@floway-dev/provider'; - -// A serve-layer attempt result counts as success when: -// - The SSE event stream actually opened (`type: 'events'`). Mid-stream -// failure is the upstream's responsibility from there on; a fresh -// attempt on a different upstream does not start once the client has -// begun consuming events. -// - The non-streaming envelope landed: `PlainResult` with a 2xx status, -// or the OpenAI-Responses-compact `{type:'result'}` envelope. -// `api-error` and `internal-error` are failures: the serve loop falls -// through to the next candidate. 4xx is on the failure side — 429 -// (rate-limit) is the responsibility of the upstream that issued it, and -// the gateway's candidate ordering exists to absorb that kind of -// transient. Passthrough serves feed in an enlarged `plain` shape that -// carries the raw upstream Response plus per-attempt telemetry alongside -// the status; the success discriminant is unchanged. -type IterableAttemptResult = - | { readonly type: 'events' } - | { readonly type: 'result' } - | { readonly type: 'plain'; readonly status: number } - | { readonly type: 'api-error' } - | { readonly type: 'internal-error' }; - -const isAttemptSuccess = (result: IterableAttemptResult): boolean => { - switch (result.type) { - case 'events': - case 'result': - return true; - case 'plain': - return result.status >= 200 && result.status < 300; - case 'api-error': - case 'internal-error': - return false; - } -}; - -// Tries each narrowed candidate in order and returns the first success. A -// per-candidate *failure result* falls through so a transient 5xx/429 on -// one upstream rolls over to the next; a thrown error leaves the loop and -// surfaces to the caller, so a dial failure does not advance. When the -// list is exhausted the most recent failure is returned so callers can -// forward it verbatim and clients still see real upstream telemetry rather -// than a synthetic gateway envelope. Callers are contractually required to -// hand in a non-empty candidate list — the empty-candidate branch renders -// each caller's own protocol-shaped "no viable candidate" envelope at the -// serve site. -// -// Owns per-attempt AttemptState: clears the two timing slots and stamps -// `ctx.attempt.telemetry` with the current candidate's -// `PerformanceTelemetryContext` synchronously BEFORE handing control to -// `run`. That way a mid-attempt throw (interceptor bug, translation -// error, provider-layer JS exception not represented as a ChatServeFailure) -// still attributes the perf error row to the throwing candidate: the -// outer catch reads `ctx.attempt.telemetry` and feeds it into -// `recordFailedRequest`. Callsites don't need to duplicate this stamp. -export const iterateCandidates = async ( - candidates: readonly ModelCandidate[], - invocationLabel: string, - ctx: GatewayCtx, - operation: PerformanceOperation, - run: (candidate: ModelCandidate) => Promise, -): Promise => { - let lastFailure: T | undefined; - for (const candidate of candidates) { - ctx.attempt.timing.upstreamCallStartedAt = null; - ctx.attempt.timing.firstOutputTokenAt = null; - ctx.attempt.telemetry = upstreamPerformanceContext(ctx, candidate, operation); - const result = await run(candidate); - if (isAttemptSuccess(result)) return result; - lastFailure = result; - } - if (lastFailure === undefined) { - throw new Error(`invariant broken: ${invocationLabel} exhausted candidates with neither success nor failure`); - } - return lastFailure; -}; diff --git a/packages/gateway/src/data-plane/shared/telemetry/performance.ts b/packages/gateway/src/data-plane/shared/telemetry/performance.ts index c81ce87b5e..866b67a84f 100644 --- a/packages/gateway/src/data-plane/shared/telemetry/performance.ts +++ b/packages/gateway/src/data-plane/shared/telemetry/performance.ts @@ -9,7 +9,7 @@ export type { PerformanceTelemetryContext }; // Structural view of the fields recordPerformance actually reads. Every chat / // passthrough call site passes its full `GatewayCtx`; the OpenAI Responses image- -// generation server tool synthesizes a per-dispatch object because each image +// generation hosted tool synthesizes a per-dispatch object because each image // call carries its own TTFT window and can't share `ctx.attempt` with the // enclosing OpenAI Responses turn. type PerformanceRecordScope = Pick; diff --git a/packages/gateway/src/data-plane/shared/telemetry/settle.ts b/packages/gateway/src/data-plane/shared/telemetry/settle.ts index 94319339cb..0be62ef336 100644 --- a/packages/gateway/src/data-plane/shared/telemetry/settle.ts +++ b/packages/gateway/src/data-plane/shared/telemetry/settle.ts @@ -10,7 +10,7 @@ import type { TelemetryModelIdentity } from '@floway-dev/provider'; // the caller sample the stream-end monotonic timestamp at exactly the // moment the token stream terminates, so TPOT measures the stream // itself rather than the settle path. Callers with no such distinction -// (passthrough JSON, the image-generation server tool) omit it and get +// (passthrough JSON, the image-generation hosted tool) omit it and get // the call-time stamp. // // The usage-record D1 write is fire-and-forget. A transient repo failure diff --git a/packages/gateway/src/data-plane/shared/upstream-call-options.ts b/packages/gateway/src/data-plane/shared/upstream-call-options.ts index 6eed9add8c..9f6dd9db9a 100644 --- a/packages/gateway/src/data-plane/shared/upstream-call-options.ts +++ b/packages/gateway/src/data-plane/shared/upstream-call-options.ts @@ -10,7 +10,7 @@ export const buildUpstreamCallOptions = ( ctx: GatewayCtx, headers: Headers, ): UpstreamCallOptions => ({ - fetcher: ctx.dump?.http?.wrapFetcher(candidate.fetcher, candidate.provider.upstreamId) ?? candidate.fetcher, + fetcher: candidate.fetcher, waitUntil: ctx.backgroundScheduler, headers: filterInboundHeadersForProvider(headers, candidate.provider), wrapUpstreamCall: stampUpstreamCallStart(ctx.attempt.timing), diff --git a/packages/gateway/src/data-plane/tools/web-search/alpha-search/execution.ts b/packages/gateway/src/data-plane/tools/web-search/alpha-search/execution.ts index 4636bf0223..4daba6de2a 100644 --- a/packages/gateway/src/data-plane/tools/web-search/alpha-search/execution.ts +++ b/packages/gateway/src/data-plane/tools/web-search/alpha-search/execution.ts @@ -60,8 +60,8 @@ export const executeAlphaSearch = async ({ let parsed: unknown; try { parsed = JSON.parse(raw); - } catch { - throw new Error('OpenAI search upstream returned a non-JSON success body'); + } catch (cause) { + throw new Error('OpenAI search upstream returned a non-JSON success body', { cause }); } if (parsed === null || typeof parsed !== 'object' || typeof (parsed as { output?: unknown }).output !== 'string') { throw new Error('OpenAI search upstream response must include an output string'); diff --git a/packages/gateway/src/data-plane/tools/web-search/alpha-search/upstream.ts b/packages/gateway/src/data-plane/tools/web-search/alpha-search/upstream.ts index d63adec713..81b16d1ac5 100644 --- a/packages/gateway/src/data-plane/tools/web-search/alpha-search/upstream.ts +++ b/packages/gateway/src/data-plane/tools/web-search/alpha-search/upstream.ts @@ -1,12 +1,11 @@ import { enumerateModelCandidates } from '../../../providers/resolution.ts'; -import { filterInboundHeadersForProvider } from '../../../shared/inbound-headers.ts'; import type { WebSearchConfig } from '../types.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; -import { identityWrapUpstreamCall, providerModelOf } from '@floway-dev/provider'; +import type { ModelCandidate } from '@floway-dev/provider'; export type AlphaSearchDispatcher = (body: Record, signal: AbortSignal | undefined, headers: Headers) => Promise; -export const resolveAlphaSearchDispatcher = async ({ +export const resolveAlphaSearchCandidate = async ({ config, upstreamIds, scheduler, @@ -16,7 +15,7 @@ export const resolveAlphaSearchDispatcher = async ({ upstreamIds: readonly string[] | null; scheduler: BackgroundScheduler; runtimeLocation: string; -}): Promise => { +}): Promise => { if (upstreamIds !== null && !upstreamIds.includes(config.upstreamId)) { throw new Error('Selected OpenAI search upstream is outside this API key scope'); } @@ -35,21 +34,5 @@ export const resolveAlphaSearchDispatcher = async ({ throw new Error('Selected upstream does not support OpenAI search passthrough'); } - return async (body, signal, headers) => { - const { model: _callerModel, ...request } = body; - // TODO: pin SearchRequest.id to one provider account when Codex upstreams - // support account pools. The current Codex provider has one active account. - const result = await candidate.provider.instance.callAlphaSearch( - providerModelOf(candidate), - request, - signal, - { - fetcher: candidate.fetcher, - waitUntil: scheduler, - headers: filterInboundHeadersForProvider(headers, candidate.provider), - wrapUpstreamCall: identityWrapUpstreamCall, - }, - ); - return result.response; - }; + return candidate; }; diff --git a/packages/gateway/src/data-plane/tools/web-search/providers/shared.ts b/packages/gateway/src/data-plane/tools/web-search/providers/shared.ts index 1af1100865..110f29b166 100644 --- a/packages/gateway/src/data-plane/tools/web-search/providers/shared.ts +++ b/packages/gateway/src/data-plane/tools/web-search/providers/shared.ts @@ -1,7 +1,7 @@ import { isJsonObject } from '../../../../shared/json-helpers.ts'; import { sleep } from '../../../../shared/sleep.ts'; import type { WebSearchProviderErrorCode, WebSearchProviderResult } from '../types.ts'; -import { discardUpstreamResponse } from '@floway-dev/provider'; +import { discardUpstreamResponse } from '@floway-dev/http'; const MAX_WEB_SEARCH_QUERY_LENGTH = 1000; const RETRY_DELAYS_MS = [1000, 2000, 4000, 8000] as const; diff --git a/packages/gateway/src/dump/accumulator.ts b/packages/gateway/src/dump/accumulator.ts deleted file mode 100644 index 213adaf3e8..0000000000 --- a/packages/gateway/src/dump/accumulator.ts +++ /dev/null @@ -1,384 +0,0 @@ -// Per-request dump pipeline. Opens the dump session (request snapshot + -// opt-in decision) and exposes the mid-flight hooks the respond layer -// calls to record outcomes and frames. When the api key has no retention -// configured, opening returns null and the data plane pays no per-request -// cost. - -import type { Context } from 'hono'; - -import { HttpCapture } from './http-capture.ts'; -import { getDumpBroker, getDumpStore } from './registry.ts'; -import type { StreamRecording } from './turn-dump.ts'; -import type { DumpStreamEvent, DumpWriteRecord, PreparedDumpRequestBody, StoredDumpResponseBody } from './types.ts'; -import { encodeBodyForWire } from './wire.ts'; -import { attemptTtftMs, type AttemptTiming } from '../data-plane/shared/attempt-timing.ts'; -import type { RequestBody } from '../data-plane/shared/request-body.ts'; -import { getRepo } from '../repo/index.ts'; -import type { ApiKey, TokenUsage } from '../repo/types.ts'; -import { ulid } from '../shared/ulid.ts'; -import type { DumpErrorMeta, DumpMetadata, DumpUpstreamRef } from '@floway-dev/dump/types'; -import type { BackgroundScheduler } from '@floway-dev/platform'; -import { type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ChatTargetApi, TelemetryModelIdentity } from '@floway-dev/provider'; - -// Frozen at ctx construction so `finalize` never has to re-read a stream -// the handler already consumed. -interface RequestSnapshot { - readonly method: string; - readonly path: string; - readonly headers: ReadonlyArray; - readonly bodyByteLength: number; - readonly streamError: string | null; -} - -interface ResponseSnapshot { - readonly status: number; - readonly headers: ReadonlyArray; - readonly rawCaptured: boolean; - readonly bytes: Uint8Array; - readonly payloadBytes: number; - readonly streamError: string | null; -} - -// Four independent attribution slots the mid-flight hooks fill: `model` and -// `upstreamId` identify what the turn was about, `inputTokens` / -// `outputTokens` quantify what the upstream reported. They're independent -// because different outcomes set different subsets: -// -// • Every protocol handler calls `requestedModel(model)` immediately after -// parsing the payload, so `model` is set regardless of outcome. -// • `success(identity, usage)` fills all four; the upstream-resolved model -// id may overwrite what `requestedModel` had. -// • `error(kind, upstream?)` records a categorized api-error envelope -// (`kind` matches `ApiErrorResult.source`). Real upstream non-2xx pass -// `upstream` so a 4xx/5xx row in the dashboard names the upstream that -// rejected the call; the gateway arm may also pass it when a candidate -// was already chosen (item-not-found rewrite, server-tool input -// rejection). -// • `failed(reason)` records an uncategorized terminal failure: a thrown -// exception (caught by the respond layer or passthrough-serve), a -// source-emitted error frame, a downstream cancel, or a writer error. -// Caller passes a string or Error; the accumulator one-line-formats -// it (`.message` only — never the stack, which lives in the response -// body's debug envelope). -// -// `requestedModel`-set model survives across both error variants so even an -// outright-failed turn carries model attribution. - -// Anthropic-style disjoint per-category counts: input excludes cache reads -// and cache writes; sum the present ones onto the dump's single inputTokens -// column. Missing categories stay null (not measured) instead of zero so a -// recorded zero genuinely means "upstream said zero". -const tokenUsageInput = (usage: TokenUsage | null): number | null => { - if (!usage) return null; - const { input, input_cache_read, input_cache_write } = usage; - if (input === undefined && input_cache_read === undefined && input_cache_write === undefined) return null; - return (input ?? 0) + (input_cache_read ?? 0) + (input_cache_write ?? 0); -}; - -const oneLineError = (err: unknown): string => { - const msg = (err instanceof Error ? err.message : String(err)).replace(/\s+/g, ' ').trim(); - return msg.length > 500 ? `${msg.slice(0, 497)}…` : msg; -}; - -const headerPairs = (headers: Headers): Array<[string, string]> => { - const pairs: Array<[string, string]> = []; - headers.forEach((value, name) => { pairs.push([name, value]); }); - return pairs; -}; - -const resolveUpstreamRef = async (id: string | null): Promise => { - if (!id) return null; - const upstream = await getRepo().upstreams.getById(id); - if (!upstream) return null; - return { id: upstream.id, name: upstream.name, kind: upstream.kind, hue: upstream.hue }; -}; - -export class DumpAccumulator { - readonly http = new HttpCapture(); - private readonly events: DumpStreamEvent[] = []; - private sentPayloadBytes = 0; - private model: string | null = null; - private upstreamId: string | null = null; - private inputTokens: number | null = null; - private outputTokens: number | null = null; - private errorMeta: DumpErrorMeta | null = null; - private readonly preparedRequestBody: Promise; - // Pre-translation (target-protocol) view. Populated only on translated turns - // via `traverseTranslation`'s capture hook; stays empty on native turns. - private readonly upstreamEvents: DumpStreamEvent[] = []; - private upstreamTargetApi: ChatTargetApi | null = null; - private upstreamApiErrorEnvelope: { status: number; headers: Array<[string, string]>; body: Uint8Array } | null = null; - - constructor( - private readonly apiKey: ApiKey, - private readonly requestSnapshot: RequestSnapshot, - requestBody: Uint8Array, - private readonly startedAt: number, - private readonly backgroundScheduler: BackgroundScheduler, - private readonly wantsStream: boolean = false, - private readonly timing?: AttemptTiming, - ) { - this.preparedRequestBody = getDumpStore().prepareRequestBody(requestBody); - // Preparation starts eagerly and is awaited at terminal persistence. Mark - // a rejection handled immediately so a long upstream wait cannot surface - // it as an unhandled promise before `write()` records the dump failure. - void this.preparedRequestBody.catch(() => {}); - } - - // --- mid-flight hooks (called from per-protocol respond layer) --- - - requestedModel(model: string): void { - this.model = model; - } - - error(kind: 'upstream' | 'gateway', upstream?: string): void { - this.errorMeta = { kind }; - if (upstream !== undefined) this.upstreamId = upstream; - } - - failed(reason: unknown): void { - this.errorMeta = { kind: 'failed', reason: typeof reason === 'string' ? reason : oneLineError(reason) }; - } - - // Records one protocol frame. Stored as the canonical ProtocolFrame so - // neither serialization nor parsing happens on this path; the dashboard - // derives the SSE wire view on demand via the per-protocol - // frame-to-SSE encoder + reducer. - frame(frame: ProtocolFrame): void { - this.events.push({ frame, ts: Date.now() - this.startedAt }); - } - - openStream(): StreamRecording { - return { frame: frame => { this.frame(frame); }, end: () => {}, fact: null }; - } - - // --- pre-translation upstream hooks (called from `traverseTranslation`) --- - - // Stamped eagerly at capture construction so `meta.targetApi` is set even - // when the upstream stream produces zero frames (e.g. immediate done). - setUpstreamTargetApi(api: ChatTargetApi): void { - this.upstreamTargetApi = api; - } - - // Records one ORIGINAL target-protocol frame, before Floway translates it - // into the source protocol. Same shape as `frame()` so the dashboard renders - // the upstream view with the same collected+events experience, dispatched - // by `meta.targetApi` instead of `meta.path`. - upstreamFrame(frame: ProtocolFrame): void { - this.upstreamEvents.push({ frame, ts: Date.now() - this.startedAt }); - } - - // Captures the verbatim upstream api-error envelope (status/headers/body) - // BEFORE the optional `trip.apiError` rewrite. The bytes variant of the - // upstream body; the dashboard renders it like any non-stream response body. - upstreamApiError(error: { status: number; headers: Headers; body: Uint8Array }): void { - this.upstreamApiErrorEnvelope = { status: error.status, headers: headerPairs(error.headers), body: error.body }; - } - - recordSentPayloadBytes(byteLength: number): void { - this.sentPayloadBytes += byteLength; - } - - success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void { - this.model = identity.model; - this.upstreamId = identity.upstream; - this.inputTokens = tokenUsageInput(usage); - this.outputTokens = usage?.output ?? null; - } - - // --- response-side: handler exit --- - - // Schedules the dump-record write at the turn's terminal point. Two input - // shapes: - // - // • `(status, headers)` — no HTTP Response object to tee. The WebSocket - // OpenAI Responses path uses this: its "response" is the stream of frames - // already captured via `frame()`, while the send seam records their - // actual UTF-8 payload bytes via `recordSentPayloadBytes()`. - // • `(response)` — tees the response body so the client gets bytes - // flowing while a background reader accumulates the other half. The - // returned Response streams the client-side bytes; status, statusText, - // and headers pass through verbatim so the tee is invisible to the - // client. A null body falls through to the bare form. - // - // The background drain → record assembly → store put → broker publish is - // scheduled through the runtime's BackgroundScheduler so dump write - // failures cannot turn a successful upstream call into a 502. - finalize(status: number, headers: ReadonlyArray): void; - finalize(response: Response): Response; - finalize(...args: [number, ReadonlyArray] | [Response]): void | Response { - if (args.length === 2) { - const [status, headers] = args; - this.backgroundScheduler(this.write({ - status, - headers: headers.map(([k, v]) => [k, v]), - rawCaptured: this.requestSnapshot.method !== 'WS', - bytes: new Uint8Array(), - payloadBytes: this.sentPayloadBytes, - streamError: null, - })); - return; - } - - const [response] = args; - const responseStatus = response.status; - const responseHeaders = headerPairs(response.headers); - - if (response.body === null) { - this.finalize(responseStatus, responseHeaders); - return response; - } - - const [forClient, forCapture] = response.body.tee(); - this.backgroundScheduler((async () => { - const reader = forCapture.getReader(); - const chunks: Uint8Array[] = []; - let total = 0; - let streamError: string | null = null; - try { - for (;;) { - const { value, done } = await reader.read(); - if (done) break; - chunks.push(value); - total += value.byteLength; - } - } catch (err) { - streamError = oneLineError(err); - } - const bytes = new Uint8Array(total); - let offset = 0; - for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.byteLength; } - await this.write({ - status: responseStatus, - headers: responseHeaders, - rawCaptured: true, - bytes, - payloadBytes: bytes.byteLength, - streamError, - }); - })()); - - return new Response(forClient, { - status: response.status, - statusText: response.statusText, - headers: response.headers, - }); - } - - // --- private: persist --- - - private async write(response: ResponseSnapshot): Promise { - // ULID-from-completedAt keeps ids increasing with row creation time; the - // random tail provides the deterministic tie-breaker for one millisecond. - const completedAt = Date.now(); - const recordId = ulid(completedAt); - - // Prefer the accumulator's frame log so dumps reflect the gateway's - // frame sequence regardless of negotiated wire shape; passthrough - // endpoints with no frames fall back to captured bytes. - const responseBody: StoredDumpResponseBody = this.events.length > 0 - ? { type: 'stream', events: this.events } - : response.bytes.byteLength > 0 || response.streamError !== null - ? { type: 'bytes', body: response.bytes } - : { type: 'none' }; - - const ttftMs = this.wantsStream ? attemptTtftMs(this.timing) : null; - - const meta: DumpMetadata = { - id: recordId, - startedAt: this.startedAt, - completedAt, - method: this.requestSnapshot.method, - path: this.requestSnapshot.path, - status: response.status, - upstream: await resolveUpstreamRef(this.upstreamId ?? this.http.exchanges.at(-1)?.upstreamId ?? null), - model: this.model, - inputTokens: this.inputTokens, - outputTokens: this.outputTokens, - requestBytes: this.requestSnapshot.bodyByteLength, - responseBytes: response.payloadBytes, - durationMs: completedAt - this.startedAt, - ttftMs, - // Precedence: an explicit error stamp from the respond path wins; - // otherwise a request-body read failure (operator-side payload didn't - // arrive intact) outranks a response-body read failure. Both stream- - // read failures surface as `kind: 'failed'`. - error: this.errorMeta - ?? (this.requestSnapshot.streamError !== null ? { kind: 'failed', reason: this.requestSnapshot.streamError } : null) - ?? (response.streamError !== null ? { kind: 'failed', reason: response.streamError } : null), - targetApi: this.upstreamTargetApi, - }; - - // Build the parallel pre-translation upstream body ONLY when upstream - // data was captured. An empty upstream stream with no api-error means the - // turn produced nothing to show (or was native) — omit the field so old - // records and native turns share the same shape. - const hasUpstream = this.upstreamEvents.length > 0 || this.upstreamApiErrorEnvelope !== null; - const upstream = !hasUpstream ? undefined : { - status: this.upstreamApiErrorEnvelope?.status ?? null, - headers: this.upstreamApiErrorEnvelope?.headers ?? [], - body: (this.upstreamApiErrorEnvelope !== null - ? { type: 'bytes' as const, body: this.upstreamApiErrorEnvelope.body } - : { type: 'stream' as const, events: this.upstreamEvents }) satisfies StoredDumpResponseBody, - }; - - // Commit the row before publishing so subscribers fetching detail off the meta frame find it. - try { - const record: DumpWriteRecord = { - shape: 'edge', - capture: { - exchanges: this.http.exchanges, - ...(response.rawCaptured ? { - response: { - body: encodeBodyForWire(response.bytes, response.headers.find(([name]) => name.toLowerCase() === 'content-type')?.[1] ?? ''), - complete: response.streamError === null, - error: response.streamError, - }, - } : {}), - }, - meta, - request: { - method: this.requestSnapshot.method, - path: this.requestSnapshot.path, - headers: this.requestSnapshot.headers.map(([k, v]) => [k, v]), - body: await this.preparedRequestBody, - }, - response: { - status: response.status, - headers: response.headers.map(([k, v]) => [k, v]), - body: responseBody, - ...(upstream !== undefined ? { upstream } : {}), - }, - }; - await getDumpStore().put(this.apiKey.id, record); - await getDumpBroker().publish(this.apiKey.id, meta); - } catch (err) { - console.error(`[dump] write failed for key=${this.apiKey.id} record=${recordId}`, oneLineError(err)); - } - } -} - -// Returns null when the api key opts out of dumps; callers then skip all -// per-request dump work. `method` is passed explicitly rather than read -// off the request so the WebSocket OpenAI Responses path can record each turn -// as `WS /v1/responses` rather than the upgrade's `GET`. -export const openDumpAccumulator = ( - c: Context, - method: string, - apiKey: ApiKey, - requestBody: RequestBody, - backgroundScheduler: BackgroundScheduler, - wantsStream: boolean = false, - timing?: AttemptTiming, -): DumpAccumulator | null => { - if (apiKey.dumpRetentionSeconds === null) return null; - const requestSnapshot: RequestSnapshot = { - method, - path: c.req.path, - headers: headerPairs(c.req.raw.headers), - bodyByteLength: requestBody.bytes.byteLength, - streamError: requestBody.streamError, - }; - return new DumpAccumulator(apiKey, requestSnapshot, requestBody.bytes, Date.now(), backgroundScheduler, wantsStream, timing); -}; diff --git a/packages/gateway/src/dump/attribution.ts b/packages/gateway/src/dump/attribution.ts index f9d3d438eb..f9326881da 100644 --- a/packages/gateway/src/dump/attribution.ts +++ b/packages/gateway/src/dump/attribution.ts @@ -1,34 +1,7 @@ -// What a turn's record says about the turn, and what fills it. -// -// `DumpMetadata` is common to both record shapes — the dashboard lists a run -// and a pair of edges in the same list, and a turn's model, upstream and token -// counts do not depend on which mechanism served it — so the hooks that stamp -// attribution and the assembly that turns it into metadata are one thing rather -// than one per shape. -// -// Four independent slots the mid-flight hooks fill: `model` and `upstreamId` -// identify what the turn was about, `inputTokens` / `outputTokens` quantify -// what the upstream reported. They're independent because different outcomes -// set different subsets: -// -// • Every protocol handler calls `requestedModel(model)` immediately after -// parsing the payload, so `model` is set regardless of outcome. -// • `success(identity, usage)` fills all four; the upstream-resolved model -// id may overwrite what `requestedModel` had. -// • `error(kind, upstream?)` records a categorized api-error envelope -// (`kind` matches `ApiErrorResult.source`). Real upstream non-2xx pass -// `upstream` so a 4xx/5xx row in the dashboard names the upstream that -// rejected the call; the gateway arm may also pass it when a candidate -// was already chosen (item-not-found rewrite, server-tool input -// rejection). -// • `failed(reason)` records an uncategorized terminal failure: a thrown -// exception, a source-emitted error frame, a downstream cancel, or a -// writer error. Caller passes a string or Error; this one-line-formats it -// (`.message` only — never the stack, which lives in the response body's -// debug envelope). -// -// `requestedModel`-set model survives across both error variants so even an -// outright-failed turn carries model attribution. +// Listing metadata summarizes the run's observed model, upstream and quantities. +// The requested model survives failures before resolution; successful observations +// replace its identity and add only quantities that were actually reported. +// Explicit protocol or stage failures outrank transport and settlement fallbacks. import { getRepo } from '../repo/index.ts'; import type { TokenUsage } from '../repo/types.ts'; @@ -58,7 +31,7 @@ const resolveUpstreamRef = async (id: string | null): Promise error instanceof Error ? error.stack ?? error.message : String(error); - -export class HttpCapture { - readonly exchanges: DumpCapture['exchanges'] = []; - - wrapFetcher(fetcher: Fetcher, upstreamId: string): Fetcher { - return async (url, init) => { - const body = isReplayableBody(init.body) ? init.body.open() : init.body; - const request = new Request(url, { ...init, body, duplex: 'half' } as RequestInit); - const bytes = new Uint8Array(await request.arrayBuffer()); - const exchange: DumpCapture['exchanges'][number] = { - upstreamId, - request: { - url, - method: request.method, - headers: [...request.headers], - body: encodeBodyForWire(bytes, request.headers.get('content-type') ?? ''), - }, - response: null, - error: null, - }; - this.exchanges.push(exchange); - try { - const response = await fetcher(url, { ...init, headers: request.headers, body: body == null ? null : bytes }); - const capture = new RawResponseCapture(response); - Object.defineProperty(exchange, 'response', { enumerable: true, get: () => capture.snapshot() }); - return capture.wrap(); - } catch (error) { - exchange.error = errorText(error); - throw error; - } - }; - } -} - -class RawResponseCapture { - private readonly chunks: Uint8Array[] = []; - private complete = false; - private cancelled = false; - private error: string | null = null; - - constructor(private readonly response: Response) {} - - snapshot(): NonNullable { - const bytes = new Uint8Array(this.chunks.reduce((size, chunk) => size + chunk.byteLength, 0)); - let offset = 0; - for (const chunk of this.chunks) { - bytes.set(chunk, offset); - offset += chunk.byteLength; - } - return { - status: this.response.status, - headers: [...this.response.headers], - body: encodeBodyForWire(bytes, this.response.headers.get('content-type') ?? ''), - complete: this.complete, - error: this.error, - }; - } - - wrap(): Response { - if (this.response.body === null) { - this.complete = true; - return this.response; - } - const reader = this.response.body.getReader(); - let released = false; - const finish = () => { - if (!released) { released = true; reader.releaseLock(); } - }; - const stream = new ReadableStream({ - pull: async controller => { - try { - const { value, done } = await reader.read(); - if (this.cancelled) return; - if (done) { - this.complete = true; - finish(); - controller.close(); - } else { - this.chunks.push(value.slice()); - controller.enqueue(value); - } - } catch (error) { - if (this.cancelled) return; - this.error = errorText(error); - finish(); - controller.error(error); - } - }, - cancel: async reason => { - this.cancelled = true; - if (reason !== undefined) this.error = errorText(reason); - try { await reader.cancel(reason); } finally { finish(); } - }, - }, { highWaterMark: 0 }); - return new Response(stream, { status: this.response.status, statusText: this.response.statusText, headers: this.response.headers }); - } -} diff --git a/packages/gateway/src/dump/registry.ts b/packages/gateway/src/dump/registry.ts index 8032fc6d0a..8162b7fdaa 100644 --- a/packages/gateway/src/dump/registry.ts +++ b/packages/gateway/src/dump/registry.ts @@ -1,5 +1,5 @@ -import type { DumpStore } from './store-contract.ts'; import { DUMP_DISABLED_REASON, type DumpBroker } from '@floway-dev/dump'; +import type { DumpStore } from '@floway-dev/dump/types'; let _store: DumpStore | null = null; let _broker: DumpBroker | null = null; diff --git a/packages/gateway/src/dump/run-sink.ts b/packages/gateway/src/dump/run-sink.ts index 5d5c0ab632..67f46c6690 100644 --- a/packages/gateway/src/dump/run-sink.ts +++ b/packages/gateway/src/dump/run-sink.ts @@ -25,7 +25,7 @@ export class RunDump { private sentPayloadBytes = 0; constructor( - apiKey: ApiKey, + private readonly apiKey: ApiKey, private readonly requestSnapshot: RequestSnapshot, private readonly startedAt: number, private readonly backgroundScheduler: BackgroundScheduler, @@ -33,7 +33,7 @@ export class RunDump { private readonly timing: AttemptTiming, ) { this.recorder = createRunRecorder({ - write: record => getDumpStore().put(apiKey.id, { shape: 'run', ...record }), + write: record => getDumpStore().put(apiKey.id, record), publish: meta => getDumpBroker().publish(apiKey.id, meta), }); this.sink = this.recorder.sink; @@ -59,6 +59,17 @@ export class RunDump { openStream(): StreamRecording { return this.recorder.openStream(); } + openSubRequest(turn: { readonly method: string; readonly path: string }, wantsStream: boolean, timing: AttemptTiming): RunDump { + return new RunDump( + this.apiKey, + { method: turn.method, path: turn.path, bodyByteLength: 0, streamError: null }, + Date.now(), + this.backgroundScheduler, + wantsStream, + timing, + ); + } + success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void { this.attribution.success(identity, usage); } diff --git a/packages/gateway/src/dump/schemas.ts b/packages/gateway/src/dump/schemas.ts deleted file mode 100644 index 307b5e5557..0000000000 --- a/packages/gateway/src/dump/schemas.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { z } from 'zod'; - -import { dumpMetadataSchema } from '@floway-dev/dump/schemas'; - -export const persistedDumpMetadataSchema = dumpMetadataSchema.omit({ upstream: true }); - -export const dumpHeadersSchema = z.array(z.tuple([z.string(), z.string()])); - -export const dumpBodyDescriptorSchema = z.object({ - key: z.string(), - type: z.enum(['bytes', 'events', 'run', 'capture']), -}).strict(); - -const dumpProtocolFrameSchema = z.discriminatedUnion('type', [ - z.object({ type: z.literal('event'), event: z.unknown() }).strict(), - z.object({ type: z.literal('done') }).strict(), -]); - -export const dumpStreamEventSchema = z.object({ - frame: dumpProtocolFrameSchema, - ts: z.number(), -}).strict(); - -export const dumpStreamEventsSchema = z.array(dumpStreamEventSchema); - -const rawBodySchema = z.object({ - encoding: z.enum(['utf8', 'base64']), - data: z.string(), -}).strict(); - -const rawCaptureSchema = z.object({ - body: rawBodySchema, - complete: z.boolean(), - error: z.string().nullable(), -}).strict(); - -export const dumpCaptureSchema = z.object({ - exchanges: z.array(z.object({ - upstreamId: z.string(), - request: z.object({ - url: z.string(), - method: z.string(), - headers: dumpHeadersSchema, - body: rawBodySchema, - }).strict(), - response: rawCaptureSchema.extend({ status: z.number(), headers: dumpHeadersSchema }).nullable(), - error: z.string().nullable(), - }).strict()), - response: rawCaptureSchema.optional(), -}).strict(); - -export const dumpCaptureEnvelopeSchema = z.object({ - version: z.literal(1), - capture: dumpCaptureSchema, - upstream: z.object({ - status: z.number().nullable(), - headers: dumpHeadersSchema, - body: z.discriminatedUnion('type', [ - z.object({ type: z.literal('stream'), events: dumpStreamEventsSchema }).strict(), - z.object({ type: z.literal('bytes'), body: rawBodySchema }).strict(), - z.object({ type: z.literal('none') }).strict(), - ]), - }).optional(), -}).strict(); diff --git a/packages/gateway/src/dump/storage-codec.ts b/packages/gateway/src/dump/storage-codec.ts index 576a876fd6..aa63506a96 100644 --- a/packages/gateway/src/dump/storage-codec.ts +++ b/packages/gateway/src/dump/storage-codec.ts @@ -1,14 +1,11 @@ -import type { z } from 'zod'; - -import { - dumpBodyDescriptorSchema, - dumpHeadersSchema, - persistedDumpMetadataSchema, - dumpStreamEventsSchema, -} from './schemas.ts'; -import type { DumpStreamEvent } from './types.ts'; +import { z } from 'zod'; + +import { dumpMetadataSchema } from '@floway-dev/dump/schemas'; import type { DumpMetadata } from '@floway-dev/dump/types'; +const persistedDumpMetadataSchema = dumpMetadataSchema.omit({ upstream: true }); +const dumpBodyDescriptorSchema = z.object({ key: z.string(), type: z.literal('run') }).strict(); + export type DumpBodyDescriptor = z.infer; type PersistedDumpMetadata = z.infer; @@ -38,20 +35,8 @@ export const encodePersistedDumpMetadata = (metadata: DumpMetadata, context: str export const decodePersistedDumpMetadata = (text: string, context: string): PersistedDumpMetadata => parseJson(text, context, persistedDumpMetadataSchema); -export const encodeDumpHeaders = (headers: Array<[string, string]>, context: string): string => - encodeJson(headers, context, dumpHeadersSchema); - -export const decodeDumpHeaders = (text: string, context: string): Array<[string, string]> => - parseJson(text, context, dumpHeadersSchema); - export const encodeDumpBodyDescriptor = (descriptor: DumpBodyDescriptor, context: string): string => encodeJson(descriptor, context, dumpBodyDescriptorSchema); export const decodeDumpBodyDescriptor = (text: string, context: string): DumpBodyDescriptor => parseJson(text, context, dumpBodyDescriptorSchema); - -export const encodeDumpStreamEvents = (events: DumpStreamEvent[], context: string): string => - encodeJson(events, context, dumpStreamEventsSchema); - -export const decodeDumpStreamEvents = (text: string, context: string): DumpStreamEvent[] => - parseJson(text, context, dumpStreamEventsSchema); diff --git a/packages/gateway/src/dump/store-contract.ts b/packages/gateway/src/dump/store-contract.ts deleted file mode 100644 index b652fa8e71..0000000000 --- a/packages/gateway/src/dump/store-contract.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { DumpWriteRecord, PreparedDumpRequestBody, StoredDumpRecord } from './types.ts'; -import type { DumpMetadata, DumpRecordId } from '@floway-dev/dump/types'; - -// Per-API-key request dump storage contract: metadata in SQL, bodies in the -// FileStore. Request bytes are prepared before the terminal write; reads -// always rehydrate raw bytes for the control plane. - -export interface DumpListOptions { - before?: DumpRecordId; - q?: string; - failures?: boolean; - limit: number; -} - -export interface DumpStore { - // Starts body preparation while the request is in flight. Implementations - // may return identity bytes, but persistent stores compress here so the - // accumulator can release the original request buffer before terminal IO. - prepareRequestBody(body: Uint8Array): Promise; - - // Write body files BEFORE the metadata row so a partial failure leaves - // orphan files (sweep-collectable), not orphan rows (broken records). - put(keyId: string, record: DumpWriteRecord): Promise; - - // Newest-first, paginated by ULID cursor. Reads enforce the API key's - // current rolling retention even before queued physical deletion runs. - list(keyId: string, opts: DumpListOptions): Promise; - - get(keyId: string, recordId: DumpRecordId): Promise; - - deleteExpiredBatch(keyId: string, now: number, limit: number): Promise; - findOldestCreatedAt(keyId: string): Promise; -} diff --git a/packages/gateway/src/dump/turn-dump.ts b/packages/gateway/src/dump/turn-dump.ts deleted file mode 100644 index dc00d0250a..0000000000 --- a/packages/gateway/src/dump/turn-dump.ts +++ /dev/null @@ -1,50 +0,0 @@ -// What a turn stamps on its recording, whichever shape that recording is. -// -// Two shapes are alive while the migration runs: the edge record, which holds what the client -// sent and what it got back, and the run record, which holds every stage of the pipeline as -// an event stream. The shape follows the endpoint — a pipelined one produces the run shape — -// and the stages in between neither know nor care which they are stamping. - -import type { HttpCapture } from './http-capture.ts'; -import type { TokenUsage } from '../repo/types.ts'; -import type { StreamFact } from '@floway-dev/pipeline'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { TelemetryModelIdentity } from '@floway-dev/provider'; - -/** - * One stream, as the recording knows it. - * - * A run record identifies its streams, because their content arrives over time and after the - * fact that holds them: the fact carries `{"$stream": n}` and the frames arrive afterwards - * naming that id. `end` is what says the record of this stream is complete — a client that - * stopped reading leaves it short, and the absence of the terminator is how a reader tells a - * stream that ended from one that was cut off. - * - * The edge record has one frame log and no way to name anything in it, so its `fact` is null - * and its terminator has nothing to write. That is the shape's limit rather than an omission, - * and it goes away with the shape. - */ -export interface StreamRecording { - frame(frame: ProtocolFrame): void | Promise; - end(): void | Promise; - readonly fact: StreamFact | null; -} - -export interface TurnDump { - readonly http?: HttpCapture; - requestedModel(model: string): void; - success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void; - error(kind: 'upstream' | 'gateway', upstream?: string): void; - failed(reason: unknown, options?: { readonly fallback: boolean }): void; - frame(frame: ProtocolFrame): void | Promise; - /** Begins recording one stream. Every call is a new one, which is what lets a turn that - * opens two — a sub-request beside the answer — keep them apart. */ - openStream(): StreamRecording; - /** How much of the answer actually went out. A transport that writes its own frames counts - * them itself, because nothing downstream of it can. */ - recordSentPayloadBytes(byteLength: number): void; - /** Closes the recording. A transport that owns its own response — the WebSocket turn — - * states what it sent; an HTTP one hands over the response so the bytes can be teed. */ - finalize(status: number | null, responseBytes: number | readonly unknown[]): void; - finalize(response: Response): Response; -} diff --git a/packages/gateway/src/dump/types.ts b/packages/gateway/src/dump/types.ts deleted file mode 100644 index d7fa6c869b..0000000000 --- a/packages/gateway/src/dump/types.ts +++ /dev/null @@ -1,158 +0,0 @@ -// Per-API-key request-dump types. Three shapes split by lifecycle: -// -// - the write shape (`DumpWrite*`) carries a request body prepared while the -// upstream is running, so persistence does not need the original bytes; -// - the storage/read shape (`Stored*`, with `body: Uint8Array`) is what the -// store rehydrates and what flows in-process to the dashboard's reader; -// - the wire shape (`Dump*`, with `body: DumpBody`) is the JSON-friendly -// view served to the dashboard by `dumpRecordToWire`. -// -// `DumpMetadata` and `DumpStreamEvent` are body-free and shared verbatim. - -import type { z } from 'zod'; - -import type { - dumpCaptureSchema, - dumpStreamEventSchema, -} from './schemas.ts'; -import type { DumpMetadata } from '@floway-dev/dump/types'; - -export type DumpCapture = z.infer; - -// Canonical protocol frame the gateway's respond layer fans out to every -// dump-enabled key. Stored as ProtocolFrame (not the SSE-serialized form) -// so the gateway's live fold and the dashboard's cold fold can share the -// same `collectXProtocolEventsToResult` reducer; the SSE wire view is -// derived on demand by the dashboard via `XProtocolFrameToSSEFrame`. -// -// `unknown` for the event payload because the storage layer is protocol- -// agnostic — the dashboard dispatches the right per-protocol serializer -// based on `meta.path`. -export type DumpStreamEvent = z.infer; - -// --- Storage shape (in-process, never serialized) --- - -export interface StoredDumpRequest { - method: string; - path: string; - // Captured verbatim with no redaction: the dump only surfaces to the - // owning API key's operator, who already holds the key. - headers: Array<[string, string]>; - body: Uint8Array; -} - -export type PreparedDumpRequestBody = { - readonly encoding: 'identity' | 'gzip'; - readonly bytes: Uint8Array; - readonly decodedByteLength: number; -}; - -export interface DumpWriteRequest { - method: string; - path: string; - headers: Array<[string, string]>; - body: PreparedDumpRequestBody; -} - -export type StoredDumpResponseBody = - | { type: 'stream'; events: DumpStreamEvent[] } - | { type: 'bytes'; body: Uint8Array } - | { type: 'none' }; - -// The pre-translation (target-protocol) view of what the upstream returned. -// Present only for turns that traversed a translation (`traverseTranslation`) -// and only when the inner attempt produced events or an upstream api-error. -// Native turns and gateway-synthesized errors have no upstream view. -export interface StoredDumpUpstreamResponse { - status: number | null; - headers: Array<[string, string]>; - body: StoredDumpResponseBody; -} - -export interface StoredDumpResponse { - status: number | null; - headers: Array<[string, string]>; - body: StoredDumpResponseBody; - // Parallel pre-translation upstream body. Absent on native turns and on - // records written before this field existed. Named `upstream` here (the - // response body — what the upstream sent before Floway translated it) - // independently of `DumpMetadata.upstream` (a provider identity); the two - // live on different objects. - readonly upstream?: StoredDumpUpstreamResponse; -} - -export type StoredDumpEdgeRecord = { - shape: 'edge'; - capture?: DumpCapture; - meta: DumpMetadata; - request: StoredDumpRequest; - response: StoredDumpResponse; -}; - -export type DumpWriteEdgeRecord = { - shape: 'edge'; - capture?: DumpCapture; - meta: DumpMetadata; - request: DumpWriteRequest; - response: StoredDumpResponse; -}; - -// --- Wire shape (serialized JSON over the dashboard's control plane) --- - -// `utf8` is chosen from the upstream content-type, with a UTF-8-fatal -// fallback to `base64` when a textual content-type carried non-UTF-8 bytes. -// JSON cannot carry `Uint8Array` directly, so this discriminator lives at -// the HTTP boundary and nowhere else. -export type DumpBody = - | { encoding: 'utf8'; data: string } - | { encoding: 'base64'; data: string }; - -interface DumpRequest { - method: string; - path: string; - headers: Array<[string, string]>; - body: DumpBody; -} - -export type DumpResponseBody = - | { type: 'stream'; events: DumpStreamEvent[] } - | { type: 'bytes'; body: DumpBody } - | { type: 'none' }; - -export interface DumpUpstreamResponse { - status: number | null; - headers: Array<[string, string]>; - body: DumpResponseBody; -} - -interface DumpResponse { - status: number | null; - headers: Array<[string, string]>; - body: DumpResponseBody; - readonly upstream?: DumpUpstreamResponse; -} - -export type DumpEdgeRecord = { - shape: 'edge'; - capture?: DumpCapture; - meta: DumpMetadata; - request: DumpRequest; - response: DumpResponse; -}; - -// A run's event stream records every stage and both directions. -export type StoredDumpRunRecord = { - shape: 'run'; - meta: DumpMetadata; - events: Uint8Array; -}; - -export type DumpRunRecord = { - shape: 'run'; - meta: DumpMetadata; - events: string; -}; - -export type StoredDumpRecord = StoredDumpEdgeRecord | StoredDumpRunRecord; -export type DumpWriteRecord = DumpWriteEdgeRecord | StoredDumpRunRecord; -export type DumpRecord = DumpEdgeRecord | DumpRunRecord; diff --git a/packages/gateway/src/dump/wire.ts b/packages/gateway/src/dump/wire.ts deleted file mode 100644 index fe74a0320e..0000000000 --- a/packages/gateway/src/dump/wire.ts +++ /dev/null @@ -1,76 +0,0 @@ -import type { - DumpBody, - DumpRecord, - DumpResponseBody, - DumpUpstreamResponse, - StoredDumpRecord, - StoredDumpResponseBody, - StoredDumpUpstreamResponse, -} from './types.ts'; -import { encodeBase64, isTextualMediaType } from '@floway-dev/protocols/common'; - -const contentTypeOf = (headers: ReadonlyArray): string => - headers.find(([name]) => name.toLowerCase() === 'content-type')?.[1] ?? ''; - -// Wire encoding decision: textual content-types try UTF-8 first and fall -// back to base64 when the bytes do not decode cleanly (a content-type -// that lied about being text). -export const encodeBodyForWire = (bytes: Uint8Array, contentType: string): DumpBody => { - if (isTextualMediaType(contentType)) { - try { - return { encoding: 'utf8', data: new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(bytes) }; - } catch {} - } - return { encoding: 'base64', data: encodeBase64(bytes) }; -}; - -const responseBodyToWire = (body: StoredDumpResponseBody, contentType: string): DumpResponseBody => { - switch (body.type) { - case 'stream': return { type: 'stream', events: body.events }; - case 'bytes': return { type: 'bytes', body: encodeBodyForWire(body.body, contentType) }; - case 'none': return { type: 'none' }; - } -}; - -export const upstreamResponseToWire = ( - upstream: StoredDumpUpstreamResponse | undefined, -): DumpUpstreamResponse | undefined => { - if (upstream === undefined) return undefined; - // The upstream body's content-type comes from the upstream response headers - // (api-error path); the `stream` branch ignores it. - return { - status: upstream.status, - headers: upstream.headers, - body: responseBodyToWire(upstream.body, contentTypeOf(upstream.headers)), - }; -}; - -// Converts stored bytes to the JSON representation served by the control plane. -export const dumpRecordToWire = (record: StoredDumpRecord): DumpRecord => { - if (record.shape === 'run') { - return { - shape: 'run', - meta: record.meta, - events: new TextDecoder('utf-8', { fatal: true }).decode(record.events), - }; - } - return { - shape: 'edge', - ...(record.capture === undefined ? {} : { capture: record.capture }), - meta: record.meta, - request: { - method: record.request.method, - path: record.request.path, - headers: record.request.headers, - body: encodeBodyForWire(record.request.body, contentTypeOf(record.request.headers)), - }, - response: { - status: record.response.status, - headers: record.response.headers, - body: responseBodyToWire(record.response.body, contentTypeOf(record.response.headers)), - ...(record.response.upstream !== undefined ? { - upstream: upstreamResponseToWire(record.response.upstream), - } : {}), - }, - }; -}; diff --git a/packages/gateway/src/repo/dump-store.ts b/packages/gateway/src/repo/dump-store.ts index 102bd166cc..ab9d66f9ad 100644 --- a/packages/gateway/src/repo/dump-store.ts +++ b/packages/gateway/src/repo/dump-store.ts @@ -1,26 +1,23 @@ import { DUMP_FILE_PREFIX, SPILLED_FILE_STAGE_GRACE_MS } from './spilled-files-policy.ts'; import { parseUpstreamHue, parseUpstreamKind } from './upstream-parse.ts'; -import { dumpCaptureEnvelopeSchema } from '../dump/schemas.ts'; import { decodeDumpBodyDescriptor, - decodeDumpHeaders, - decodeDumpStreamEvents, decodePersistedDumpMetadata, encodeDumpBodyDescriptor, - encodeDumpHeaders, - encodeDumpStreamEvents, encodePersistedDumpMetadata, } from '../dump/storage-codec.ts'; import type { DumpBodyDescriptor } from '../dump/storage-codec.ts'; -import type { DumpListOptions, DumpStore } from '../dump/store-contract.ts'; -import type { DumpWriteRecord, PreparedDumpRequestBody, StoredDumpRecord, StoredDumpEdgeRecord, StoredDumpRequest, StoredDumpResponse, StoredDumpResponseBody, StoredDumpUpstreamResponse } from '../dump/types.ts'; -import { upstreamResponseToWire } from '../dump/wire.ts'; import { gunzipBytes, gzipBytes } from '../shared/gzip.ts'; -import type { DumpMetadata, DumpRecordId, DumpUpstreamRef } from '@floway-dev/dump/types'; +import type { + DumpListOptions, DumpStore, + DumpMetadata, + DumpRecordId, + DumpUpstreamRef, + StoredDumpRecord, +} from '@floway-dev/dump/types'; import type { FileStore, SqlDatabase } from '@floway-dev/platform'; -import { decodeForgivingBase64 } from '@floway-dev/protocols/common'; -// Bodies live at `dumps/v1/{keyId}/{YYYYMMDDHH}/{recordId}-{uniqueSuffix}.{req|resp|run}.gz`. +// Bodies live at `dumps/v1/{keyId}/{YYYYMMDDHH}/{recordId}-{uniqueSuffix}.run.gz`. // The hour segment remains useful for operator inspection; lifecycle and // collection are driven by the shared spilled_files registry. @@ -33,14 +30,10 @@ interface DumpRow { upstream_kind: string | null; upstream_hue: number | null; meta_json: string; - request_headers_json: string; - response_headers_json: string | null; - request_body_descriptor: string | null; response_body_descriptor: string | null; - response_upstream_body_descriptor: string | null; } -// A null `upstream_id` means no upstream was identified at capture time +// A null `upstream_id` means no upstream was identified for the run // (auth/validation reject, no candidate matched); a non-null id with a null // joined `upstream_name` means the referenced upstream was since deleted. // `upstreams.name`/`provider` are NOT NULL so checking name alone suffices. @@ -67,29 +60,8 @@ const hourBucket = (ms: number): string => { return `${y}${m}${d}${h}`; }; -const bodyPath = (keyId: string, bucket: string, recordId: string, side: 'req' | 'resp' | 'resp.up' | 'run'): string => - `${DUMP_FILE_PREFIX}${keyId}/${bucket}/${recordId}-${crypto.randomUUID()}.${side}.gz`; - -const putRawBody = async ( - files: FileStore, - key: string, - rawBytes: Uint8Array, - type: DumpBodyDescriptor['type'], -): Promise => { - const gz = await gzipBytes(rawBytes); - await files.put(key, gz); - return { key, type }; -}; - -const putPreparedBody = async ( - files: FileStore, - key: string, - prepared: PreparedDumpRequestBody, -): Promise => { - const gz = prepared.encoding === 'gzip' ? prepared.bytes : await gzipBytes(prepared.bytes); - await files.put(key, gz); - return { key, type: 'bytes' }; -}; +const bodyPath = (keyId: string, bucket: string, recordId: string): string => + `${DUMP_FILE_PREFIX}${keyId}/${bucket}/${recordId}-${crypto.randomUUID()}.run.gz`; const fetchBody = async (files: FileStore, descriptor: DumpBodyDescriptor): Promise => { const gz = await files.get(descriptor.key); @@ -97,143 +69,28 @@ const fetchBody = async (files: FileStore, descriptor: DumpBodyDescriptor): Prom return await gunzipBytes(gz); }; -// A run record has no edge halves at row level — its request and response are -// events inside the stream — and `request_headers_json` is NOT NULL. The empty -// list is how the row spells "this shape has none"; nothing reads it back, -// because `get` dispatches on the body descriptor first. -const NO_EDGE_HEADERS = '[]'; +// The SQL row requires request_headers_json; headers themselves live in run facts. +const EMPTY_HEADERS_JSON = '[]'; export class FileDumpStore implements DumpStore { constructor(private readonly db: SqlDatabase, private readonly files: FileStore) {} - async prepareRequestBody(body: Uint8Array): Promise { - return { - encoding: 'gzip', - bytes: await gzipBytes(body), - decodedByteLength: body.byteLength, - }; - } - - // Both shapes take the same route: files are staged in the registry, written, - // and only then pointed at by a row. A run's NDJSON is one more body file - // under that contract, carried by the response descriptor — which is what - // leaves retention, the sweep and the files-before-row ordering untouched by - // its arrival. - async put(keyId: string, record: DumpWriteRecord): Promise { - const bucket = hourBucket(record.meta.completedAt); - const requestFileKey = record.shape === 'run' || record.request.body.decodedByteLength === 0 - ? null - : bodyPath(keyId, bucket, record.meta.id, 'req'); - const responseFileKey = record.shape === 'run' - ? bodyPath(keyId, bucket, record.meta.id, 'run') - : record.response.body.type === 'none' - || (record.response.body.type === 'bytes' && record.response.body.body.byteLength === 0) - ? null - : bodyPath(keyId, bucket, record.meta.id, 'resp'); - // The pre-translation upstream body, when present. Same descriptor shape - // as the downstream response body (`{key, type}`), spilled under a - // distinct `resp.up` side so the sweep can own it independently. - const capture = record.shape === 'edge' ? record.capture : undefined; - const upstream = record.shape === 'edge' ? record.response.upstream : undefined; - const upstreamBody = upstream?.body; - const upstreamFileKey = capture !== undefined - ? bodyPath(keyId, bucket, record.meta.id, 'resp.up') - : upstreamBody === undefined - ? null - : upstreamBody.type === 'bytes' && upstreamBody.body.byteLength === 0 - ? null - : upstreamBody.type === 'none' - ? null - : bodyPath(keyId, bucket, record.meta.id, 'resp.up'); - const staged = [ - ...(requestFileKey === null ? [] : [{ fileKey: requestFileKey, ownerKind: 'dump-request' }]), - ...(responseFileKey === null ? [] : [{ fileKey: responseFileKey, ownerKind: 'dump-response' }]), - ...(upstreamFileKey === null ? [] : [{ fileKey: upstreamFileKey, ownerKind: 'dump-response-upstream' }]), - ]; - if (staged.length > 0) { - await this.db - .prepare( - `INSERT INTO spilled_files (file_key, owner_kind, owner_key, state, collect_after) - SELECT - json_extract(value, '$.fileKey'), - json_extract(value, '$.ownerKind'), - json_array(?, ?), - 'staged', - ? - FROM json_each(?)`, - ) - .bind(keyId, record.meta.id, Date.now() + SPILLED_FILE_STAGE_GRACE_MS, JSON.stringify(staged)) - .run(); - } - - let requestDescriptor: DumpBodyDescriptor | null = null; - let responseDescriptor: DumpBodyDescriptor | null = null; - if (record.shape === 'run') { - responseDescriptor = await putRawBody(this.files, responseFileKey!, record.events, 'run'); - } else { - if (requestFileKey !== null) { - requestDescriptor = await putPreparedBody(this.files, requestFileKey, record.request.body); - } - if (record.response.body.type === 'bytes') { - if (record.response.body.body.byteLength > 0) { - responseDescriptor = await putRawBody(this.files, responseFileKey!, record.response.body.body, 'bytes'); - } - } else if (record.response.body.type === 'stream') { - responseDescriptor = await putRawBody( - this.files, - responseFileKey!, - new TextEncoder().encode(encodeDumpStreamEvents(record.response.body.events, `dump record ${record.meta.id} response events`)), - 'events', - ); - } - } - - let upstreamDescriptor: DumpBodyDescriptor | null = null; - if (capture !== undefined) { - const envelope = dumpCaptureEnvelopeSchema.parse({ version: 1, capture, upstream: upstreamResponseToWire(upstream) }); - upstreamDescriptor = await putRawBody(this.files, upstreamFileKey!, new TextEncoder().encode(JSON.stringify(envelope)), 'capture'); - } else if (upstreamBody !== undefined) { - if (upstreamBody.type === 'bytes') { - if (upstreamBody.body.byteLength > 0) { - upstreamDescriptor = await putRawBody(this.files, upstreamFileKey!, upstreamBody.body, 'bytes'); - } - } else if (upstreamBody.type === 'stream') { - upstreamDescriptor = await putRawBody( - this.files, - upstreamFileKey!, - new TextEncoder().encode(encodeDumpStreamEvents(upstreamBody.events, `dump record ${record.meta.id} upstream response events`)), - 'events', - ); - } - } - - // Files before row — a partial failure leaves orphan files the sweep - // collects, never an orphan row whose detail fetch would 404. + async put(keyId: string, record: StoredDumpRecord): Promise { + const { meta, events } = record; + const fileKey = bodyPath(keyId, hourBucket(meta.completedAt), meta.id); + await this.db.prepare( + `INSERT INTO spilled_files (file_key, owner_kind, owner_key, state, collect_after) + VALUES (?, 'dump-response', ?, 'staged', ?)`, + ).bind(fileKey, JSON.stringify([keyId, meta.id]), Date.now() + SPILLED_FILE_STAGE_GRACE_MS).run(); + await this.files.put(fileKey, await gzipBytes(events)); await this.db.prepare( `INSERT INTO dump_records - (key_id, id, created_at, upstream_id, meta_json, request_headers_json, response_headers_json, request_body_descriptor, response_body_descriptor, response_upstream_body_descriptor) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + (key_id, id, created_at, upstream_id, meta_json, request_headers_json, response_headers_json, request_body_descriptor, response_body_descriptor) + VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, ?)`, ).bind( - keyId, - record.meta.id, - record.meta.completedAt, - record.meta.upstream?.id ?? null, - encodePersistedDumpMetadata(record.meta, `dump record ${record.meta.id} metadata`), - record.shape === 'run' - ? NO_EDGE_HEADERS - : encodeDumpHeaders(record.request.headers, `dump record ${record.meta.id} request headers`), - record.shape === 'run' || record.response.body.type === 'none' - ? null - : encodeDumpHeaders(record.response.headers, `dump record ${record.meta.id} response headers`), - requestDescriptor === null - ? null - : encodeDumpBodyDescriptor(requestDescriptor, `dump record ${record.meta.id} request body descriptor`), - responseDescriptor === null - ? null - : encodeDumpBodyDescriptor(responseDescriptor, `dump record ${record.meta.id} response body descriptor`), - upstreamDescriptor === null - ? null - : encodeDumpBodyDescriptor(upstreamDescriptor, `dump record ${record.meta.id} upstream response body descriptor`), + keyId, meta.id, meta.completedAt, meta.upstream?.id ?? null, + encodePersistedDumpMetadata(meta, `dump record ${meta.id} metadata`), EMPTY_HEADERS_JSON, + encodeDumpBodyDescriptor({ key: fileKey, type: 'run' }, `dump record ${meta.id} run descriptor`), ).run(); } @@ -276,7 +133,7 @@ export class FileDumpStore implements DumpStore { async get(keyId: string, recordId: DumpRecordId): Promise { const row = await this.db.prepare( 'SELECT d.id, d.upstream_id, u.name AS upstream_name, u.provider AS upstream_kind, u.hue AS upstream_hue, ' - + 'd.meta_json, d.request_headers_json, d.response_headers_json, d.request_body_descriptor, d.response_body_descriptor, d.response_upstream_body_descriptor ' + + 'd.meta_json, d.response_body_descriptor ' + 'FROM dump_records d LEFT JOIN upstreams u ON u.id = d.upstream_id ' + 'JOIN api_keys k ON k.id = d.key_id AND k.deleted_at IS NULL AND k.dump_retention_seconds IS NOT NULL ' + 'WHERE d.key_id = ? AND d.id = ? AND d.created_at >= ? - k.dump_retention_seconds * 1000', @@ -287,93 +144,13 @@ export class FileDumpStore implements DumpStore { ...decodePersistedDumpMetadata(row.meta_json, `dump record ${recordId} metadata`), upstream: hydrateUpstream(row), }; - const requestDescriptor = row.request_body_descriptor === null - ? null - : decodeDumpBodyDescriptor(row.request_body_descriptor, `dump record ${recordId} request body descriptor`); const responseDescriptor = row.response_body_descriptor === null ? null : decodeDumpBodyDescriptor(row.response_body_descriptor, `dump record ${recordId} response body descriptor`); - const upstreamDescriptor = row.response_upstream_body_descriptor === null - ? null - : decodeDumpBodyDescriptor(row.response_upstream_body_descriptor, `dump record ${recordId} upstream response body descriptor`); - - // The body kind is the shape: a run was written as one NDJSON stream and - // has no edge halves to rebuild. - if (responseDescriptor?.type === 'run') { - return { shape: 'run', meta, events: await fetchBody(this.files, responseDescriptor) }; - } - - const requestHeaders = decodeDumpHeaders(row.request_headers_json, `dump record ${recordId} request headers`); - const responseHeaders = row.response_headers_json === null - ? null - : decodeDumpHeaders(row.response_headers_json, `dump record ${recordId} response headers`); - - const request: StoredDumpRequest = { - method: meta.method, - path: meta.path, - headers: requestHeaders, - body: requestDescriptor ? await fetchBody(this.files, requestDescriptor) : new Uint8Array(), - }; - - // Headers null iff `type: 'none'`; a null descriptor with headers is a - // legitimate empty-body `bytes` response (nothing to gzip), reconstructed - // here from a zero-length buffer so the discriminator round-trips. - let responseBody: StoredDumpResponseBody; - if (responseHeaders === null) { - responseBody = { type: 'none' }; - } else if (responseDescriptor === null) { - responseBody = { type: 'bytes', body: new Uint8Array() }; - } else if (responseDescriptor.type === 'events') { - const text = new TextDecoder().decode(await fetchBody(this.files, responseDescriptor)); - responseBody = { - type: 'stream', - events: decodeDumpStreamEvents(text, `dump record ${recordId} response events at key=${responseDescriptor.key}`), - }; - } else { - responseBody = { type: 'bytes', body: await fetchBody(this.files, responseDescriptor) }; + if (responseDescriptor?.type !== 'run') { + throw new Error(`dump record ${recordId} has no run stream to read`); } - - // The pre-translation upstream body, when a descriptor was persisted. - // Same rehydration rules as the downstream body; absent on native turns - // and on records written before the upstream column existed (NULL). - let upstream: StoredDumpUpstreamResponse | undefined; - let capture: StoredDumpEdgeRecord['capture']; - if (upstreamDescriptor?.type === 'capture') { - const envelope = dumpCaptureEnvelopeSchema.parse(JSON.parse(new TextDecoder().decode(await fetchBody(this.files, upstreamDescriptor)))); - capture = envelope.capture; - const stored = envelope.upstream; - if (stored !== undefined) { - upstream = { - ...stored, body: stored.body.type === 'bytes' - ? { type: 'bytes', body: stored.body.body.encoding === 'utf8' ? new TextEncoder().encode(stored.body.body.data) : decodeForgivingBase64(stored.body.body.data) } - : stored.body, - }; - } - } else if (upstreamDescriptor !== null) { - let upstreamBody: StoredDumpResponseBody; - if (upstreamDescriptor.type === 'events') { - const text = new TextDecoder().decode(await fetchBody(this.files, upstreamDescriptor)); - upstreamBody = { - type: 'stream', - events: decodeDumpStreamEvents(text, `dump record ${recordId} upstream response events at key=${upstreamDescriptor.key}`), - }; - } else { - upstreamBody = { type: 'bytes', body: await fetchBody(this.files, upstreamDescriptor) }; - } - // Upstream headers/status were not persisted separately (the bytes case - // carries an api-error envelope the dashboard renders as a body, not as - // a headered response). An empty header set + null status is the honest - // representation for what was captured. - upstream = { status: null, headers: [], body: upstreamBody }; - } - - const response: StoredDumpResponse = { - status: meta.status, - headers: responseHeaders ?? [], - body: responseBody, - ...(upstream !== undefined ? { upstream } : {}), - }; - return { shape: 'edge', meta, request, response, ...(capture === undefined ? {} : { capture }) }; + return { meta, events: await fetchBody(this.files, responseDescriptor) }; } async deleteExpiredBatch(keyId: string, now: number, limit: number): Promise { diff --git a/packages/gateway/src/repo/expiration-sweeps-sql.ts b/packages/gateway/src/repo/expiration-sweeps-sql.ts index 5cb9028c4d..c18a5035d7 100644 --- a/packages/gateway/src/repo/expiration-sweeps-sql.ts +++ b/packages/gateway/src/repo/expiration-sweeps-sql.ts @@ -19,14 +19,12 @@ interface BackfillRow { interface DumpBackfillRow extends BackfillRow { id: string; - request_body_descriptor: string | null; response_body_descriptor: string | null; - response_upstream_body_descriptor: string | null; } interface DumpFileOwner { fileKey: string; - ownerKind: 'dump-request' | 'dump-response' | 'dump-response-upstream'; + ownerKind: 'dump-response'; keyId: string; recordId: string; } @@ -62,7 +60,7 @@ export class SqlExpirationSweepsRepo implements ExpirationSweepsRepo { private async backfillCleanupSource(source: CleanupBackfillSource, cursor: number, limit: number): Promise { const config = CLEANUP_BACKFILL_SOURCES[source]; const descriptorColumns = source === 'dump_records' - ? ', id, request_body_descriptor, response_body_descriptor, response_upstream_body_descriptor' + ? ', id, response_body_descriptor' : ''; const { results } = await this.db .prepare( @@ -102,35 +100,14 @@ export class SqlExpirationSweepsRepo implements ExpirationSweepsRepo { } private async registerDumpFiles(rows: readonly DumpBackfillRow[]): Promise { - const files: DumpFileOwner[] = rows.flatMap(row => [ - ...(row.request_body_descriptor === null ? [] : [{ - fileKey: decodeDumpBodyDescriptor( - row.request_body_descriptor, - `dump record ${row.key_id}/${row.id} request body descriptor during expiration backfill`, - ).key, - ownerKind: 'dump-request' as const, - keyId: row.key_id, - recordId: row.id, - }]), - ...(row.response_body_descriptor === null ? [] : [{ - fileKey: decodeDumpBodyDescriptor( - row.response_body_descriptor, - `dump record ${row.key_id}/${row.id} response body descriptor during expiration backfill`, - ).key, - ownerKind: 'dump-response' as const, - keyId: row.key_id, - recordId: row.id, - }]), - ...(row.response_upstream_body_descriptor === null ? [] : [{ - fileKey: decodeDumpBodyDescriptor( - row.response_upstream_body_descriptor, - `dump record ${row.key_id}/${row.id} upstream response body descriptor during expiration backfill`, - ).key, - ownerKind: 'dump-response-upstream' as const, - keyId: row.key_id, - recordId: row.id, - }]), - ]); + const files: DumpFileOwner[] = rows.map(row => { + if (row.response_body_descriptor === null) throw new Error(`dump record ${row.key_id}/${row.id} has no run descriptor during expiration backfill`); + return { + fileKey: decodeDumpBodyDescriptor(row.response_body_descriptor, + `dump record ${row.key_id}/${row.id} response body descriptor during expiration backfill`).key, + ownerKind: 'dump-response', keyId: row.key_id, recordId: row.id, + }; + }); if (files.length === 0) return; await this.db .prepare( @@ -145,16 +122,7 @@ export class SqlExpirationSweepsRepo implements ExpirationSweepsRepo { JOIN dump_records AS records ON records.key_id = json_extract(incoming.value, '$.keyId') AND records.id = json_extract(incoming.value, '$.recordId') - WHERE ( - json_extract(incoming.value, '$.ownerKind') = 'dump-request' - AND json_extract(records.request_body_descriptor, '$.key') = json_extract(incoming.value, '$.fileKey') - ) OR ( - json_extract(incoming.value, '$.ownerKind') = 'dump-response' - AND json_extract(records.response_body_descriptor, '$.key') = json_extract(incoming.value, '$.fileKey') - ) OR ( - json_extract(incoming.value, '$.ownerKind') = 'dump-response-upstream' - AND json_extract(records.response_upstream_body_descriptor, '$.key') = json_extract(incoming.value, '$.fileKey') - ) + WHERE json_extract(records.response_body_descriptor, '$.key') = json_extract(incoming.value, '$.fileKey') ON CONFLICT (file_key) DO NOTHING`, ) .bind(JSON.stringify(files)) diff --git a/packages/http/src/index.ts b/packages/http/src/index.ts index 3935f33271..47eb4ea6cc 100644 --- a/packages/http/src/index.ts +++ b/packages/http/src/index.ts @@ -39,3 +39,5 @@ export { HttpProtocolError } from './errors.ts'; export type { HttpProtocolErrorCode } from './errors.ts'; export { STATUS_LINE } from './grammar.ts'; + +export { discardUpstreamResponse } from './response.ts'; diff --git a/packages/http/src/response.ts b/packages/http/src/response.ts index 8b0a1a3dbd..f5d73b1f26 100644 --- a/packages/http/src/response.ts +++ b/packages/http/src/response.ts @@ -19,3 +19,7 @@ export const exchangeResponse = (exchange: HttpResponseExchange): Response => statusText: exchange.statusText, headers: exchange.headers.map(([name, value]): [string, string] => [name, value]), }); + +export const discardUpstreamResponse = async (response: Response): Promise => { + await response.body?.cancel(); +}; diff --git a/packages/interceptor/__tests__/index_test.ts b/packages/interceptor/__tests__/index_test.ts deleted file mode 100644 index c638f43dad..0000000000 --- a/packages/interceptor/__tests__/index_test.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { test } from 'vitest'; - -import { type Interceptor, runInterceptors } from '../src/index.ts'; -import { assertEquals, assertRejects } from '@floway-dev/test-utils'; - -type TestCtx = { payload: { value: string } }; -type TestEnv = { traceId: string }; - -test('composes interceptors outermost-first and unwinds epilogues inside-out', async () => { - const calls: string[] = []; - - const outer: Interceptor = async (_ctx, _env, run) => { - calls.push('outer-before'); - const result = await run(); - calls.push('outer-after'); - return result; - }; - const inner: Interceptor = async (_ctx, _env, run) => { - calls.push('inner-before'); - const result = await run(); - calls.push('inner-after'); - return result; - }; - - await runInterceptors({ payload: { value: 'ok' } }, { traceId: 't' }, [outer, inner], () => { - calls.push('terminal'); - return Promise.resolve('done'); - }); - - assertEquals(calls, ['outer-before', 'inner-before', 'terminal', 'inner-after', 'outer-after']); -}); - -test('lets an interceptor retry by calling run() again — each call reruns the inner chain', async () => { - const ctx: TestCtx = { payload: { value: 'broken' } }; - let attempts = 0; - - const interceptor: Interceptor = async (current, _env, run) => { - const first = await run(); - if (first !== 'fail') return first; - current.payload.value = 'fixed'; - return await run(); - }; - - const result = await runInterceptors(ctx, { traceId: 't' }, [interceptor], () => { - attempts += 1; - return Promise.resolve(ctx.payload.value === 'broken' ? 'fail' : ctx.payload.value); - }); - - assertEquals(attempts, 2); - assertEquals(result, 'fixed'); -}); - -test('propagates an inner throw past each enclosing run() call site without swallowing', async () => { - const seen: string[] = []; - - const wrap = (label: string): Interceptor => async (_ctx, _env, run) => { - seen.push(`${label}-before`); - try { - return await run(); - } finally { - seen.push(`${label}-after`); - } - }; - - const boom = new Error('upstream blew up'); - await assertRejects( - () => runInterceptors({ payload: { value: 'x' } }, { traceId: 't' }, [wrap('outer'), wrap('inner')], () => Promise.reject(boom)), - Error, - 'upstream blew up', - ); - - assertEquals(seen, ['outer-before', 'inner-before', 'inner-after', 'outer-after']); -}); - -test('lets an interceptor patch context before run and transform the result after run', async () => { - const ctx: TestCtx = { payload: { value: 'original' } }; - - const interceptor: Interceptor = async (current, _env, run) => { - current.payload.value = 'patched'; - const result = await run(); - return `${result}:${current.payload.value}`; - }; - - const result = await runInterceptors(ctx, { traceId: 't' }, [interceptor], () => Promise.resolve(ctx.payload.value)); - - assertEquals(ctx.payload.value, 'patched'); - assertEquals(result, 'patched:patched'); -}); diff --git a/packages/interceptor/package.json b/packages/interceptor/package.json deleted file mode 100644 index ba6e7f7397..0000000000 --- a/packages/interceptor/package.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "@floway-dev/interceptor", - "private": true, - "version": "0.0.0", - "type": "module", - "exports": { - ".": { "import": "./src/index.ts", "types": "./src/index.ts" } - }, - "scripts": { - "typecheck": "tsc --noEmit" - }, - "devDependencies": { - "@floway-dev/test-utils": "workspace:*" - } -} diff --git a/packages/interceptor/src/index.ts b/packages/interceptor/src/index.ts deleted file mode 100644 index 7c56eea7e2..0000000000 --- a/packages/interceptor/src/index.ts +++ /dev/null @@ -1,48 +0,0 @@ -// Interceptors wrap a single typed call. Each interceptor receives the call's -// own invocation state, the ambient environment around it, and a `run` to -// delegate to the next interceptor (the innermost run executes the call -// itself). Interceptors may inspect or mutate that invocation state before -// `run`, await `run` and transform the result, short-circuit by returning -// without calling `run`, or retry by invoking `run` again. The shape is -// intentionally generic in Ctx/Env/Result so it works for any kind of call — -// provider-side wire shaping, source-side translation, retry policy — wired by -// the caller into concrete chains. -// -// `Ctx` carries the call itself — payload, headers, chosen target — and is the -// slot interceptors write to. `Env` carries what surrounds the call and does -// not belong to it: the gateway's chat chains pass the request-scoped gateway -// context there, while the provider-boundary chains have nothing ambient to -// hand down and pass `{}`. -// -// ## Mutation convention -// -// Mutations applied to `ctx` or `env` before `run()` propagate forward through -// every downstream interceptor and into the terminal call. They are **one-way**: -// the interceptor that wrote a field does not restore it on the way out, and -// the framework does not snapshot/rewind state for it. Whatever consumes the -// chain's output post-run (the caller that invoked `runInterceptors`, an outer -// interceptor's after-`run()` code) must keep its own captured copy of any input -// it still needs. -// -// The convention exists because partial adoption is the worst case: if some -// interceptors restore and others don't, there is no honest invariant the -// rest of the codebase can rely on — readers can no longer tell what `ctx` -// will look like at any given seam without auditing every interceptor in the -// chain. Forbidding restore everywhere is the only way to get a single -// predictable shape. -// -// The framework does not enforce this; reviewers do. A new interceptor that -// writes `ctx.foo = bar` in `try` and `ctx.foo = original` in `finally` is a -// convention violation, not a feature. -export type InterceptorRun = () => Promise; -export type Interceptor = (ctx: Ctx, env: Env, run: InterceptorRun) => Promise; - -export const runInterceptors = async ( - ctx: Ctx, - env: Env, - interceptors: readonly Interceptor[], - terminal: InterceptorRun, -): Promise => { - const run = (index: number): Promise => (index < interceptors.length ? interceptors[index](ctx, env, () => run(index + 1)) : terminal()); - return await run(0); -}; diff --git a/packages/interceptor/tsconfig.json b/packages/interceptor/tsconfig.json deleted file mode 100644 index 4cffc03465..0000000000 --- a/packages/interceptor/tsconfig.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "include": ["vitest.config.ts", "src/**/*.ts", "__tests__/**/*.ts"] -} diff --git a/packages/interceptor/vitest.config.ts b/packages/interceptor/vitest.config.ts deleted file mode 100644 index a94d8e60e5..0000000000 --- a/packages/interceptor/vitest.config.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { defineConfig } from 'vitest/config'; - -export default defineConfig({ - test: { - include: ['__tests__/**/*_test.ts'], - environment: 'node', - }, -}); diff --git a/packages/pipeline/src/dump.ts b/packages/pipeline/src/dump.ts index e83c006479..75e85d7574 100644 --- a/packages/pipeline/src/dump.ts +++ b/packages/pipeline/src/dump.ts @@ -341,3 +341,60 @@ export const encodeRun = (events: readonly Event[], options?: { readonly shareSt * so the stored file and the live protocol are the same bytes. */ export const toNdjson = (events: readonly DumpEvent[]): string => events.map(event => `${JSON.stringify(event)}\n`).join(''); + +// --- The reading ------------------------------------------------------------------- + +/** + * Resolves a run's own object space as its events arrive. + * + * The encoder interns: a value that appears twice is written once as a node under an `object` + * event and pointed at by `{"$": n}` everywhere after. A reader folds the same way — it keeps + * every node it has been given and resolves a reference against them — and it can do so line by + * line, because the format's own invariant is that a reference never points forward. + * + * What comes back is the value as the run held it, with the two things JSON cannot carry + * restored: `undefined` and the numeric edges. A secret stays redacted, because redaction is + * what was stored; bytes stay base64, because reading them back out is a decision about what + * they are and this knows only that they are bytes. + */ +export const createRunReader = () => { + const nodes = new Map(); + const resolved = new Map(); + + const read = (stored: Stored): unknown => { + if (stored === null || typeof stored !== 'object') return stored; + if (Array.isArray(stored)) return stored.map(read); + if ('$' in stored) { + const id = stored.$ as number; + if (resolved.has(id)) return resolved.get(id); + const node = nodes.get(id); + if (node === undefined) throw new Error(`dump reference $${id} points at a node no event carried`); + const value = read(node); + resolved.set(id, value); + return value; + } + if ('$stream' in stored) return { stream: stored.$stream }; + if ('$secret' in stored) return stored.$secret; + if ('$bytes' in stored) return { bytes: stored.$bytes }; + if ('$undefined' in stored) return undefined; + if ('$number' in stored) return stored.$number === 'NaN' ? NaN : stored.$number === 'Infinity' ? Infinity : -Infinity; + if ('$bigint' in stored) return BigInt(stored.$bigint as string); + return Object.fromEntries(Object.entries(stored).map(([key, child]) => [decodeKey(key), read(child)])); + }; + + /** What one event says. An `object` event says nothing on its own — it is the space the + * events around it are written against — so it reads as `null`. */ + return (event: DumpEvent): { readonly facts?: Record; readonly frames?: readonly unknown[] } | null => { + if (event.type === 'object') { + event.nodes.forEach((node, index) => { nodes.set(event.fromObjectId + index, node); }); + return null; + } + if (event.type === 'stream.frame') return { frames: event.frames.map(read) }; + if (event.type === 'stage.entered' || event.type === 'stage.leaved') { + return event.facts === undefined + ? {} + : { facts: Object.fromEntries(Object.entries(event.facts).map(([key, value]) => [decodeKey(key), read(value)])) }; + } + return {}; + }; +}; diff --git a/packages/pipeline/src/stage.ts b/packages/pipeline/src/stage.ts index f82b42ec62..42189d49a6 100644 --- a/packages/pipeline/src/stage.ts +++ b/packages/pipeline/src/stage.ts @@ -206,8 +206,8 @@ export function defineStage(s: object): Stage { // `transform` fills the `execute` slot. Two layers: `open` holds the per-invocation state // that spans both directions, and each direction is one function returning the record it // hands on. Either side may be omitted, and an omitted one passes through unchanged — -// which is the point, because today's interceptors call the continuation 107 times across -// 68 files and 50 of those calls are a bare `return run();`. +// which is the point, because most rules shape one direction and would otherwise spend a +// line handing the other one back untouched. // // A stream, a value and a failure sit at one key, so telling them apart is reading a // value, and the stage does that where it needs to. There is no outcome-shape dispatch. diff --git a/packages/platform/src/image-processor.ts b/packages/platform/src/image-processor.ts index 03569b5f62..8488ac8274 100644 --- a/packages/platform/src/image-processor.ts +++ b/packages/platform/src/image-processor.ts @@ -91,10 +91,9 @@ export const compressBytesToWebp = async ( // In-memory passthrough used by tests. There is no WebP codec available under // the test runtime, so this stub returns the input bytes unchanged; it exists -// only to satisfy the ImageProcessor contract so the egress interceptors run -// end-to-end. Interceptor behaviour (which images are rewritten, what target -// is computed) is asserted against dedicated spy processors in the -// interceptor tests, not against this stub. +// only to satisfy the ImageProcessor contract so the provider stages run +// end-to-end. Image selection and target dimensions are asserted against +// dedicated spy processors in the owning stage tests. export const createInMemoryImageProcessor = (): ImageProcessor => ({ compressToWebp: input => Promise.resolve(input), }); diff --git a/packages/protocols/__tests__/anthropic-messages/usage_test.ts b/packages/protocols/__tests__/anthropic-messages/usage_test.ts index af07c87099..5b852c1325 100644 --- a/packages/protocols/__tests__/anthropic-messages/usage_test.ts +++ b/packages/protocols/__tests__/anthropic-messages/usage_test.ts @@ -97,46 +97,31 @@ test('Anthropic Messages usage snapshots keep the served tier a later null does .toEqual({ output_tokens: 2, speed: undefined, service_tier: 'standard' }); }); -test('Anthropic Messages usage snapshots preserve nullable iterations and isolate all nested iteration data', () => { +test('Anthropic Messages usage snapshots carry the iterations union through, nulls included', () => { + // The array is opaque: nothing in this gateway reads inside an entry, so what a snapshot owes + // it is to arrive whole and to keep saying what the upstream said — including `null`, which is + // the upstream stating it has no iterations rather than not mentioning them. expect(anthropicMessagesUsageSnapshot({ output_tokens: 0, iterations: null })).toEqual({ output_tokens: 0, iterations: null }); - const source = [{ + const iterations = [{ type: 'compaction', input_tokens: 7, cache_creation: { ephemeral_5m_input_tokens: 3 }, - provider_metadata: { - attempts: [{ regions: ['us-east', 'us-west'] }], - }, + provider_metadata: { attempts: [{ regions: ['us-east', 'us-west'] }] }, }]; - const snapshot = anthropicMessagesUsageSnapshot({ output_tokens: 0, iterations: source }); - source[0].cache_creation.ephemeral_5m_input_tokens = 9; - source[0].provider_metadata.attempts[0].regions.push('eu-west'); + const snapshot = anthropicMessagesUsageSnapshot({ output_tokens: 0, iterations }); + expect(snapshot.iterations).toEqual(iterations); - expect(snapshot.iterations).toEqual([{ - type: 'compaction', - input_tokens: 7, - cache_creation: { ephemeral_5m_input_tokens: 3 }, - provider_metadata: { - attempts: [{ regions: ['us-east', 'us-west'] }], - }, - }]); + // A later reading replaces the array rather than merging into it, so a `null` delta is a + // statement and not an omission. expect(mergeAnthropicMessagesUsageSnapshot(snapshot, { output_tokens: 1, iterations: null }).iterations).toBeNull(); }); -test('Anthropic Messages usage snapshot merges isolate opaque nested iteration data from the delta', () => { - const iterations = [{ - type: 'model', - provider_metadata: { - attempts: [{ warnings: ['slow'] }], - }, - }]; - const merged = mergeAnthropicMessagesUsageSnapshot({ output_tokens: 0 }, { output_tokens: 1, iterations }); - iterations[0].provider_metadata.attempts[0].warnings[0] = 'mutated'; +test('Anthropic Messages usage snapshot merges take the delta-s iterations over the current-s', () => { + const current = anthropicMessagesUsageSnapshot({ output_tokens: 0, iterations: [{ type: 'model' }] }); + const iterations = [{ type: 'model', provider_metadata: { attempts: [{ warnings: ['slow'] }] } }]; - expect(merged.iterations).toEqual([{ - type: 'model', - provider_metadata: { - attempts: [{ warnings: ['slow'] }], - }, - }]); + expect(mergeAnthropicMessagesUsageSnapshot(current, { output_tokens: 1, iterations }).iterations).toEqual(iterations); + // And a delta that says nothing about them leaves what the run already had. + expect(mergeAnthropicMessagesUsageSnapshot(current, { output_tokens: 1 }).iterations).toEqual([{ type: 'model' }]); }); diff --git a/packages/protocols/__tests__/openai-images/response_test.ts b/packages/protocols/__tests__/openai-images/response_test.ts index d1b590b086..4fef2fe02d 100644 --- a/packages/protocols/__tests__/openai-images/response_test.ts +++ b/packages/protocols/__tests__/openai-images/response_test.ts @@ -63,14 +63,15 @@ describe('OpenAI Images usage', () => { }); describe('OpenAI Images failures', () => { - test('an upstream error object is what the client is sent, and anything else becomes the gateway envelope', () => { + test('an upstream error object yields its message, and the gateway envelope carries the gateway-s', () => { + // Which of the two a client is actually sent is `renderFailure`'s decision, over all three + // tiers at once; what this protocol owns is reading an upstream's words and writing its own. const upstream = { error: { message: 'Your request was rejected', type: 'image_generation_user_error', code: 'moderation_blocked' } }; expect(upstreamErrorMessage(upstream)).toBe('Your request was rejected'); - expect(renderErrorEnvelope('Your request was rejected', upstream)).toBe(upstream); expect(upstreamErrorMessage('upstream is down')).toBeUndefined(); - expect(renderErrorEnvelope('Model gpt-image-1 is not available on any configured upstream.', undefined)) + expect(renderErrorEnvelope('Model gpt-image-1 is not available on any configured upstream.')) .toEqual({ error: { message: 'Model gpt-image-1 is not available on any configured upstream.', type: 'api_error' } }); }); }); diff --git a/packages/protocols/src/anthropic-messages/index.ts b/packages/protocols/src/anthropic-messages/index.ts index f61a05c576..082ca4303d 100644 --- a/packages/protocols/src/anthropic-messages/index.ts +++ b/packages/protocols/src/anthropic-messages/index.ts @@ -397,5 +397,5 @@ export const parseAnthropicBetaHeader = (raw: string | null | undefined): readon export { ANTHROPIC_MESSAGES_MISSING_TERMINAL_MESSAGE, collectAnthropicMessagesProtocolEventsToResult } from './to-result.ts'; export { generateAnthropicId } from './id.ts'; export { reassembleAnthropicMessagesEvents } from './reassemble.ts'; -export { anthropicMessagesProtocolFrameToSSEFrame } from './to-sse.ts'; +export { anthropicMessagesEventToSsePayload, anthropicMessagesProtocolFrameToSSEFrame } from './to-sse.ts'; export { PROMPT_TOO_LONG_MESSAGE, buildPromptTooLongBody } from './context-window-error.ts'; diff --git a/packages/protocols/src/anthropic-messages/reassemble.ts b/packages/protocols/src/anthropic-messages/reassemble.ts index 2e9fbe74d8..c9383c33ce 100644 --- a/packages/protocols/src/anthropic-messages/reassemble.ts +++ b/packages/protocols/src/anthropic-messages/reassemble.ts @@ -13,7 +13,6 @@ import type { AnthropicMessagesUsageDelta, AnthropicMessagesWebSearchToolResultBlock, } from './index.ts'; -import { cloneAnthropicMessagesUsageIterations } from './usage.ts'; import { isJsonObject } from '../common/json.ts'; import { captureExtras } from '../common/reassemble-extras.ts'; @@ -115,9 +114,7 @@ const applyAnthropicMessagesUsage = (usage: AnthropicMessagesUsage, update: Anth if (update.server_tool_use != null) { usage.server_tool_use = update.server_tool_use; } - if (update.iterations !== undefined) { - usage.iterations = cloneAnthropicMessagesUsageIterations(update.iterations); - } + if (update.iterations !== undefined) usage.iterations = update.iterations; }; const createBlockAccumulator = (event: Extract): AnthropicMessagesBlockAccumulator => { diff --git a/packages/protocols/src/anthropic-messages/to-sse.ts b/packages/protocols/src/anthropic-messages/to-sse.ts index 378b133a1f..8275695478 100644 --- a/packages/protocols/src/anthropic-messages/to-sse.ts +++ b/packages/protocols/src/anthropic-messages/to-sse.ts @@ -51,7 +51,7 @@ const citationToSsePayload = (citation: AnthropicMessagesTextCitation): Anthropi } : citation; -const anthropicMessagesEventToSsePayload = (event: AnthropicMessagesStreamEvent): AnthropicMessagesSseEventPayload => { +export const anthropicMessagesEventToSsePayload = (event: AnthropicMessagesStreamEvent): AnthropicMessagesSseEventPayload => { if (event.type === 'content_block_start') { const { content_block } = event; if (content_block.type !== 'text' || !content_block.citations) return event as AnthropicMessagesSseEventPayload; diff --git a/packages/protocols/src/anthropic-messages/usage.ts b/packages/protocols/src/anthropic-messages/usage.ts index aa8834ed22..fab3b81a91 100644 --- a/packages/protocols/src/anthropic-messages/usage.ts +++ b/packages/protocols/src/anthropic-messages/usage.ts @@ -2,6 +2,12 @@ export interface AnthropicMessagesUsageServerToolUse { web_search_requests?: number; } +// The beta usage union includes model attempts, advisor attempts, and +// compaction entries, and remains additively extensible. Floway carries it +// opaquely: nothing reads inside an entry and nothing writes one, so the array +// an upstream sent travels by reference from the event to the snapshot rather +// than being deep-copied at each of the three hops that touch it. +// https://github.com/anthropics/anthropic-sdk-typescript/blob/3b45cd3b69c956ac63384fdb09ce1d8109f3fa80/src/resources/beta/messages/messages.ts#L1724-L1829 export interface AnthropicMessagesUsageIteration { type: string; model?: string; @@ -16,13 +22,6 @@ export interface AnthropicMessagesUsageIteration { [key: string]: unknown; } -// The beta usage union includes model attempts, advisor attempts, and -// compaction entries, and remains additively extensible. Floway only needs an -// isolated opaque snapshot, not a closed projection of those variants. -// https://github.com/anthropics/anthropic-sdk-typescript/blob/3b45cd3b69c956ac63384fdb09ce1d8109f3fa80/src/resources/beta/messages/messages.ts#L1724-L1829 -export const cloneAnthropicMessagesUsageIterations = (iterations: AnthropicMessagesUsageIteration[] | null): AnthropicMessagesUsageIteration[] | null => - iterations === null ? null : structuredClone(iterations); - export interface AnthropicMessagesCacheCreationTtlTokens { ephemeral_5m_input_tokens?: number; ephemeral_1h_input_tokens?: number; @@ -101,7 +100,7 @@ export const anthropicMessagesUsageSnapshot = (usage?: AnthropicMessagesUsageDel ...(present(usage.output_tokens_details) ? { output_tokens_details: { ...usage.output_tokens_details } } : {}), ...(present(usage.service_tier) ? { service_tier: usage.service_tier } : {}), ...(present(usage.speed) ? { speed: usage.speed } : {}), - ...(usage.iterations === undefined ? {} : { iterations: cloneAnthropicMessagesUsageIterations(usage.iterations) }), + ...(usage.iterations === undefined ? {} : { iterations: usage.iterations }), }; export const mergeAnthropicMessagesUsageSnapshot = ( diff --git a/packages/protocols/src/common/error-envelope.ts b/packages/protocols/src/common/error-envelope.ts index c92fbd7408..222b60d551 100644 --- a/packages/protocols/src/common/error-envelope.ts +++ b/packages/protocols/src/common/error-envelope.ts @@ -1,11 +1,18 @@ // What a client is sent when a run produced no answer, and how an upstream's own words are // read out of what it sent. // -// An upstream that refused in its own words is handed on in them — a client reads the code -// and type inside, and the gateway has nothing truer to say than what the upstream said. This -// is why the test is whether an upstream answered at all rather than which shape it answered -// in: an OpenAI family's error is `{error:{...}}` and a rerank client's is `{message}`, and -// both are already the shape that client reads. +// An upstream that refused in its own words is handed on in them — a client reads the code and +// type inside, and the gateway has nothing truer to say than what the upstream said. The test +// is only that the upstream sent a JSON object, because the families that reach here differ in +// what that object looks like — an OpenAI family's error is `{error:{...}}` and a rerank +// client's is `{message}` — and a body that is not an object at all is one no JSON protocol's +// client can read, so it falls through to the envelope. +// +// The test is deliberately not "is this the client's own protocol". It cannot be, here: this +// function is shared and does not know which client is reading. Where a body can be the *wrong* +// protocol's — a translated wire, where the upstream spoke to a different protocol than the +// client did — the handoff drops it before it ever arrives, so what reaches here is only ever a +// body the client's own protocol produced. // // A refusal that never reached an upstream has only the gateway's own words, and those go in // the envelope every protocol here writes errors in. @@ -17,5 +24,10 @@ export const upstreamErrorMessage = (body: unknown): string | undefined => { return typeof body.error.message === 'string' ? body.error.message : undefined; }; -export const renderErrorEnvelope = (message: string, upstreamBody?: unknown): Record => - isJsonObject(upstreamBody) ? upstreamBody : { error: { message, type: 'api_error' } }; +/** The envelope the OpenAI-shaped protocols write when a refusal is the gateway's own words. + * + * Which of an upstream's body, a stage's own envelope and this one a client is actually sent is + * not decided here: that is one policy over all three tiers, and it lives with the `Failure` it + * reads. This is only the shape those protocols state a gateway-authored refusal in. */ +export const renderErrorEnvelope = (message: string): Record => + ({ error: { message, type: 'api_error' } }); diff --git a/packages/protocols/src/common/index.ts b/packages/protocols/src/common/index.ts index 2b7a1293d2..f146029528 100644 --- a/packages/protocols/src/common/index.ts +++ b/packages/protocols/src/common/index.ts @@ -16,3 +16,4 @@ export * from './parse-events.ts'; export { renderErrorEnvelope, upstreamErrorMessage } from './error-envelope.ts'; export { isJsonObject, type JsonObject } from './json.ts'; export { captureExtras } from './reassemble-extras.ts'; +export { mapKeepingIdentity, withIndexesChanged, withKeysChanged } from './rebuild.ts'; diff --git a/packages/protocols/src/common/rebuild.ts b/packages/protocols/src/common/rebuild.ts new file mode 100644 index 0000000000..5fda71756a --- /dev/null +++ b/packages/protocols/src/common/rebuild.ts @@ -0,0 +1,49 @@ +// Rebuilding a value instead of writing into one. +// +// A record's values are frozen — a fact that could be edited after the event is not a fact — so +// anything that shapes a payload it was handed returns a new one. What these two add is the +// discipline one level down, without paying for a copy of the whole tree: a rewrite that changed +// nothing hands the same object back, so an untouched message or block rides through by identity +// and only the path that actually changed is new. That is the same rule the record itself is +// folded by, and it is what keeps a long conversation costing a handful of objects rather than a +// clone per candidate. + +/** Applies `rewrite` to every item, and hands back the original array when none of them + * changed — so "nothing happened here" is the same test at every level. */ +export const mapKeepingIdentity = (items: readonly T[], rewrite: (item: T) => T): readonly T[] => { + let changed = false; + const next = items.map(item => { + const rewritten = rewrite(item); + if (rewritten !== item) changed = true; + return rewritten; + }); + return changed ? next : items; +}; + +/** The object form: rebuild with `patch` applied, or hand back the original when the patch is + * empty. `undefined` in the patch removes the key, which is what a rest-destructuring would + * do and what `delete` used to do in place. */ +export const withKeysChanged = (source: T, patch: Readonly>): T => { + const keys = Object.keys(patch); + if (keys.length === 0) return source; + const next: Record = { ...(source as Record) }; + for (const key of keys) { + if (patch[key] === undefined) delete next[key]; + else next[key] = patch[key]; + } + return next as T; +}; + +/** An array with one index replaced, or removed where the replacement is `null`. Every other + * element rides through by identity, and an index nobody replaced leaves the array itself + * untouched. */ +export const withIndexesChanged = (items: readonly T[], replacements: ReadonlyMap): readonly T[] => { + if (replacements.size === 0) return items; + const next: T[] = []; + items.forEach((item, index) => { + if (!replacements.has(index)) { next.push(item); return; } + const replacement = replacements.get(index)!; + if (replacement !== null) next.push(replacement); + }); + return next; +}; diff --git a/packages/protocols/src/openai-chat-completions/index.ts b/packages/protocols/src/openai-chat-completions/index.ts index ed7ede10ab..28cd408e6a 100644 --- a/packages/protocols/src/openai-chat-completions/index.ts +++ b/packages/protocols/src/openai-chat-completions/index.ts @@ -1,5 +1,11 @@ // OpenAI Chat Completions type definitions (subset needed for translation) +export interface OpenAIChatCompletionsErrorEvent { + error: { message: string; type?: string; name?: string; stack?: string; cause?: unknown; target_api?: string }; +} + +export type ClientOpenAIChatCompletionsStreamEvent = OpenAIChatCompletionsStreamEvent | OpenAIChatCompletionsErrorEvent; + export interface OpenAIChatCompletionsPayload { model: string; messages: OpenAIChatCompletionsMessage[]; diff --git a/packages/protocols/src/openai-chat-completions/to-sse.ts b/packages/protocols/src/openai-chat-completions/to-sse.ts index 632cfc37a9..b00d9d2a3c 100644 --- a/packages/protocols/src/openai-chat-completions/to-sse.ts +++ b/packages/protocols/src/openai-chat-completions/to-sse.ts @@ -1,12 +1,14 @@ -import type { OpenAIChatCompletionsStreamEvent } from './index.ts'; +import type { ClientOpenAIChatCompletionsStreamEvent } from './index.ts'; import { isOpenAIUsageOnlyEventShape, type ProtocolFrame, type SseFrame, sseFrame } from '../common/index.ts'; interface OpenAIChatCompletionsSseFrameOptions { includeUsageChunk: boolean; } -export const openaiChatCompletionsProtocolFrameToSSEFrame = (frame: ProtocolFrame, options: OpenAIChatCompletionsSseFrameOptions): SseFrame | null => { +export function openaiChatCompletionsProtocolFrameToSSEFrame(frame: ProtocolFrame, options: { includeUsageChunk: true }): SseFrame; +export function openaiChatCompletionsProtocolFrameToSSEFrame(frame: ProtocolFrame, options: OpenAIChatCompletionsSseFrameOptions): SseFrame | null; +export function openaiChatCompletionsProtocolFrameToSSEFrame(frame: ProtocolFrame, options: OpenAIChatCompletionsSseFrameOptions): SseFrame | null { if (frame.type === 'done') return sseFrame('[DONE]'); if (!options.includeUsageChunk && isOpenAIUsageOnlyEventShape(frame.event)) return null; return sseFrame(JSON.stringify(frame.event)); -}; +} diff --git a/packages/protocols/src/openai-images/stream.ts b/packages/protocols/src/openai-images/stream.ts index 0596177163..7f8d1ed002 100644 --- a/packages/protocols/src/openai-images/stream.ts +++ b/packages/protocols/src/openai-images/stream.ts @@ -40,7 +40,7 @@ export const parseOpenAIImagesStream = ( /** The event's own name, which the specification writes twice: as the SSE `event:` label and * as `type` inside the payload. Upstreams that write only the label have been seen on the - * Responses protocol, and every reader here takes the name off the payload, so the two are + * OpenAI Responses protocol, and every reader here takes the name off the payload, so the two are * reconciled once at the boundary rather than at each reader. An event that carries neither * is not one this protocol can place — it could be the terminal or a partial — so it ends the * read rather than being passed on as an unknown. */ diff --git a/packages/protocols/src/openai-responses/compact.ts b/packages/protocols/src/openai-responses/compact.ts index 463e5b4f65..482ab134cc 100644 --- a/packages/protocols/src/openai-responses/compact.ts +++ b/packages/protocols/src/openai-responses/compact.ts @@ -34,7 +34,7 @@ export type CanonicalOpenAIResponsesCompactPayload = Omit = {}): UpstreamRecord => { const config = { @@ -120,8 +120,8 @@ test('createAzureProvider sends upstream model ids in OpenAI-shaped request bodi return sseResponse(); }, async () => { - const chat = await instance.instance.callOpenAIChatCompletions(providerModel, { messages: [{ role: 'user', content: 'hello' }] }, undefined, noopUpstreamCallOptions()); - const responses = await instance.instance.callOpenAIResponses(providerModel, { + const chat = await collectChatProviderPipeline(instance, 'openaiChatCompletions', providerModel, { messages: [{ role: 'user', content: 'hello' }] }, undefined, noopUpstreamCallOptions()); + const responses = await collectChatProviderPipeline(instance, 'openaiResponses', providerModel, { input: [{ type: 'additional_tools', role: 'developer', @@ -132,11 +132,11 @@ test('createAzureProvider sends upstream model ids in OpenAI-shaped request bodi tools: [{ type: 'function', name: 'lookup', description: 'Look up a record.', parameters: { type: 'object', properties: {} } }], }], }], - }, 'generate', undefined, noopUpstreamCallOptions()); + }, undefined, noopUpstreamCallOptions()); const embeddings = await callProviderPipeline(instance, 'openaiEmbeddings', providerModel, { input: 'hello' }, undefined, noopUpstreamCallOptions()); - assertEquals(chat.modelKey, 'gpt-prod'); - assertEquals(responses.modelKey, 'gpt-prod'); + assertEquals(chat.facts['response.provider.modelKey'], 'gpt-prod'); + assertEquals(responses.facts['response.provider.modelKey'], 'gpt-prod'); assertEquals(embeddings.modelKey, 'gpt-prod'); }, ); @@ -177,7 +177,7 @@ test('createAzureProvider runs the OpenAI Responses boundary on compact requests }), { status: 200, headers: { 'content-type': 'application/json' } }); }, async () => { - const result = await instance.instance.callOpenAIResponses(providerModel, { + const result = await collectChatProviderPipeline(instance, 'openaiResponsesCompact', providerModel, { input: [{ type: 'additional_tools', role: 'developer', @@ -188,9 +188,9 @@ test('createAzureProvider runs the OpenAI Responses boundary on compact requests tools: [{ type: 'function', name: 'lookup', description: 'Look up a record.', parameters: { type: 'object', properties: {} } }], }], }], - }, 'compact', undefined, noopUpstreamCallOptions()); - assertEquals(result.action, 'compact'); - assertEquals(result.ok, true); + }, undefined, noopUpstreamCallOptions()); + assertEquals(result.facts['request.provider.responsesAction'], 'compact'); + assertEquals((result.output !== null && 'kind' in result.output), true); }, ); @@ -212,21 +212,15 @@ test.each(['generate', 'compact'] as const)('createAzureProvider preserves %s up await withMockedFetch( () => Promise.resolve(upstreamResponse), async () => { - const result = await instance.instance.callOpenAIResponses( - providerModel, - { input: [{ type: 'message', role: 'user', content: 'hello' }] }, - action, - undefined, - noopUpstreamCallOptions(), - ); - assertEquals(result.action, action); - assertEquals(result.ok, false); - if (result.ok) throw new Error('expected upstream error'); - assertEquals(result.modelKey, 'gpt-prod'); - assertEquals(result.response, upstreamResponse); - assertEquals(result.response.status, 418); - assertEquals(result.response.headers.get('x-upstream-error'), action); - assertEquals(await result.response.text(), '{"error":"upstream rejected the request"}'); + const result = await collectChatProviderPipeline(instance, action === 'compact' ? 'openaiResponsesCompact' : 'openaiResponses', providerModel, { input: [{ type: 'message', role: 'user', content: 'hello' }] }, undefined, noopUpstreamCallOptions()); + assertEquals(result.facts['request.provider.responsesAction'], action); + assertEquals((result.output !== null && 'kind' in result.output), false); + if ((result.output !== null && 'kind' in result.output)) throw new Error('expected upstream error'); + assertEquals(result.facts['response.provider.modelKey'], 'gpt-prod'); + if (result.response === null) throw new Error('expected HTTP failure body'); + assertEquals(result.response!.status, 418); + assertEquals(result.response!.headers.get('x-upstream-error'), action); + assertEquals(await result.response!.text(), '{"error":"upstream rejected the request"}'); }, ); }); @@ -271,10 +265,10 @@ test('createAzureProvider supports Azure AI cross-provider models with explicit return sseResponse(); }, async () => { - const chat = await instance.instance.callOpenAIChatCompletions(chatProviderModel, { messages: [{ role: 'user', content: 'hello' }] }, undefined, chatOpts); - const responses = await instance.instance.callOpenAIResponses(openaiResponsesProviderModel, { input: [{ type: 'message', role: 'user', content: 'hello' }] }, 'generate', undefined, openaiResponsesOpts); - assertEquals(chat.modelKey, 'deepseek-v4-pro'); - assertEquals(responses.modelKey, 'gpt-5.4-pro'); + const chat = await collectChatProviderPipeline(instance, 'openaiChatCompletions', chatProviderModel, { messages: [{ role: 'user', content: 'hello' }] }, undefined, chatOpts); + const responses = await collectChatProviderPipeline(instance, 'openaiResponses', openaiResponsesProviderModel, { input: [{ type: 'message', role: 'user', content: 'hello' }] }, undefined, openaiResponsesOpts); + assertEquals(chat.facts['response.provider.modelKey'], 'deepseek-v4-pro'); + assertEquals(responses.facts['response.provider.modelKey'], 'gpt-5.4-pro'); }, ); @@ -333,10 +327,10 @@ test('createAzureProvider supports native Azure Anthropic Messages models', asyn return sseResponse(); }, async () => { - const messages = await instance.instance.callAnthropicMessages(providerModel, { max_tokens: 16, messages: [{ role: 'user', content: 'hello' }] }, undefined, { ...noopUpstreamCallOptions(), anthropicBeta: ['context-1m'] }); - const count = await instance.instance.callAnthropicMessagesCountTokens(providerModel, { max_tokens: 16, messages: [{ role: 'user', content: 'hello' }] }, undefined, { ...noopUpstreamCallOptions(), anthropicBeta: ['context-1m', 'advanced-tool-use'] }); - assertEquals(messages.modelKey, 'claude-prod'); - assertEquals(count.modelKey, 'claude-prod'); + const messages = await collectChatProviderPipeline(instance, 'anthropicMessages', providerModel, { max_tokens: 16, messages: [{ role: 'user', content: 'hello' }] }, undefined, { ...noopUpstreamCallOptions(), anthropicBeta: ['context-1m'] }); + const count = await collectChatProviderPipeline(instance, 'anthropicMessagesCountTokens', providerModel, { max_tokens: 16, messages: [{ role: 'user', content: 'hello' }] }, undefined, { ...noopUpstreamCallOptions(), anthropicBeta: ['context-1m', 'advanced-tool-use'] }); + assertEquals(messages.facts['response.provider.modelKey'], 'claude-prod'); + assertEquals(count.facts['response.provider.modelKey'], 'claude-prod'); }, ); @@ -476,7 +470,7 @@ test('createAzureProvider exposes image models and routes generations with api-v assertEquals(models[0].endpoints, { openaiImagesGenerations: {}, openaiImagesEdits: {} }); const result = await callProviderPipeline(provider, 'openaiImagesGenerations', models[0], { prompt: 'hello' }, undefined, noopUpstreamCallOptions()); assertEquals(result.modelKey, 'gpt-image-2'); - assertEquals(result.response.status, 200); + assertEquals(result.response!.status, 200); }, ); assertEquals(observedUrl, 'https://example.openai.azure.com/openai/v1/images/generations?api-version=preview'); @@ -529,7 +523,7 @@ test('createAzureProvider callOpenAIImagesEdits posts multipart with model repla }], }, undefined, noopUpstreamCallOptions()); assertEquals(result.modelKey, 'gpt-image-2'); - assertEquals(result.response.status, 200); + assertEquals(result.response!.status, 200); }, ); assertEquals(observedUrl, 'https://example.openai.azure.com/openai/v1/images/edits?api-version=preview'); diff --git a/packages/provider-azure/__tests__/stages/audio-request_test.ts b/packages/provider-azure/__tests__/stages/audio-request_test.ts new file mode 100644 index 0000000000..87f00935e2 --- /dev/null +++ b/packages/provider-azure/__tests__/stages/audio-request_test.ts @@ -0,0 +1,19 @@ +import { expect, test } from 'vitest'; + +import { prepareAzureRequest } from '../../src/stages/prepare-request.ts'; +import { http } from '@floway-dev/http/pipeline'; +import type { HttpMultipartBody } from '@floway-dev/http/request-content'; +import { compose } from '@floway-dev/pipeline'; +import { observeProviderCall, providerModelFacts, type ProviderRequest, type ProviderResponse } from '@floway-dev/provider'; +import { collectHttpPipeline, noopUpstreamCallOptions, stubProviderModel } from '@floway-dev/test-utils'; + +test('audio prepare omits model fields selected by the deployment URL', async () => { + const file = { bytes: new TextEncoder().encode('audio'), name: 'meeting.wav', type: 'audio/wav' }; + const entries = [{ name: 'model', value: 'public-model' }, { name: 'file', value: file }, { name: 'response_format', value: 'json' }]; + const pipeline = compose, ProviderResponse & Record>('audio.deployment', [prepareAzureRequest({ endpoint: 'https://azure.example', apiKey: 'key', models: [] }, 'openaiAudioTranscriptions'), observeProviderCall, http]); + const executed = await collectHttpPipeline(pipeline, { 'request.provider.model': providerModelFacts(stubProviderModel({ providerData: { upstreamModelId: 'deployment' } })), 'request.provider.payload': { entries }, 'request.http.callId': 0, 'request.http.headers': [] }, { httpCall: () => ({ ...noopUpstreamCallOptions(), fetcher: async () => new Response('{}'), signal: undefined }) }); + const body = executed.facts['request.http.body'] as HttpMultipartBody; + expect(body.entries.map(entry => entry.name)).toEqual(['file', 'response_format']); + expect(body.entries[0]!.value).toBe(file); + expect(executed.facts['request.http.url']).toBe('https://azure.example/openai/deployments/deployment/audio/transcriptions?api-version=2025-04-01-preview'); +}); diff --git a/packages/provider-azure/__tests__/interceptors/openai-responses/fill-empty-namespace-descriptions_test.ts b/packages/provider-azure/__tests__/stages/openai-responses/fill-empty-namespace-descriptions_test.ts similarity index 81% rename from packages/provider-azure/__tests__/interceptors/openai-responses/fill-empty-namespace-descriptions_test.ts rename to packages/provider-azure/__tests__/stages/openai-responses/fill-empty-namespace-descriptions_test.ts index 56854cf36d..c8653731ed 100644 --- a/packages/provider-azure/__tests__/interceptors/openai-responses/fill-empty-namespace-descriptions_test.ts +++ b/packages/provider-azure/__tests__/stages/openai-responses/fill-empty-namespace-descriptions_test.ts @@ -1,11 +1,11 @@ import { test } from 'vitest'; -import { withEmptyNamespaceDescriptionsFilled } from '../../../src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; +import { fillAzureNamespaceDescriptions } from '../../../src/stages/fill-namespace-descriptions.ts'; import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; +import { applyProviderStage, type OpenAIResponsesProbe } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; -const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBoundaryCtx => ({ +const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesProbe => ({ payload, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiResponses: {} } }), @@ -36,7 +36,7 @@ test('fills Codex OpenAI Responses Lite namespace descriptions before Azure disp }], }); - await withEmptyNamespaceDescriptionsFilled(ctx, {}, async () => {}); + await applyProviderStage(fillAzureNamespaceDescriptions, ctx, async () => {}); const [item] = ctx.payload.input; if (item?.type !== 'additional_tools') throw new Error('expected additional_tools input'); diff --git a/packages/provider-azure/__tests__/fetch_test.ts b/packages/provider-azure/__tests__/stages/prepare-request_test.ts similarity index 67% rename from packages/provider-azure/__tests__/fetch_test.ts rename to packages/provider-azure/__tests__/stages/prepare-request_test.ts index 0b9a7dfbfa..bbce50ebc2 100644 --- a/packages/provider-azure/__tests__/fetch_test.ts +++ b/packages/provider-azure/__tests__/stages/prepare-request_test.ts @@ -1,19 +1,10 @@ import { test } from 'vitest'; -import { assertAzureUpstreamRecord } from '../src/config.ts'; -import { - azureFetchOpenAIChatCompletions, - azureFetchOpenAIAudioTranscriptions, - azureFetchOpenAIEmbeddings, - azureFetchOpenAIImagesGenerations, - azureFetchAnthropicMessages, - azureFetchAnthropicMessagesCountTokens, - azureFetchOpenAIResponses, - azureFetchOpenAIResponsesCompact, -} from '../src/fetch.ts'; +import { assertAzureUpstreamRecord } from '../../src/config.ts'; +import { prepareAzureRequest } from '../../src/stages/prepare-request.ts'; import type { UpstreamRecord } from '@floway-dev/provider'; -import { directFetcher, identityWrapUpstreamCall } from '@floway-dev/provider'; -import { assertEquals, withMockedFetch } from '@floway-dev/test-utils'; +import { identityWrapUpstreamCall } from '@floway-dev/provider'; +import { collectPreparedProviderRequest, testFetcher, assertEquals, withMockedFetch } from '@floway-dev/test-utils'; const baseRecord: UpstreamRecord = { id: 'up_azure', @@ -57,10 +48,10 @@ test('OpenAI v1 transports apply api-key auth and the canonical paths', async () return new Response('{}', { status: 200 }); }, async () => { - await azureFetchOpenAIChatCompletions(config, { method: 'POST', body: JSON.stringify({ model: 'set-by-provider' }) }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await azureFetchOpenAIResponses(config, { method: 'POST', body: JSON.stringify({ model: 'set-by-provider' }) }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await azureFetchOpenAIResponsesCompact(config, { method: 'POST', body: JSON.stringify({ model: 'set-by-provider' }) }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await azureFetchOpenAIEmbeddings(config, { method: 'POST', body: JSON.stringify({ model: 'set-by-provider' }) }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'openaiChatCompletions'), 'openaiChatCompletions', key => ({ upstreamModelId: key }), { method: 'POST', body: JSON.stringify({ model: 'set-by-provider' }) }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'openaiResponses'), 'openaiResponses', key => ({ upstreamModelId: key }), { method: 'POST', body: JSON.stringify({ model: 'set-by-provider' }) }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'openaiResponsesCompact'), 'openaiResponsesCompact', key => ({ upstreamModelId: key }), { method: 'POST', body: JSON.stringify({ model: 'set-by-provider' }) }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'openaiEmbeddings'), 'openaiEmbeddings', key => ({ upstreamModelId: key }), { method: 'POST', body: JSON.stringify({ model: 'set-by-provider' }) }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -81,7 +72,7 @@ test('OpenAI v1 transports apply api-key auth and the canonical paths', async () seen.map(item => item.contentType), ['application/json', 'application/json', 'application/json', 'application/json'], ); - assertEquals(seen[0].body, { model: 'set-by-provider' }); + assertEquals(seen[0].body, { model: 'set-by-provider', stream: true }); }); test('image transports append the Azure preview api-version', async () => { @@ -94,7 +85,7 @@ test('image transports append the Azure preview api-version', async () => { return new Response('{}', { status: 200 }); }, async () => { - await azureFetchOpenAIImagesGenerations(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'openaiImagesGenerations'), 'openaiImagesGenerations', key => ({ upstreamModelId: key }), { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -121,7 +112,7 @@ test('audio transcription selects the deployment on every admitted endpoint shap return new Response('{}', { status: 200 }); }, async () => { - await azureFetchOpenAIAudioTranscriptions(config, 'transcribe deployment', { method: 'POST', body: new FormData() }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'openaiAudioTranscriptions'), 'openaiAudioTranscriptions', key => ({ upstreamModelId: key }), { method: 'POST', body: new FormData() }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }, 'transcribe deployment'); }, ); assertEquals(seenUrl, expected); @@ -144,7 +135,7 @@ test('endpoint that already includes /openai/v1 routes through unchanged', async return new Response('{}', { status: 200 }); }, async () => { - await azureFetchOpenAIResponses(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'openaiResponses'), 'openaiResponses', key => ({ upstreamModelId: key }), { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -173,7 +164,7 @@ test('Foundry project endpoints route OpenAI v1 calls under the project base', a return new Response('{}', { status: 200 }); }, async () => { - await azureFetchOpenAIResponses(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'openaiResponses'), 'openaiResponses', key => ({ upstreamModelId: key }), { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -202,8 +193,8 @@ test('Foundry project endpoints split OpenAI v1 vs Anthropic surfaces', async () return new Response('{}', { status: 200 }); }, async () => { - await azureFetchOpenAIResponses(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await azureFetchAnthropicMessages(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'openaiResponses'), 'openaiResponses', key => ({ upstreamModelId: key }), { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'anthropicMessages'), 'anthropicMessages', key => ({ upstreamModelId: key }), { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -235,7 +226,7 @@ test('native Anthropic calls land on the resource Anthropic base when a project return new Response('{}', { status: 200 }); }, async () => { - await azureFetchAnthropicMessages(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'anthropicMessages'), 'anthropicMessages', key => ({ upstreamModelId: key }), { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -270,8 +261,8 @@ test('Azure Foundry Anthropic surface uses x-api-key + anthropic-version', async return new Response('{}', { status: 200 }); }, async () => { - await azureFetchAnthropicMessages(config, { method: 'POST', body: '{}' }, { extraHeaders: [['anthropic-beta', 'context-1m']], fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await azureFetchAnthropicMessagesCountTokens(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'anthropicMessages'), 'anthropicMessages', key => ({ upstreamModelId: key }), { method: 'POST', body: '{}' }, { extraHeaders: [['anthropic-beta', 'context-1m']], fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'anthropicMessagesCountTokens'), 'anthropicMessagesCountTokens', key => ({ upstreamModelId: key }), { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -315,7 +306,7 @@ test('Foundry Anthropic messages target URI is accepted and splits per surface', return new Response('{}', { status: 200 }); }, async () => { - await azureFetchAnthropicMessages(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareAzureRequest(config, 'anthropicMessages'), 'anthropicMessages', key => ({ upstreamModelId: key }), { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); diff --git a/packages/provider-azure/package.json b/packages/provider-azure/package.json index 432f72a585..9b92bc8f0b 100644 --- a/packages/provider-azure/package.json +++ b/packages/provider-azure/package.json @@ -4,7 +4,10 @@ "version": "0.0.0", "type": "module", "exports": { - ".": { "import": "./src/index.ts", "types": "./src/index.ts" } + ".": { + "import": "./src/index.ts", + "types": "./src/index.ts" + } }, "scripts": { "typecheck": "tsc --noEmit" @@ -12,7 +15,6 @@ "dependencies": { "@floway-dev/http": "workspace:*", "@floway-dev/pipeline": "workspace:*", - "@floway-dev/interceptor": "workspace:*", "@floway-dev/protocols": "workspace:*", "@floway-dev/provider": "workspace:*" }, diff --git a/packages/provider-azure/src/fetch.ts b/packages/provider-azure/src/fetch.ts deleted file mode 100644 index 9d4572b7ff..0000000000 --- a/packages/provider-azure/src/fetch.ts +++ /dev/null @@ -1,85 +0,0 @@ -import type { AzureUpstreamConfig } from './config.ts'; -import { azureAnthropicBaseUrl, azureOpenAiV1BaseUrl } from './endpoint.ts'; -import { type FetchInit, type UpstreamFetchOptions, joinBaseAndPath } from '@floway-dev/provider'; - -const azureFetchUrl = async ( - config: AzureUpstreamConfig, - surface: 'openai' | 'anthropic', - url: string, - init: FetchInit, - options: UpstreamFetchOptions, -): Promise => { - const headers = new Headers(init.headers); - if (surface === 'anthropic') { - headers.set('x-api-key', config.apiKey); - headers.set('anthropic-version', '2023-06-01'); - } else { - headers.set('api-key', config.apiKey); - } - if (init.body && !headers.has('Content-Type') && !(init.body instanceof FormData)) { - headers.set('Content-Type', 'application/json'); - } - if (options.extraHeaders) { - for (const [key, value] of options.extraHeaders) headers.set(key, value); - } - return await options.wrapUpstreamCall(() => options.fetcher(url, { ...init, headers })); -}; - -const azureFetchInternal = async ( - config: AzureUpstreamConfig, - surface: 'openai' | 'anthropic', - path: string, - init: FetchInit, - options: UpstreamFetchOptions, - query?: string, -): Promise => { - const baseUrl = surface === 'openai' ? azureOpenAiV1BaseUrl(config.endpoint) : azureAnthropicBaseUrl(config.endpoint); - const url = joinBaseAndPath(baseUrl, path); - if (!query) { - return await azureFetchUrl(config, surface, url, init, options); - } - // Append per-endpoint query through URL.searchParams so a future path - // that itself carries a query suffix does not produce `path?a?b`. - const parsed = new URL(url); - for (const [key, value] of new URLSearchParams(query).entries()) parsed.searchParams.append(key, value); - return await azureFetchUrl(config, surface, parsed.href, init, options); -}; - -const azureDeploymentScopedOpenAIAudioTranscriptionUrl = (config: AzureUpstreamConfig, deployment: string): string => { - const url = new URL(config.endpoint); - url.pathname = `/openai/deployments/${encodeURIComponent(deployment)}/audio/transcriptions`; - url.search = ''; - url.hash = ''; - url.searchParams.set('api-version', '2025-04-01-preview'); - return url.href; -}; - -export const azureFetchOpenAIChatCompletions = (config: AzureUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - azureFetchInternal(config, 'openai', '/chat/completions', init, options); -export const azureFetchOpenAIResponses = (config: AzureUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - azureFetchInternal(config, 'openai', '/responses', init, options); -export const azureFetchOpenAIResponsesCompact = (config: AzureUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - azureFetchInternal(config, 'openai', '/responses/compact', init, options); -export const azureFetchOpenAIEmbeddings = (config: AzureUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - azureFetchInternal(config, 'openai', '/embeddings', init, options); -export const azureFetchOpenAICompletions = (config: AzureUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - azureFetchInternal(config, 'openai', '/completions', init, options); -// gpt-image-2 (released 2026-04-21) and the gpt-image-1 family are exposed -// only under Azure's preview lifecycle today. We will drop the query suffix -// once Azure promotes the image endpoints to the GA default. -export const azureFetchOpenAIImagesGenerations = (config: AzureUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - azureFetchInternal(config, 'openai', '/images/generations', init, options, 'api-version=preview'); -export const azureFetchOpenAIImagesEdits = (config: AzureUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - azureFetchInternal(config, 'openai', '/images/edits', init, options, 'api-version=preview'); -// Azure selects the transcription deployment in the operation path, so the -// multipart body must omit `model`. The 2025-04 preview route covers both -// Whisper and GPT transcription deployments; all configured endpoint shapes -// reduce to their resource host before this operation-specific path is set. -// https://github.com/Azure/azure-rest-api-specs/blob/b0a48bcbffead733affe03944ef09f5e8d12f8c8/specification/cognitiveservices/OpenAI.Inference/models/audio/audio_transcription.tsp#L119-L126 -// https://github.com/Azure/azure-rest-api-specs/blob/928047803788f7377fa003a26ba2bdc2e0fcccc0/specification/cognitiveservices/OpenAI.Inference/routes/audio_transcription.tsp#L19-L49 -export const azureFetchOpenAIAudioTranscriptions = (config: AzureUpstreamConfig, deployment: string, init: FetchInit, options: UpstreamFetchOptions): Promise => - azureFetchUrl(config, 'openai', azureDeploymentScopedOpenAIAudioTranscriptionUrl(config, deployment), init, options); -export const azureFetchAnthropicMessages = (config: AzureUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - azureFetchInternal(config, 'anthropic', '/v1/messages', init, options); -export const azureFetchAnthropicMessagesCountTokens = (config: AzureUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - azureFetchInternal(config, 'anthropic', '/v1/messages/count_tokens', init, options); diff --git a/packages/provider-azure/src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts b/packages/provider-azure/src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts deleted file mode 100644 index b44ac95749..0000000000 --- a/packages/provider-azure/src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts +++ /dev/null @@ -1,22 +0,0 @@ -import type { OpenAIResponsesBoundaryCtx } from './types.ts'; -import { mapOpenAIResponsesTools } from '@floway-dev/protocols/openai-responses'; - -/** - * Azure OpenAI still enforces the former namespace-description minLength even - * though current OpenAI Responses permits an empty required string. Supply a - * deterministic model-facing description only on Azure-bound requests. - * - * References: - * - https://github.com/openai/openai-openapi/commit/466c74a42f51c02f1927bc666815251dc53845dc - * - https://github.com/openai/codex/issues/37380 - */ -export const withEmptyNamespaceDescriptionsFilled = async ( - ctx: OpenAIResponsesBoundaryCtx, - _env: object, - run: () => Promise, -): Promise => { - ctx.payload = mapOpenAIResponsesTools(ctx.payload, tool => tool.type === 'namespace' && tool.description === '' - ? { ...tool, description: `Tools in the ${tool.name} namespace.` } - : tool); - return await run(); -}; diff --git a/packages/provider-azure/src/interceptors/openai-responses/index.ts b/packages/provider-azure/src/interceptors/openai-responses/index.ts deleted file mode 100644 index 53e8066faf..0000000000 --- a/packages/provider-azure/src/interceptors/openai-responses/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { withEmptyNamespaceDescriptionsFilled } from './fill-empty-namespace-descriptions.ts'; -import type { AzureOpenAIResponsesBoundaryInterceptor } from './types.ts'; - -// Azure-specific OpenAI Responses workarounds stay behind the provider boundary so -// the gateway and OpenAI-compatible providers retain the canonical wire value. -export const AZURE_OPENAI_RESPONSES_BOUNDARY = [ - withEmptyNamespaceDescriptionsFilled, -] as const satisfies readonly AzureOpenAIResponsesBoundaryInterceptor[]; diff --git a/packages/provider-azure/src/interceptors/openai-responses/types.ts b/packages/provider-azure/src/interceptors/openai-responses/types.ts deleted file mode 100644 index 13747397ff..0000000000 --- a/packages/provider-azure/src/interceptors/openai-responses/types.ts +++ /dev/null @@ -1,16 +0,0 @@ -import type { Interceptor } from '@floway-dev/interceptor'; -import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; -import type { ProviderModel, ProviderOpenAIResponsesResult, OpenAIResponsesAction } from '@floway-dev/provider'; - -export interface OpenAIResponsesBoundaryCtx { - payload: CanonicalOpenAIResponsesPayload; - headers: Headers; - readonly model: ProviderModel; - action: OpenAIResponsesAction; -} - -export type AzureOpenAIResponsesBoundaryInterceptor = Interceptor< - OpenAIResponsesBoundaryCtx, - object, - ProviderOpenAIResponsesResult ->; diff --git a/packages/provider-azure/src/pipelines.ts b/packages/provider-azure/src/pipelines.ts index 61803e73d2..c257d86ce5 100644 --- a/packages/provider-azure/src/pipelines.ts +++ b/packages/provider-azure/src/pipelines.ts @@ -1,13 +1,20 @@ import type { AzureUpstreamConfig } from './config.ts'; +import { fillAzureNamespaceDescriptions } from './stages/fill-namespace-descriptions.ts'; import { prepareAzureRequest } from './stages/prepare-request.ts'; import { http } from '@floway-dev/http/pipeline'; import { compose } from '@floway-dev/pipeline'; -import type { ProviderOperationPayloads, ProviderOperationResponse, ProviderPipelines, ProviderRequest } from '@floway-dev/provider'; +import { decodeProviderResponse, observeProviderCall, observeProviderResponsesCall, selectProviderResponsesAction } from '@floway-dev/provider'; +import type { ProviderOperationPayloads, ProviderOperationRequest, ProviderOperationResponse, ProviderPipelines, ProviderRequest } from '@floway-dev/provider'; export const createAzurePipelines = (config: AzureUpstreamConfig): ProviderPipelines => ({ - openaiCompletions: compose, ProviderOperationResponse<'openaiCompletions'>>('azure.openaiCompletions', [prepareAzureRequest(config, 'openaiCompletions'), http]), - openaiEmbeddings: compose, ProviderOperationResponse<'openaiEmbeddings'>>('azure.openaiEmbeddings', [prepareAzureRequest(config, 'openaiEmbeddings'), http]), - openaiImagesGenerations: compose, ProviderOperationResponse<'openaiImagesGenerations'>>('azure.openaiImagesGenerations', [prepareAzureRequest(config, 'openaiImagesGenerations'), http]), - openaiImagesEdits: compose, ProviderOperationResponse<'openaiImagesEdits'>>('azure.openaiImagesEdits', [prepareAzureRequest(config, 'openaiImagesEdits'), http]), - openaiAudioTranscriptions: compose, ProviderOperationResponse<'openaiAudioTranscriptions'>>('azure.openaiAudioTranscriptions', [prepareAzureRequest(config, 'openaiAudioTranscriptions'), http]), + openaiChatCompletions: compose, ProviderOperationResponse<'openaiChatCompletions'>>('azure.openaiChatCompletions', [decodeProviderResponse('openaiChatCompletions'), prepareAzureRequest(config, 'openaiChatCompletions'), observeProviderCall, http]), + openaiResponses: compose, ProviderOperationResponse<'openaiResponses'>>('azure.openaiResponses', [selectProviderResponsesAction('generate'), fillAzureNamespaceDescriptions, decodeProviderResponse('openaiResponses'), prepareAzureRequest(config, 'openaiResponses'), observeProviderResponsesCall, http]), + openaiResponsesCompact: compose, ProviderOperationResponse<'openaiResponsesCompact'>>('azure.openaiResponsesCompact', [selectProviderResponsesAction('compact'), fillAzureNamespaceDescriptions, decodeProviderResponse('openaiResponsesCompact'), prepareAzureRequest(config, 'openaiResponsesCompact'), observeProviderResponsesCall, http]), + anthropicMessages: compose, ProviderOperationResponse<'anthropicMessages'>>('azure.anthropicMessages', [decodeProviderResponse('anthropicMessages'), prepareAzureRequest(config, 'anthropicMessages'), observeProviderCall, http]), + anthropicMessagesCountTokens: compose, ProviderOperationResponse<'anthropicMessagesCountTokens'>>('azure.anthropicMessagesCountTokens', [decodeProviderResponse('anthropicMessagesCountTokens'), prepareAzureRequest(config, 'anthropicMessagesCountTokens'), observeProviderCall, http]), + openaiCompletions: compose, ProviderOperationResponse<'openaiCompletions'>>('azure.openaiCompletions', [prepareAzureRequest(config, 'openaiCompletions'), observeProviderCall, http]), + openaiEmbeddings: compose, ProviderOperationResponse<'openaiEmbeddings'>>('azure.openaiEmbeddings', [prepareAzureRequest(config, 'openaiEmbeddings'), observeProviderCall, http]), + openaiImagesGenerations: compose, ProviderOperationResponse<'openaiImagesGenerations'>>('azure.openaiImagesGenerations', [prepareAzureRequest(config, 'openaiImagesGenerations'), observeProviderCall, http]), + openaiImagesEdits: compose, ProviderOperationResponse<'openaiImagesEdits'>>('azure.openaiImagesEdits', [prepareAzureRequest(config, 'openaiImagesEdits'), observeProviderCall, http]), + openaiAudioTranscriptions: compose, ProviderOperationResponse<'openaiAudioTranscriptions'>>('azure.openaiAudioTranscriptions', [prepareAzureRequest(config, 'openaiAudioTranscriptions'), observeProviderCall, http]), }); diff --git a/packages/provider-azure/src/provider.ts b/packages/provider-azure/src/provider.ts index 047a00c23f..3a9b4df46f 100644 --- a/packages/provider-azure/src/provider.ts +++ b/packages/provider-azure/src/provider.ts @@ -1,53 +1,13 @@ import { assertAzureUpstreamRecord } from './config.ts'; import { AZURE_DEFAULT_FLAGS } from './defaults.ts'; -import { azureFetchOpenAIAudioTranscriptions, azureFetchOpenAIChatCompletions, azureFetchOpenAICompletions, azureFetchOpenAIEmbeddings, azureFetchOpenAIImagesEdits, azureFetchOpenAIImagesGenerations, azureFetchAnthropicMessages, azureFetchAnthropicMessagesCountTokens, azureFetchOpenAIResponses, azureFetchOpenAIResponsesCompact } from './fetch.ts'; -import { AZURE_OPENAI_RESPONSES_BOUNDARY } from './interceptors/openai-responses/index.ts'; -import type { OpenAIResponsesBoundaryCtx } from './interceptors/openai-responses/types.ts'; import { createAzurePipelines } from './pipelines.ts'; -import { runInterceptors } from '@floway-dev/interceptor'; -import { parseAnthropicMessagesStream } from '@floway-dev/protocols/anthropic-messages'; import { kindForEndpoints } from '@floway-dev/protocols/common'; -import { parseOpenAIChatCompletionsStream } from '@floway-dev/protocols/openai-chat-completions'; -import { parseOpenAIResponsesStream, type OpenAIResponsesCompactionResult, toCompactPayloadShape } from '@floway-dev/protocols/openai-responses'; -import { headersForAnthropicMessagesCall, jsonRequestBody, serializeModelPathOpenAIAudioTranscriptionRequest, serializeOpenAIImagesEditsRequest, type FetchInit, type HttpHeaderLines, type ProviderInstance, type Provider, type ProviderModel, type ProviderOpenAIResponsesResult, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord, publicModelId, resolveEffectiveFlags, streamingProviderCall } from '@floway-dev/provider'; - -const upstreamModelIdOf = (model: ProviderModel): string => (model.providerData as { upstreamModelId: string }).upstreamModelId; - -type AzureTypedFetch = (config: ReturnType['config'], init: FetchInit, options: UpstreamFetchOptions) => Promise; +import { publicModelId, resolveEffectiveFlags, type ProviderInstance, type Provider, type UpstreamRecord } from '@floway-dev/provider'; export const createAzureProvider = (record: UpstreamRecord): Provider => { const azure = assertAzureUpstreamRecord(record); - const callStreaming = ( - transport: AzureTypedFetch, - model: ProviderModel, - body: Record, - signal: AbortSignal | undefined, - headers: HttpHeaderLines, - parser: ProviderStreamParser, - opts: UpstreamCallOptions, - ) => { - const upstreamModelId = upstreamModelIdOf(model); - return streamingProviderCall( - transport( - azure.config, - { method: 'POST', body: jsonRequestBody({ ...body, stream: true, model: upstreamModelId }), signal }, - { extraHeaders: headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, - ), - parser, - upstreamModelId, - signal, - ); - }; - - const callNonStreaming = async (transport: AzureTypedFetch, model: ProviderModel, body: Record, signal: AbortSignal | undefined, headers: HttpHeaderLines, opts: UpstreamCallOptions) => { - const upstreamModelId = upstreamModelIdOf(model); - const response = await transport(azure.config, { method: 'POST', body: jsonRequestBody({ ...body, model: upstreamModelId }), signal }, { extraHeaders: headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }); - return { response, modelKey: upstreamModelId }; - }; - const instance: ProviderInstance = { - callAlphaSearch: () => Promise.reject(new Error('Azure provider does not support callAlphaSearch')), getProvidedModels() { return Promise.resolve(azure.config.models.map(model => { const effective = resolveEffectiveFlags([AZURE_DEFAULT_FLAGS, azure.flagOverrides, model.flagOverrides]); @@ -68,60 +28,6 @@ export const createAzureProvider = (record: UpstreamRecord): Provider => { }; })); }, - callOpenAICompletions: (model, body, signal, opts) => callNonStreaming(azureFetchOpenAICompletions, model, body, signal, [...opts.headers], opts), - callOpenAIChatCompletions: (model, body, signal, opts) => callStreaming(azureFetchOpenAIChatCompletions, model, body, signal, [...opts.headers], parseOpenAIChatCompletionsStream, opts), - callOpenAIResponses: async (model, body, action, signal, opts) => { - const ctx: OpenAIResponsesBoundaryCtx = { - payload: { ...body, model: model.id }, - headers: new Headers(opts.headers), - model, - action, - }; - return await runInterceptors( - ctx, {}, AZURE_OPENAI_RESPONSES_BOUNDARY, async () => { - const { model: _ignored, ...wireBody } = ctx.payload; - switch (ctx.action) { - case 'generate': { - const stream = await callStreaming(azureFetchOpenAIResponses, model, wireBody, signal, [...ctx.headers], parseOpenAIResponsesStream, opts); - return stream.ok - ? { action: 'generate', ok: true, events: stream.events, modelKey: stream.modelKey, ...(stream.headers ? { headers: stream.headers } : {}) } - : { action: 'generate', ok: false, response: stream.response, modelKey: stream.modelKey }; - } - case 'compact': { - const upstreamModelId = upstreamModelIdOf(model); - const response = await azureFetchOpenAIResponsesCompact( - azure.config, - { method: 'POST', body: jsonRequestBody({ ...toCompactPayloadShape(wireBody), model: upstreamModelId }), signal }, - { extraHeaders: [...ctx.headers], fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, - ); - return response.ok - ? { action: 'compact', ok: true, result: (await response.json()) as OpenAIResponsesCompactionResult, modelKey: upstreamModelId } - : { action: 'compact', ok: false, response, modelKey: upstreamModelId }; - } - default: - ctx.action satisfies never; - throw new Error(`Unhandled OpenAIResponsesAction: ${ctx.action as string}`); - } - }, - ); - }, - callAnthropicMessages: (model, body, signal, opts) => callStreaming(azureFetchAnthropicMessages, model, body, signal, headersForAnthropicMessagesCall([...opts.headers], opts.anthropicBeta), parseAnthropicMessagesStream, opts), - callAnthropicMessagesCountTokens: (model, body, signal, opts) => callNonStreaming(azureFetchAnthropicMessagesCountTokens, model, body, signal, headersForAnthropicMessagesCall([...opts.headers], opts.anthropicBeta), opts), - callOpenAIEmbeddings: (model, body, signal, opts) => callNonStreaming(azureFetchOpenAIEmbeddings, model, body, signal, [...opts.headers], opts), - callOpenAIImagesGenerations: (model, body, signal, opts) => callNonStreaming(azureFetchOpenAIImagesGenerations, model, body, signal, [...opts.headers], opts), - callOpenAIImagesEdits: async (model, request, signal, opts) => { - const upstreamModelId = upstreamModelIdOf(model); - const body = await serializeOpenAIImagesEditsRequest(request, upstreamModelId); - const response = await azureFetchOpenAIImagesEdits(azure.config, { method: 'POST', body, signal }, { extraHeaders: [...opts.headers], fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }); - return { response, modelKey: upstreamModelId }; - }, - callOpenAIAudioTranscriptions: async (model, request, signal, opts) => { - const upstreamModelId = upstreamModelIdOf(model); - const body = serializeModelPathOpenAIAudioTranscriptionRequest(request); - const response = await azureFetchOpenAIAudioTranscriptions(azure.config, upstreamModelId, { method: 'POST', body, signal }, { extraHeaders: [...opts.headers], fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }); - return { response, modelKey: upstreamModelId }; - }, - callRerank: () => Promise.reject(new Error('Azure provider does not support callRerank')), }; return { diff --git a/packages/provider-azure/src/stages/fill-namespace-descriptions.ts b/packages/provider-azure/src/stages/fill-namespace-descriptions.ts new file mode 100644 index 0000000000..7a8387c7ce --- /dev/null +++ b/packages/provider-azure/src/stages/fill-namespace-descriptions.ts @@ -0,0 +1,21 @@ +import { defineStage, move } from '@floway-dev/pipeline'; +import { mapOpenAIResponsesTools, type CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; +import type { ProviderModelFacts } from '@floway-dev/provider'; + +type Facts = { 'request.provider.payload': Omit; 'request.provider.model': ProviderModelFacts }; + +// Azure retains the former namespace-description minimum length. +// https://github.com/openai/openai-openapi/commit/466c74a42f51c02f1927bc666815251dc53845dc +// https://github.com/openai/codex/issues/37380 +export const fillAzureNamespaceDescriptions = defineStage({ + name: 'fillAzureNamespaceDescriptions', + through: { + request: { needs: ['request.provider.payload', 'request.provider.model'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const { model: _, ...payload } = mapOpenAIResponsesTools({ ...facts['request.provider.payload'], model: facts['request.provider.model'].id }, tool => tool.type === 'namespace' && tool.description === '' + ? { ...tool, description: `Tools in the ${tool.name} namespace.` } : tool); + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + }, +}); diff --git a/packages/provider-azure/src/stages/prepare-request.ts b/packages/provider-azure/src/stages/prepare-request.ts index b0b1ed9ae4..5dbdfec276 100644 --- a/packages/provider-azure/src/stages/prepare-request.ts +++ b/packages/provider-azure/src/stages/prepare-request.ts @@ -1,17 +1,24 @@ import type { AzureUpstreamConfig } from '../config.ts'; -import { azureOpenAiV1BaseUrl } from '../endpoint.ts'; -import { type HttpRequestFacts, type HttpResponseFacts } from '@floway-dev/http/pipeline'; +import { azureAnthropicBaseUrl, azureOpenAiV1BaseUrl } from '../endpoint.ts'; +import { type HttpRequestFacts } from '@floway-dev/http/pipeline'; import { multipartBody, type HttpBody, type HttpBodyEncoding } from '@floway-dev/http/request-content'; import { defineStage, move, secret } from '@floway-dev/pipeline'; +import { toCompactPayloadShape } from '@floway-dev/protocols/openai-responses'; import { joinBaseAndPath, mergeHttpHeaders, prepareOpenAIImagesEditsBody, withHttpContentType, type ProviderOperationPayloads, type ProviderRequest, type ProviderResponse, type ProviderServices } from '@floway-dev/provider'; -type AzureOperation = 'openaiCompletions' | 'openaiEmbeddings' | 'openaiImagesGenerations' | 'openaiImagesEdits' | 'openaiAudioTranscriptions'; +type AzureOperation = 'openaiChatCompletions' | 'openaiResponses' | 'openaiResponsesCompact' | 'anthropicMessages' | 'anthropicMessagesCountTokens' | 'openaiCompletions' | 'openaiEmbeddings' | 'openaiImagesGenerations' | 'openaiImagesEdits' | 'openaiAudioTranscriptions'; interface AzureHttpFacts extends HttpRequestFacts { + 'request.provider.modelKey': string; 'request.azure.deployment': string; } const PATHS: Record, string> = { + openaiChatCompletions: '/chat/completions', + openaiResponses: '/responses', + openaiResponsesCompact: '/responses/compact', + anthropicMessages: '/v1/messages', + anthropicMessagesCountTokens: '/v1/messages/count_tokens', openaiCompletions: '/completions', openaiEmbeddings: '/embeddings', openaiImagesGenerations: '/images/generations', @@ -19,19 +26,20 @@ const PATHS: Record, string }; export const prepareAzureRequest = (config: AzureUpstreamConfig, operation: O) => { - const prepare = defineStage, AzureHttpFacts, HttpResponseFacts, ProviderResponse, ProviderServices>({ + const prepare = defineStage & { 'request.provider.anthropicBeta'?: readonly string[]; 'request.provider.responsesAction': 'generate' | 'compact' }, AzureHttpFacts, ProviderResponse, ProviderResponse, ProviderServices>({ name: `prepareAzure${operation.replace(/^openai/, 'OpenAI').replace(/^alpha/, 'Alpha').replace(/^rerank/, 'Rerank')}`, through: { request: { - needs: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'], + needs: [...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? ['request.provider.responsesAction' as const] : []), 'request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers', ...(operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens' ? ['request.provider.anthropicBeta' as const] : [])], consumes: ['request.provider.model', 'request.provider.payload', 'request.http.headers'], - provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.azure.deployment'], + provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.provider.modelKey', 'request.azure.deployment'], }, - response: { needs: ['response.http.exchange'], consumes: [], provides: ['response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'] }, + response: { needs: [], consumes: [], provides: [] }, }, execute: async (facts, next) => { const { 'request.provider.model': model, 'request.provider.payload': payload, ...rest } = facts; const deployment = (model.providerData as { upstreamModelId: string }).upstreamModelId; + const anthropic = operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens'; let body: HttpBody; let encoding: HttpBodyEncoding = 'json'; let url: URL; @@ -45,20 +53,34 @@ export const prepareAzureRequest = (config: AzureUpstr // https://github.com/Azure/azure-rest-api-specs/blob/928047803788f7377fa003a26ba2bdc2e0fcccc0/specification/cognitiveservices/OpenAI.Inference/routes/audio_transcription.tsp#L19-L49 url.searchParams.set('api-version', '2025-04-01-preview'); } else { - url = new URL(joinBaseAndPath(azureOpenAiV1BaseUrl(config.endpoint), PATHS[operation as Exclude])); + const baseUrl = anthropic ? azureAnthropicBaseUrl(config.endpoint) : azureOpenAiV1BaseUrl(config.endpoint); + const path = operation === 'openaiResponses' || operation === 'openaiResponsesCompact' + ? PATHS[facts['request.provider.responsesAction'] === 'compact' ? 'openaiResponsesCompact' : 'openaiResponses'] + : PATHS[operation as Exclude]; + url = new URL(joinBaseAndPath(baseUrl, path)); if (operation === 'openaiImagesEdits') { const prepared = await prepareOpenAIImagesEditsBody(payload as ProviderOperationPayloads['openaiImagesEdits'], deployment); body = prepared.body; encoding = prepared.encoding; } else { - body = { ...payload, model: deployment }; + body = (operation === 'openaiResponses' || operation === 'openaiResponsesCompact') && facts['request.provider.responsesAction'] === 'compact' + ? { ...toCompactPayloadShape(payload as ProviderOperationPayloads['openaiResponsesCompact']), model: deployment } + : { ...payload, ...(operation === 'openaiChatCompletions' || operation === 'openaiResponses' || operation === 'openaiResponsesCompact' || operation === 'anthropicMessages' ? { stream: true } : {}), model: deployment }; } if (operation === 'openaiImagesGenerations' || operation === 'openaiImagesEdits') url.searchParams.append('api-version', 'preview'); } - const base = withHttpContentType([['api-key', secret(config.apiKey)]], body, encoding); - const headers = mergeHttpHeaders(base, facts['request.http.headers']); + // https://learn.microsoft.com/en-us/azure/ai-foundry/foundry-models/how-to/use-foundry-models + // https://docs.anthropic.com/en/api/versioning + const base = withHttpContentType(anthropic ? [['x-api-key', secret(config.apiKey)], ['anthropic-version', '2023-06-01']] : [['api-key', secret(config.apiKey)]], body, encoding); + let headers = mergeHttpHeaders(base, facts['request.http.headers']); + if (anthropic) { + const beta = facts['request.provider.anthropicBeta'] as readonly string[]; + headers = headers.filter(([name]) => name.toLowerCase() !== 'anthropic-beta'); + if (beta.length > 0) headers = [...headers, ['anthropic-beta', beta.join(',')]]; + } const back = await next(move({ ...rest, + 'request.provider.modelKey': deployment, 'request.azure.deployment': deployment, 'request.http.url': url.href, 'request.http.method': 'POST', @@ -66,7 +88,7 @@ export const prepareAzureRequest = (config: AzureUpstr 'request.http.body': body, 'request.http.encoding': encoding, })); - return move({ ...back, 'response.provider.modelKey': deployment, 'response.provider.called': back['response.http.exchange'].type === 'response', 'response.provider.previousCalls': [] }); + return move({ ...back }); }, }); return prepare; diff --git a/packages/provider-claude-code/__tests__/fetch_test.ts b/packages/provider-claude-code/__tests__/fetch_test.ts index 9d8161faea..79c6f9bec0 100644 --- a/packages/provider-claude-code/__tests__/fetch_test.ts +++ b/packages/provider-claude-code/__tests__/fetch_test.ts @@ -1,6 +1,6 @@ import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; -import { callClaudeCodeAnthropicMessages } from '../src/fetch.ts'; +import { callClaudeCodeAnthropicMessages } from './test-utils/call.ts'; import { CLAUDE_CODE_HEADERS_HAIKU, CLAUDE_CODE_HEADERS_SONNET_OPUS } from '../src/headers.ts'; import type { ClaudeCodeAccessTokenEntry, @@ -84,8 +84,6 @@ const seedAccount = (overrides: Partial = {}): void const readQuotaEntry = (): ClaudeCodeQuotaSnapshotEntry | null => currentState().accounts[0]!.quotaSnapshot; -// Yield to the queue so the .catch chain from fireAndForgetPersist completes -// before assertions inspect the persisted state. const flushAsyncQueue = (): Promise => new Promise(resolve => setTimeout(resolve, 0)); beforeEach(() => { @@ -304,7 +302,7 @@ describe('callClaudeCodeAnthropicMessages — wire body', () => { describe('callClaudeCodeAnthropicMessages — incomplete stream diagnostics', () => { test('logs upstream trace headers and last raw SSE frames when message_stop is absent', async () => { seedAccount({ accessToken: freshAccessTokenEntry }); - const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const warn = vi.fn(); vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response( new ReadableStream({ start(controller) { @@ -324,22 +322,18 @@ describe('callClaudeCodeAnthropicMessages — incomplete stream diagnostics', () )); const result = await callClaudeCodeAnthropicMessages({ - upstreamId, model: sonnetModel, body: minimalBody, shaped: false, call: noopUpstreamCallOptions(), + upstreamId, model: sonnetModel, body: minimalBody, shaped: false, call: noopUpstreamCallOptions(), observers: { log: { debug: () => {}, info: () => {}, warn, error: () => {} } }, }); expect(result.ok).toBe(true); if (!result.ok) return; await drain(result.events); expect(warn).toHaveBeenCalledTimes(1); - const line = warn.mock.calls[0]![0] as string; - expect(line).toContain('claude_code_messages_stream_incomplete'); - expect(line).toContain('upstream_id=up_cc'); - expect(line).toContain('request_id=req_trace'); - expect(line).toContain('cf_ray=ray_trace'); - expect(line).toContain('trace_response=trace_response'); - expect(line).toContain('raw_sse_frames=2'); - expect(line).toContain('terminal_event=null'); - expect(line).toContain('message_delta'); + expect(warn.mock.calls[0]![0]).toBe('claude_code_messages_stream_incomplete'); + const fields = warn.mock.calls[0]![1] as Record; + expect(fields).toMatchObject({ upstream_id: upstreamId, request_id: 'req_trace', cf_ray: 'ray_trace', trace_response: 'trace_response', raw_sse_frames: 2, terminal_event: null }); + expect(fields.last_sse_frames).toContain('message_delta'); + }); test('does not log a complete Messages stream', async () => { @@ -444,12 +438,7 @@ describe('callClaudeCodeAnthropicMessages — quota persistence', () => { expect(data.status).toBe('rejected'); }); - test('2xx → registers persist promise via opts.call.waitUntil exactly once', async () => { - // On Cloudflare Workers the runtime cancels orphan promises the moment - // the response is sent. The gateway threads `waitUntil` through - // UpstreamCallOptions so the persist can extend the worker's lifetime; - // assert we hand the persist promise to it exactly once and that the - // promise actually mutates state when awaited. + test('2xx quota write completes through the run deferred lifecycle', async () => { seedAccount({ accessToken: freshAccessTokenEntry }); vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); const waitUntil = vi.fn<(promise: Promise) => void>(); @@ -458,10 +447,7 @@ describe('callClaudeCodeAnthropicMessages — quota persistence', () => { upstreamId, model: sonnetModel, body: minimalBody, shaped: false, call, }); expect(result.ok).toBe(true); - expect(waitUntil).toHaveBeenCalledTimes(1); - const handed = waitUntil.mock.calls[0]![0]; - expect(handed).toBeInstanceOf(Promise); - await expect(handed).resolves.toBeUndefined(); + expect(waitUntil).not.toHaveBeenCalled(); const stored = readQuotaEntry(); expect(stored).not.toBeNull(); }); diff --git a/packages/provider-claude-code/__tests__/provider_test.ts b/packages/provider-claude-code/__tests__/provider_test.ts index 35c7007fd3..1ab2199914 100644 --- a/packages/provider-claude-code/__tests__/provider_test.ts +++ b/packages/provider-claude-code/__tests__/provider_test.ts @@ -4,9 +4,10 @@ import { buildClaudeCodeCatalog, type ClaudeCodeApiModel } from '../src/models.t import { pricingForClaudeCodeModelKey } from '../src/pricing.ts'; import { createClaudeCodeProvider } from '../src/provider.ts'; import type { ClaudeCodeAccessTokenEntry, ClaudeCodeAccountCredential, ClaudeCodeUpstreamState } from '../src/state.ts'; +import { getFailureFacts, type Event } from '@floway-dev/pipeline'; import type { AnthropicMessagesPayload, AnthropicMessagesTextBlock } from '@floway-dev/protocols/anthropic-messages'; import { initProviderRepo, type FlagId, type AnthropicMessagesUpstreamCallOptions, type UpstreamRecord } from '@floway-dev/provider'; -import { noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, readJsonRequest } from '@floway-dev/test-utils'; +import { collectChatProviderPipeline, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, readJsonRequest } from '@floway-dev/test-utils'; const upstreamId = 'up_cc_provider'; @@ -175,17 +176,86 @@ describe('createClaudeCodeProvider — factory surface', () => { }); }); -describe('createClaudeCodeProvider — callAnthropicMessages routes through chain', () => { +describe('createClaudeCodeProvider — Messages operation stages', () => { + test.each([false, true])('the actual Messages pipeline preserves shaped=%s content and dated model identity', async shaped => { + const provider = createClaudeCodeProvider(currentRecord); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const userId = JSON.stringify({ device_id: 'd'.repeat(32), account_uuid: '', session_id: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' }); + const payload = { + max_tokens: 16, messages: [{ role: 'user' as const, content: 'hello' }], + ...(shaped ? { system: [{ type: 'text' as const, text: "You are Claude Code, Anthropic's official CLI for Claude." }], metadata: { user_id: userId } } : {}), + }; + const result = await collectChatProviderPipeline(provider, 'anthropicMessages', sonnetProviderModel, payload, undefined, shaped ? cliClientCallOpts() : noopAnthropicMessagesUpstreamCallOptions()); + expect(result.facts['response.provider.called']).toBe(true); + expect(result.facts['response.provider.modelKey']).toBe('claude-sonnet-4-5-20250929'); + expect(result.frames.some(frame => frame.type === 'event' && frame.event.type === 'message_start')).toBe(true); + const init = fetchSpy.mock.calls[0]![1] as RequestInit; + const body = await readJsonRequest(init) as WireAnthropicMessagesPayload; + expect(body.model).toBe('claude-sonnet-4-5-20250929'); + expect(body.stream).toBe(true); + expect(body.system).toHaveLength(shaped ? 1 : 3); + if (shaped) expect(body.metadata.user_id).toBe(userId); + else expect(body.system[0]!.text).toMatch(/^x-anthropic-billing-header:/); + expect(new Headers(init.headers).get('authorization')).toBe('Bearer at_cached'); + expect(new Headers(init.headers).get('anthropic-beta')).toEqual(shaped ? 'oauth-2025-04-20' : expect.stringContaining('claude-code-20250219')); + }); + + test('production session identity derives from the original user turn before system hoisting', async () => { + const provider = createClaudeCodeProvider(currentRecord); + const bodies: WireAnthropicMessagesPayload[] = []; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (_url, init) => { + bodies.push(await readJsonRequest(init!) as WireAnthropicMessagesPayload); + return sseResponse(); + }); + for (const prompt of ['What is the capital of France?', 'Who wrote The Great Gatsby?']) { + await collectChatProviderPipeline(provider, 'anthropicMessages', sonnetProviderModel, { max_tokens: 16, system: 'Shared system instructions', messages: [{ role: 'user', content: prompt }] }, undefined, noopAnthropicMessagesUpstreamCallOptions()); + } + const ids = bodies.map(body => (JSON.parse(body.metadata.user_id) as { session_id: string }).session_id); + expect(ids[0]).not.toBe(ids[1]); + expect(bodies.every(body => JSON.stringify(body.messages[0]).includes('Shared system instructions'))).toBe(true); + }); + + test.each([200, 403])('finite body readers preserve an original stream failure at status%s', async status => { + const original = new Error('Claude body broke'); + const body = new ReadableStream({ start(controller) { controller.error(original); } }); + vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(body, { status, headers: { 'content-type': 'application/json' } })); + const events: Event[] = []; + const caught = await collectChatProviderPipeline(createClaudeCodeProvider(currentRecord), 'anthropicMessages', sonnetProviderModel, { max_tokens: 16, messages: [] }, undefined, noopAnthropicMessagesUpstreamCallOptions(), { dump: event => { events.push(event); } }).catch((error: unknown) => error); + expect(caught).toBe(original); + expect(getFailureFacts(caught)).toMatchObject({ 'response.provider.called': true, 'response.provider.modelKey': 'claude-sonnet-4-5-20250929' }); + expect(events.find(event => event.type === 'stage.failed')).toMatchObject({ error: original }); + }); + + test('observing a JSON error preserves its original UTF8 BOM bytes', async () => { + const body = { error: { type: 'permission_error', message: 'A beta feature is unavailable' } }; + const bytes = new TextEncoder().encode(`\uFEFF${JSON.stringify(body)}`); + vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(bytes, { status: 403, headers: { 'content-type': 'application/json', 'x-upstream': 'retained' } })); + const result = await collectChatProviderPipeline(createClaudeCodeProvider(currentRecord), 'anthropicMessages', sonnetProviderModel, { max_tokens: 16, messages: [] }, undefined, noopAnthropicMessagesUpstreamCallOptions()); + expect(result.facts['response.claudeCode.failureBody']).toEqual(body); + expect(new Uint8Array(await result.response!.arrayBuffer())).toEqual(bytes); + }); + + test('the Messages pipeline preserves a complete terminal refusal while recording state persistence', async () => { + const body = { error: { type: 'permission_error', message: 'OAuth authentication is currently not allowed for this organization' }, diagnostic: 'x'.repeat(800) }; + vi.spyOn(globalThis, 'fetch').mockResolvedValue(Response.json(body, { status: 403, headers: { 'x-upstream': 'retained' } })); + const result = await collectChatProviderPipeline(createClaudeCodeProvider(currentRecord), 'anthropicMessages', sonnetProviderModel, { max_tokens: 16, messages: [] }, undefined, noopAnthropicMessagesUpstreamCallOptions()); + expect(result.output).toBeNull(); + expect(result.response?.status).toBe(403); + expect(result.response?.headers.get('x-upstream')).toBe('retained'); + expect(await result.response!.json()).toEqual(body); + expect(result.facts['response.claudeCode.failureBody']).toEqual(body); + expect((currentRecord.state as ClaudeCodeUpstreamState).accounts[0]!.state).toBe('refresh_failed'); + }); + test('unshaped request runs the re-mimicry chain (3-block system, pinned UA, metadata.user_id)', async () => { const instance = createClaudeCodeProvider(currentRecord); const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); - await instance.instance.callAnthropicMessages( + await collectChatProviderPipeline(instance, 'anthropicMessages', sonnetProviderModel, { max_tokens: 16, messages: [{ role: 'user', content: 'hello' }] }, undefined, - noopAnthropicMessagesUpstreamCallOptions(), - ); + noopAnthropicMessagesUpstreamCallOptions()); const init = fetchSpy.mock.calls[0]![1] as RequestInit; const wireHeaders = new Headers(init.headers); @@ -207,7 +277,7 @@ describe('createClaudeCodeProvider — callAnthropicMessages routes through chai const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); const userId = JSON.stringify({ device_id: 'd'.repeat(32), account_uuid: '', session_id: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' }); - await instance.instance.callAnthropicMessages( + await collectChatProviderPipeline(instance, 'anthropicMessages', sonnetProviderModel, { max_tokens: 16, @@ -216,8 +286,7 @@ describe('createClaudeCodeProvider — callAnthropicMessages routes through chai metadata: { user_id: userId }, }, undefined, - cliClientCallOpts(), - ); + cliClientCallOpts()); const init = fetchSpy.mock.calls[0]![1] as RequestInit; const body = await readJsonRequest(init) as WireAnthropicMessagesPayload; @@ -242,12 +311,11 @@ describe('createClaudeCodeProvider — callAnthropicMessages routes through chai const instance = createClaudeCodeProvider(currentRecord); const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); - await instance.instance.callAnthropicMessages( + await collectChatProviderPipeline(instance, 'anthropicMessages', sonnetProviderModel, { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }] }, undefined, - { ...noopAnthropicMessagesUpstreamCallOptions(), headers: new Headers({ 'user-agent': 'claude-cli/2.1.181' }) }, - ); + { ...noopAnthropicMessagesUpstreamCallOptions(), headers: new Headers({ 'user-agent': 'claude-cli/2.1.181' }) }); const init = fetchSpy.mock.calls[0]![1] as RequestInit; const body = await readJsonRequest(init) as WireAnthropicMessagesPayload; diff --git a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/backfill-required-fields_test.ts b/packages/provider-claude-code/__tests__/stages/backfill-required-fields_test.ts similarity index 74% rename from packages/provider-claude-code/__tests__/interceptors/anthropic-messages/backfill-required-fields_test.ts rename to packages/provider-claude-code/__tests__/stages/backfill-required-fields_test.ts index fdf3e5090e..4495129725 100644 --- a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/backfill-required-fields_test.ts +++ b/packages/provider-claude-code/__tests__/stages/backfill-required-fields_test.ts @@ -1,16 +1,20 @@ import { test } from 'vitest'; -import { backfillRequiredFields } from '../../../src/interceptors/anthropic-messages/backfill-required-fields.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { backfillRequiredFields } from '../../src/stages/backfill-required-fields.ts'; import { ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS, type AnthropicMessagesPayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProviderModel, ProviderStreamResult } from '@floway-dev/provider'; -import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; +import type { ProviderModel } from '@floway-dev/provider'; +import { type AnthropicMessagesProbe, type ProviderStreamResult } from '@floway-dev/test-utils'; +import { applyProviderStage, assertEquals, stubProviderModel } from '@floway-dev/test-utils'; + +type ClaudeCodeStageProbe = AnthropicMessagesProbe & { upstreamId: string }; const okEvents = (): Promise> => Promise.resolve({ ok: true, events: (async function* () {})(), modelKey: 'test' }); -const invocation = (payload: AnthropicMessagesPayload, model: ProviderModel = stubProviderModel({ endpoints: { anthropicMessages: {} } })): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload, model: ProviderModel = stubProviderModel({ endpoints: { anthropicMessages: {} } })): ClaudeCodeStageProbe => ({ payload, + headers: new Headers(), + anthropicBeta: [], model, upstreamId: 'up_test', }); @@ -26,7 +30,7 @@ test('backfills max_tokens from ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS when both temperature: 0.5, } as Partial as AnthropicMessagesPayload); - await backfillRequiredFields(ctx, {}, okEvents); + await applyProviderStage(backfillRequiredFields, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.max_tokens, ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS); }); @@ -42,7 +46,7 @@ test('prefers model.limits.max_output_tokens over the gateway fallback when set' model, ); - await backfillRequiredFields(ctx, {}, okEvents); + await applyProviderStage(backfillRequiredFields, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.max_tokens, 64000); }); @@ -59,7 +63,7 @@ test('preserves caller-supplied max_tokens', async () => { model, ); - await backfillRequiredFields(ctx, {}, okEvents); + await applyProviderStage(backfillRequiredFields, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.max_tokens, 100); }); @@ -74,7 +78,7 @@ test('backfills temperature to 1 when missing', async () => { messages: [{ role: 'user', content: 'hi' }], }); - await backfillRequiredFields(ctx, {}, okEvents); + await applyProviderStage(backfillRequiredFields, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.temperature, 1); }); @@ -87,7 +91,7 @@ test('preserves caller-supplied temperature', async () => { temperature: 0.7, }); - await backfillRequiredFields(ctx, {}, okEvents); + await applyProviderStage(backfillRequiredFields, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.temperature, 0.7); }); @@ -102,7 +106,7 @@ test('preserves caller-supplied temperature: 0', async () => { temperature: 0, }); - await backfillRequiredFields(ctx, {}, okEvents); + await applyProviderStage(backfillRequiredFields, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.temperature, 0); }); diff --git a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/hoist-user-system-to-messages_test.ts b/packages/provider-claude-code/__tests__/stages/hoist-user-system-to-messages_test.ts similarity index 71% rename from packages/provider-claude-code/__tests__/interceptors/anthropic-messages/hoist-user-system-to-messages_test.ts rename to packages/provider-claude-code/__tests__/stages/hoist-user-system-to-messages_test.ts index f8bb662f00..c10438980d 100644 --- a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/hoist-user-system-to-messages_test.ts +++ b/packages/provider-claude-code/__tests__/stages/hoist-user-system-to-messages_test.ts @@ -1,16 +1,19 @@ import { test } from 'vitest'; -import { hoistUserSystemToMessages } from '../../../src/interceptors/anthropic-messages/hoist-user-system-to-messages.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { hoistUserSystemToMessages } from '../../src/stages/hoist-user-system-to-messages.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProviderStreamResult } from '@floway-dev/provider'; -import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; +import { type AnthropicMessagesProbe, type ProviderStreamResult } from '@floway-dev/test-utils'; +import { applyProviderStage, assertEquals, stubProviderModel } from '@floway-dev/test-utils'; + +type ClaudeCodeStageProbe = AnthropicMessagesProbe & { upstreamId: string }; const okEvents = (): Promise> => Promise.resolve({ ok: true, events: (async function* () {})(), modelKey: 'test' }); -const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload): ClaudeCodeStageProbe => ({ payload, + headers: new Headers(), + anthropicBeta: [], model: stubProviderModel({ endpoints: { anthropicMessages: {} } }), upstreamId: 'up_test', }); @@ -23,7 +26,7 @@ test('captures a string system into a synthetic user/assistant pair and drops `s system: 'You are a pirate.', }); - await hoistUserSystemToMessages(ctx, {}, okEvents); + await applyProviderStage(hoistUserSystemToMessages, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.system, undefined); assertEquals(ctx.payload.messages, [ @@ -44,7 +47,7 @@ test('joins multi-block system into one synthetic turn with blank-line separator ], }); - await hoistUserSystemToMessages(ctx, {}, okEvents); + await applyProviderStage(hoistUserSystemToMessages, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.system, undefined); assertEquals(ctx.payload.messages[0], { role: 'user', content: [{ type: 'text', text: '[System Instructions]\nfirst rule\n\nsecond rule' }] }); @@ -57,7 +60,7 @@ test('drops system entirely when caller did not send one and leaves messages unt messages: [{ role: 'user', content: 'hi' }], }); - await hoistUserSystemToMessages(ctx, {}, okEvents); + await applyProviderStage(hoistUserSystemToMessages, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.system, undefined); assertEquals(ctx.payload.messages.length, 1); @@ -72,7 +75,7 @@ test('drops system but does not inject a synthetic turn when system text is empt system: [{ type: 'text', text: '' }], }); - await hoistUserSystemToMessages(ctx, {}, okEvents); + await applyProviderStage(hoistUserSystemToMessages, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.system, undefined); assertEquals(ctx.payload.messages.length, 1); diff --git a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-billing-block_test.ts b/packages/provider-claude-code/__tests__/stages/inject-billing-block_test.ts similarity index 72% rename from packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-billing-block_test.ts rename to packages/provider-claude-code/__tests__/stages/inject-billing-block_test.ts index ca62714629..643d4d8413 100644 --- a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-billing-block_test.ts +++ b/packages/provider-claude-code/__tests__/stages/inject-billing-block_test.ts @@ -1,17 +1,20 @@ import { test } from 'vitest'; -import { CLAUDE_CLI_VERSION } from '../../../src/headers.ts'; -import { injectBillingBlock } from '../../../src/interceptors/anthropic-messages/inject-billing-block.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { CLAUDE_CLI_VERSION } from '../../src/headers.ts'; +import { injectBillingBlock } from '../../src/stages/inject-billing-block.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent, AnthropicMessagesTextBlock } from '@floway-dev/protocols/anthropic-messages'; -import type { ProviderStreamResult } from '@floway-dev/provider'; -import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; +import { type AnthropicMessagesProbe, type ProviderStreamResult } from '@floway-dev/test-utils'; +import { applyProviderStage, assertEquals, stubProviderModel } from '@floway-dev/test-utils'; + +type ClaudeCodeStageProbe = AnthropicMessagesProbe & { upstreamId: string }; const okEvents = (): Promise> => Promise.resolve({ ok: true, events: (async function* () {})(), modelKey: 'test' }); -const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload): ClaudeCodeStageProbe => ({ payload, + headers: new Headers(), + anthropicBeta: [], model: stubProviderModel({ endpoints: { anthropicMessages: {} } }), upstreamId: 'up_test', }); @@ -23,7 +26,7 @@ test('drops a single billing block as system[0] with the pinned CLI version and messages: [{ role: 'user', content: 'hello world' }], }); - await injectBillingBlock(ctx, {}, okEvents); + await applyProviderStage(injectBillingBlock, ctx, okEvents, { 'request.claudeCode.shaped': false }); const system = ctx.payload.system; if (!Array.isArray(system)) throw new Error('expected system to be an array'); @@ -46,7 +49,7 @@ test('overwrites any pre-existing system array (hoist already ran)', async () => system: [{ type: 'text', text: 'stale leftover' } satisfies AnthropicMessagesTextBlock], }); - await injectBillingBlock(ctx, {}, okEvents); + await applyProviderStage(injectBillingBlock, ctx, okEvents, { 'request.claudeCode.shaped': false }); const system = ctx.payload.system; if (!Array.isArray(system)) throw new Error('expected system to be an array'); diff --git a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-default-template_test.ts b/packages/provider-claude-code/__tests__/stages/inject-default-template_test.ts similarity index 82% rename from packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-default-template_test.ts rename to packages/provider-claude-code/__tests__/stages/inject-default-template_test.ts index 628f3290b7..d880a86446 100644 --- a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-default-template_test.ts +++ b/packages/provider-claude-code/__tests__/stages/inject-default-template_test.ts @@ -1,17 +1,20 @@ import { test } from 'vitest'; -import { injectDefaultTemplate } from '../../../src/interceptors/anthropic-messages/inject-default-template.ts'; -import { DEFAULT_TEMPLATE_BLOCK, IDENTITY_BLOCK } from '../../../src/interceptors/anthropic-messages/system-blocks.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { injectDefaultTemplate } from '../../src/stages/inject-default-template.ts'; +import { DEFAULT_TEMPLATE_BLOCK, IDENTITY_BLOCK } from '../../src/system-blocks.ts'; import type { AnthropicMessagesClientTool, AnthropicMessagesPayload, AnthropicMessagesStreamEvent, AnthropicMessagesTextBlock } from '@floway-dev/protocols/anthropic-messages'; -import type { ProviderStreamResult } from '@floway-dev/provider'; -import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; +import { type AnthropicMessagesProbe, type ProviderStreamResult } from '@floway-dev/test-utils'; +import { applyProviderStage, assertEquals, stubProviderModel } from '@floway-dev/test-utils'; + +type ClaudeCodeStageProbe = AnthropicMessagesProbe & { upstreamId: string }; const okEvents = (): Promise> => Promise.resolve({ ok: true, events: (async function* () {})(), modelKey: 'test' }); -const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload): ClaudeCodeStageProbe => ({ payload, + headers: new Headers(), + anthropicBeta: [], model: stubProviderModel({ endpoints: { anthropicMessages: {} } }), upstreamId: 'up_test', }); @@ -26,7 +29,7 @@ test('appends DEFAULT_TEMPLATE_BLOCK as system[2] with ephemeral cache_control i system: [billingBlock, IDENTITY_BLOCK], }); - await injectDefaultTemplate(ctx, {}, okEvents); + await applyProviderStage(injectDefaultTemplate, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.system, [billingBlock, IDENTITY_BLOCK, DEFAULT_TEMPLATE_BLOCK]); if (!Array.isArray(ctx.payload.system)) throw new Error('expected system to be an array'); @@ -57,7 +60,7 @@ test('preserves ephemeral cache_control when caller already holds 3 breakpoints tools: [cachedTool], }); - await injectDefaultTemplate(ctx, {}, okEvents); + await applyProviderStage(injectDefaultTemplate, ctx, okEvents, { 'request.claudeCode.shaped': false }); if (!Array.isArray(ctx.payload.system)) throw new Error('expected system to be an array'); assertEquals(ctx.payload.system.length, 4); @@ -94,7 +97,7 @@ test('demotes our cache_control when caller already holds 4 breakpoints (would b tools: [cachedTool], }); - await injectDefaultTemplate(ctx, {}, okEvents); + await applyProviderStage(injectDefaultTemplate, ctx, okEvents, { 'request.claudeCode.shaped': false }); if (!Array.isArray(ctx.payload.system)) throw new Error('expected system to be an array'); assertEquals(ctx.payload.system.length, 4); @@ -124,7 +127,7 @@ test('demotes our cache_control when caller already exceeds the cap', async () = tools: [cachedTool], }); - await injectDefaultTemplate(ctx, {}, okEvents); + await applyProviderStage(injectDefaultTemplate, ctx, okEvents, { 'request.claudeCode.shaped': false }); if (!Array.isArray(ctx.payload.system)) throw new Error('expected system to be an array'); const injected = ctx.payload.system.at(-1)!; diff --git a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-identity-block_test.ts b/packages/provider-claude-code/__tests__/stages/inject-identity-block_test.ts similarity index 62% rename from packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-identity-block_test.ts rename to packages/provider-claude-code/__tests__/stages/inject-identity-block_test.ts index f51a9a40aa..d3ea13a2ff 100644 --- a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/inject-identity-block_test.ts +++ b/packages/provider-claude-code/__tests__/stages/inject-identity-block_test.ts @@ -1,17 +1,20 @@ import { test } from 'vitest'; -import { injectIdentityBlock } from '../../../src/interceptors/anthropic-messages/inject-identity-block.ts'; -import { IDENTITY_BLOCK } from '../../../src/interceptors/anthropic-messages/system-blocks.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { injectIdentityBlock } from '../../src/stages/inject-identity-block.ts'; +import { IDENTITY_BLOCK } from '../../src/system-blocks.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProviderStreamResult } from '@floway-dev/provider'; -import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; +import { type AnthropicMessagesProbe, type ProviderStreamResult } from '@floway-dev/test-utils'; +import { applyProviderStage, assertEquals, stubProviderModel } from '@floway-dev/test-utils'; + +type ClaudeCodeStageProbe = AnthropicMessagesProbe & { upstreamId: string }; const okEvents = (): Promise> => Promise.resolve({ ok: true, events: (async function* () {})(), modelKey: 'test' }); -const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload): ClaudeCodeStageProbe => ({ payload, + headers: new Headers(), + anthropicBeta: [], model: stubProviderModel({ endpoints: { anthropicMessages: {} } }), upstreamId: 'up_test', }); @@ -25,7 +28,7 @@ test('appends IDENTITY_BLOCK after an existing system[0] block', async () => { system: [billing], }); - await injectIdentityBlock(ctx, {}, okEvents); + await applyProviderStage(injectIdentityBlock, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.system, [billing, IDENTITY_BLOCK]); }); @@ -39,7 +42,7 @@ test('appends IDENTITY_BLOCK onto a one-block system array regardless of block c system: [existing], }); - await injectIdentityBlock(ctx, {}, okEvents); + await applyProviderStage(injectIdentityBlock, ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.system, [existing, IDENTITY_BLOCK]); }); diff --git a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/synthesize-metadata-user-id_test.ts b/packages/provider-claude-code/__tests__/stages/synthesize-metadata-user-id_test.ts similarity index 69% rename from packages/provider-claude-code/__tests__/interceptors/anthropic-messages/synthesize-metadata-user-id_test.ts rename to packages/provider-claude-code/__tests__/stages/synthesize-metadata-user-id_test.ts index 93a273dffa..306e32d231 100644 --- a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/synthesize-metadata-user-id_test.ts +++ b/packages/provider-claude-code/__tests__/stages/synthesize-metadata-user-id_test.ts @@ -1,20 +1,22 @@ import { parse, validate, version } from 'uuid'; import { test } from 'vitest'; -import { parseMetadataUserID } from '../../../src/detection.ts'; -import { hoistUserSystemToMessages } from '../../../src/interceptors/anthropic-messages/hoist-user-system-to-messages.ts'; -import { CLAUDE_CODE_ANTHROPIC_MESSAGES_BOUNDARY } from '../../../src/interceptors/anthropic-messages/index.ts'; -import { synthesizeMetadataUserId } from '../../../src/interceptors/anthropic-messages/synthesize-metadata-user-id.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { parseMetadataUserID } from '../../src/detection.ts'; +import { hoistUserSystemToMessages } from '../../src/stages/hoist-user-system-to-messages.ts'; +import { synthesizeMetadataUserId } from '../../src/stages/synthesize-metadata-user-id.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProviderStreamResult } from '@floway-dev/provider'; -import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; +import { type AnthropicMessagesProbe, type ProviderStreamResult } from '@floway-dev/test-utils'; +import { applyProviderStage, assertEquals, stubProviderModel } from '@floway-dev/test-utils'; + +type ClaudeCodeStageProbe = AnthropicMessagesProbe & { upstreamId: string }; const okEvents = (): Promise> => Promise.resolve({ ok: true, events: (async function* () {})(), modelKey: 'test' }); -const invocation = (payload: AnthropicMessagesPayload, upstreamId = 'up_test'): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload, upstreamId = 'up_test'): ClaudeCodeStageProbe => ({ payload, + headers: new Headers(), + anthropicBeta: [], model: stubProviderModel({ endpoints: { anthropicMessages: {} } }), upstreamId, }); @@ -26,7 +28,7 @@ test('fills metadata.user_id with the new JSON shape when absent', async () => { messages: [{ role: 'user', content: 'hello' }], }); - await synthesizeMetadataUserId(ctx, {}, okEvents); + await applyProviderStage(synthesizeMetadataUserId(ctx.upstreamId), ctx, okEvents, { 'request.claudeCode.shaped': false }); const userId = ctx.payload.metadata?.user_id; if (typeof userId !== 'string') throw new Error('expected user_id to be a string'); @@ -43,8 +45,8 @@ test('fills metadata.user_id with the new JSON shape when absent', async () => { test('device_id is stable per upstream id', async () => { const a = invocation({ model: 'claude-sonnet-4-5-20250929', max_tokens: 1, messages: [{ role: 'user', content: 'a' }] }); const b = invocation({ model: 'claude-sonnet-4-5-20250929', max_tokens: 1, messages: [{ role: 'user', content: 'b' }] }); - await synthesizeMetadataUserId(a, {}, okEvents); - await synthesizeMetadataUserId(b, {}, okEvents); + await applyProviderStage(synthesizeMetadataUserId(a.upstreamId), a, okEvents, { 'request.claudeCode.shaped': false }); + await applyProviderStage(synthesizeMetadataUserId(b.upstreamId), b, okEvents, { 'request.claudeCode.shaped': false }); const ad = parseMetadataUserID(a.payload.metadata!.user_id!)!; const bd = parseMetadataUserID(b.payload.metadata!.user_id!)!; assertEquals(ad.deviceId, bd.deviceId); @@ -53,8 +55,8 @@ test('device_id is stable per upstream id', async () => { test('device_id differs across upstreams', async () => { const a = invocation({ model: 'm', max_tokens: 1, messages: [{ role: 'user', content: 'x' }] }, 'up_a'); const b = invocation({ model: 'm', max_tokens: 1, messages: [{ role: 'user', content: 'x' }] }, 'up_b'); - await synthesizeMetadataUserId(a, {}, okEvents); - await synthesizeMetadataUserId(b, {}, okEvents); + await applyProviderStage(synthesizeMetadataUserId(a.upstreamId), a, okEvents, { 'request.claudeCode.shaped': false }); + await applyProviderStage(synthesizeMetadataUserId(b.upstreamId), b, okEvents, { 'request.claudeCode.shaped': false }); const ad = parseMetadataUserID(a.payload.metadata!.user_id!)!; const bd = parseMetadataUserID(b.payload.metadata!.user_id!)!; if (ad.deviceId === bd.deviceId) throw new Error('expected different device ids per upstream'); @@ -63,8 +65,8 @@ test('device_id differs across upstreams', async () => { test('session_id is stable for same upstream + same first-user prefix', async () => { const a = invocation({ model: 'm', max_tokens: 1, messages: [{ role: 'user', content: 'prefix' }, { role: 'assistant', content: 'reply1' }] }); const b = invocation({ model: 'm', max_tokens: 1, messages: [{ role: 'user', content: 'prefix' }, { role: 'assistant', content: 'reply2' }] }); - await synthesizeMetadataUserId(a, {}, okEvents); - await synthesizeMetadataUserId(b, {}, okEvents); + await applyProviderStage(synthesizeMetadataUserId(a.upstreamId), a, okEvents, { 'request.claudeCode.shaped': false }); + await applyProviderStage(synthesizeMetadataUserId(b.upstreamId), b, okEvents, { 'request.claudeCode.shaped': false }); const ad = parseMetadataUserID(a.payload.metadata!.user_id!)!; const bd = parseMetadataUserID(b.payload.metadata!.user_id!)!; assertEquals(ad.sessionId, bd.sessionId); @@ -78,7 +80,7 @@ test('preserves a caller-supplied user_id verbatim', async () => { messages: [{ role: 'user', content: 'x' }], metadata: { user_id: explicit }, }); - await synthesizeMetadataUserId(ctx, {}, okEvents); + await applyProviderStage(synthesizeMetadataUserId(ctx.upstreamId), ctx, okEvents, { 'request.claudeCode.shaped': false }); assertEquals(ctx.payload.metadata?.user_id, explicit); }); @@ -105,8 +107,10 @@ test('session_id differs when system prompt is shared but user message differs ( // Drive the same step pair the production chain does: synthesize first, // then hoist. Synthesize sees the operator's real first user message; // hoist runs after and rewrites `messages` for the wire shape. - await synthesizeMetadataUserId(a, {}, () => hoistUserSystemToMessages(a, {}, okEvents)); - await synthesizeMetadataUserId(b, {}, () => hoistUserSystemToMessages(b, {}, okEvents)); + await applyProviderStage(synthesizeMetadataUserId(a.upstreamId), a, okEvents, { 'request.claudeCode.shaped': false }); + await applyProviderStage(hoistUserSystemToMessages, a, okEvents, { 'request.claudeCode.shaped': false }); + await applyProviderStage(synthesizeMetadataUserId(b.upstreamId), b, okEvents, { 'request.claudeCode.shaped': false }); + await applyProviderStage(hoistUserSystemToMessages, b, okEvents, { 'request.claudeCode.shaped': false }); const ad = parseMetadataUserID(a.payload.metadata!.user_id!)!; const bd = parseMetadataUserID(b.payload.metadata!.user_id!)!; @@ -114,11 +118,3 @@ test('session_id differs when system prompt is shared but user message differs ( throw new Error('expected different session_ids for distinct user prompts sharing a system prompt'); } }); - -test('chain registers synthesize before hoist', () => { - const chain = CLAUDE_CODE_ANTHROPIC_MESSAGES_BOUNDARY; - const synthIdx = chain.indexOf(synthesizeMetadataUserId); - const hoistIdx = chain.indexOf(hoistUserSystemToMessages); - if (synthIdx === -1 || hoistIdx === -1) throw new Error('chain missing required step'); - if (synthIdx >= hoistIdx) throw new Error(`synthesize (${synthIdx}) must run before hoist (${hoistIdx}) so session_id derives from the real user message`); -}); diff --git a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/system-blocks_test.ts b/packages/provider-claude-code/__tests__/system-blocks_test.ts similarity index 98% rename from packages/provider-claude-code/__tests__/interceptors/anthropic-messages/system-blocks_test.ts rename to packages/provider-claude-code/__tests__/system-blocks_test.ts index 6642ddd92e..e9428118e4 100644 --- a/packages/provider-claude-code/__tests__/interceptors/anthropic-messages/system-blocks_test.ts +++ b/packages/provider-claude-code/__tests__/system-blocks_test.ts @@ -5,7 +5,7 @@ import { computeCcVersionFingerprint, DEFAULT_TEMPLATE_BLOCK, IDENTITY_BLOCK, -} from '../../../src/interceptors/anthropic-messages/system-blocks.ts'; +} from '../src/system-blocks.ts'; import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; const minimalBody = (firstUserText: string): AnthropicMessagesPayload => ({ diff --git a/packages/provider-claude-code/__tests__/test-utils/call.ts b/packages/provider-claude-code/__tests__/test-utils/call.ts new file mode 100644 index 0000000000..fcf95c6e34 --- /dev/null +++ b/packages/provider-claude-code/__tests__/test-utils/call.ts @@ -0,0 +1,24 @@ +import { authenticateClaudeCode } from '../../src/stages/authenticate.ts'; +import { decodeClaudeCodeResponse } from '../../src/stages/decode-response.ts'; +import { observeClaudeCodeResponse } from '../../src/stages/observe-response.ts'; +import { prepareClaudeCodeRequest } from '../../src/stages/prepare-request.ts'; +import { retryClaudeCodeAccess } from '../../src/stages/retry-access.ts'; +import { http } from '@floway-dev/http/pipeline'; +import { compose, defineStage, move, type RunServices } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import { observeProviderCall, type Provider, type ProviderModel, type AnthropicMessagesUpstreamCallOptions } from '@floway-dev/provider'; +import { collectChatProviderPipeline, type ProviderStreamResult } from '@floway-dev/test-utils'; + +interface Options { upstreamId: string; model: ProviderModel; body: Omit; shaped: boolean; signal?: AbortSignal; call: AnthropicMessagesUpstreamCallOptions; observers?: Pick } +export const callClaudeCodeAnthropicMessages = async (opts: Options): Promise> => { + const prepare = defineStage, Record, Record, Record>({ + name: 'supplyClaudeCodePreparedFixture', + through: { request: { needs: [], consumes: [], provides: ['request.claudeCode.shaped', 'request.provider.modelKey'] }, response: { needs: [], consumes: [], provides: [] } }, + execute: async (facts, next) => move({ ...await next(move({ ...facts, 'request.claudeCode.shaped': opts.shaped, 'request.provider.modelKey': (opts.model.providerData as { upstreamModelId: string }).upstreamModelId })) }), + }); + const provider: Provider = { upstreamId: opts.upstreamId, kind: 'claude-code', name: 'Claude Code fixture', inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, instance: { getProvidedModels: async () => [] }, pipelines: { anthropicMessages: compose('claudeCode.transportFixture', [prepare, authenticateClaudeCode(opts.upstreamId), prepareClaudeCodeRequest, decodeClaudeCodeResponse(opts.upstreamId), retryClaudeCodeAccess(opts.upstreamId), observeClaudeCodeResponse(opts.upstreamId), observeProviderCall, http]) } }; + const result = await collectChatProviderPipeline(provider, 'anthropicMessages', opts.model, opts.body, opts.signal, opts.call, opts.observers); + if (result.output !== null && 'kind' in result.output && result.output.kind === 'stream') return { ok: true, modelKey: result.facts['response.provider.modelKey'], events: (async function* () { yield* result.frames; })() }; + if (result.response === null) throw new Error('Expected Messages stream or HTTP failure'); + return { ok: false, modelKey: result.facts['response.provider.modelKey'], response: result.response }; +}; diff --git a/packages/provider-claude-code/package.json b/packages/provider-claude-code/package.json index 70d9de00a1..27124730a1 100644 --- a/packages/provider-claude-code/package.json +++ b/packages/provider-claude-code/package.json @@ -10,7 +10,8 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@floway-dev/interceptor": "workspace:*", + "@floway-dev/http": "workspace:*", + "@floway-dev/pipeline": "workspace:*", "@floway-dev/protocols": "workspace:*", "@floway-dev/provider": "workspace:*", "@noble/hashes": "^2.2.0", diff --git a/packages/provider-claude-code/src/access-token.ts b/packages/provider-claude-code/src/access-token.ts index 91145cb0c9..a02946fa56 100644 --- a/packages/provider-claude-code/src/access-token.ts +++ b/packages/provider-claude-code/src/access-token.ts @@ -198,7 +198,7 @@ const ensureClaudeCodeAccessTokenInner = async ( return { entry: newAccessTokenEntry, freshlyMinted: true }; }; -// Terminal flip from the oauth-error path. Distinct from fetch.ts's +// Terminal flip from the oauth-error path. Distinct from backend.ts's // `persistTerminalAccountState`: the caller has already read the account and // established that it is active, so the log's identity and `from_state` come // from that read instead of a second one, and the trigger is an oauth-protocol diff --git a/packages/provider-claude-code/src/backend.ts b/packages/provider-claude-code/src/backend.ts new file mode 100644 index 0000000000..047725942a --- /dev/null +++ b/packages/provider-claude-code/src/backend.ts @@ -0,0 +1,242 @@ +import { logWarn, logInfo, type LogFields } from './log.ts'; +import { type ClaudeCodeQuotaSnapshot } from './quota.ts'; +import { + readClaudeCodeUpstreamState, + replaceSoleAccount, + type ClaudeCodeAccountCredential, +} from './state.ts'; +import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import { getProviderRepo } from '@floway-dev/provider'; + +export const ANTHROPIC_MESSAGES_ENDPOINT = 'https://api.anthropic.com/v1/messages?beta=true'; +export const STREAM_DIAGNOSTIC_FRAME_LIMIT = 3; +export const STREAM_DIAGNOSTIC_FRAME_DATA_CHARS = 256; + +export const synthetic503 = (message: string): Response => + new Response( + JSON.stringify({ error: { type: 'claude_code_upstream_unavailable', message } }), + { status: 503, headers: { 'content-type': 'application/json' } }, + ); + +export const synthetic429 = (message: string, retryAtIso: string | null, now: Date): Response => { + const retryAfterSeconds = retryAtIso === null + ? 60 + : Math.max(0, Math.ceil((new Date(retryAtIso).getTime() - now.getTime()) / 1000)); + return new Response( + JSON.stringify({ error: { type: 'claude_code_rate_limited', message, retry_at: retryAtIso } }), + { + status: 429, + headers: { 'content-type': 'application/json', 'retry-after': String(retryAfterSeconds) }, + }, + ); +}; + +export interface StreamDiagnosticFrame { + event: string | null; + data: string; +} + +export const observedClaudeCodeMessagesStream = async function* ( + events: AsyncIterable>, + options: { + upstreamId: string; + model: string; + headers: Headers; + signal: AbortSignal | undefined; + frames: StreamDiagnosticFrame[]; + rawFrameCount: () => number; + warn: (message: string, fields: Readonly>) => Promise; + }, +): AsyncGenerator> { + let terminalEvent: 'message_stop' | 'error' | null = null; + let streamError: unknown; + try { + for await (const frame of events) { + if (frame.type === 'event') { + if (frame.event.type === 'message_stop') terminalEvent = 'message_stop'; + else if (frame.event.type === 'error') terminalEvent = 'error'; + } + yield frame; + } + } catch (error) { + streamError = error; + throw error; + } finally { + if (!options.signal?.aborted && !(terminalEvent !== null && streamError === undefined)) { + const fields = { + upstream_id: options.upstreamId, + model: options.model, + request_id: options.headers.get('request-id'), + cf_ray: options.headers.get('cf-ray'), + trace_response: options.headers.get('traceresponse'), + raw_sse_frames: options.rawFrameCount(), + terminal_event: terminalEvent, + error: streamError === undefined ? null : streamError, + last_sse_frames: JSON.stringify(options.frames), + }; + try { await options.warn('claude_code_messages_stream_incomplete', fields); } catch (recordingError) { + if (streamError === undefined || recordingError === streamError) throw recordingError; + throw new AggregateError([streamError, recordingError], 'Claude Code stream failed and recording also failed', { cause: streamError }); + } + } + } +}; + +// `anthropic-ratelimit-unified-status: rejected` paired with a future +// `unified-reset` timestamp means the upstream's primary plan window is +// exhausted and a fresh request would 429 right away; short-circuit at +// the gate so we don't burn an OAuth refresh on a request that has no +// chance. +// +// Note 1: `overage.status: rejected` (typically paired with +// `overage-disabled-reason: out_of_credits`) is NOT a short-circuit +// signal. It only reports that the account has no extra-usage credits +// to spill into once the primary window runs out — which is the steady +// state for any plan-tier account that hasn't bought extra credits, so +// blocking on it would refuse every request to such accounts. The +// primary `status` already reflects whether the upstream will actually +// reject the next request. +// +// Note 2: a primary `status: rejected` WITHOUT a `reset` is treated as +// non-gating. Sub2api `ratelimit_service.go:953-961` flags this exact +// shape as "likely not a real rate limit" (e.g. an "Extra usage required" +// body sentinel) and passes it through verbatim — without a reset we'd +// otherwise lock the account out indefinitely because the next request +// never fires to refresh the snapshot. +export const isRateLimitedNow = ( + snapshot: ClaudeCodeQuotaSnapshot | null, + now: Date, +): snapshot is ClaudeCodeQuotaSnapshot => { + if (!snapshot) return false; + if (snapshot.status !== 'rejected') return false; + if (!snapshot.reset) return false; + return new Date(snapshot.reset).getTime() > now.getTime(); +}; + +export const persistQuotaSnapshot = async (upstreamId: string, snapshot: ClaudeCodeQuotaSnapshot, info: (message: string, fields: LogFields) => void | Promise = logInfo): Promise => { + // Stamped before the write: the mutator is replayed on a lost race and must + // return the same document each time, and this records when the snapshot was + // observed rather than which attempt landed it. + const fetchedAt = Date.now(); + // The prior status comes from the state the mutator was handed: the write is + // retried against whoever won the row, so only that view describes the + // snapshot this write actually replaced. + let previousAccount!: ClaudeCodeAccountCredential; + await getProviderRepo().upstreams.saveState(upstreamId, current => { + const state = readClaudeCodeUpstreamState(current); + previousAccount = state.accounts[0]; + return replaceSoleAccount(state, account => ({ + ...account, + quotaSnapshot: { fetchedAt, data: snapshot }, + })); + }); + const priorStatus = previousAccount.quotaSnapshot === null ? null : previousAccount.quotaSnapshot.data.status; + // Emit only on transition. Persisting every response would flood the log + // with one event per request; the dashboard already reads the snapshot + // verbatim. Operators care about the moment the upstream flipped from + // `allowed` to `rejected` (or back), not the steady state. + if (priorStatus !== snapshot.status) { + await info('claude_code_quota_state_transition', { + upstream_id: upstreamId, + account_uuid: previousAccount.accountUuid, + from_status: priorStatus, + to_status: snapshot.status, + reset_at_iso: snapshot.reset, + representative_claim: snapshot.representativeClaim, + }); + } +}; + +// Credential-class terminal sentinels on the data-plane response body. Both +// signal a permanently disabled org that no retry, refresh, or re-import +// can recover — the operator must contact Anthropic. Matching the +// lowercased `error.message` substring mirrors sub2api +// `ratelimit_service.go:208-214` (400 path) and CRS +// `claudeRelayService.js:140-153` (`_isOrganizationDisabledError`, both +// 400 and 403). We match on "organization has been disabled" rather than +// CRS's slightly longer "this organization has been disabled" so a body +// that omits the leading "this" still matches (sub2api uses the shorter +// form too). +const ORG_DISABLED_400_SENTINEL = 'organization has been disabled'; +const ORG_BANNED_403_SENTINEL = 'oauth authentication is currently not allowed'; + +interface AnthropicErrorBody { + error?: { type?: unknown; message?: unknown }; +} + +// Returns the operator-facing terminal message when the response matches +// one of the credential-class sentinels, or `null` otherwise. The body +// parse is intentionally defensive: a 400/403 that isn't JSON (or whose +// JSON shape doesn't match `{error:{type,message}}`) is the common case +// for unrelated errors (`max_tokens` validation, beta-feature gating, +// etc.) and must NOT trigger a terminal flip. +export const detectTerminalSentinel = (status: number, bodyText: string): string | null => { + if (status !== 400 && status !== 403) return null; + let parsed: unknown; + try { + parsed = JSON.parse(bodyText); + } catch { + logWarn('claude_code_unparseable_error_body', { status, body_snippet: bodyText.slice(0, 256) }); + return null; + } + if (typeof parsed !== 'object' || parsed === null) return null; + const error = (parsed as AnthropicErrorBody).error; + if (typeof error !== 'object' || error === null) return null; + const type = error.type; + const message = error.message; + if (typeof type !== 'string' || typeof message !== 'string') return null; + const lowered = message.toLowerCase(); + if (status === 400 && type === 'invalid_request_error' && lowered.includes(ORG_DISABLED_400_SENTINEL)) { + return `Organization disabled by Anthropic — re-import will not recover; contact support: ${message}`; + } + if (status === 403 && type === 'permission_error' && lowered.includes(ORG_BANNED_403_SENTINEL)) { + return `Organization banned from OAuth by Anthropic — re-import will not recover; contact support: ${message}`; + } + return null; +}; + +// Terminal flip from the data-plane sentinel detector. Distinct from +// access-token.ts's `persistTerminalState`: this path runs in a +// deferred response-observation context, so the account it logs +// comes from the mutator's own view; the flip is body-sentinel-triggered (org +// disabled/banned), not oauth-error-triggered, so the log carries +// `upstream_status` instead of `oauth_code`; and a sibling oauth-side flip may +// have already happened, so an account that is no longer `active` is returned +// untouched — the repo reads that as "nothing to do" and the dashboard keeps +// the first signal. Merging the two helpers would force conditional dispatch +// on every one of these axes. +export const persistTerminalAccountState = async ( + upstreamId: string, + terminalMessage: string, + reason: string, + upstreamStatus: number, + warn: (message: string, fields: LogFields) => void | Promise = logWarn, +): Promise => { + // Stamped before the write for the same reason as the quota snapshot: a + // replay must produce the same document. + const flippedAt = new Date().toISOString(); + let previousAccount!: ClaudeCodeAccountCredential; + await getProviderRepo().upstreams.saveState(upstreamId, current => { + const state = readClaudeCodeUpstreamState(current); + previousAccount = state.accounts[0]; + if (previousAccount.state !== 'active') return state; + return replaceSoleAccount(state, account => ({ + ...account, + state: 'refresh_failed', + stateMessage: terminalMessage, + stateUpdatedAt: flippedAt, + accessToken: null, + })); + }); + if (previousAccount.state !== 'active') return; + await warn('claude_code_account_state_flip', { + upstream_id: upstreamId, + account_uuid: previousAccount.accountUuid, + from_state: previousAccount.state, + to_state: 'refresh_failed', + reason, + upstream_status: upstreamStatus, + message: terminalMessage, + }); +}; diff --git a/packages/provider-claude-code/src/fetch.ts b/packages/provider-claude-code/src/fetch.ts deleted file mode 100644 index c96ce3dea6..0000000000 --- a/packages/provider-claude-code/src/fetch.ts +++ /dev/null @@ -1,516 +0,0 @@ -import { ensureClaudeCodeAccessToken, invalidateClaudeCodeAccessToken, type EnsuredAccessToken } from './access-token.ts'; -import { ClaudeCodeOAuthSessionTerminatedError } from './auth/oauth.ts'; -import { pickClaudeCodeHeaders } from './headers.ts'; -import { logWarn, logInfo } from './log.ts'; -import type { ClaudeCodeProviderData } from './models.ts'; -import { parseClaudeCodeQuotaHeaders, type ClaudeCodeQuotaSnapshot } from './quota.ts'; -import { - readClaudeCodeUpstreamState, - replaceSoleAccount, - type ClaudeCodeAccountCredential, -} from './state.ts'; -import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import { parseAnthropicMessagesStream } from '@floway-dev/protocols/anthropic-messages'; -import type { ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; -import { - getProviderRepo, - headersForAnthropicMessagesCall, - jsonRequestBody, - streamingProviderCall, - type AnthropicMessagesUpstreamCallOptions, - type ProviderModel, - type ProviderStreamResult, -} from '@floway-dev/provider'; - -const ANTHROPIC_ANTHROPIC_MESSAGES_ENDPOINT = 'https://api.anthropic.com/v1/messages?beta=true'; -const STREAM_DIAGNOSTIC_FRAME_LIMIT = 3; -const STREAM_DIAGNOSTIC_FRAME_DATA_CHARS = 256; - -export interface CallClaudeCodeAnthropicMessagesOptions { - upstreamId: string; - model: ProviderModel; - body: Omit; - // `shaped: true` means the inbound request already looks like real CC - // traffic. The gateway has reduced ordinary headers to the provider module's - // allowlist and carries anthropic-beta as typed Anthropic Messages metadata, so the - // wire path preserves the complete genuine fingerprint. It supplies a - // default Content-Type when absent and sets cached OAuth auth. - // `shaped: false` means the gateway's re-mimicry chain rebuilt the - // payload's system blocks / metadata / model id — replace headers with - // the pinned CC set so the wire shape matches end-to-end. - shaped: boolean; - signal?: AbortSignal; - call: AnthropicMessagesUpstreamCallOptions; -} - -const synthetic503 = (message: string): Response => - new Response( - JSON.stringify({ error: { type: 'claude_code_upstream_unavailable', message } }), - { status: 503, headers: { 'content-type': 'application/json' } }, - ); - -const synthetic429 = (message: string, retryAtIso: string | null, now: Date): Response => { - const retryAfterSeconds = retryAtIso === null - ? 60 - : Math.max(0, Math.ceil((new Date(retryAtIso).getTime() - now.getTime()) / 1000)); - return new Response( - JSON.stringify({ error: { type: 'claude_code_rate_limited', message, retry_at: retryAtIso } }), - { - status: 429, - headers: { 'content-type': 'application/json', 'retry-after': String(retryAfterSeconds) }, - }, - ); -}; - -const oneLineError = (error: unknown): string => { - const message = (error instanceof Error ? error.message : String(error)).replace(/\s+/g, ' ').trim(); - return message.length > 512 ? `${message.slice(0, 509)}...` : message; -}; - -interface StreamDiagnosticFrame { - event: string | null; - data: string; -} - -const observedClaudeCodeMessagesStream = async function* ( - events: AsyncIterable>, - options: { - upstreamId: string; - model: string; - headers: Headers; - signal: AbortSignal | undefined; - frames: StreamDiagnosticFrame[]; - rawFrameCount: () => number; - }, -): AsyncGenerator> { - let terminalEvent: 'message_stop' | 'error' | null = null; - let streamError: unknown; - try { - for await (const frame of events) { - if (frame.type === 'event') { - if (frame.event.type === 'message_stop') terminalEvent = 'message_stop'; - else if (frame.event.type === 'error') terminalEvent = 'error'; - } - yield frame; - } - } catch (error) { - streamError = error; - throw error; - } finally { - if (options.signal?.aborted || (terminalEvent !== null && streamError === undefined)) return; - logWarn('claude_code_messages_stream_incomplete', { - upstream_id: options.upstreamId, - model: options.model, - request_id: options.headers.get('request-id'), - cf_ray: options.headers.get('cf-ray'), - trace_response: options.headers.get('traceresponse'), - raw_sse_frames: options.rawFrameCount(), - terminal_event: terminalEvent, - error: streamError === undefined ? null : oneLineError(streamError), - last_sse_frames: JSON.stringify(options.frames), - }); - } -}; - -// `anthropic-ratelimit-unified-status: rejected` paired with a future -// `unified-reset` timestamp means the upstream's primary plan window is -// exhausted and a fresh request would 429 right away; short-circuit at -// the gate so we don't burn an OAuth refresh on a request that has no -// chance. -// -// Note 1: `overage.status: rejected` (typically paired with -// `overage-disabled-reason: out_of_credits`) is NOT a short-circuit -// signal. It only reports that the account has no extra-usage credits -// to spill into once the primary window runs out — which is the steady -// state for any plan-tier account that hasn't bought extra credits, so -// blocking on it would refuse every request to such accounts. The -// primary `status` already reflects whether the upstream will actually -// reject the next request. -// -// Note 2: a primary `status: rejected` WITHOUT a `reset` is treated as -// non-gating. Sub2api `ratelimit_service.go:953-961` flags this exact -// shape as "likely not a real rate limit" (e.g. an "Extra usage required" -// body sentinel) and passes it through verbatim — without a reset we'd -// otherwise lock the account out indefinitely because the next request -// never fires to refresh the snapshot. -const isRateLimitedNow = ( - snapshot: ClaudeCodeQuotaSnapshot | null, - now: Date, -): snapshot is ClaudeCodeQuotaSnapshot => { - if (!snapshot) return false; - if (snapshot.status !== 'rejected') return false; - if (!snapshot.reset) return false; - return new Date(snapshot.reset).getTime() > now.getTime(); -}; - -const persistQuotaSnapshot = async (upstreamId: string, snapshot: ClaudeCodeQuotaSnapshot): Promise => { - // Stamped before the write: the mutator is replayed on a lost race and must - // return the same document each time, and this records when the snapshot was - // observed rather than which attempt landed it. - const fetchedAt = Date.now(); - // The prior status comes from the state the mutator was handed: the write is - // retried against whoever won the row, so only that view describes the - // snapshot this write actually replaced. - let previousAccount!: ClaudeCodeAccountCredential; - await getProviderRepo().upstreams.saveState(upstreamId, current => { - const state = readClaudeCodeUpstreamState(current); - previousAccount = state.accounts[0]; - return replaceSoleAccount(state, account => ({ - ...account, - quotaSnapshot: { fetchedAt, data: snapshot }, - })); - }); - const priorStatus = previousAccount.quotaSnapshot === null ? null : previousAccount.quotaSnapshot.data.status; - // Emit only on transition. Persisting every response would flood the log - // with one event per request; the dashboard already reads the snapshot - // verbatim. Operators care about the moment the upstream flipped from - // `allowed` to `rejected` (or back), not the steady state. - if (priorStatus !== snapshot.status) { - logInfo('claude_code_quota_state_transition', { - upstream_id: upstreamId, - account_uuid: previousAccount.accountUuid, - from_status: priorStatus, - to_status: snapshot.status, - reset_at_iso: snapshot.reset, - representative_claim: snapshot.representativeClaim, - }); - } -}; - -// Best-effort persist: the hot path must not block on the write completing or -// surface its failures to the caller. Skip writing when the response carries -// no rate-limit signal at all — that would erase the prior snapshot for no -// upside. -// -// On Cloudflare Workers the runtime cancels orphan promises the moment the -// response is sent to the client, so a bare fire-and-forget would lose the -// write on the hot path. The `waitUntil` callback (when supplied by the -// gateway) extends the worker's lifetime past the response so the persist -// completes. When `waitUntil` is undefined (Node target / tests), the -// promise still runs to completion under the host event loop -// — and tests can observe it by awaiting a microtask flush. -const persistQuotaFromHeadersFireAndForget = ( - upstreamId: string, - headers: Headers, - waitUntil: ((promise: Promise) => void) | undefined, -): void => { - const snapshot = parseClaudeCodeQuotaHeaders(headers); - if (Object.keys(snapshot.raw).length === 0) return; - const persist = persistQuotaSnapshot(upstreamId, snapshot).catch(error => { - logWarn('claude_code_quota_persist_failed', { - upstream_id: upstreamId, - error: String(error), - }); - }); - waitUntil?.(persist); -}; - -// Credential-class terminal sentinels on the data-plane response body. Both -// signal a permanently disabled org that no retry, refresh, or re-import -// can recover — the operator must contact Anthropic. Matching the -// lowercased `error.message` substring mirrors sub2api -// `ratelimit_service.go:208-214` (400 path) and CRS -// `claudeRelayService.js:140-153` (`_isOrganizationDisabledError`, both -// 400 and 403). We match on "organization has been disabled" rather than -// CRS's slightly longer "this organization has been disabled" so a body -// that omits the leading "this" still matches (sub2api uses the shorter -// form too). -const ORG_DISABLED_400_SENTINEL = 'organization has been disabled'; -const ORG_BANNED_403_SENTINEL = 'oauth authentication is currently not allowed'; - -interface AnthropicErrorBody { - error?: { type?: unknown; message?: unknown }; -} - -// Returns the operator-facing terminal message when the response matches -// one of the credential-class sentinels, or `null` otherwise. The body -// parse is intentionally defensive: a 400/403 that isn't JSON (or whose -// JSON shape doesn't match `{error:{type,message}}`) is the common case -// for unrelated errors (`max_tokens` validation, beta-feature gating, -// etc.) and must NOT trigger a terminal flip. -const detectTerminalSentinel = (status: number, bodyText: string): string | null => { - if (status !== 400 && status !== 403) return null; - let parsed: unknown; - try { - parsed = JSON.parse(bodyText); - } catch { - logWarn('claude_code_unparseable_error_body', { status, body_snippet: bodyText.slice(0, 256) }); - return null; - } - if (typeof parsed !== 'object' || parsed === null) return null; - const error = (parsed as AnthropicErrorBody).error; - if (typeof error !== 'object' || error === null) return null; - const type = error.type; - const message = error.message; - if (typeof type !== 'string' || typeof message !== 'string') return null; - const lowered = message.toLowerCase(); - if (status === 400 && type === 'invalid_request_error' && lowered.includes(ORG_DISABLED_400_SENTINEL)) { - return `Organization disabled by Anthropic — re-import will not recover; contact support: ${message}`; - } - if (status === 403 && type === 'permission_error' && lowered.includes(ORG_BANNED_403_SENTINEL)) { - return `Organization banned from OAuth by Anthropic — re-import will not recover; contact support: ${message}`; - } - return null; -}; - -// Terminal flip from the data-plane sentinel detector. Distinct from -// access-token.ts's `persistTerminalState`: this path runs in a -// fire-and-forget context with no caller-side state, so the account it logs -// comes from the mutator's own view; the flip is body-sentinel-triggered (org -// disabled/banned), not oauth-error-triggered, so the log carries -// `upstream_status` instead of `oauth_code`; and a sibling oauth-side flip may -// have already happened, so an account that is no longer `active` is returned -// untouched — the repo reads that as "nothing to do" and the dashboard keeps -// the first signal. Merging the two helpers would force conditional dispatch -// on every one of these axes. -const persistTerminalAccountState = async ( - upstreamId: string, - terminalMessage: string, - reason: string, - upstreamStatus: number, -): Promise => { - // Stamped before the write for the same reason as the quota snapshot: a - // replay must produce the same document. - const flippedAt = new Date().toISOString(); - let previousAccount!: ClaudeCodeAccountCredential; - await getProviderRepo().upstreams.saveState(upstreamId, current => { - const state = readClaudeCodeUpstreamState(current); - previousAccount = state.accounts[0]; - if (previousAccount.state !== 'active') return state; - return replaceSoleAccount(state, account => ({ - ...account, - state: 'refresh_failed', - stateMessage: terminalMessage, - stateUpdatedAt: flippedAt, - accessToken: null, - })); - }); - if (previousAccount.state !== 'active') return; - logWarn('claude_code_account_state_flip', { - upstream_id: upstreamId, - account_uuid: previousAccount.accountUuid, - from_state: previousAccount.state, - to_state: 'refresh_failed', - reason, - upstream_status: upstreamStatus, - message: terminalMessage, - }); -}; - -// Fire-and-forget: clones the response so the original body still streams -// to the caller intact (we surface upstream verbatim — the terminal flip -// is purely additional dashboard signal). -// -// Same lifecycle dance as `persistQuotaFromHeadersFireAndForget`: under -// Workers the runtime cancels orphan promises the moment the response is -// sent, so we register the persist with `waitUntil` when the gateway -// supplied it. -const maybePersistTerminalFromBodyFireAndForget = ( - upstreamId: string, - response: Response, - waitUntil: ((promise: Promise) => void) | undefined, -): void => { - if (response.status !== 400 && response.status !== 403) return; - const cloned = response.clone(); - const task = (async () => { - let bodyText: string; - try { - bodyText = await cloned.text(); - } catch (error) { - logWarn('claude_code_terminal_sentinel_body_read_failed', { - upstream_id: upstreamId, - upstream_status: response.status, - error: String(error), - }); - return; - } - const terminalMessage = detectTerminalSentinel(response.status, bodyText); - if (terminalMessage === null) return; - const reason = response.status === 400 ? 'org_disabled_400_sentinel' : 'org_banned_403_sentinel'; - logWarn('claude_code_terminal_sentinel_detected', { - upstream_id: upstreamId, - upstream_status: response.status, - reason, - }); - await persistTerminalAccountState(upstreamId, terminalMessage, reason, response.status); - })().catch(error => { - logWarn('claude_code_terminal_sentinel_persist_failed', { - upstream_id: upstreamId, - error: String(error), - }); - }); - waitUntil?.(task); -}; - -const syntheticReturn = ( - upstreamModelId: string, - response: Response, -): ProviderStreamResult => ({ - ok: false, - modelKey: upstreamModelId, - response, -}); - -// Either ensures a usable access token or returns a 503 wrap for terminal -// refresh failures; other errors propagate. Used at both the cold-start -// call site and the 401-retry branch so the catch shape lives in one place. -const ensureOrSession503 = async ( - opts: CallClaudeCodeAnthropicMessagesOptions, - upstreamModelId: string, -): Promise> => { - try { - return await ensureClaudeCodeAccessToken({ - upstreamId: opts.upstreamId, - repo: getProviderRepo().upstreams, - fetcher: opts.call.fetcher, - }); - } catch (err) { - if (err instanceof ClaudeCodeOAuthSessionTerminatedError) { - // ensureClaudeCodeAccessToken already persisted the terminal state. - return syntheticReturn(upstreamModelId, synthetic503(`Claude Code refresh failed: ${err.upstreamMessage}`)); - } - throw err; - } -}; - -export const callClaudeCodeAnthropicMessages = async ( - opts: CallClaudeCodeAnthropicMessagesOptions, -): Promise> => { - // `opts.model.id` is the public alias on the catalog; the dated upstream id - // Anthropic expects on the wire — and that the pricing table keys by — rides - // on `opts.model.providerData.upstreamModelId`. Resolve once so synthetic - // gates, the wire body, and the streaming-call modelKey all surface the - // same dated id. - const upstreamModelId = (opts.model.providerData as ClaudeCodeProviderData).upstreamModelId; - - const fresh = await getProviderRepo().upstreams.getById(opts.upstreamId); - if (!fresh) throw new Error(`Claude Code upstream ${opts.upstreamId} disappeared mid-request`); - const state = readClaudeCodeUpstreamState(fresh.state); - const account = state.accounts[0]; - - if (account.state !== 'active') { - return syntheticReturn(upstreamModelId, synthetic503( - `Claude Code account is ${account.state}: ${account.stateMessage}`, - )); - } - - const now = new Date(); - const quotaData = account.quotaSnapshot === null ? null : account.quotaSnapshot.data; - if (isRateLimitedNow(quotaData, now)) { - const resetIso = quotaData.reset; - return syntheticReturn(upstreamModelId, synthetic429( - resetIso ? `Claude Code upstream rate-limited until ${resetIso}` : 'Claude Code upstream rate-limited', - resetIso, - now, - )); - } - - const ensured = await ensureOrSession503(opts, upstreamModelId); - if ('modelKey' in ensured) return ensured; - - return await performUpstreamCall(opts, upstreamModelId, ensured, false); -}; - -const performUpstreamCall = async ( - opts: CallClaudeCodeAnthropicMessagesOptions, - upstreamModelId: string, - accessToken: EnsuredAccessToken, - alreadyRetried: boolean, -): Promise> => { - let responseHeaders: Headers | undefined; - let rawSseFrameCount = 0; - const rawSseFrames: StreamDiagnosticFrame[] = []; - const observeRawSseFrame = (frame: SseFrame): void => { - rawSseFrameCount += 1; - rawSseFrames.push({ - event: frame.event ?? null, - data: frame.data.length > STREAM_DIAGNOSTIC_FRAME_DATA_CHARS - ? `${frame.data.slice(0, STREAM_DIAGNOSTIC_FRAME_DATA_CHARS - 3)}...` - : frame.data, - }); - if (rawSseFrames.length > STREAM_DIAGNOSTIC_FRAME_LIMIT) rawSseFrames.shift(); - }; - let headers: Record; - if (opts.shaped) { - // The gateway already reduced ordinary headers to the Claude Code module's - // allowlist. This path restores typed Anthropic Messages metadata, preserves the - // resulting fingerprint, fills Content-Type when absent, and sets - // provider-owned OAuth auth. - const passthrough = Object.fromEntries(headersForAnthropicMessagesCall([...opts.call.headers], opts.call.anthropicBeta)); - // Sub2api always sets Content-Type when the inbound omits it - // (`gateway_service.go` request-forwarding path), so the upstream - // never receives a body-bearing request without a media type. - if (!('content-type' in passthrough)) passthrough['content-type'] = 'application/json'; - headers = { ...passthrough, authorization: `Bearer ${accessToken.entry.token}` }; - } else { - headers = { ...pickClaudeCodeHeaders(upstreamModelId), authorization: `Bearer ${accessToken.entry.token}` }; - } - - // Force stream:true regardless of caller intent. The streaming envelope is - // what the gateway boundary expects; non-streaming Anthropic Messages is routed - // elsewhere. Safe in the shaped passthrough path too: shaped detection - // requires CC client headers + system blocks + a valid metadata.user_id, - // and the real Claude Code client always sets `stream: true`. - const wireBody: AnthropicMessagesPayload = { ...opts.body, model: upstreamModelId, stream: true }; - - const upstreamFetch = opts.call.wrapUpstreamCall(() => opts.call.fetcher(ANTHROPIC_ANTHROPIC_MESSAGES_ENDPOINT, { - method: 'POST', - headers, - body: jsonRequestBody(wireBody), - signal: opts.signal, - })).then(response => { - responseHeaders = response.headers; - // `opts.call.waitUntil` is set by the gateway on Workers so the - // runtime keeps the worker alive past the response (without it, the - // persist promise gets cancelled the moment the response returns). - // Undefined under hosts that don't supply it (Node target / tests). - const { waitUntil } = opts.call; - // Every Anthropic response (2xx or 429) ships an - // `anthropic-ratelimit-unified-*` snapshot; capture both so the rate- - // limited gate above stays accurate as the window evolves. Other - // statuses (4xx/5xx outside 429) carry no quota signal so we skip them. - if (response.ok || response.status === 429) { - persistQuotaFromHeadersFireAndForget(opts.upstreamId, response.headers, waitUntil); - } - // 400 / 403 may carry the credential-class terminal sentinels — the - // detector is body-shape-defensive and only flips on a real match, so - // unrelated 400s (`max_tokens` validation, etc.) pass straight through. - maybePersistTerminalFromBodyFireAndForget(opts.upstreamId, response, waitUntil); - return response; - }); - - const result = await streamingProviderCall( - upstreamFetch, - (body, parserOptions) => parseAnthropicMessagesStream(body, { ...parserOptions, onSseFrame: observeRawSseFrame }), - upstreamModelId, - opts.signal, - ); - - if (!result.ok && result.response.status === 401 && !accessToken.freshlyMinted && !alreadyRetried) { - // Cached token rejected; invalidate so the next mint reads stale=null, - // then re-enter with a fresh-minted token. A second 401 (alreadyRetried - // == true) means the refresh_token itself is the problem and the - // operator has to re-import — surface the 401 verbatim so the gateway - // boundary reports the real upstream message rather than masking it. - await invalidateClaudeCodeAccessToken({ - upstreamId: opts.upstreamId, - repo: getProviderRepo().upstreams, - }); - const ensured = await ensureOrSession503(opts, upstreamModelId); - if ('modelKey' in ensured) return ensured; - return await performUpstreamCall(opts, upstreamModelId, ensured, true); - } - - if (!result.ok) return result; - return { - ...result, - events: observedClaudeCodeMessagesStream(result.events, { - upstreamId: opts.upstreamId, - model: upstreamModelId, - headers: responseHeaders ?? result.headers ?? new Headers(), - signal: opts.signal, - frames: rawSseFrames, - rawFrameCount: () => rawSseFrameCount, - }), - }; -}; diff --git a/packages/provider-claude-code/src/headers.ts b/packages/provider-claude-code/src/headers.ts index 782ebd555a..06498143a4 100644 --- a/packages/provider-claude-code/src/headers.ts +++ b/packages/provider-claude-code/src/headers.ts @@ -29,7 +29,7 @@ const STAINLESS_PACKAGE_VERSION = '0.94.0'; // // `X-Stainless-Helper-Method: stream` is always set: every /v1/messages // call we make to Anthropic is a streaming call (see `wireBody.stream = -// true` in fetch.ts), so this matches real CC's wire output captured in +// true` in backend.ts), so this matches real CC's wire output captured in // `@anthropic-ai/claude-code@2.1.10` cli.js and sub2api's pinned set // (gateway_service.go:7427-7429 / allowedHeaders list at :432). const STAINLESS_BASE = { diff --git a/packages/provider-claude-code/src/index.ts b/packages/provider-claude-code/src/index.ts index ec630466d5..fee1f4fa67 100644 --- a/packages/provider-claude-code/src/index.ts +++ b/packages/provider-claude-code/src/index.ts @@ -19,7 +19,7 @@ export * from './detection.ts'; export * from './headers.ts'; export * from './log.ts'; export * from './quota.ts'; -export * from './interceptors/anthropic-messages/system-blocks.ts'; +export * from './system-blocks.ts'; export * from './pricing.ts'; -export * from './fetch.ts'; +export * from './backend.ts'; export * from './provider.ts'; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/backfill-required-fields.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/backfill-required-fields.ts deleted file mode 100644 index 63231fbccb..0000000000 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/backfill-required-fields.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { AnthropicMessagesBoundaryCtx } from './types.ts'; -import { ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS } from '@floway-dev/protocols/anthropic-messages'; - -// Real Claude Code always sends `max_tokens` and `temperature` on every -// request body. Anthropic's `/v1/messages` requires `max_tokens` and 422s -// without it; `temperature` is technically optional upstream, but its -// absence is a CC-shape fingerprint failure that the plan-billing detector -// keys on. Third-party callers (cline, aider, custom integrations) routinely -// omit one or both, expecting the gateway to backfill. -// -// Sub2api (`backend/internal/service/gateway_service.go:1301-1314`, -// rev 4a5665da5b2c6b83c4597844ea6e573746c821b1) unconditionally backfills -// both: `max_tokens` to 128000 and `temperature` to 1. We mirror the same -// unconditional fill but cap `max_tokens` to the model's advertised output -// limit when present (`limits.max_output_tokens`), falling back to the -// gateway-wide ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS (8192) — sub2api's hardcoded -// 128000 ignores per-model output caps which we don't want to reproduce. -// -// Positioned at the head of the chain so the rest of the re-mimicry steps -// see a fully-formed payload. Caller-supplied values are never overwritten. -export const backfillRequiredFields = async ( - ctx: AnthropicMessagesBoundaryCtx, - _env: object, - run: () => Promise, -): Promise => { - const next = { ...ctx.payload }; - - next.max_tokens ??= ctx.model.limits.max_output_tokens ?? ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS; - next.temperature ??= 1; - - ctx.payload = next; - return await run(); -}; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/index.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/index.ts deleted file mode 100644 index 5dc9da6a1f..0000000000 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/index.ts +++ /dev/null @@ -1,55 +0,0 @@ -// Claude Code re-mimicry chain. Runs only on the re-mimicry path (callAnthropicMessages -// short-circuits to passthrough when the inbound request is already CC-shaped). -// -// Order matters: -// 1. backfill-required-fields fills max_tokens and temperature when -// the third-party caller omitted them. -// Runs first so the rest of the chain -// (and downstream fingerprint compute -// in inject-billing-block) sees the -// fully-formed CC wire shape. -// 2. synthesize-metadata-user-id runs early so session_id derives from -// the operator's real first user -// message rather than the synthetic -// ... pair that hoist -// injects when system text is present. -// Two conversations sharing a system -// prompt must NOT share a session id, -// or prompt-cache routing and -// rate-limit accounting collapse. -// 3. hoist-user-system-to-messages captures the caller's system text into -// a synthetic user/assistant pair so the -// three mimicry blocks below own -// `payload.system`. -// 4. inject-billing-block system[0]: per-request cc_version / -// cch=00000 fingerprint. -// 5. inject-identity-block system[1]: canonical CC identity text. -// 6. inject-default-template system[2]: cached boilerplate template -// (carries cache_control:ephemeral). -// -// The on-wire `model` field is set in `fetch.ts` from -// `opts.model.providerData.upstreamModelId` so the chain never has to rewrite -// it — the catalog id is already Anthropic's public alias and the dated -// upstream id is read straight off `opts.model.providerData`. - -import { backfillRequiredFields } from './backfill-required-fields.ts'; -import { hoistUserSystemToMessages } from './hoist-user-system-to-messages.ts'; -import { injectBillingBlock } from './inject-billing-block.ts'; -import { injectDefaultTemplate } from './inject-default-template.ts'; -import { injectIdentityBlock } from './inject-identity-block.ts'; -import { synthesizeMetadataUserId } from './synthesize-metadata-user-id.ts'; -import type { AnthropicMessagesBoundaryCtx } from './types.ts'; -import type { Interceptor } from '@floway-dev/interceptor'; -import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProviderStreamResult } from '@floway-dev/provider'; - -export type { AnthropicMessagesBoundaryCtx } from './types.ts'; - -export const CLAUDE_CODE_ANTHROPIC_MESSAGES_BOUNDARY: readonly Interceptor>[] = [ - backfillRequiredFields, - synthesizeMetadataUserId, - hoistUserSystemToMessages, - injectBillingBlock, - injectIdentityBlock, - injectDefaultTemplate, -]; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-billing-block.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-billing-block.ts deleted file mode 100644 index 2b7d13e17e..0000000000 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-billing-block.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { buildBillingBlock, computeCcVersionFingerprint } from './system-blocks.ts'; -import type { AnthropicMessagesBoundaryCtx } from './types.ts'; -import { CLAUDE_CLI_VERSION } from '../../headers.ts'; - -// Drops the per-request `cc_version=${VERSION}.${FP}` billing block at the -// head of `system`. This must run BEFORE inject-identity-block / -// inject-default-template so the order on the wire matches the byte-for-byte -// CC shape: system[0] billing, system[1] identity, system[2] default -// template (the cached one). -// -// Hoist must have run first so any caller-supplied system text is already -// captured into `messages`; this interceptor unconditionally starts a fresh -// `system` array. -// -// The fingerprint runs on the post-hoist payload deliberately. That is the -// shape Anthropic will actually see on the wire, so the fingerprint must -// reflect it — fingerprinting the pre-hoist shape would compute a different -// value than what the request body settles to and break CC mimicry. -export const injectBillingBlock = async ( - ctx: AnthropicMessagesBoundaryCtx, - _env: object, - run: () => Promise, -): Promise => { - const fingerprint = computeCcVersionFingerprint(CLAUDE_CLI_VERSION, ctx.payload); - const block = buildBillingBlock(CLAUDE_CLI_VERSION, fingerprint); - ctx.payload = { ...ctx.payload, system: [block] }; - return await run(); -}; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-identity-block.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-identity-block.ts deleted file mode 100644 index a6a768933f..0000000000 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-identity-block.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { IDENTITY_BLOCK } from './system-blocks.ts'; -import type { AnthropicMessagesBoundaryCtx } from './types.ts'; -import type { AnthropicMessagesTextBlock } from '@floway-dev/protocols/anthropic-messages'; - -// system[1]; relies on injectBillingBlock having materialized payload.system as an array (see ./index.ts chain order). -export const injectIdentityBlock = async ( - ctx: AnthropicMessagesBoundaryCtx, - _env: object, - run: () => Promise, -): Promise => { - const system = ctx.payload.system as AnthropicMessagesTextBlock[]; - ctx.payload = { ...ctx.payload, system: [...system, IDENTITY_BLOCK] }; - return await run(); -}; diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/types.ts b/packages/provider-claude-code/src/interceptors/anthropic-messages/types.ts deleted file mode 100644 index ddd1ed502c..0000000000 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/types.ts +++ /dev/null @@ -1,14 +0,0 @@ -import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; -import type { ProviderModel } from '@floway-dev/provider'; - -// Boundary ctx for Claude Code Anthropic Messages interceptors. The chain runs only on -// the re-mimicry path; callAnthropicMessages decides shaped-vs-unshaped before -// entering the chain. `upstreamId` is required by synthesize-metadata-user-id -// to derive deterministic device/session ids that stay stable per upstream -// across requests (so prompt-cache hits depend on conversation content only, -// not on per-call randomness). -export interface AnthropicMessagesBoundaryCtx { - payload: AnthropicMessagesPayload; - readonly model: ProviderModel; - readonly upstreamId: string; -} diff --git a/packages/provider-claude-code/src/models.ts b/packages/provider-claude-code/src/models.ts index c931c3b067..71dcf23454 100644 --- a/packages/provider-claude-code/src/models.ts +++ b/packages/provider-claude-code/src/models.ts @@ -10,7 +10,7 @@ // 4.6+ and Fable 5+ return with the alias already (no date), // so the alias derivation is the identity. The catalog id we publish is // always the alias; the original /v1/models id rides on -// `providerData.upstreamModelId` so the wire fetch in `fetch.ts` and the +// `providerData.upstreamModelId` so the wire fetch in `backend.ts` and the // pricing table key by the per-revision id. import { CLAUDE_CODE_HEADERS_SONNET_OPUS } from './headers.ts'; diff --git a/packages/provider-claude-code/src/pipeline-facts.ts b/packages/provider-claude-code/src/pipeline-facts.ts new file mode 100644 index 0000000000..c30e02ff55 --- /dev/null +++ b/packages/provider-claude-code/src/pipeline-facts.ts @@ -0,0 +1,26 @@ +import type { EnsuredAccessToken } from './access-token.ts'; +import type { ClaudeCodeAccountCredential } from './state.ts'; +import type { HttpRequestFacts } from '@floway-dev/http/pipeline'; +import type { Secret } from '@floway-dev/pipeline'; +import type { ProviderModelFacts, ProviderOperationRequest } from '@floway-dev/provider'; + +export type ClaudeCodeRequest = ProviderOperationRequest<'anthropicMessages'>; +export type ClaudeCodeAccountFacts = Omit & { + readonly refreshToken: Secret | null; + readonly accessToken: (Omit, 'token'> & { readonly token: Secret }) | null; +}; +export type ClaudeCodeAccessFacts = Omit & { readonly entry: Omit & { readonly token: Secret } }; +export interface ClaudeCodePreparedRequest extends ClaudeCodeRequest { + 'request.provider.modelKey': string; + 'request.claudeCode.shaped': boolean; +} +export interface ClaudeCodeAuthenticatedRequest extends ClaudeCodePreparedRequest { + 'request.claudeCode.account': ClaudeCodeAccountFacts; + 'request.claudeCode.access': ClaudeCodeAccessFacts; +} +export interface ClaudeCodeHttpRequest extends HttpRequestFacts { + 'request.provider.modelKey': string; + 'request.claudeCode.model': ProviderModelFacts; + 'request.claudeCode.account': ClaudeCodeAccountFacts; + 'request.claudeCode.access': ClaudeCodeAccessFacts; +} diff --git a/packages/provider-claude-code/src/pipelines.ts b/packages/provider-claude-code/src/pipelines.ts new file mode 100644 index 0000000000..9d9ff0d330 --- /dev/null +++ b/packages/provider-claude-code/src/pipelines.ts @@ -0,0 +1,23 @@ +import { authenticateClaudeCode } from './stages/authenticate.ts'; +import { backfillRequiredFields } from './stages/backfill-required-fields.ts'; +import { decodeClaudeCodeResponse } from './stages/decode-response.ts'; +import { hoistUserSystemToMessages } from './stages/hoist-user-system-to-messages.ts'; +import { injectBillingBlock } from './stages/inject-billing-block.ts'; +import { injectDefaultTemplate } from './stages/inject-default-template.ts'; +import { injectIdentityBlock } from './stages/inject-identity-block.ts'; +import { observeClaudeCodeResponse } from './stages/observe-response.ts'; +import { prepareClaudeCodeRequest } from './stages/prepare-request.ts'; +import { recognizeClaudeCodeShape } from './stages/recognize-shape.ts'; +import { retryClaudeCodeAccess } from './stages/retry-access.ts'; +import { synthesizeMetadataUserId } from './stages/synthesize-metadata-user-id.ts'; +import { http } from '@floway-dev/http/pipeline'; +import { compose } from '@floway-dev/pipeline'; +import { observeProviderCall, type ProviderOperationRequest, type ProviderChatResponse, type ProviderPipelines } from '@floway-dev/provider'; + +export const createClaudeCodePipelines = (upstreamId: string): ProviderPipelines => ({ + anthropicMessages: compose, ProviderChatResponse<'anthropicMessages'>>('claudeCode.anthropicMessages', [ + recognizeClaudeCodeShape, backfillRequiredFields, synthesizeMetadataUserId(upstreamId), hoistUserSystemToMessages, + injectBillingBlock, injectIdentityBlock, injectDefaultTemplate, authenticateClaudeCode(upstreamId), + prepareClaudeCodeRequest, decodeClaudeCodeResponse(upstreamId), retryClaudeCodeAccess(upstreamId), observeClaudeCodeResponse(upstreamId), observeProviderCall, http, + ]), +}); diff --git a/packages/provider-claude-code/src/provider.ts b/packages/provider-claude-code/src/provider.ts index e98b443753..f4685796d8 100644 --- a/packages/provider-claude-code/src/provider.ts +++ b/packages/provider-claude-code/src/provider.ts @@ -1,20 +1,14 @@ import { ensureClaudeCodeAccessToken } from './access-token.ts'; import { assertClaudeCodeUpstreamRecord } from './config.ts'; import { CLAUDE_CODE_DEFAULT_FLAGS } from './defaults.ts'; -import { isClaudeCodeShapedRequest } from './detection.ts'; -import { callClaudeCodeAnthropicMessages } from './fetch.ts'; -import { CLAUDE_CODE_ANTHROPIC_MESSAGES_BOUNDARY, type AnthropicMessagesBoundaryCtx } from './interceptors/anthropic-messages/index.ts'; import { buildClaudeCodeCatalog, fetchClaudeCodeModelsList } from './models.ts'; +import { createClaudeCodePipelines } from './pipelines.ts'; import { assertClaudeCodeUpstreamState } from './state.ts'; -import { runInterceptors } from '@floway-dev/interceptor'; -import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { getProviderRepo, - headersForAnthropicMessagesCall, resolveEffectiveFlags, type ProviderInstance, type Provider, - type ProviderStreamResult, type UpstreamRecord, } from '@floway-dev/provider'; @@ -41,7 +35,6 @@ export const createClaudeCodeProvider = (record: UpstreamRecord): Provider => { const enabledFlags = resolveEffectiveFlags([CLAUDE_CODE_DEFAULT_FLAGS, record.flagOverrides]); const instance: ProviderInstance = { - callAlphaSearch: rejectUnsupported('callAlphaSearch'), // Catalog refresh mints an access token and hits /v1/models on every // dispatcher poll. `ensureClaudeCodeAccessToken` flips the row to // `refresh_failed` and throws `ClaudeCodeOAuthSessionTerminatedError` @@ -56,66 +49,6 @@ export const createClaudeCodeProvider = (record: UpstreamRecord): Provider => { const apiModels = await fetchClaudeCodeModelsList(access.entry.token, fetcher); return buildClaudeCodeCatalog(apiModels, enabledFlags); }, - - callAnthropicMessages: async (model, body, signal: AbortSignal | undefined, opts) => { - const ctx: AnthropicMessagesBoundaryCtx = { - payload: { ...body, model: model.id }, - model, - upstreamId: record.id, - }; - - // Detection runs on the unmodified payload plus the Claude Code - // fingerprint admitted by the provider module and Anthropic Messages boundaries. - // The re-mimicry chain would clobber operator-supplied `system` content - // and overwrite the wire shape — exactly what a CC-shaped passthrough - // needs to preserve. So the chain only runs on the unshaped path; the - // shaped path skips straight to the terminal call, which preserves the - // caller's own system blocks, metadata and tool shape rather than - // re-deriving them. The call preserves that filtered fingerprint, - // supplies the provider-owned OAuth auth, and restamps the resolved model - // id. - const looksShaped = isClaudeCodeShapedRequest({ - headers: new Headers(headersForAnthropicMessagesCall([...opts.headers], opts.anthropicBeta).map(([name, value]) => [name, value])), - body: ctx.payload, - }); - - const terminal = async (): Promise> => { - // Drop `model` from the payload: callClaudeCodeAnthropicMessages re-attaches the - // dated upstream id (from `opts.model.providerData.upstreamModelId`) on - // the wire so Anthropic sees a stable per-revision id rather than the - // public alias the catalog exposes to clients. - const { model: _ignored, ...wireBody } = ctx.payload; - return await callClaudeCodeAnthropicMessages({ - upstreamId: record.id, - model, - body: wireBody, - shaped: looksShaped, - signal, - call: opts, - }); - }; - - if (looksShaped) return await terminal(); - - return await runInterceptors>( - ctx, - {}, - CLAUDE_CODE_ANTHROPIC_MESSAGES_BOUNDARY, - terminal, - ); - }, - - // Only /v1/messages is supported; reject any other endpoint loudly so a - // dispatcher routing bug surfaces instead of a silent shape mismatch. - callAnthropicMessagesCountTokens: rejectUnsupported('callAnthropicMessagesCountTokens'), - callOpenAICompletions: rejectUnsupported('callOpenAICompletions'), - callOpenAIChatCompletions: rejectUnsupported('callOpenAIChatCompletions'), - callOpenAIResponses: rejectUnsupported('callOpenAIResponses'), - callOpenAIEmbeddings: rejectUnsupported('callOpenAIEmbeddings'), - callOpenAIImagesGenerations: rejectUnsupported('callOpenAIImagesGenerations'), - callOpenAIImagesEdits: rejectUnsupported('callOpenAIImagesEdits'), - callOpenAIAudioTranscriptions: rejectUnsupported('callOpenAIAudioTranscriptions'), - callRerank: rejectUnsupported('callRerank'), }; return { @@ -126,10 +59,7 @@ export const createClaudeCodeProvider = (record: UpstreamRecord): Provider => { disabledPublicModelIds: record.disabledPublicModelIds, modelPrefix: record.modelPrefix, modelsCache: record.modelsCache, - pipelines: {}, + pipelines: createClaudeCodePipelines(record.id), instance, }; }; - -const rejectUnsupported = (capability: string) => (): Promise => - Promise.reject(new Error(`Claude Code provider does not implement ${capability}`)); diff --git a/packages/provider-claude-code/src/stages/authenticate.ts b/packages/provider-claude-code/src/stages/authenticate.ts new file mode 100644 index 0000000000..ec50782d61 --- /dev/null +++ b/packages/provider-claude-code/src/stages/authenticate.ts @@ -0,0 +1,47 @@ +import { ensureClaudeCodeAccessToken } from '../access-token.ts'; +import { ClaudeCodeOAuthSessionTerminatedError } from '../auth/oauth.ts'; +import { isRateLimitedNow, synthetic429, synthetic503 } from '../backend.ts'; +import type { ClaudeCodeAuthenticatedRequest, ClaudeCodePreparedRequest } from '../pipeline-facts.ts'; +import { readClaudeCodeUpstreamState } from '../state.ts'; +import { takeHttpResponse } from '@floway-dev/http/pipeline'; +import { defineStage, move, secret } from '@floway-dev/pipeline'; +import { getProviderRepo, type ProviderChatResponse, type ProviderChatServices } from '@floway-dev/provider'; + +export const authenticateClaudeCode = (upstreamId: string) => defineStage, ProviderChatResponse<'anthropicMessages'>, ProviderChatResponse<'anthropicMessages'>, ProviderChatServices>({ + name: 'authenticateClaudeCode', + through: { + request: { needs: ['request.provider.modelKey', 'request.http.callId'], consumes: [], provides: ['request.claudeCode.account', 'request.claudeCode.access'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + return: { provides: ['response.http.exchange', 'response.http.body', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.provider.output'] }, + execute: async (facts, next, use) => { + const fresh = await getProviderRepo().upstreams.getById(upstreamId); + if (fresh === null) throw new Error(`Claude Code upstream ${upstreamId} disappeared mid-request`); + const account = readClaudeCodeUpstreamState(fresh.state).accounts[0]; + const accountFacts = { + ...account, refreshToken: account.refreshToken === null ? null : secret(account.refreshToken), + accessToken: account.accessToken === null ? null : { ...account.accessToken, token: secret(account.accessToken.token) }, + }; + const reply = (response: Response) => { + const exchange = takeHttpResponse(response); + return move({ + ...facts, 'request.claudeCode.account': accountFacts, 'response.http.exchange': exchange, 'response.http.body': exchange.body, + 'response.provider.modelKey': facts['request.provider.modelKey'], 'response.provider.called': false, + 'response.provider.previousCalls': [], 'response.provider.output': null, + }); + }; + if (account.state !== 'active') return reply(synthetic503(`Claude Code account is ${account.state}: ${account.stateMessage}`)); + const now = new Date(); + const quota = account.quotaSnapshot === null ? null : account.quotaSnapshot.data; + if (isRateLimitedNow(quota, now)) return reply(synthetic429(quota.reset ? `Claude Code upstream rate-limited until ${quota.reset}` : 'Claude Code upstream rate-limited', quota.reset, now)); + const call = use.httpCall(facts['request.http.callId']); + let access; + try { + access = await ensureClaudeCodeAccessToken({ upstreamId, repo: getProviderRepo().upstreams, fetcher: call.fetcher }); + } catch (error) { + if (error instanceof ClaudeCodeOAuthSessionTerminatedError) return reply(synthetic503(`Claude Code refresh failed: ${error.upstreamMessage}`)); + throw error; + } + return move({ ...await next(move({ ...facts, 'request.claudeCode.account': accountFacts, 'request.claudeCode.access': { ...access, entry: { ...access.entry, token: secret(access.entry.token) } } })) }); + }, +}); diff --git a/packages/provider-claude-code/src/stages/backfill-required-fields.ts b/packages/provider-claude-code/src/stages/backfill-required-fields.ts new file mode 100644 index 0000000000..3a047932af --- /dev/null +++ b/packages/provider-claude-code/src/stages/backfill-required-fields.ts @@ -0,0 +1,37 @@ +import type { ClaudeCodePreparedRequest } from '../pipeline-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import { ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS } from '@floway-dev/protocols/anthropic-messages'; +import type { ProviderModel, ProviderChatResponse } from '@floway-dev/provider'; + +// Real Claude Code always sends `max_tokens` and `temperature` on every +// request body. Anthropic's `/v1/messages` requires `max_tokens` and 422s +// without it; `temperature` is technically optional upstream, but its +// absence is a CC-shape fingerprint failure that the plan-billing detector +// keys on. Third-party callers (cline, aider, custom integrations) routinely +// omit one or both, expecting the gateway to backfill. +// +// Sub2api (`backend/internal/service/gateway_service.go:1301-1314`, +// rev 4a5665da5b2c6b83c4597844ea6e573746c821b1) unconditionally backfills +// both: `max_tokens` to 128000 and `temperature` to 1. We mirror the same +// unconditional fill but cap `max_tokens` to the model's advertised output +// limit when present (`limits.max_output_tokens`), falling back to the +// gateway-wide ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS (8192) — sub2api's hardcoded +// 128000 ignores per-model output caps which we don't want to reproduce. +// +// Positioned at the head of the chain so the rest of the re-mimicry steps +// see a fully-formed payload. Caller-supplied values are never overwritten. +export const backfillRequiredFieldsPayload =

>(payload: P, model: Pick): P => { + const next = { ...payload }; + + next.max_tokens ??= model.limits.max_output_tokens ?? ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS; + next.temperature ??= 1; + + return next; +}; + +export const backfillRequiredFields = defineStage, ProviderChatResponse<'anthropicMessages'>>({ + name: 'backfillRequiredFields', + through: { request: { needs: ['request.provider.payload', 'request.claudeCode.shaped', 'request.provider.model'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, response: { needs: [], consumes: [], provides: [] } }, + execute: async (facts, next) => move({ ...await next(move({ ...facts, 'request.provider.payload': facts['request.claudeCode.shaped'] ? facts['request.provider.payload'] : backfillRequiredFieldsPayload(facts['request.provider.payload'], facts['request.provider.model']) })) }), +}); diff --git a/packages/provider-claude-code/src/stages/decode-response.ts b/packages/provider-claude-code/src/stages/decode-response.ts new file mode 100644 index 0000000000..593a7c643c --- /dev/null +++ b/packages/provider-claude-code/src/stages/decode-response.ts @@ -0,0 +1,41 @@ +import { observedClaudeCodeMessagesStream, STREAM_DIAGNOSTIC_FRAME_DATA_CHARS, STREAM_DIAGNOSTIC_FRAME_LIMIT, type StreamDiagnosticFrame } from '../backend.ts'; +import type { ClaudeCodeHttpRequest } from '../pipeline-facts.ts'; +import { exchangeResponse } from '@floway-dev/http/pipeline'; +import { defineStage, move, setRelease } from '@floway-dev/pipeline'; +import { parseAnthropicMessagesStream } from '@floway-dev/protocols/anthropic-messages'; +import { isEventStreamMediaType } from '@floway-dev/protocols/common'; +import type { ProviderChatResponse, ProviderChatServices, ProviderResponse } from '@floway-dev/provider'; + +export const decodeClaudeCodeResponse = (upstreamId: string) => defineStage, ProviderChatServices>({ + name: 'decodeClaudeCodeResponse', + through: { + request: { needs: ['request.provider.modelKey', 'request.http.callId'], consumes: [], provides: [] }, + response: { needs: ['response.http.exchange', 'response.http.body'], consumes: ['response.http.body'], provides: ['response.http.body', 'response.provider.output'] }, + }, + execute: async (facts, next, use) => { + const back = await next(move({ ...facts })); + const exchange = back['response.http.exchange']; + if (exchange.type === 'transportFailure' || exchange.status < 200 || exchange.status >= 300) return move({ ...back, 'response.provider.output': null }); + const response = exchangeResponse(exchange); + if (exchange.body === null || !isEventStreamMediaType(response.headers.get('content-type'))) { + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + const text = await response.text(); + let body: unknown; + try { body = JSON.parse(text); } catch (error) { if (!(error instanceof SyntaxError)) throw error; body = text; } + const snippet = text.length === 0 ? '' : text.length > 1024 ? `${text.slice(0, 1024)}...[truncated]` : text; + return move({ ...back, 'response.provider.output': { status: 502, message: `Upstream returned ${exchange.status} with content-type "${response.headers.get('content-type') ?? 'unknown'}" but stream is required (provider must force stream=true and return text/event-stream when response.ok). Body: ${snippet}`, body } }); + } + const signal = use.httpCall(facts['request.http.callId']).signal; + let rawFrameCount = 0; + const samples: StreamDiagnosticFrame[] = []; + const parsed = use.recordProtocolFrames(parseAnthropicMessagesStream(exchange.body, { + signal, onSseFrame: frame => { + rawFrameCount++; + samples.push({ event: frame.event ?? null, data: frame.data.length > STREAM_DIAGNOSTIC_FRAME_DATA_CHARS ? `${frame.data.slice(0, STREAM_DIAGNOSTIC_FRAME_DATA_CHARS - 3)}...` : frame.data }); + if (samples.length > STREAM_DIAGNOSTIC_FRAME_LIMIT) samples.shift(); + }, + })); + const frames = use.recordProtocolFrames(observedClaudeCodeMessagesStream(parsed, { upstreamId, model: facts['request.provider.modelKey'], headers: response.headers, signal, frames: samples, rawFrameCount: () => rawFrameCount, warn: use.log.warn })); + return move({ ...back, 'response.provider.output': { kind: 'stream' as const, frames } }); + }, +}); diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/hoist-user-system-to-messages.ts b/packages/provider-claude-code/src/stages/hoist-user-system-to-messages.ts similarity index 65% rename from packages/provider-claude-code/src/interceptors/anthropic-messages/hoist-user-system-to-messages.ts rename to packages/provider-claude-code/src/stages/hoist-user-system-to-messages.ts index da8164a499..05ae158868 100644 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/hoist-user-system-to-messages.ts +++ b/packages/provider-claude-code/src/stages/hoist-user-system-to-messages.ts @@ -1,5 +1,7 @@ -import type { AnthropicMessagesBoundaryCtx } from './types.ts'; -import type { AnthropicMessagesMessage, AnthropicMessagesTextBlock } from '@floway-dev/protocols/anthropic-messages'; +import type { ClaudeCodePreparedRequest } from '../pipeline-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload, AnthropicMessagesMessage, AnthropicMessagesTextBlock } from '@floway-dev/protocols/anthropic-messages'; +import type { ProviderChatResponse } from '@floway-dev/provider'; // Synthetic assistant turn that closes the hoisted user/assistant pair so the // upstream's role-alternation guard stays satisfied. The text is visible to @@ -27,12 +29,8 @@ const SYNTHETIC_ACK = 'Understood. I will follow these instructions.'; // // References: // - https://github.com/Wei-Shaw/sub2api/blob/4a5665da5b2c6b83c4597844ea6e573746c821b1/backend/internal/service/gateway_service.go#L4480-L4486 -export const hoistUserSystemToMessages = async ( - ctx: AnthropicMessagesBoundaryCtx, - _env: object, - run: () => Promise, -): Promise => { - const system: string | AnthropicMessagesTextBlock[] | undefined = ctx.payload.system; +export const hoistUserSystemToMessagesPayload =

>(payload: P): P => { + const system: string | AnthropicMessagesTextBlock[] | undefined = payload.system; let captured = ''; if (typeof system === 'string') { captured = system; @@ -43,8 +41,8 @@ export const hoistUserSystemToMessages = async ( .join('\n\n'); } // inject-billing-block et al rebuild `system` from scratch as a three-block - // array; removing the field here keeps the boundary mutation self-contained. - const nextPayload = { ...ctx.payload }; + // array; removing the field here keeps the payload transformation self-contained. + const nextPayload = { ...payload }; delete nextPayload.system; if (captured !== '') { @@ -59,6 +57,11 @@ export const hoistUserSystemToMessages = async ( nextPayload.messages = [...synthetic, ...nextPayload.messages]; } - ctx.payload = nextPayload; - return await run(); + return nextPayload; }; + +export const hoistUserSystemToMessages = defineStage, ProviderChatResponse<'anthropicMessages'>>({ + name: 'hoistUserSystemToMessages', + through: { request: { needs: ['request.provider.payload', 'request.claudeCode.shaped'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, response: { needs: [], consumes: [], provides: [] } }, + execute: async (facts, next) => move({ ...await next(move({ ...facts, 'request.provider.payload': facts['request.claudeCode.shaped'] ? facts['request.provider.payload'] : hoistUserSystemToMessagesPayload(facts['request.provider.payload']) })) }), +}); diff --git a/packages/provider-claude-code/src/stages/inject-billing-block.ts b/packages/provider-claude-code/src/stages/inject-billing-block.ts new file mode 100644 index 0000000000..3b9d32d4b6 --- /dev/null +++ b/packages/provider-claude-code/src/stages/inject-billing-block.ts @@ -0,0 +1,33 @@ +import { CLAUDE_CLI_VERSION } from '../headers.ts'; +import type { ClaudeCodePreparedRequest } from '../pipeline-facts.ts'; +import { buildBillingBlock, computeCcVersionFingerprint } from '../system-blocks.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import type { ProviderChatResponse } from '@floway-dev/provider'; + +// Drops the per-request `cc_version=${VERSION}.${FP}` billing block at the +// head of `system`. This must run BEFORE inject-identity-block / +// inject-default-template so the order on the wire matches the byte-for-byte +// CC shape: system[0] billing, system[1] identity, system[2] default +// template (the cached one). +// +// Hoist must have run first so any caller-supplied system text is already +// captured into `messages`; this stage unconditionally starts a fresh +// `system` array. +// +// The fingerprint runs on the post-hoist payload deliberately. That is the +// shape Anthropic will actually see on the wire, so the fingerprint must +// reflect it — fingerprinting the pre-hoist shape would compute a different +// value than what the request body settles to and break CC mimicry. +export const injectBillingBlockPayload =

>(payload: P): P => { + const fingerprint = computeCcVersionFingerprint(CLAUDE_CLI_VERSION, payload); + const block = buildBillingBlock(CLAUDE_CLI_VERSION, fingerprint); + payload = { ...payload, system: [block] }; + return payload; +}; + +export const injectBillingBlock = defineStage, ProviderChatResponse<'anthropicMessages'>>({ + name: 'injectBillingBlock', + through: { request: { needs: ['request.provider.payload', 'request.claudeCode.shaped'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, response: { needs: [], consumes: [], provides: [] } }, + execute: async (facts, next) => move({ ...await next(move({ ...facts, 'request.provider.payload': facts['request.claudeCode.shaped'] ? facts['request.provider.payload'] : injectBillingBlockPayload(facts['request.provider.payload']) })) }), +}); diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-default-template.ts b/packages/provider-claude-code/src/stages/inject-default-template.ts similarity index 64% rename from packages/provider-claude-code/src/interceptors/anthropic-messages/inject-default-template.ts rename to packages/provider-claude-code/src/stages/inject-default-template.ts index 5407d8319e..e6583d1089 100644 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/inject-default-template.ts +++ b/packages/provider-claude-code/src/stages/inject-default-template.ts @@ -1,6 +1,8 @@ -import { DEFAULT_TEMPLATE_BLOCK } from './system-blocks.ts'; -import type { AnthropicMessagesBoundaryCtx } from './types.ts'; +import type { ClaudeCodePreparedRequest } from '../pipeline-facts.ts'; +import { DEFAULT_TEMPLATE_BLOCK } from '../system-blocks.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import type { ProviderChatResponse } from '@floway-dev/provider'; // Anthropic's prompt-caching API rejects requests that carry more than four // `cache_control` markers; the docs state "the API enforces a maximum of 4 @@ -17,7 +19,7 @@ const ANTHROPIC_CACHE_BREAKPOINT_CAP = 4; // Mirrors sub2api's `collectCacheControlPaths`. The billing and identity // blocks injected earlier in this chain carry no `cache_control`, so they // never contribute here. -const countCacheBreakpoints = (payload: AnthropicMessagesPayload): number => { +const countCacheBreakpoints = (payload: Pick): number => { let count = 0; if (payload.tools) { @@ -55,22 +57,24 @@ const countCacheBreakpoints = (payload: AnthropicMessagesPayload): number => { // un-cached block. The caller's own breakpoints stay where they are — if // the caller already exceeds the cap, that overage is theirs to fix. // -// Chain order in ./index.ts guarantees `system` is already a fresh array +// Chain order in ../pipelines.ts guarantees `system` is already a fresh array // when this runs. -export const injectDefaultTemplate = async ( - ctx: AnthropicMessagesBoundaryCtx, - _env: object, - run: () => Promise, -): Promise => { - if (!Array.isArray(ctx.payload.system)) { +export const injectDefaultTemplatePayload =

>(payload: P): P => { + if (!Array.isArray(payload.system)) { throw new Error('inject-default-template: expected system to be an array (inject-billing-block must run first)'); } - const system = ctx.payload.system; - const callerBreakpoints = countCacheBreakpoints(ctx.payload); + const system = payload.system; + const callerBreakpoints = countCacheBreakpoints(payload); const wouldOverflowBreakpointCap = callerBreakpoints >= ANTHROPIC_CACHE_BREAKPOINT_CAP; const templateBlock = wouldOverflowBreakpointCap ? { type: 'text' as const, text: DEFAULT_TEMPLATE_BLOCK.text } : DEFAULT_TEMPLATE_BLOCK; - ctx.payload = { ...ctx.payload, system: [...system, templateBlock] }; - return await run(); + payload = { ...payload, system: [...system, templateBlock] }; + return payload; }; + +export const injectDefaultTemplate = defineStage, ProviderChatResponse<'anthropicMessages'>>({ + name: 'injectDefaultTemplate', + through: { request: { needs: ['request.provider.payload', 'request.claudeCode.shaped'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, response: { needs: [], consumes: [], provides: [] } }, + execute: async (facts, next) => move({ ...await next(move({ ...facts, 'request.provider.payload': facts['request.claudeCode.shaped'] ? facts['request.provider.payload'] : injectDefaultTemplatePayload(facts['request.provider.payload']) })) }), +}); diff --git a/packages/provider-claude-code/src/stages/inject-identity-block.ts b/packages/provider-claude-code/src/stages/inject-identity-block.ts new file mode 100644 index 0000000000..391b49fd17 --- /dev/null +++ b/packages/provider-claude-code/src/stages/inject-identity-block.ts @@ -0,0 +1,18 @@ +import type { ClaudeCodePreparedRequest } from '../pipeline-facts.ts'; +import { IDENTITY_BLOCK } from '../system-blocks.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload, AnthropicMessagesTextBlock } from '@floway-dev/protocols/anthropic-messages'; +import type { ProviderChatResponse } from '@floway-dev/provider'; + +// system[1]; relies on injectBillingBlock having materialized payload.system as an array (see ../pipelines.ts chain order). +export const injectIdentityBlockPayload =

>(payload: P): P => { + const system = payload.system as AnthropicMessagesTextBlock[]; + payload = { ...payload, system: [...system, IDENTITY_BLOCK] }; + return payload; +}; + +export const injectIdentityBlock = defineStage, ProviderChatResponse<'anthropicMessages'>>({ + name: 'injectIdentityBlock', + through: { request: { needs: ['request.provider.payload', 'request.claudeCode.shaped'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, response: { needs: [], consumes: [], provides: [] } }, + execute: async (facts, next) => move({ ...await next(move({ ...facts, 'request.provider.payload': facts['request.claudeCode.shaped'] ? facts['request.provider.payload'] : injectIdentityBlockPayload(facts['request.provider.payload']) })) }), +}); diff --git a/packages/provider-claude-code/src/stages/observe-response.ts b/packages/provider-claude-code/src/stages/observe-response.ts new file mode 100644 index 0000000000..4d5632e9b2 --- /dev/null +++ b/packages/provider-claude-code/src/stages/observe-response.ts @@ -0,0 +1,42 @@ +import { detectTerminalSentinel, persistQuotaSnapshot, persistTerminalAccountState } from '../backend.ts'; +import type { ClaudeCodeHttpRequest } from '../pipeline-facts.ts'; +import { parseClaudeCodeQuotaHeaders } from '../quota.ts'; +import { exchangeResponse, takeHttpResponse } from '@floway-dev/http/pipeline'; +import { defer, defineStage, move, setRelease, type Deferred } from '@floway-dev/pipeline'; +import type { ProviderResponse } from '@floway-dev/provider'; + +type Observed = ProviderResponse & { 'response.claudeCode.background': Deferred; 'response.claudeCode.failureBody': unknown }; + +export const observeClaudeCodeResponse = (upstreamId: string) => defineStage({ + name: 'observeClaudeCodeResponse', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { needs: ['response.http.exchange', 'response.http.body'], consumes: ['response.http.exchange', 'response.http.body'], provides: ['response.http.exchange', 'response.http.body', 'response.claudeCode.background', 'response.claudeCode.failureBody'] }, + }, + execute: async (facts, next, use) => { + const back = await next(move({ ...facts })); + const exchange = back['response.http.exchange']; + const tasks: Promise[] = []; + if (exchange.type === 'transportFailure') return move({ ...back, 'response.claudeCode.background': defer(Promise.all(tasks)), 'response.claudeCode.failureBody': null }); + const response = exchangeResponse(exchange); + if (response.ok || response.status === 429) { + const quota = parseClaudeCodeQuotaHeaders(response.headers); + if (Object.keys(quota.raw).length > 0) tasks.push(persistQuotaSnapshot(upstreamId, quota, use.log.info)); + } + let failureBody: unknown = null; + let retained = exchange; + if (response.status === 400 || response.status === 403) { + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + const bytes = await response.arrayBuffer(); + const text = new TextDecoder().decode(bytes); + try { failureBody = JSON.parse(text); } catch (error) { if (!(error instanceof SyntaxError)) throw error; failureBody = text; } + const terminal = detectTerminalSentinel(response.status, text); + if (terminal !== null) tasks.push(persistTerminalAccountState(upstreamId, terminal, response.status === 400 ? 'org_disabled_400_sentinel' : 'org_banned_403_sentinel', response.status, use.log.warn)); + retained = takeHttpResponse(new Response(bytes, { status: exchange.status, statusText: exchange.statusText, headers: response.headers })); + } + return move({ + ...back, 'response.http.exchange': retained, 'response.http.body': retained.body, + 'response.claudeCode.background': defer(Promise.all(tasks)), 'response.claudeCode.failureBody': failureBody, + }); + }, +}); diff --git a/packages/provider-claude-code/src/stages/prepare-request.ts b/packages/provider-claude-code/src/stages/prepare-request.ts new file mode 100644 index 0000000000..b736a09a4d --- /dev/null +++ b/packages/provider-claude-code/src/stages/prepare-request.ts @@ -0,0 +1,29 @@ +import { ANTHROPIC_MESSAGES_ENDPOINT } from '../backend.ts'; +import { pickClaudeCodeHeaders } from '../headers.ts'; +import type { ClaudeCodeAuthenticatedRequest, ClaudeCodeHttpRequest } from '../pipeline-facts.ts'; +import { defineStage, isSecret, move, secret } from '@floway-dev/pipeline'; +import { headersForAnthropicMessagesCall, replaceHttpHeader, type ProviderChatResponse } from '@floway-dev/provider'; + +export const prepareClaudeCodeRequest = defineStage, ProviderChatResponse<'anthropicMessages'>>({ + name: 'prepareClaudeCodeRequest', + through: { + request: { needs: ['request.provider.model', 'request.provider.modelKey', 'request.provider.payload', 'request.provider.anthropicBeta', 'request.claudeCode.shaped', 'request.claudeCode.access', 'request.http.headers'], consumes: ['request.provider.model', 'request.provider.payload', 'request.provider.anthropicBeta', 'request.http.headers'], provides: ['request.claudeCode.model', 'request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const { 'request.provider.model': model, 'request.provider.payload': payload, 'request.provider.anthropicBeta': beta, ...rest } = facts; + let headers = facts['request.claudeCode.shaped'] + ? headersForAnthropicMessagesCall(facts['request.http.headers'].map(([name, value]): [string, string] => [name, isSecret(value) ? value.reveal() : value]), beta) + : Object.entries(pickClaudeCodeHeaders(facts['request.provider.modelKey'])); + if (!headers.some(([name]) => name.toLowerCase() === 'content-type')) headers = [...headers, ['content-type', 'application/json']]; + return move({ + ...await next(move({ + ...rest, + 'request.claudeCode.model': model, + 'request.http.url': ANTHROPIC_MESSAGES_ENDPOINT, 'request.http.method': 'POST', + 'request.http.headers': replaceHttpHeader(headers, 'authorization', secret(`Bearer ${facts['request.claudeCode.access'].entry.token.reveal()}`)), + 'request.http.body': { ...payload, model: facts['request.provider.modelKey'], stream: true }, 'request.http.encoding': 'json' as const, + })), + }); + }, +}); diff --git a/packages/provider-claude-code/src/stages/recognize-shape.ts b/packages/provider-claude-code/src/stages/recognize-shape.ts new file mode 100644 index 0000000000..6a652c4ae9 --- /dev/null +++ b/packages/provider-claude-code/src/stages/recognize-shape.ts @@ -0,0 +1,18 @@ +import { isClaudeCodeShapedRequest } from '../detection.ts'; +import type { ClaudeCodeProviderData } from '../models.ts'; +import type { ClaudeCodePreparedRequest, ClaudeCodeRequest } from '../pipeline-facts.ts'; +import { defineStage, isSecret, move } from '@floway-dev/pipeline'; +import { headersForAnthropicMessagesCall, type ProviderChatResponse } from '@floway-dev/provider'; + +export const recognizeClaudeCodeShape = defineStage, ProviderChatResponse<'anthropicMessages'>>({ + name: 'recognizeClaudeCodeShape', + through: { + request: { needs: ['request.provider.model', 'request.provider.payload', 'request.provider.anthropicBeta', 'request.http.headers'], consumes: [], provides: ['request.provider.modelKey', 'request.claudeCode.shaped'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const headers = facts['request.http.headers'].map(([name, value]): [string, string] => [name, isSecret(value) ? value.reveal() : value]); + const shaped = isClaudeCodeShapedRequest({ headers: new Headers(headersForAnthropicMessagesCall(headers, facts['request.provider.anthropicBeta']).map(([name, value]) => [name, value])), body: { ...facts['request.provider.payload'], model: facts['request.provider.model'].id } }); + return move({ ...await next(move({ ...facts, 'request.claudeCode.shaped': shaped, 'request.provider.modelKey': (facts['request.provider.model'].providerData as ClaudeCodeProviderData).upstreamModelId })) }); + }, +}); diff --git a/packages/provider-claude-code/src/stages/retry-access.ts b/packages/provider-claude-code/src/stages/retry-access.ts new file mode 100644 index 0000000000..b6766315ac --- /dev/null +++ b/packages/provider-claude-code/src/stages/retry-access.ts @@ -0,0 +1,36 @@ +import { ensureClaudeCodeAccessToken, invalidateClaudeCodeAccessToken } from '../access-token.ts'; +import { ClaudeCodeOAuthSessionTerminatedError } from '../auth/oauth.ts'; +import { synthetic503 } from '../backend.ts'; +import type { ClaudeCodeHttpRequest } from '../pipeline-facts.ts'; +import { takeHttpResponse } from '@floway-dev/http/pipeline'; +import { defineStage, move, secret } from '@floway-dev/pipeline'; +import { getProviderRepo, replaceHttpHeader, type ProviderResponse, type ProviderChatServices } from '@floway-dev/provider'; + +export const retryClaudeCodeAccess = (upstreamId: string) => defineStage({ + name: 'retryClaudeCodeAccess', + through: { + request: { needs: ['request.claudeCode.access', 'request.http.callId', 'request.http.headers', 'request.provider.modelKey'], consumes: ['request.claudeCode.access', 'request.http.headers'], provides: ['request.claudeCode.access', 'request.http.headers'] }, + response: { needs: ['response.http.exchange', 'response.http.body', 'response.provider.previousCalls'], consumes: ['response.http.exchange', 'response.http.body'], provides: ['response.http.exchange', 'response.http.body', 'response.provider.previousCalls'] }, + }, + execute: async (facts, next, use) => { + const first = await next(move({ ...facts })); + const exchange = first['response.http.exchange']; + if (exchange.type !== 'response' || exchange.status !== 401 || facts['request.claudeCode.access'].freshlyMinted) return move({ ...first }); + if (exchange.body !== null) await exchange.body[Symbol.asyncDispose](); + const previous = [...first['response.provider.previousCalls'], { modelKey: first['response.provider.modelKey'] }]; + await invalidateClaudeCodeAccessToken({ upstreamId, repo: getProviderRepo().upstreams }); + let access; + try { + access = await ensureClaudeCodeAccessToken({ upstreamId, repo: getProviderRepo().upstreams, fetcher: use.httpCall(facts['request.http.callId']).fetcher }); + } catch (error) { + if (!(error instanceof ClaudeCodeOAuthSessionTerminatedError)) throw error; + const rejected = takeHttpResponse(synthetic503(`Claude Code refresh failed: ${error.upstreamMessage}`)); + return move({ ...first, 'response.http.exchange': rejected, 'response.http.body': rejected.body, 'response.provider.called': false, 'response.provider.previousCalls': previous }); + } + const retried = await next(move({ + ...facts, 'request.claudeCode.access': { ...access, entry: { ...access.entry, token: secret(access.entry.token) } }, + 'request.http.headers': replaceHttpHeader(facts['request.http.headers'], 'authorization', secret(`Bearer ${access.entry.token}`)), + })); + return move({ ...retried, 'response.provider.previousCalls': previous }); + }, +}); diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/synthesize-metadata-user-id.ts b/packages/provider-claude-code/src/stages/synthesize-metadata-user-id.ts similarity index 70% rename from packages/provider-claude-code/src/interceptors/anthropic-messages/synthesize-metadata-user-id.ts rename to packages/provider-claude-code/src/stages/synthesize-metadata-user-id.ts index c5c4df68d4..1d32792125 100644 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/synthesize-metadata-user-id.ts +++ b/packages/provider-claude-code/src/stages/synthesize-metadata-user-id.ts @@ -2,8 +2,10 @@ import { sha256 } from '@noble/hashes/sha2.js'; import { bytesToHex } from '@noble/hashes/utils.js'; import { v4 } from 'uuid'; -import type { AnthropicMessagesBoundaryCtx } from './types.ts'; +import type { ClaudeCodePreparedRequest } from '../pipeline-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; import type { AnthropicMessagesMessage, AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import type { ProviderChatResponse } from '@floway-dev/provider'; // Real CC includes `metadata.user_id` on every /v1/messages request: a JSON // envelope `{device_id, account_uuid, session_id}` (v2.1.78+) or the legacy @@ -28,22 +30,24 @@ import type { AnthropicMessagesMessage, AnthropicMessagesPayload } from '@floway // References: // - https://github.com/Wei-Shaw/sub2api/blob/4a5665da5b2c6b83c4597844ea6e573746c821b1/backend/internal/service/metadata_userid.go#L15 -export const synthesizeMetadataUserId = async ( - ctx: AnthropicMessagesBoundaryCtx, - _env: object, - run: () => Promise, -): Promise => { - const existing = ctx.payload.metadata?.user_id; - if (typeof existing === 'string' && existing.length > 0) return await run(); +export const synthesizeMetadataUserIdPayload =

>(payload: P, upstreamId: string): P => { + const existing = payload.metadata?.user_id; + if (typeof existing === 'string' && existing.length > 0) return payload; - const deviceId = deviceIdForUpstream(ctx.upstreamId); - const sessionId = sessionIdForPayload(ctx.upstreamId, ctx.payload); + const deviceId = deviceIdForUpstream(upstreamId); + const sessionId = sessionIdForPayload(upstreamId, payload); const userId = JSON.stringify({ device_id: deviceId, account_uuid: '', session_id: sessionId }); - ctx.payload = { ...ctx.payload, metadata: { ...ctx.payload.metadata, user_id: userId } }; - return await run(); + payload = { ...payload, metadata: { ...payload.metadata, user_id: userId } }; + return payload; }; +export const synthesizeMetadataUserId = (upstreamId: string) => defineStage, ProviderChatResponse<'anthropicMessages'>>({ + name: 'synthesizeMetadataUserId', + through: { request: { needs: ['request.provider.payload', 'request.claudeCode.shaped'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, response: { needs: [], consumes: [], provides: [] } }, + execute: async (facts, next) => move({ ...await next(move({ ...facts, 'request.provider.payload': facts['request.claudeCode.shaped'] ? facts['request.provider.payload'] : synthesizeMetadataUserIdPayload(facts['request.provider.payload'], upstreamId) })) }), +}); + // 64-hex (32-byte) device_id, matching the format real CC emits. See // https://github.com/Wei-Shaw/sub2api/blob/4a5665da5b2c6b83c4597844ea6e573746c821b1/backend/internal/service/gateway_prompt_test.go#L17 const deviceIdForUpstream = (upstreamId: string): string => diff --git a/packages/provider-claude-code/src/interceptors/anthropic-messages/system-blocks.ts b/packages/provider-claude-code/src/system-blocks.ts similarity index 98% rename from packages/provider-claude-code/src/interceptors/anthropic-messages/system-blocks.ts rename to packages/provider-claude-code/src/system-blocks.ts index 167cfe133e..57598e33c6 100644 --- a/packages/provider-claude-code/src/interceptors/anthropic-messages/system-blocks.ts +++ b/packages/provider-claude-code/src/system-blocks.ts @@ -89,7 +89,7 @@ const FINGERPRINT_INDICES = [4, 7, 20] as const; const encoder = new TextEncoder(); -const extractFirstUserText = (body: AnthropicMessagesPayload): string => { +const extractFirstUserText = (body: Pick): string => { for (const msg of body.messages) { if (msg.role !== 'user') continue; if (typeof msg.content === 'string') return msg.content; @@ -106,7 +106,7 @@ const extractFirstUserText = (body: AnthropicMessagesPayload): string => { // user-message text; positions past the end are filled with 0x30 ('0'). // The output drives `${VERSION}.${FP}` in the billing block; matching // real CC's wire shape is cheap and robust to detector tightening. -export const computeCcVersionFingerprint = (version: string, body: AnthropicMessagesPayload): string => { +export const computeCcVersionFingerprint = (version: string, body: Pick): string => { const utf8 = encoder.encode(extractFirstUserText(body)); const chars = new Uint8Array(FINGERPRINT_INDICES.length); for (let i = 0; i < FINGERPRINT_INDICES.length; i++) { diff --git a/packages/provider-codex/__tests__/fetch_test.ts b/packages/provider-codex/__tests__/fetch_test.ts index 20303e0ef6..285ea84d96 100644 --- a/packages/provider-codex/__tests__/fetch_test.ts +++ b/packages/provider-codex/__tests__/fetch_test.ts @@ -2,7 +2,7 @@ import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; import { createUpstreamStateRepoStub } from './upstream-state-repo.ts'; import { CODEX_CLI_VERSION, CODEX_ORIGINATOR, CODEX_RESPONSES_LITE_CLIENT_METADATA_KEY, CODEX_RESPONSES_LITE_HEADER, CODEX_USER_AGENT } from '../src/constants.ts'; -import { callCodexAlphaSearch, callCodexOpenAIImagesGenerations, callCodexOpenAIResponses, callCodexOpenAIResponsesCompact, type CodexCallEffects } from '../src/fetch.ts'; +import { callCodexAlphaSearch, callCodexOpenAIImagesGenerations, callCodexOpenAIResponses, callCodexOpenAIResponsesCompact, type CodexCallEffects } from './test-utils/call.ts'; import * as responsesLite from '../src/responses-lite.ts'; import type { CodexAccessTokenEntry, CodexAccountCredential, CodexQuotaSnapshotEntryMap, CodexUpstreamState } from '../src/state.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; @@ -62,8 +62,6 @@ const seedAccountState = (overrides: Partial): void => { const readQuotaEntry = (): CodexQuotaSnapshotEntryMap | null => (currentRecord.state as CodexUpstreamState).accounts[0].quotaSnapshot; -// putCodexQuota fires-and-forgets via .catch(() => {}); yield to the task -// queue so the saveState promise resolves before the caller asserts on state. const flushMicrotasks = (): Promise => new Promise(resolve => setTimeout(resolve, 0)); beforeEach(() => { @@ -81,7 +79,7 @@ afterEach(() => vi.restoreAllMocks()); const sseResponse = (status = 200): Response => new Response( new ReadableStream({ start(c) { - c.enqueue(new TextEncoder().encode('event: response.created\ndata: {"type":"response.created"}\n\n')); + c.enqueue(new TextEncoder().encode('event: response.completed\ndata: {"type":"response.completed","response":{"id":"resp_test","object":"response","status":"completed","model":"gpt-5.4","output":[]}}\n\n')); c.close(); }, }), @@ -147,7 +145,7 @@ describe('callCodexOpenAIResponses — gates', () => { }, }, }); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); const result = await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { input: [], stream: true }, headers: new Headers(), effects: makeEffects(), call: noopUpstreamCallOptions(), @@ -177,7 +175,7 @@ describe('callCodexOpenAIResponses — token freshness', () => { test('reuses fresh state-cached access token without refreshing', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { input: [], stream: true }, headers: new Headers(), effects: makeEffects(), call: noopUpstreamCallOptions(), @@ -191,7 +189,7 @@ describe('callCodexOpenAIResponses — token freshness', () => { refresh_token: null, accessToken: { token: 'at_only', expiresAt: null, refreshedAt: 'now' }, }); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); const result = await callCodexOpenAIResponses({ upstreamId, account: accessOnlyAccount, model, body: { input: [], stream: true }, headers: new Headers(), effects: makeEffects(), call: noopUpstreamCallOptions(), @@ -559,7 +557,7 @@ describe('Codex private Responses wire selection', () => { bodies.push(await new Response(init?.body).text()); headers.push(new Headers(init?.headers)); if (bodies.length === 1) { - selectedModel.providerData.useResponsesLite = false; + expect(Object.isFrozen(selectedModel.providerData)).toBe(true); return errorJson(401, { error: { code: 'expired_token', message: 'expired' } }); } return action === 'generate' ? sseEventsResponse([]) : compactJsonResponse(); @@ -598,7 +596,7 @@ describe('Codex private Responses wire selection', () => { headers: new Headers({ [CODEX_RESPONSES_LITE_HEADER]: 'false' }), effects: makeEffects(), call: noopUpstreamCallOptions(), }); if (result.ok) throw new Error('expected upstream error'); - expect(result.response).toBe(upstream); + expect(result.response.status).toBe(upstream.status); expect(result.response.headers.get(CODEX_RESPONSES_LITE_HEADER)).toBe(marker ?? null); expect(result.response.headers.get('content-type')).toBe('application/problem+json'); expect(result.response.statusText).toBe('Upstream failure'); @@ -611,7 +609,7 @@ describe('Codex private Responses wire selection', () => { describe('callCodexOpenAIResponses — upstream classification', () => { test('happy path: 200 → ok:true, quota persisted', async () => { seedFreshAccessToken(); - vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); const result = await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { input: [], stream: true }, headers: new Headers(), effects: makeEffects(), call: noopUpstreamCallOptions(), @@ -625,7 +623,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('upstream body has store:false and stream:true forced even if caller passes otherwise', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { input: [], stream: false as unknown as true, store: true } as unknown as Parameters[0]['body'], @@ -639,7 +637,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('builds Codex responses headers and metadata from a clean set', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, @@ -719,7 +717,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { refresh_token: null, accessToken: { token: 'at_only', expiresAt: null, refreshedAt: 'now' }, }); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); const result = await callCodexOpenAIResponses({ upstreamId, account, @@ -843,7 +841,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('preserves a hyphenated Codex session id for prompt cache', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, @@ -860,7 +858,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('canonicalizes downstream session_id to the Codex session-id header', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, @@ -877,7 +875,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('prefers downstream session-id over session_id when both are provided', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, @@ -894,7 +892,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('generates a Codex session id when the downstream request has none', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { input: [], stream: true }, headers: new Headers(), effects: makeEffects(), call: noopUpstreamCallOptions(), @@ -908,7 +906,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('derives the same session id across turns of a stateless conversation', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); const turn = { upstreamId, account: activeAccount, model, body: { @@ -932,7 +930,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('derives distinct session ids when only the instructions differ', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); const call = (instructions: string) => callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { @@ -954,7 +952,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('derives distinct session ids when only the first user message differs', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); const call = (content: string) => callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { @@ -976,7 +974,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('uses account.openaiDeviceId as the installation id', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); const deviceId = '22222222-3333-4444-9555-666666666666'; await callCodexOpenAIResponses({ upstreamId, account: { ...activeAccount, openaiDeviceId: deviceId }, @@ -992,7 +990,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('prefers a caller-supplied installation id from client_metadata over the account device id', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: { ...activeAccount, openaiDeviceId: 'account-device-id' }, model, @@ -1014,7 +1012,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('passes through caller thread-id and x-client-request-id when distinct from session-id', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { input: [], stream: true }, @@ -1038,7 +1036,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('merges caller-supplied x-codex-turn-metadata extras over the synthesized blob', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { input: [], stream: true }, @@ -1072,7 +1070,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('reads the turn-metadata blob from the body before the header', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { @@ -1117,7 +1115,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('keeps the unbounded tool inventory in the body blob and out of the header blob', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); const toolNamespacesInfo = { namespaces: [{ name: 'shell', tools: ['exec'] }] }; await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, @@ -1147,7 +1145,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { test('preserves caller client_metadata extras while keeping identity-mirror keys gateway-owned', async () => { seedFreshAccessToken(); - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { @@ -1192,7 +1190,7 @@ describe('callCodexOpenAIResponses — upstream classification', () => { }); const upstreamBody = { error: { code: 'token_invalidated', message: 're-import required' } }; const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(errorJson(401, upstreamBody, { 'x-upstream-marker': 'kept' })); - const persistTerminalState = vi.fn(async () => { throw new Error('state write failed'); }); + const persistTerminalState = vi.fn(async () => {}); const result = await callCodexOpenAIResponses({ upstreamId, account: accessOnlyAccount, model, body: { input: [], stream: true }, headers: new Headers(), @@ -1271,22 +1269,18 @@ describe('callCodexOpenAIResponses — upstream classification', () => { }); }); -describe('callCodexOpenAIResponses — background-write registration', () => { - // Background state writes (quota snapshot on 2xx/429, access-token put on - // 401-retry) must reach the runtime's waitUntil slot so workerd does not - // cancel them the instant the streaming response returns to the client. - // Without this, freshly-minted Codex tokens and quota snapshots get dropped - // on the floor and the next request re-mints / re-races the upstream. - test('2xx persists quota snapshot via opts.call.waitUntil', async () => { +describe('Codex Responses deferred state writes', () => { + test('2xx quota persistence completes through the run deferred lifecycle', async () => { seedFreshAccessToken(); - vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); + vi.spyOn(globalThis, 'fetch').mockImplementation(async () => sseResponse()); const waitUntil = vi.fn<(promise: Promise) => void>(); await callCodexOpenAIResponses({ upstreamId, account: activeAccount, model, body: { input: [], stream: true }, headers: new Headers(), effects: makeEffects(), call: { ...noopUpstreamCallOptions(), waitUntil }, }); - expect(waitUntil).toHaveBeenCalledTimes(1); + expect(waitUntil).not.toHaveBeenCalled(); + expect(readQuotaEntry()).not.toBeNull(); }); test('401-retry persists the fresh access token before returning', async () => { @@ -1301,9 +1295,10 @@ describe('callCodexOpenAIResponses — background-write registration', () => { model, body: { input: [], stream: true }, headers: new Headers(), effects: makeEffects(), call: { ...noopUpstreamCallOptions(), waitUntil }, }); - // The access-token write is awaited because its CAS result carries the - // effective plan; only the successful retry's quota write is backgrounded. - expect(waitUntil).toHaveBeenCalledTimes(1); + // The refresh CAS supplies the retry plan; drain also completes the + // successful endpoint's quota observation. + expect(waitUntil).not.toHaveBeenCalled(); + expect(readQuotaEntry()).not.toBeNull(); expect((currentRecord.state as CodexUpstreamState).accounts[0].accessToken?.token).toBe('at2'); }); }); @@ -1459,14 +1454,6 @@ describe('callCodexOpenAIImagesGenerations', () => { }); }); -// `callCodexOpenAIResponsesCompact` shares OAuth + quota + 401-retry plumbing with -// `callCodexOpenAIResponses` (both go through `prepareCodexCall` → -// `dispatchCodexHttpCall` → `refreshAccessTokenForRetry`). The streaming -// suite above pins those shared paths; this block exercises only the -// compact-specific wire contract — endpoint URL, `Accept: application/json`, -// body shape (no `stream`, no `store`), unary JSON decoding — plus the 401 -// retry on the unary endpoint to confirm the retry decision is taken from -// the bare response status (no SSE wrap in the path). const compactJsonResponse = (overrides?: Partial): Response => new Response(JSON.stringify({ id: 'resp_x', @@ -1513,7 +1500,7 @@ describe('callCodexOpenAIResponsesCompact', () => { expect(result.result.output[0]).toMatchObject({ id: 'cmp_x', type: 'compaction', encrypted_content: 'FULL_BLOB' }); }); - test('2xx persists quota snapshot via opts.call.waitUntil', async () => { + test('2xx quota persistence completes through the run deferred lifecycle', async () => { seedFreshAccessToken(); vi.spyOn(globalThis, 'fetch').mockResolvedValue(compactJsonResponse()); const waitUntil = vi.fn<(promise: Promise) => void>(); @@ -1522,7 +1509,8 @@ describe('callCodexOpenAIResponsesCompact', () => { body: { input: [] }, headers: new Headers(), effects: makeEffects(), call: { ...noopUpstreamCallOptions(), waitUntil }, }); - expect(waitUntil).toHaveBeenCalledTimes(1); + expect(waitUntil).not.toHaveBeenCalled(); + expect(readQuotaEntry()).not.toBeNull(); }); test('401 other → refresh + retry once on the compact endpoint, succeed', async () => { diff --git a/packages/provider-codex/__tests__/provider_test.ts b/packages/provider-codex/__tests__/provider_test.ts index 5d88d8f638..e2c31854f8 100644 --- a/packages/provider-codex/__tests__/provider_test.ts +++ b/packages/provider-codex/__tests__/provider_test.ts @@ -5,9 +5,9 @@ import { CODEX_RESPONSES_LITE_CLIENT_METADATA_KEY, CODEX_RESPONSES_LITE_HEADER } import { createCodexProvider } from '../src/provider.ts'; import type { CodexAccessTokenEntry, CodexUpstreamState } from '../src/state.ts'; import { exchangeResponse } from '@floway-dev/http/pipeline'; -import { move, run, setRelease, type Event } from '@floway-dev/pipeline'; -import { directFetcher, initProviderRepo, providerModelFacts, type UpstreamRecord } from '@floway-dev/provider'; -import { callProviderPipeline, noopUpstreamCallOptions, readJsonRequest, stubProviderModel } from '@floway-dev/test-utils'; +import { getFailureFacts, move, run, setRelease, type Event } from '@floway-dev/pipeline'; +import { directFetcher, initProviderRepo, providerModelFacts, type ProviderOperationPayloads, type UpstreamRecord } from '@floway-dev/provider'; +import { callProviderPipeline, collectChatProviderPipeline, noopUpstreamCallOptions, readJsonRequest, stubProviderModel } from '@floway-dev/test-utils'; const farFutureMs = Date.now() + 24 * 60 * 60 * 1000; @@ -85,14 +85,100 @@ const idToken = (planType = 'plus'): string => [ Buffer.from('signature').toString('base64url'), ].join('.'); -const oauthTokenResponse = (overrides: Partial<{ access_token: string; refresh_token: string; expires_in: number }> = {}): Response => new Response(JSON.stringify({ +const oauthTokenResponse = (overrides: Partial<{ access_token: string; refresh_token: string; expires_in: number; id_token: string }> = {}): Response => new Response(JSON.stringify({ access_token: overrides.access_token ?? 'at_minted', refresh_token: overrides.refresh_token ?? 'rt_v2', - id_token: idToken(), + id_token: overrides.id_token ?? idToken(), expires_in: overrides.expires_in ?? 3600, }), { status: 200, headers: new Headers({ 'content-type': 'application/json' }) }); describe('createCodexProvider', () => { + test('the401 finite observer preserves an original stream failure without canceling its locked body', async () => { + const original = new Error('Codex refusal body broke'); + const body = new ReadableStream({ start(controller) { controller.error(original); } }); + vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(body, { status: 401 })); + const events: Event[] = []; + const caught = await collectChatProviderPipeline(createCodexProvider(current!), 'openaiResponses', stubProviderModel({ id: 'gpt-5.4', endpoints: { openaiResponses: {} } }), { input: [] }, undefined, noopUpstreamCallOptions(), { dump: event => { events.push(event); } }).catch((error: unknown) => error); + expect(caught).toBe(original); + expect(getFailureFacts(caught)).toMatchObject({ 'response.provider.called': true, 'response.provider.modelKey': 'gpt-5.4' }); + expect(events.find(event => event.type === 'stage.failed')).toMatchObject({ error: original }); + }); + + test('access-only401 remains readable while terminal persistence failure reaches the run outcome', async () => { + current = accessOnlyRecord(freshAccessToken); + const writeError = new Error('terminal state write failed'); + repo.saveState.mockRejectedValue(writeError); + const body = { error: { code: 'token_invalidated', message: 're-import required' }, diagnostic: 'full upstream details' }; + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(Response.json(body, { status: 401, headers: { 'x-upstream': 'retained' } })); + const provider = createCodexProvider(current); + const events: Event[] = []; + const executed = await run(provider.pipelines.openaiResponses!, move({ + 'request.provider.model': providerModelFacts(stubProviderModel({ id: 'gpt-5.4', endpoints: { openaiResponses: {} } })), + 'request.provider.payload': { input: [], stream: true }, + 'request.http.callId': 0, 'request.http.headers': [], + }), { httpCall: () => noopUpstreamCallOptions(), recordProtocolFrames: (frames: AsyncIterable) => frames, dump: (event: Event) => { events.push(event); } }); + const exchange = executed.facts['response.http.exchange']; + if (exchange.type === 'transportFailure') throw exchange.error; + const response = exchangeResponse(exchange); + expect(response.status).toBe(401); + expect(response.headers.get('x-upstream')).toBe('retained'); + expect(await response.json()).toEqual(body); + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + await expect(executed.drain()).rejects.toBe(writeError); + expect(fetchSpy).toHaveBeenCalledTimes(1); + expect(events.find(event => event.type === 'deferred.settled' && event.outcome.status === 'rejected')).toMatchObject({ + deferred: (executed.facts as typeof executed.facts & Record)['response.codex.background'], outcome: { status: 'rejected', reason: writeError }, + }); + }); + + test.each(['openaiResponses', 'openaiResponsesCompact'] as const)('%s pipeline keeps parsed private wire content and actual endpoint observations', async operation => { + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => operation === 'openaiResponses' + ? new Response(sseResponse().body, { headers: { [CODEX_RESPONSES_LITE_HEADER]: 'true' } }) + : Response.json({ id: 'compact', object: 'response.compaction', output: [] })); + const provider = createCodexProvider(current!); + const model = stubProviderModel({ id: 'future-lite', endpoints: { openaiResponses: {} }, providerData: { useResponsesLite: true } }); + const options = noopUpstreamCallOptions(); + const result = await collectChatProviderPipeline(provider, operation, model, { + input: [{ type: 'message', role: 'user', content: 'hi' }], tools: [{ type: 'function', name: 'lookup', parameters: { type: 'object' } }], text: { verbosity: 'low' }, + }, undefined, options); + const wire = await readJsonRequest(fetchSpy.mock.calls[0]![1] as RequestInit) as Record; + expect(wire).not.toHaveProperty('instructions'); + expect(wire).not.toHaveProperty('tools'); + expect(wire.text).toEqual({ verbosity: 'low' }); + expect(typeof result.facts['request.http.body']).toBe('object'); + expect(result.facts['response.provider.called']).toBe(true); + expect(result.facts['response.provider.modelKey']).toBe('future-lite'); + const output = result.facts['response.provider.output']; + if (output === null || !('kind' in output)) throw new Error('Expected protocol output'); + if (output.kind === 'stream') { + expect(result.frames.filter(frame => frame.type === 'event').map(frame => frame.event.type)).toEqual(['response.created', 'response.in_progress', 'response.completed']); + const exchange = result.facts['response.http.exchange']; + if (exchange.type !== 'response') throw new Error('Expected response'); + expect(exchange.headers.some(([name]) => name === CODEX_RESPONSES_LITE_HEADER)).toBe(false); + expect(exchange.headers).toContainEqual(['content-type', 'text/event-stream']); + } else expect(output.body).toMatchObject({ object: 'response.compaction' }); + }); + + test('Responses pipeline retains the first 401 observation and retries without applying the image plan gate', async () => { + const state = current!.state as CodexUpstreamState; + current = { ...current!, config: { accounts: [{ email: 'a@b.com', chatgptAccountId: 'acc', chatgptUserId: 'usr', planType: 'free' }] }, state: { accounts: [{ ...state.accounts[0]!, accessToken: { ...freshAccessToken, planType: 'free' } }] } }; + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => { + if (String(input) === 'https://auth.openai.com/oauth/token') return oauthTokenResponse({ id_token: idToken('free') }); + return new Headers(init?.headers).get('authorization') === 'Bearer at' + ? Response.json({ error: { code: 'invalid_token', message: 'expired' } }, { status: 401 }) + : sseResponse(); + }); + const provider = createCodexProvider(current!); + const model = stubProviderModel({ id: 'gpt-5.4', endpoints: { openaiResponses: {} } }); + const options = noopUpstreamCallOptions(); + const result = await collectChatProviderPipeline(provider, 'openaiResponses', model, { input: [] }, undefined, options); + expect(result.facts['response.provider.previousCalls']).toEqual([{ modelKey: 'gpt-5.4' }]); + expect(fetchSpy.mock.calls.filter(([url]) => String(url).includes('/responses'))).toHaveLength(2); + const output = result.facts['response.provider.output']; + if (output === null || !('kind' in output) || output.kind !== 'stream') throw new Error('Expected stream'); + expect(result.frames.some(frame => frame.type === 'event' && frame.event.type === 'response.completed')).toBe(true); + }); + test('Codex Images records the complete parsed 401 body and preserves its preceding endpoint observation', async () => { const refusal = { error: { code: 'invalid_token', message: 'expired bearer' }, diagnostic: 'x'.repeat(600) }; vi.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => { @@ -310,7 +396,7 @@ describe('createCodexProvider', () => { test('callOpenAIResponses preserves developer messages on the Codex wire', async () => { const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(sseResponse()); const instance = createCodexProvider(baseRecord); - const result = await instance.instance.callOpenAIResponses( + const result = await collectChatProviderPipeline(instance, 'openaiResponses', stubProviderModel({ id: 'gpt-5.4', display_name: 'gpt-5.4', endpoints: { openaiResponses: {} } }), { input: [ @@ -320,12 +406,10 @@ describe('createCodexProvider', () => { ], stream: true, }, - 'generate', undefined, - noopUpstreamCallOptions(), - ); - expect(result.ok).toBe(true); - expect(result.action).toBe('generate'); + noopUpstreamCallOptions()); + expect(result.output).not.toBeNull(); + expect(result.facts['response.provider.responsesAction']).toBe('generate'); const init = fetchSpy.mock.calls[0]?.[1] as RequestInit | undefined; if (init === undefined) throw new Error('expected a Codex upstream request'); const body = await readJsonRequest(init) as Record; @@ -337,7 +421,7 @@ describe('createCodexProvider', () => { ]); }); - test.each(['generate', 'compact'] as const)('%s retains the full Standard body through interceptors before private Lite encoding', async action => { + test.each(['generate', 'compact'] as const)('%s retains the full Standard body through stages before private Lite encoding', async action => { const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async () => action === 'generate' ? sseResponse() : new Response(JSON.stringify({ id: 'cmp_1', object: 'response.compaction', output: [] }))); @@ -351,14 +435,14 @@ describe('createCodexProvider', () => { ]; const options = noopUpstreamCallOptions(); options.headers.set(CODEX_RESPONSES_LITE_HEADER, 'true'); - const result = await provider.instance.callOpenAIResponses(model, { + const result = await collectChatProviderPipeline(provider, action === 'generate' ? 'openaiResponses' : 'openaiResponsesCompact', model, { input, tools: [{ type: 'custom', name: 'patch' }], text: { verbosity: 'low' }, client_metadata: { [CODEX_RESPONSES_LITE_CLIENT_METADATA_KEY]: 'true' }, - } as Parameters[1], action, undefined, options); - expect(result.ok).toBe(true); - expect(result.action).toBe(action); + } as ProviderOperationPayloads['openaiResponses'], undefined, options); + expect(result.output).not.toBeNull(); + expect(result.facts['response.provider.responsesAction']).toBe(action); const wire = await readJsonRequest(fetchSpy.mock.calls[0]![1] as RequestInit) as Record; expect(wire).not.toHaveProperty('instructions'); expect(wire).not.toHaveProperty('tools'); @@ -382,15 +466,13 @@ describe('createCodexProvider', () => { test('callOpenAIResponses re-reads state per request (operator re-import takes effect)', async () => { repo.getById.mockResolvedValueOnce({ ...baseRecord, state: { accounts: [{ chatgptAccountId: 'acc', refresh_token: 'rt_v1', state: 'session_terminated', state_updated_at: '2026-01-02T00:00:00Z', openaiDeviceId: '11111111-2222-4333-8444-555555555555', accessToken: null, quotaSnapshot: null }] } as CodexUpstreamState }); const instance = createCodexProvider(baseRecord); - const result = await instance.instance.callOpenAIResponses( + const result = await collectChatProviderPipeline(instance, 'openaiResponses', stubProviderModel({ id: 'gpt-5.4', display_name: 'gpt-5.4', endpoints: { openaiResponses: {} } }), { input: [], stream: true }, - 'generate', undefined, - noopUpstreamCallOptions(), - ); - expect(result.ok).toBe(false); - if (!result.ok) expect(result.response.status).toBe(503); + noopUpstreamCallOptions()); + expect(result.output).toBeNull(); + expect(result.response!.status).toBe(503); }); test('callOpenAIImagesGenerations posts gpt-image-2 through the ChatGPT Codex endpoint', async () => { @@ -468,21 +550,7 @@ describe('createCodexProvider', () => { }); }); - test.each([ - 'callOpenAIEmbeddings', - 'callOpenAIAudioTranscriptions', - 'callOpenAIChatCompletions', - 'callAnthropicMessagesCountTokens', - 'callAnthropicMessages', - ] as const)('%s returns a synthetic 405 (data plane never dispatches these to Codex)', async method => { - const instance = createCodexProvider(baseRecord); - const model = stubProviderModel({ id: 'gpt-5.4', display_name: 'gpt-5.4', endpoints: { openaiResponses: {} } }); - // @ts-expect-error: each method has a different body type; we only assert - // the synthetic 405 envelope is what comes back. - const result = await instance.instance[method](model, {}, undefined, noopUpstreamCallOptions()) as { response: Response }; - expect(result.response.status).toBe(405); - const body = await result.response.json() as { error: { type: string; message: string } }; - expect(body.error.type).toBe('method_not_allowed'); - expect(body.error.message).toMatch(/codex/i); + test.each(['openaiEmbeddings', 'openaiAudioTranscriptions', 'openaiChatCompletions', 'anthropicMessagesCountTokens', 'anthropicMessages'] as const)('%s has no Codex dispatch pipeline', operation => { + expect(createCodexProvider(baseRecord).pipelines[operation]).toBeUndefined(); }); }); diff --git a/packages/provider-codex/__tests__/interceptors/openai-responses/action-pivot_test.ts b/packages/provider-codex/__tests__/stages/action-pivot_test.ts similarity index 68% rename from packages/provider-codex/__tests__/interceptors/openai-responses/action-pivot_test.ts rename to packages/provider-codex/__tests__/stages/action-pivot_test.ts index 1a537cf3ae..95f75db047 100644 --- a/packages/provider-codex/__tests__/interceptors/openai-responses/action-pivot_test.ts +++ b/packages/provider-codex/__tests__/stages/action-pivot_test.ts @@ -1,35 +1,30 @@ import { afterEach, beforeEach, expect, test, vi } from 'vitest'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; -import { createUpstreamStateRepoStub } from '../../upstream-state-repo.ts'; -import type { Interceptor } from '@floway-dev/interceptor'; -import { initProviderRepo, type ProviderOpenAIResponsesResult, type UpstreamRecord } from '@floway-dev/provider'; +import { createUpstreamStateRepoStub } from '../upstream-state-repo.ts'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; +import { initProviderRepo, type UpstreamRecord } from '@floway-dev/provider'; import { assertEquals, readJsonRequest } from '@floway-dev/test-utils'; -// Terminal dispatch must use the post-chain action and preserve late Standard -// tool/instruction changes until the private wire encoder and compact projector. -const pivotGenerateToCompact: Interceptor = async (ctx, _env, run) => { - ctx.action = 'compact'; - ctx.payload = { - ...ctx.payload, - instructions: 'Late instructions', - tools: [...ctx.payload.tools ?? [], { type: 'custom', name: 'late_tool' }], - }; - return await run(); -}; - -vi.mock('../../../src/interceptors/openai-responses/index.ts', async () => { - const original = await vi.importActual('../../../src/interceptors/openai-responses/index.ts'); - return { - ...original, - CODEX_OPENAI_RESPONSES_BOUNDARY: [...original.CODEX_OPENAI_RESPONSES_BOUNDARY, pivotGenerateToCompact], - }; +// A provider stage can pivot an initially selected generate operation before private +// wire preparation. Both Standard and Lite compaction projection must see its new content. +vi.mock('../../src/stages/inject-default-instructions.ts', async () => { + const original = await vi.importActual('../../src/stages/inject-default-instructions.ts'); + const { defineStage, move } = await import('@floway-dev/pipeline'); + const pivot = defineStage, Record, Record, Record>({ + name: 'pivotCodexResponsesToCompact', + through: { request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload', 'request.provider.responsesAction'], provides: ['request.provider.payload', 'request.provider.responsesAction'] }, response: { needs: [], consumes: [], provides: [] } }, + execute: async (facts, next) => { + const payload = facts['request.provider.payload'] as CanonicalOpenAIResponsesPayload; + return move({ ...await next(move({ ...facts, 'request.provider.responsesAction': 'compact', 'request.provider.payload': { ...payload, instructions: 'Late instructions', tools: [...payload.tools ?? [], { type: 'custom', name: 'late_tool' }] } })) }); + }, + }); + return { ...original, injectCodexDefaultInstructions: pivot }; }); // Imports below MUST follow the vi.mock so the provider module resolves // against the mocked chain on first import. -const { createCodexProvider } = await import('../../../src/provider.ts'); -const { noopUpstreamCallOptions, stubProviderModel } = await import('@floway-dev/test-utils'); +const { createCodexProvider } = await import('../../src/provider.ts'); +const { collectChatProviderPipeline, noopUpstreamCallOptions, stubProviderModel } = await import('@floway-dev/test-utils'); const farFutureMs = Date.now() + 24 * 60 * 60 * 1000; @@ -86,7 +81,7 @@ test.each([true, false])('Codex projects a post-chain compact pivot after catalo }); const instance = createCodexProvider(baseRecord); - const result = await instance.instance.callOpenAIResponses( + const result = await collectChatProviderPipeline(instance, 'openaiResponses', stubProviderModel({ id: 'gpt-5.4', display_name: 'gpt-5.4', endpoints: { openaiResponses: {} }, providerData: { useResponsesLite } }), { input: [{ type: 'message', role: 'user', content: 'hi' }], @@ -97,13 +92,11 @@ test.each([true, false])('Codex projects a post-chain compact pivot after catalo stream: true, parallel_tool_calls: false, }, - 'generate', undefined, - noopUpstreamCallOptions(), - ); + noopUpstreamCallOptions()); - if (!result.ok) throw new Error('expected ok result'); - if (result.action !== 'compact') throw new Error(`expected compact variant after pivot, got ${result.action}`); + expect(result.output).toMatchObject({ kind: 'value' }); + expect(result.facts['response.provider.responsesAction']).toBe('compact'); if (compactUrl === undefined) throw new Error('expected /codex/responses/compact to be hit'); if (compactBody === undefined) throw new Error('expected compact body capture'); diff --git a/packages/provider-codex/__tests__/interceptors/openai-responses/inject-default-instructions_test.ts b/packages/provider-codex/__tests__/stages/inject-default-instructions_test.ts similarity index 77% rename from packages/provider-codex/__tests__/interceptors/openai-responses/inject-default-instructions_test.ts rename to packages/provider-codex/__tests__/stages/inject-default-instructions_test.ts index d1aa644ab4..f78a945651 100644 --- a/packages/provider-codex/__tests__/interceptors/openai-responses/inject-default-instructions_test.ts +++ b/packages/provider-codex/__tests__/stages/inject-default-instructions_test.ts @@ -1,17 +1,13 @@ import { test } from 'vitest'; -import { injectDefaultInstructions } from '../../../src/interceptors/openai-responses/inject-default-instructions.ts'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; +import { injectCodexDefaultInstructions } from '../../src/stages/inject-default-instructions.ts'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import type { ProviderStreamResult } from '@floway-dev/provider'; -import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; - -const stubRequest = {}; +import { applyProviderStage, type OpenAIResponsesProbe, assertEquals, stubProviderModel, type ProviderStreamResult } from '@floway-dev/test-utils'; const okEvents = (): Promise> => Promise.resolve({ ok: true, events: (async function* () {})(), modelKey: 'test', headers: new Headers() }); -const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBoundaryCtx => ({ +const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesProbe => ({ payload, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiResponses: {} } }), @@ -21,7 +17,7 @@ const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBo test('injects the default when instructions is absent', async () => { const ctx = invocation({ model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }] }); - await injectDefaultInstructions(ctx, stubRequest, okEvents); + await applyProviderStage(injectCodexDefaultInstructions, ctx, okEvents); assertEquals(ctx.payload.instructions, "You're a helpful assistant."); }); @@ -29,7 +25,7 @@ test('injects the default when instructions is absent', async () => { test('injects the default when instructions is an empty string', async () => { const ctx = invocation({ model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }], instructions: '' }); - await injectDefaultInstructions(ctx, stubRequest, okEvents); + await applyProviderStage(injectCodexDefaultInstructions, ctx, okEvents); assertEquals(ctx.payload.instructions, "You're a helpful assistant."); }); @@ -37,7 +33,7 @@ test('injects the default when instructions is an empty string', async () => { test('injects the default when instructions is null', async () => { const ctx = invocation({ model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }], instructions: null }); - await injectDefaultInstructions(ctx, stubRequest, okEvents); + await applyProviderStage(injectCodexDefaultInstructions, ctx, okEvents); assertEquals(ctx.payload.instructions, "You're a helpful assistant."); }); @@ -45,7 +41,7 @@ test('injects the default when instructions is null', async () => { test('preserves a caller-supplied instructions string', async () => { const ctx = invocation({ model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }], instructions: 'You are a pirate.' }); - await injectDefaultInstructions(ctx, stubRequest, okEvents); + await applyProviderStage(injectCodexDefaultInstructions, ctx, okEvents); assertEquals(ctx.payload.instructions, 'You are a pirate.'); }); @@ -64,7 +60,7 @@ test.each([ instructions: value as unknown as string, }); - await injectDefaultInstructions(ctx, stubRequest, okEvents); + await applyProviderStage(injectCodexDefaultInstructions, ctx, okEvents); assertEquals(ctx.payload.instructions, value); }, @@ -80,7 +76,7 @@ test('injects the default and preserves input items it does not own', async () = ], }); - await injectDefaultInstructions(ctx, stubRequest, okEvents); + await applyProviderStage(injectCodexDefaultInstructions, ctx, okEvents); assertEquals(ctx.payload.instructions, "You're a helpful assistant."); assertEquals(ctx.payload.input, [ diff --git a/packages/provider-codex/__tests__/interceptors/openai-responses/strip-unsupported-fields_test.ts b/packages/provider-codex/__tests__/stages/strip-unsupported-response-fields_test.ts similarity index 82% rename from packages/provider-codex/__tests__/interceptors/openai-responses/strip-unsupported-fields_test.ts rename to packages/provider-codex/__tests__/stages/strip-unsupported-response-fields_test.ts index 87ff5c15ea..3e8b305b8c 100644 --- a/packages/provider-codex/__tests__/interceptors/openai-responses/strip-unsupported-fields_test.ts +++ b/packages/provider-codex/__tests__/stages/strip-unsupported-response-fields_test.ts @@ -1,17 +1,13 @@ import { test } from 'vitest'; -import { stripUnsupportedFields } from '../../../src/interceptors/openai-responses/strip-unsupported-fields.ts'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; +import { stripCodexUnsupportedFields } from '../../src/stages/strip-unsupported-response-fields.ts'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import type { ProviderStreamResult } from '@floway-dev/provider'; -import { assertEquals, assertFalse, stubProviderModel } from '@floway-dev/test-utils'; - -const stubRequest = {}; +import { applyProviderStage, type OpenAIResponsesProbe, assertEquals, assertFalse, stubProviderModel, type ProviderStreamResult } from '@floway-dev/test-utils'; const okEvents = (): Promise> => Promise.resolve({ ok: true, events: (async function* () {})(), modelKey: 'test', headers: new Headers() }); -const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBoundaryCtx => ({ +const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesProbe => ({ payload, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiResponses: {} } }), @@ -21,7 +17,7 @@ const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBo test('drops every field Codex rejects with Unsupported parameter', async () => { // The fields enumerated below are the full set Codex's ChatGPT-subscription // path rejects; keeping the assertion exhaustive guards against silent - // drift if the constant inside the interceptor is edited without updating + // drift if the constant inside the stage is edited without updating // its rationale. Several entries (frequency_penalty, presence_penalty, // user, stream_options) are not on the canonical payload and reach Codex only // through a permissive caller. `prompt_cache_retention` is modeled but @@ -42,7 +38,7 @@ test('drops every field Codex rejects with Unsupported parameter', async () => { stream_options: { include_usage: true }, } as unknown as CanonicalOpenAIResponsesPayload); - await stripUnsupportedFields(ctx, stubRequest, okEvents); + await applyProviderStage(stripCodexUnsupportedFields, ctx, okEvents); assertFalse('max_output_tokens' in ctx.payload); assertFalse('temperature' in ctx.payload); @@ -67,7 +63,7 @@ test('leaves supported fields intact', async () => { temperature: 0.7, }); - await stripUnsupportedFields(ctx, stubRequest, okEvents); + await applyProviderStage(stripCodexUnsupportedFields, ctx, okEvents); assertEquals(ctx.payload.model, 'gpt-test'); assertEquals(ctx.payload.input, [{ type: 'message', role: 'user', content: 'hello' }]); @@ -82,7 +78,7 @@ test('payload without any unsupported fields is preserved as-is', async () => { const payload: CanonicalOpenAIResponsesPayload = { model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }] }; const ctx = invocation(payload); - await stripUnsupportedFields(ctx, stubRequest, okEvents); + await applyProviderStage(stripCodexUnsupportedFields, ctx, okEvents); assertEquals(ctx.payload, { model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }] }); }); diff --git a/packages/provider-codex/__tests__/test-utils/call.ts b/packages/provider-codex/__tests__/test-utils/call.ts new file mode 100644 index 0000000000..5563c026ef --- /dev/null +++ b/packages/provider-codex/__tests__/test-utils/call.ts @@ -0,0 +1,36 @@ +import type { CodexBackendCallBase, CodexCallEffects } from '../../src/backend.ts'; +import { createCodexPipelines } from '../../src/pipelines.ts'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import type { OpenAIImagesGenerationsPayload } from '@floway-dev/protocols/openai-images'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesCompactionResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import type { Provider, ProviderModel, ProviderOperationPayloads, UpstreamCallOptions } from '@floway-dev/provider'; +import { callProviderPipeline, collectChatProviderPipeline, type ProviderStreamResult } from '@floway-dev/test-utils'; + +interface CallOptions extends CodexBackendCallBase { model: ProviderModel } +interface ResponsesOptions extends CallOptions { body: Omit } +const providerFor = (opts: CallOptions, fallbackPlanType: string | undefined = undefined): Provider => ({ + upstreamId: opts.upstreamId, kind: 'codex', name: 'Codex test', inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + instance: { getProvidedModels: async () => [] }, + pipelines: createCodexPipelines({ upstreamId: opts.upstreamId, readAccount: async () => opts.account, effects: opts.effects, fallbackPlanType }), +}); +const optionsFor = (opts: CallOptions): UpstreamCallOptions => ({ ...opts.call, headers: opts.headers }); +const replay = async function* (frames: readonly ProtocolFrame[]) { yield* frames; }; +export const callCodexOpenAIResponses = async (opts: ResponsesOptions): Promise> => { + const result = await collectChatProviderPipeline(providerFor(opts), 'openaiResponses', opts.model, opts.body, opts.signal, optionsFor(opts)); + if (result.output !== null && 'kind' in result.output && result.output.kind === 'stream') { + const exchange = result.facts['response.http.exchange']; + if (exchange.type === 'transportFailure') throw exchange.error; + return { ok: true, modelKey: result.facts['response.provider.modelKey'], events: replay(result.frames), headers: new Headers(exchange.headers.map(([name, value]): [string, string] => [name, value])) }; + } + if (result.response === null) throw new Error('Expected Responses stream or HTTP failure'); + return { ok: false, modelKey: result.facts['response.provider.modelKey'], response: result.response }; +}; +export const callCodexOpenAIResponsesCompact = async (opts: ResponsesOptions): Promise<{ ok: true; result: OpenAIResponsesCompactionResult; modelKey: string } | { ok: false; response: Response; modelKey: string }> => { + const result = await collectChatProviderPipeline(providerFor(opts), 'openaiResponsesCompact', opts.model, opts.body, opts.signal, optionsFor(opts)); + if (result.output !== null && 'kind' in result.output && result.output.kind === 'value') return { ok: true, modelKey: result.facts['response.provider.modelKey'], result: result.output.body }; + if (result.response === null) throw new Error('Expected compaction value or HTTP failure'); + return { ok: false, modelKey: result.facts['response.provider.modelKey'], response: result.response }; +}; +export const callCodexAlphaSearch = (opts: CallOptions & { body: Record }) => callProviderPipeline(providerFor(opts), 'alphaSearch', opts.model, opts.body, opts.signal, optionsFor(opts)); +export const callCodexOpenAIImagesGenerations = (opts: CallOptions & { body: Omit; fallbackPlanType: string | undefined }) => callProviderPipeline(providerFor(opts, opts.fallbackPlanType), 'openaiImagesGenerations', opts.model, opts.body as ProviderOperationPayloads['openaiImagesGenerations'], opts.signal, optionsFor(opts)); +export type { CodexCallEffects }; diff --git a/packages/provider-codex/package.json b/packages/provider-codex/package.json index 3e70f76ebf..c75f8b4b12 100644 --- a/packages/provider-codex/package.json +++ b/packages/provider-codex/package.json @@ -12,7 +12,6 @@ "dependencies": { "@floway-dev/http": "workspace:*", "@floway-dev/pipeline": "workspace:*", - "@floway-dev/interceptor": "workspace:*", "@floway-dev/protocols": "workspace:*", "@floway-dev/provider": "workspace:*", "uuid": "^14.0.1" diff --git a/packages/provider-codex/src/backend.ts b/packages/provider-codex/src/backend.ts new file mode 100644 index 0000000000..343a838b6d --- /dev/null +++ b/packages/provider-codex/src/backend.ts @@ -0,0 +1,481 @@ +import { CodexAccessOnlyCredentialError, codexPlanObservation, ensureCodexAccessToken, invalidateCodexAccessToken, mintCodexAccessToken, type CodexPlanObservation } from './access-token.ts'; +import { isObject } from './auth/guards.ts'; +import { CodexOAuthSessionTerminatedError } from './auth/oauth.ts'; +import { CODEX_RESPONSES_LITE_CLIENT_METADATA_KEY } from './constants.ts'; +import { sha256JsonUuid, uuidV7 } from './ids.ts'; +import { codexModelUsesResponsesLite } from './models.ts'; +import { + hasCodexQuotaReading, + parseCodexQuotaHeaders, + putCodexQuota, +} from './quota.ts'; +import { + encodeCodexResponsesLiteRequest, + type CodexResponsesBody, + type CodexResponsesLiteRequest, +} from './responses-lite.ts'; +import type { CodexAccessTokenEntry, CodexAccountCredential } from './state.ts'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesInputItem } from '@floway-dev/protocols/openai-responses'; +import { toCompactPayloadShape } from '@floway-dev/protocols/openai-responses'; +import type { ProviderModel, UpstreamCallOptions } from '@floway-dev/provider'; + +// Hooks for repo-side state transitions. Refresh-token rotations and +// terminal-state transitions go through the repo; access-token and quota +// persistence are handled inside their own helpers, which write the same +// state_json row the same way. +export interface CodexCallEffects { + persistRefreshTokenRotation(newRefreshToken: string): Promise; + persistTerminalState(state: 'session_terminated' | 'refresh_failed', message: string): Promise; +} + +// Account selection shared by Codex backend calls. Every surface uses the same +// OAuth credential, quota state, terminal-session classification, and refresh +// retry contract; each operation owns its wire body and response decoding. +export interface CodexBackendCallBase { + upstreamId: string; + account: CodexAccountCredential; + model: Pick; + headers: Headers; + signal?: AbortSignal; + effects: CodexCallEffects; + call: UpstreamCallOptions; +} + +interface CodexResponsesContentInput extends Pick { + body: Omit; +} + +type CodexOpenAIResponsesBody = CodexResponsesBody; + +export const prepareCodexCall = async (opts: CodexBackendCallBase): Promise<{ ok: true; accessToken: CodexAccessTokenEntry } | { ok: false; response: Response }> => { + if (opts.account.state !== 'active') { + return { ok: false, response: synthetic503(`Codex upstream is ${opts.account.state}`) }; + } + + try { + const entry = await ensureCodexAccessToken(opts.upstreamId, opts.account.chatgptAccountId, refresh => mintAccessToken(opts, refresh)); + return { ok: true, accessToken: entry }; + } catch (err) { + if (err instanceof CodexOAuthSessionTerminatedError) return await codexRefreshFailed(opts, err); + // An access-only credential with nothing usable left is a configuration + // problem, not an upstream one, so it reaches the client as our 503 with + // the re-import instruction rather than as a bare failure. + if (err instanceof CodexAccessOnlyCredentialError) { + return { ok: false, response: synthetic503(err.message) }; + } + throw err; + } +}; + +const mintAccessToken = (opts: CodexBackendCallBase, refreshToken: string) => + mintCodexAccessToken(refreshToken, opts.call.fetcher, opts.effects.persistRefreshTokenRotation); + +interface CodexRequestIdentity { + installationId: string; + sessionId: string; + threadId: string; + clientRequestId: string; + turnId: string; + windowId: string; +} + +export interface CodexCompactionTurnMetadata { + trigger: 'manual' | 'auto'; + reason: 'user_requested' | 'context_limit'; + implementation: 'responses_compact' | 'responses_compaction_v2'; + phase: 'standalone_turn' | 'mid_turn'; + strategy: 'memento'; +} + +export interface CodexTurnMetadataOptions { + requestKind: 'turn' | 'compaction'; + compaction?: CodexCompactionTurnMetadata; +} + +export const CODEX_OPENAI_RESPONSES_COMPACTION_V2_TURN_METADATA: CodexTurnMetadataOptions = { + requestKind: 'compaction', + compaction: { + trigger: 'manual', + reason: 'user_requested', + implementation: 'responses_compaction_v2', + phase: 'standalone_turn', + strategy: 'memento', + }, +}; + +const trimHeader = (headers: Headers, name: string): string | null => { + const value = headers.get(name)?.trim() ?? ''; + return value.length > 0 ? value : null; +}; + +const stringField = (record: Record | null, key: string): string | null => { + if (record === null) return null; + const value = record[key]; + if (typeof value !== 'string') return null; + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : null; +}; + +const clientCodexClientMetadata = (body: unknown): Record => { + if (!isObject(body)) return {}; + const candidate = body.client_metadata; + return isObject(candidate) ? candidate : {}; +}; + +const parseClientTurnMetadataJson = (raw: string | null): Record | null => { + if (raw === null) return null; + try { + const parsed = JSON.parse(raw) as unknown; + return isObject(parsed) ? parsed : null; + } catch { + return null; + } +}; + +// Codex owns one metadata snapshot per turn and projects it onto three +// surfaces — the request headers, the body's flat `client_metadata` keys, and +// the body's `client_metadata["x-codex-turn-metadata"]` blob — with the blob +// declared canonical and the other two declared "compatibility projections of +// this snapshot, not separate sources of truth": +// https://github.com/openai/codex/blob/a16863f8704831d13e041ed7dba2c4a57a2a940b/codex-rs/core/src/responses_metadata.rs#L184-L189 +// +// Only the body surfaces are rebuilt per turn on every transport. The +// WebSocket transport writes its headers once, during the upgrade, and then +// carries many turns of differing `request_kind` and `window_id` over that one +// socket without reconnecting, so reading a header there yields the +// handshake's value for the life of the connection. Resolve the body first and +// keep the header as the fallback for callers that only speak the header +// projection. +const callerTurnMetadata = (opts: Pick, clientMetadata: Record): Record | null => + parseClientTurnMetadataJson(stringField(clientMetadata, 'x-codex-turn-metadata')) + ?? parseClientTurnMetadataJson(trimHeader(opts.headers, 'x-codex-turn-metadata')); + +// Identity-mirror keys live on `identity` and are projected onto every +// surface (headers, body's `client_metadata`, body's `x-codex-turn-metadata` +// blob). Drop them from caller spreads so a caller that supplies the same +// key on a different surface than identity already absorbed can't force the +// three projections to disagree. +const IDENTITY_MIRRORED_TURN_METADATA_KEYS = new Set([ + 'installation_id', 'session_id', 'thread_id', 'turn_id', 'window_id', +]); + +const IDENTITY_MIRRORED_CLIENT_METADATA_KEYS = new Set([ + 'x-codex-installation-id', 'session_id', 'thread_id', 'x-codex-window-id', 'turn_id', 'x-codex-turn-metadata', +]); + +const buildCodexRequestIdentity = ( + opts: Pick, + body: CodexOpenAIResponsesBody, + clientMetadata: Record, + clientTurnMetadata: Record | null, +): CodexRequestIdentity => { + // Identity priority for every mirrored id follows the same per-turn rule as + // `callerTurnMetadata`: caller body `client_metadata` key → parsed + // `x-codex-turn-metadata` key → caller-supplied header → gateway default. So + // a caller can split its identity across surfaces and we still emit + // consistent values everywhere, and a long-lived socket's frozen handshake + // headers never outrank the current turn's body. + const sessionId = stringField(clientMetadata, 'session_id') + ?? stringField(clientTurnMetadata, 'session_id') + ?? trimHeader(opts.headers, 'session-id') + ?? trimHeader(opts.headers, 'session_id') + ?? deriveSessionIdFromInput(body) + ?? uuidV7(); + const threadId = stringField(clientMetadata, 'thread_id') + ?? stringField(clientTurnMetadata, 'thread_id') + ?? trimHeader(opts.headers, 'thread-id') + ?? sessionId; + // Codex has no `client_metadata` counterpart for this one — both transports + // send it as a header carrying the thread id, which is immutable for the + // life of a connection anyway: + // https://github.com/openai/codex/blob/a16863f8704831d13e041ed7dba2c4a57a2a940b/codex-rs/codex-api/src/endpoint/responses.rs#L87-L91 + // https://github.com/openai/codex/blob/a16863f8704831d13e041ed7dba2c4a57a2a940b/codex-rs/core/src/client.rs#L1134-L1136 + const clientRequestId = trimHeader(opts.headers, 'x-client-request-id') ?? threadId; + const installationId = stringField(clientMetadata, 'x-codex-installation-id') + ?? stringField(clientTurnMetadata, 'installation_id') + ?? opts.account.openaiDeviceId; + // Codex advances the window on every auto-compaction — the id is + // `{thread_id}:{auto_compact_window_number}` — and a reused socket carries + // the advanced value in the frame body alone: + // https://github.com/openai/codex/blob/a16863f8704831d13e041ed7dba2c4a57a2a940b/codex-rs/core/src/session/mod.rs#L3684-L3689 + const windowId = stringField(clientMetadata, 'x-codex-window-id') + ?? stringField(clientTurnMetadata, 'window_id') + ?? trimHeader(opts.headers, 'x-codex-window-id') + ?? `${sessionId}:0`; + const turnId = stringField(clientMetadata, 'turn_id') + ?? stringField(clientTurnMetadata, 'turn_id') + ?? uuidV7(); + return { installationId, sessionId, threadId, clientRequestId, turnId, windowId }; +}; + +// A stateless caller that re-sends the full conversation every turn would +// otherwise mint a fresh UUIDv7 per request and never hit chatgpt.com's +// prompt cache. Hash `instructions` + every item up to and including the +// first user message so the id is stable across turns of the same +// conversation (subsequent turns append tail items after the first user +// message, so the seed shape is unchanged) and different conversations get +// different ids. Stateful callers using `previous_response_id` reach this +// code path with the input already expanded from the snapshot in +// attempt.ts, so they hash the same prefix as the original turn and get +// the same session id — no server-side session map required. +const deriveSessionIdFromInput = (body: CodexOpenAIResponsesBody): string | null => { + const seed = seedUpToFirstUserMessage(body.input); + if (seed === null) return null; + const instructions = typeof body.instructions === 'string' ? body.instructions : ''; + // U+0001 keeps the instructions and JSON seed components unambiguous in the + // hash input. + return sha256JsonUuid(seed, `${instructions}`); +}; + +const seedUpToFirstUserMessage = (input: readonly OpenAIResponsesInputItem[]): readonly OpenAIResponsesInputItem[] | null => { + const collected: OpenAIResponsesInputItem[] = []; + for (const item of input) { + collected.push(item); + if (isUserMessageItem(item)) return collected; + } + return null; +}; + +const isUserMessageItem = (item: OpenAIResponsesInputItem): boolean => + item.type === 'message' && item.role === 'user'; + +const buildCodexTurnMetadata = ( + identity: CodexRequestIdentity, + options: CodexTurnMetadataOptions, + clientOverrides: Record | null, +): Record => { + const base: Record = { + installation_id: identity.installationId, + session_id: identity.sessionId, + thread_id: identity.threadId, + turn_id: identity.turnId, + window_id: identity.windowId, + request_kind: options.requestKind, + }; + if (options.compaction !== undefined) base.compaction = options.compaction; + if (clientOverrides === null) return base; + // Identity-mirror keys already came from `identity`; only carry the + // caller's extras (turn_started_at_unix_ms, sandbox, workspaces, + // parent_thread_id, …) into the outgoing blob. + for (const [k, v] of Object.entries(clientOverrides)) { + if (!IDENTITY_MIRRORED_TURN_METADATA_KEYS.has(k)) base[k] = v; + } + return base; +}; + +// The blob rides both the body and a header. Codex keeps the unbounded tool +// inventory in the body copy only, "so HTTP and WebSocket compatibility +// headers remain bounded": +// https://github.com/openai/codex/blob/a16863f8704831d13e041ed7dba2c4a57a2a940b/codex-rs/core/src/responses_metadata.rs#L291-L300 +const HEADER_OMITTED_TURN_METADATA_KEYS = new Set(['tool_namespaces_info']); + +interface CodexTurnMetadataJson { + body: string; + header: string; +} + +const buildCodexTurnMetadataJson = ( + identity: CodexRequestIdentity, + options: CodexTurnMetadataOptions, + clientOverrides: Record | null, +): CodexTurnMetadataJson => { + const turnMetadata = buildCodexTurnMetadata(identity, options, clientOverrides); + return { + body: JSON.stringify(turnMetadata), + header: JSON.stringify(Object.fromEntries( + Object.entries(turnMetadata).filter(([key]) => !HEADER_OMITTED_TURN_METADATA_KEYS.has(key)), + )), + }; +}; + +const buildCodexClientMetadata = (identity: CodexRequestIdentity, turnMetadataJson: string): Record => ({ + 'x-codex-installation-id': identity.installationId, + session_id: identity.sessionId, + thread_id: identity.threadId, + 'x-codex-window-id': identity.windowId, + turn_id: identity.turnId, + 'x-codex-turn-metadata': turnMetadataJson, +}); + +export interface PreparedCodexResponsesRequest { + identity: CodexRequestIdentity; + turnMetadataJson: CodexTurnMetadataJson; + lite?: CodexResponsesLiteRequest; + body: Record; +} + +export const prepareCodexResponsesContent = ( + opts: CodexResponsesContentInput, + action: 'generate' | 'compact', +): Omit & { body: Record } => { + const clientMetadata = { ...clientCodexClientMetadata(opts.body) }; + delete clientMetadata[CODEX_RESPONSES_LITE_CLIENT_METADATA_KEY]; + const clientTurnMetadata = callerTurnMetadata(opts, clientMetadata); + const identity = buildCodexRequestIdentity(opts, opts.body, clientMetadata, clientTurnMetadata); + const metadata: CodexTurnMetadataOptions = action === 'compact' + ? { requestKind: 'compaction' } + : opts.body.input.some(item => item.type === 'compaction_trigger') + ? CODEX_OPENAI_RESPONSES_COMPACTION_V2_TURN_METADATA + : { requestKind: 'turn' }; + const turnMetadataJson = buildCodexTurnMetadataJson(identity, metadata, clientTurnMetadata); + const standard = { ...opts.body, client_metadata: clientMetadata }; + const lite = codexModelUsesResponsesLite(opts.model) + ? encodeCodexResponsesLiteRequest(standard, identity.threadId) + : undefined; + const wire = lite?.body ?? standard; + return { + identity, + turnMetadataJson, + lite, + body: action === 'compact' + ? buildCodexOpenAIResponsesCompactBody(wire, opts.model.id) + : buildCodexOpenAIResponsesBody(wire, opts.model.id, identity, turnMetadataJson.body), + }; +}; + +const buildCodexOpenAIResponsesBody = ( + wire: CodexResponsesBody, + modelKey: string, + identity: CodexRequestIdentity, + turnMetadataJson: string, +): Record => { + const callerExtras: Record = {}; + for (const [k, v] of Object.entries(clientCodexClientMetadata(wire))) { + if (!IDENTITY_MIRRORED_CLIENT_METADATA_KEYS.has(k)) callerExtras[k] = v; + } + const body: Record = { + ...wire, + model: modelKey, + store: false, + stream: true, + client_metadata: { + ...buildCodexClientMetadata(identity, turnMetadataJson), + ...callerExtras, + }, + }; + if (body.prompt_cache_key === undefined) body.prompt_cache_key = identity.threadId; + return body; +}; + +// Codex's private compact endpoint accepts these create-only fields in addition +// to the public compact payload. Encode before projecting so relocated tool +// and instruction carriers reach the wire. +// https://github.com/openai/codex/blob/3d2ee51ca2d5db578f328aa75e20aa22c0197c9a/codex-rs/core/src/client.rs#L651-L678 +const CODEX_COMPACT_EXTENSION_FIELDS = ['tools', 'parallel_tool_calls', 'reasoning', 'text'] as const; + +const buildCodexOpenAIResponsesCompactBody = ( + wire: CodexResponsesBody, + modelKey: string, +): Record => { + const body: Record = { ...toCompactPayloadShape(wire), model: modelKey }; + for (const field of CODEX_COMPACT_EXTENSION_FIELDS) { + const value = wire[field]; + if (value !== undefined) body[field] = value; + } + return body; +}; + +export const writeCodexQuotaObservation = ( + opts: CodexBackendCallBase, + response: Response, + isRateLimited: boolean, + policy: 'always' | 'when-present', +): Promise | null => { + const snapshot = parseCodexQuotaHeaders(response.headers, { now: new Date(), isRateLimited }); + return policy === 'when-present' && !hasCodexQuotaReading(snapshot) + ? null + : putCodexQuota(opts.upstreamId, opts.account.chatgptAccountId, snapshot); +}; + +// Recover from a 401 without deleting a sibling's newer credential: invalidate +// only the exact token that failed, reuse a winner already stored by another +// request, otherwise force a fresh coalesced mint. The resulting CAS write is +// awaited because it also resolves the latest plan observation for the retry. +// +// Every call site gates this behind `refresh_token !== null`: an access-only +// credential has nothing to re-mint from, its 401 was already classified as +// terminal in `observeCodexResponse`, and the verbatim upstream response +// is what reaches the client. +export const refreshAccessTokenForRetry = async ( + opts: CodexBackendCallBase, + failedEntry: CodexAccessTokenEntry, + fallbackPlan?: CodexPlanObservation, +): Promise<{ ok: true; accessToken: CodexAccessTokenEntry } | { ok: false; response: Response }> => { + try { + const retained = await invalidateCodexAccessToken( + opts.upstreamId, + opts.account.chatgptAccountId, + failedEntry.token, + ); + if (retained !== null) return { ok: true, accessToken: retained }; + const effective = await ensureCodexAccessToken( + opts.upstreamId, + opts.account.chatgptAccountId, + async refreshToken => { + const minted = await mintAccessToken(opts, refreshToken); + return mergeRetryPlan(minted, fallbackPlan ?? codexPlanObservation(failedEntry) ?? undefined); + }, + true, + ); + return { ok: true, accessToken: effective }; + } catch (err) { + if (err instanceof CodexOAuthSessionTerminatedError) return await codexRefreshFailed(opts, err); + throw err; + } +}; + +const mergeRetryPlan = ( + entry: CodexAccessTokenEntry, + fallback: CodexPlanObservation | undefined, +): CodexAccessTokenEntry => { + if (entry.planType !== undefined || fallback === undefined) return entry; + return { + ...entry, + planType: fallback.planType, + ...(fallback.observedAt === undefined ? {} : { planObservedAt: fallback.observedAt }), + }; +}; + +export interface CodexUpstreamError { + readonly rawText: string; + readonly body: unknown; + readonly code: string | null; + readonly message: string; +} + +export const decodeCodexUpstreamError = (rawText: string): CodexUpstreamError => { + let body: unknown; + try { + body = JSON.parse(rawText); + } catch { + return { rawText, body: rawText, code: null, message: rawText.slice(0, 256) }; + } + const obj = typeof body === 'object' && body !== null ? body as { error?: { code?: unknown; message?: unknown }; detail?: unknown } : undefined; + const code = obj?.error && typeof obj.error === 'object' && typeof obj.error.code === 'string' ? obj.error.code : null; + const message = obj?.error && typeof obj.error === 'object' && typeof obj.error.message === 'string' + ? obj.error.message + : typeof obj?.detail === 'string' ? obj.detail : rawText.slice(0, 256); + return { rawText, body, code, message }; +}; + +export const classifyCodexUnauthorizedResponse = async (opts: CodexBackendCallBase, response: Response, parsed: CodexUpstreamError): Promise => { + const { rawText, code, message } = parsed; + if (opts.account.refresh_token === null) { + return new Response(rawText, { status: 401, statusText: response.statusText, headers: response.headers }); + } + if (code === 'token_invalidated') { + await opts.effects.persistTerminalState('session_terminated', message); + return synthetic503(`Codex session terminated: ${message}`); + } + return new Response(rawText, { status: 401, statusText: response.statusText, headers: response.headers }); +}; + +const synthetic503 = (message: string): Response => new Response(JSON.stringify({ error: { type: 'codex_upstream_unavailable', message } }), { + status: 503, + headers: { 'content-type': 'application/json' }, +}); + +const codexRefreshFailed = async (opts: CodexBackendCallBase, err: CodexOAuthSessionTerminatedError): Promise<{ ok: false; response: Response }> => { + await opts.effects.persistTerminalState('refresh_failed', err.upstreamMessage); + return { ok: false, response: synthetic503(`Codex refresh failed: ${err.upstreamMessage}`) }; +}; diff --git a/packages/provider-codex/src/fetch.ts b/packages/provider-codex/src/fetch.ts deleted file mode 100644 index cd87a49328..0000000000 --- a/packages/provider-codex/src/fetch.ts +++ /dev/null @@ -1,926 +0,0 @@ -import { CodexAccessOnlyCredentialError, codexPlanObservation, ensureCodexAccessToken, invalidateCodexAccessToken, mintCodexAccessToken, type CodexPlanObservation } from './access-token.ts'; -import { isObject } from './auth/guards.ts'; -import { CodexOAuthSessionTerminatedError } from './auth/oauth.ts'; -import { - CODEX_BACKEND_BASE, - CODEX_CLI_VERSION, - CODEX_ALPHA_SEARCH_PATH, - CODEX_OPENAI_IMAGES_EDITS_PATH, - CODEX_OPENAI_IMAGES_GENERATIONS_PATH, - CODEX_ORIGINATOR, - CODEX_OPENAI_RESPONSES_COMPACT_PATH, - CODEX_OPENAI_RESPONSES_PATH, - CODEX_RESPONSES_LITE_CLIENT_METADATA_KEY, - CODEX_RESPONSES_LITE_HEADER, - CODEX_USER_AGENT, -} from './constants.ts'; -import { sha256JsonUuid, uuidV7 } from './ids.ts'; -import { codexModelUsesResponsesLite, codexPlanSupportsImages } from './models.ts'; -import { - hasCodexQuotaReading, - parseCodexQuotaHeaders, - putCodexQuota, -} from './quota.ts'; -import { - encodeCodexResponsesLiteRequest, - restoreCodexResponsesCompactionResult, - restoreCodexResponsesFrames, - type CodexResponsesBody, - type CodexResponsesLiteRequest, -} from './responses-lite.ts'; -import { restoreCodexResponsesOutput } from './responses-output.ts'; -import type { CodexAccessTokenEntry, CodexAccountCredential } from './state.ts'; -import { isEventStreamMediaType } from '@floway-dev/protocols/common'; -import type { OpenAIImagesGenerationsPayload } from '@floway-dev/protocols/openai-images'; -import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesCompactionResult, OpenAIResponsesInputItem, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import { parseOpenAIResponsesStream, toCompactPayloadShape } from '@floway-dev/protocols/openai-responses'; -import { jsonRequestBody, serializeOpenAIImagesEditsJsonPayload, type OpenAIImagesEditsRequest, type ProviderCallResult, type ProviderModel, type ProviderStreamResult, type ReplayableBody, streamingProviderCall, type UpstreamCallOptions } from '@floway-dev/provider'; - -export type ProviderCompactionResult = - | { ok: true; result: OpenAIResponsesCompactionResult; modelKey: string } - | { ok: false; response: Response; modelKey: string }; - -// Hooks for repo-side state transitions. Refresh-token rotations and -// terminal-state transitions go through the repo; access-token and quota -// persistence are handled inside their own helpers, which write the same -// state_json row the same way. -export interface CodexCallEffects { - persistRefreshTokenRotation(newRefreshToken: string): Promise; - persistTerminalState(state: 'session_terminated' | 'refresh_failed', message: string): Promise; -} - -// Account selection shared by Codex backend calls. Every surface uses the same -// OAuth credential, quota state, terminal-session classification, and refresh -// retry contract; each operation owns its wire body and response decoding. -export interface CodexBackendCallBase { - upstreamId: string; - account: CodexAccountCredential; - model: Pick; - headers: Headers; - signal?: AbortSignal; - effects: CodexCallEffects; - call: UpstreamCallOptions; -} - -export interface CallCodexOpenAIResponsesOptions extends CodexBackendCallBase { - body: Omit; -} - -export interface CallCodexOpenAIResponsesCompactOptions extends CodexBackendCallBase { - body: Omit; -} - -export interface CallCodexAlphaSearchOptions extends CodexBackendCallBase { - body: Record; -} - -export interface CallCodexOpenAIImagesGenerationsOptions extends CodexBackendCallBase { - body: Omit; - // Null when the account identity carries no plan claim; the image gate - // treats an unknown plan as allowed (fail open). - fallbackPlanType: string | undefined; -} - -export interface CallCodexOpenAIImagesEditsOptions extends CodexBackendCallBase { - request: OpenAIImagesEditsRequest; - fallbackPlanType: string | undefined; -} - -type CodexOpenAIResponsesBody = CodexResponsesBody; - -export const callCodexOpenAIResponses = async (opts: CallCodexOpenAIResponsesOptions): Promise> => { - const prepared = prepareCodexResponsesRequest(opts, 'generate'); - const ready = await prepareCodexCall(opts); - if (!ready.ok) return { ok: false, modelKey: opts.model.id, response: ready.response }; - const result = await performStreamingOpenAIResponsesCall(opts, prepared, ready.accessToken, false); - if (!result.ok || !result.headers?.has(CODEX_RESPONSES_LITE_HEADER)) return result; - const headers = new Headers(result.headers); - headers.delete(CODEX_RESPONSES_LITE_HEADER); - return { ...result, headers }; -}; - -export const callCodexOpenAIResponsesCompact = async (opts: CallCodexOpenAIResponsesCompactOptions): Promise => { - const prepared = prepareCodexResponsesRequest(opts, 'compact'); - const ready = await prepareCodexCall(opts); - if (!ready.ok) return { ok: false, modelKey: opts.model.id, response: ready.response }; - return await performUnaryCompactCall(opts, prepared, ready.accessToken, false); -}; - -export const callCodexAlphaSearch = async (opts: CallCodexAlphaSearchOptions): Promise => { - const requestId = stringField(opts.body, 'id') ?? uuidV7(); - const normalized = { ...opts, body: { ...opts.body, id: requestId } }; - const ready = await prepareCodexCall(normalized); - if (!ready.ok) return { modelKey: normalized.model.id, response: ready.response }; - return await performAlphaSearchCall(normalized, ready.accessToken, false); -}; - -const prepareCodexImageCall = async (opts: CodexBackendCallBase & { fallbackPlanType: string | undefined }): Promise<{ ok: true; accessToken: CodexAccessTokenEntry; effectivePlan: CodexPlanObservation; turnId: string } | { ok: false; response: Response }> => { - const ready = await prepareCodexCall(opts); - if (!ready.ok) return { ok: false, response: ready.response }; - // An account whose plan is unknown falls back to a fail-open sentinel: the - // upstream gate only withholds images from an explicitly `free` plan. - const effectivePlan = codexPlanObservation(ready.accessToken) - ?? (opts.fallbackPlanType === undefined ? { planType: '' } : { planType: opts.fallbackPlanType }); - if (!codexPlanSupportsImages(effectivePlan.planType)) return { ok: false, response: imageUnavailableResult(opts.model.id).response }; - const turnId = trimHeader(opts.headers, 'x-codex-image-turn-id') ?? uuidV7(); - return { ok: true, accessToken: ready.accessToken, effectivePlan, turnId }; -}; - -export const callCodexOpenAIImagesGenerations = async (opts: CallCodexOpenAIImagesGenerationsOptions): Promise => { - const prepared = await prepareCodexImageCall(opts); - if (!prepared.ok) return { modelKey: opts.model.id, response: prepared.response }; - const body = { ...opts.body, model: opts.model.id }; - const request: CodexImageCallRequest = { path: CODEX_OPENAI_IMAGES_GENERATIONS_PATH, body, turnId: prepared.turnId }; - return await performImageCall(opts, request, prepared.accessToken, prepared.effectivePlan, false); -}; - -export const callCodexOpenAIImagesEdits = async (opts: CallCodexOpenAIImagesEditsOptions): Promise => { - const prepared = await prepareCodexImageCall(opts); - if (!prepared.ok) return { modelKey: opts.model.id, response: prepared.response }; - const body = await serializeOpenAIImagesEditsJsonPayload(opts.request, opts.model.id); - const request: CodexImageCallRequest = { path: CODEX_OPENAI_IMAGES_EDITS_PATH, body, turnId: prepared.turnId }; - return await performImageCall(opts, request, prepared.accessToken, prepared.effectivePlan, false); -}; - -export const prepareCodexCall = async (opts: CodexBackendCallBase): Promise<{ ok: true; accessToken: CodexAccessTokenEntry } | { ok: false; response: Response }> => { - if (opts.account.state !== 'active') { - return { ok: false, response: synthetic503(`Codex upstream is ${opts.account.state}`) }; - } - - try { - const entry = await ensureCodexAccessToken(opts.upstreamId, opts.account.chatgptAccountId, refresh => mintAccessToken(opts, refresh)); - return { ok: true, accessToken: entry }; - } catch (err) { - if (err instanceof CodexOAuthSessionTerminatedError) return await codexRefreshFailed(opts, err); - // An access-only credential with nothing usable left is a configuration - // problem, not an upstream one, so it reaches the client as our 503 with - // the re-import instruction rather than as a bare failure. - if (err instanceof CodexAccessOnlyCredentialError) { - return { ok: false, response: synthetic503(err.message) }; - } - throw err; - } -}; - -const mintAccessToken = (opts: CodexBackendCallBase, refreshToken: string) => - mintCodexAccessToken(refreshToken, opts.call.fetcher, opts.effects.persistRefreshTokenRotation); - -interface CodexRequestIdentity { - installationId: string; - sessionId: string; - threadId: string; - clientRequestId: string; - turnId: string; - windowId: string; -} - -export interface CodexCompactionTurnMetadata { - trigger: 'manual' | 'auto'; - reason: 'user_requested' | 'context_limit'; - implementation: 'responses_compact' | 'responses_compaction_v2'; - phase: 'standalone_turn' | 'mid_turn'; - strategy: 'memento'; -} - -export interface CodexTurnMetadataOptions { - requestKind: 'turn' | 'compaction'; - compaction?: CodexCompactionTurnMetadata; -} - -export const CODEX_OPENAI_RESPONSES_COMPACTION_V2_TURN_METADATA: CodexTurnMetadataOptions = { - requestKind: 'compaction', - compaction: { - trigger: 'manual', - reason: 'user_requested', - implementation: 'responses_compaction_v2', - phase: 'standalone_turn', - strategy: 'memento', - }, -}; - -const trimHeader = (headers: Headers, name: string): string | null => { - const value = headers.get(name)?.trim() ?? ''; - return value.length > 0 ? value : null; -}; - -const stringField = (record: Record | null, key: string): string | null => { - if (record === null) return null; - const value = record[key]; - if (typeof value !== 'string') return null; - const trimmed = value.trim(); - return trimmed.length > 0 ? trimmed : null; -}; - -const clientCodexClientMetadata = (body: unknown): Record => { - if (!isObject(body)) return {}; - const candidate = body.client_metadata; - return isObject(candidate) ? candidate : {}; -}; - -const parseClientTurnMetadataJson = (raw: string | null): Record | null => { - if (raw === null) return null; - try { - const parsed = JSON.parse(raw) as unknown; - return isObject(parsed) ? parsed : null; - } catch { - return null; - } -}; - -// Codex owns one metadata snapshot per turn and projects it onto three -// surfaces — the request headers, the body's flat `client_metadata` keys, and -// the body's `client_metadata["x-codex-turn-metadata"]` blob — with the blob -// declared canonical and the other two declared "compatibility projections of -// this snapshot, not separate sources of truth": -// https://github.com/openai/codex/blob/a16863f8704831d13e041ed7dba2c4a57a2a940b/codex-rs/core/src/responses_metadata.rs#L184-L189 -// -// Only the body surfaces are rebuilt per turn on every transport. The -// WebSocket transport writes its headers once, during the upgrade, and then -// carries many turns of differing `request_kind` and `window_id` over that one -// socket without reconnecting, so reading a header there yields the -// handshake's value for the life of the connection. Resolve the body first and -// keep the header as the fallback for callers that only speak the header -// projection. -const callerTurnMetadata = (opts: CodexBackendCallBase, clientMetadata: Record): Record | null => - parseClientTurnMetadataJson(stringField(clientMetadata, 'x-codex-turn-metadata')) - ?? parseClientTurnMetadataJson(trimHeader(opts.headers, 'x-codex-turn-metadata')); - -// Identity-mirror keys live on `identity` and are projected onto every -// surface (headers, body's `client_metadata`, body's `x-codex-turn-metadata` -// blob). Drop them from caller spreads so a caller that supplies the same -// key on a different surface than identity already absorbed can't force the -// three projections to disagree. -const IDENTITY_MIRRORED_TURN_METADATA_KEYS = new Set([ - 'installation_id', 'session_id', 'thread_id', 'turn_id', 'window_id', -]); - -const IDENTITY_MIRRORED_CLIENT_METADATA_KEYS = new Set([ - 'x-codex-installation-id', 'session_id', 'thread_id', 'x-codex-window-id', 'turn_id', 'x-codex-turn-metadata', -]); - -const buildCodexRequestIdentity = ( - opts: CodexBackendCallBase, - body: CodexOpenAIResponsesBody, - clientMetadata: Record, - clientTurnMetadata: Record | null, -): CodexRequestIdentity => { - // Identity priority for every mirrored id follows the same per-turn rule as - // `callerTurnMetadata`: caller body `client_metadata` key → parsed - // `x-codex-turn-metadata` key → caller-supplied header → gateway default. So - // a caller can split its identity across surfaces and we still emit - // consistent values everywhere, and a long-lived socket's frozen handshake - // headers never outrank the current turn's body. - const sessionId = stringField(clientMetadata, 'session_id') - ?? stringField(clientTurnMetadata, 'session_id') - ?? trimHeader(opts.headers, 'session-id') - ?? trimHeader(opts.headers, 'session_id') - ?? deriveSessionIdFromInput(body) - ?? uuidV7(); - const threadId = stringField(clientMetadata, 'thread_id') - ?? stringField(clientTurnMetadata, 'thread_id') - ?? trimHeader(opts.headers, 'thread-id') - ?? sessionId; - // Codex has no `client_metadata` counterpart for this one — both transports - // send it as a header carrying the thread id, which is immutable for the - // life of a connection anyway: - // https://github.com/openai/codex/blob/a16863f8704831d13e041ed7dba2c4a57a2a940b/codex-rs/codex-api/src/endpoint/responses.rs#L87-L91 - // https://github.com/openai/codex/blob/a16863f8704831d13e041ed7dba2c4a57a2a940b/codex-rs/core/src/client.rs#L1134-L1136 - const clientRequestId = trimHeader(opts.headers, 'x-client-request-id') ?? threadId; - const installationId = stringField(clientMetadata, 'x-codex-installation-id') - ?? stringField(clientTurnMetadata, 'installation_id') - ?? opts.account.openaiDeviceId; - // Codex advances the window on every auto-compaction — the id is - // `{thread_id}:{auto_compact_window_number}` — and a reused socket carries - // the advanced value in the frame body alone: - // https://github.com/openai/codex/blob/a16863f8704831d13e041ed7dba2c4a57a2a940b/codex-rs/core/src/session/mod.rs#L3684-L3689 - const windowId = stringField(clientMetadata, 'x-codex-window-id') - ?? stringField(clientTurnMetadata, 'window_id') - ?? trimHeader(opts.headers, 'x-codex-window-id') - ?? `${sessionId}:0`; - const turnId = stringField(clientMetadata, 'turn_id') - ?? stringField(clientTurnMetadata, 'turn_id') - ?? uuidV7(); - return { installationId, sessionId, threadId, clientRequestId, turnId, windowId }; -}; - -// A stateless caller that re-sends the full conversation every turn would -// otherwise mint a fresh UUIDv7 per request and never hit chatgpt.com's -// prompt cache. Hash `instructions` + every item up to and including the -// first user message so the id is stable across turns of the same -// conversation (subsequent turns append tail items after the first user -// message, so the seed shape is unchanged) and different conversations get -// different ids. Stateful callers using `previous_response_id` reach this -// code path with the input already expanded from the snapshot in -// attempt.ts, so they hash the same prefix as the original turn and get -// the same session id — no server-side session map required. -const deriveSessionIdFromInput = (body: CodexOpenAIResponsesBody): string | null => { - const seed = seedUpToFirstUserMessage(body.input); - if (seed === null) return null; - const instructions = typeof body.instructions === 'string' ? body.instructions : ''; - // U+0001 keeps the instructions and JSON seed components unambiguous in the - // hash input. - return sha256JsonUuid(seed, `${instructions}`); -}; - -const seedUpToFirstUserMessage = (input: readonly OpenAIResponsesInputItem[]): readonly OpenAIResponsesInputItem[] | null => { - const collected: OpenAIResponsesInputItem[] = []; - for (const item of input) { - collected.push(item); - if (isUserMessageItem(item)) return collected; - } - return null; -}; - -const isUserMessageItem = (item: OpenAIResponsesInputItem): boolean => - item.type === 'message' && item.role === 'user'; - -const buildCodexTurnMetadata = ( - identity: CodexRequestIdentity, - options: CodexTurnMetadataOptions, - clientOverrides: Record | null, -): Record => { - const base: Record = { - installation_id: identity.installationId, - session_id: identity.sessionId, - thread_id: identity.threadId, - turn_id: identity.turnId, - window_id: identity.windowId, - request_kind: options.requestKind, - }; - if (options.compaction !== undefined) base.compaction = options.compaction; - if (clientOverrides === null) return base; - // Identity-mirror keys already came from `identity`; only carry the - // caller's extras (turn_started_at_unix_ms, sandbox, workspaces, - // parent_thread_id, …) into the outgoing blob. - for (const [k, v] of Object.entries(clientOverrides)) { - if (!IDENTITY_MIRRORED_TURN_METADATA_KEYS.has(k)) base[k] = v; - } - return base; -}; - -// The blob rides both the body and a header. Codex keeps the unbounded tool -// inventory in the body copy only, "so HTTP and WebSocket compatibility -// headers remain bounded": -// https://github.com/openai/codex/blob/a16863f8704831d13e041ed7dba2c4a57a2a940b/codex-rs/core/src/responses_metadata.rs#L291-L300 -const HEADER_OMITTED_TURN_METADATA_KEYS = new Set(['tool_namespaces_info']); - -interface CodexTurnMetadataJson { - body: string; - header: string; -} - -const buildCodexTurnMetadataJson = ( - identity: CodexRequestIdentity, - options: CodexTurnMetadataOptions, - clientOverrides: Record | null, -): CodexTurnMetadataJson => { - const turnMetadata = buildCodexTurnMetadata(identity, options, clientOverrides); - return { - body: JSON.stringify(turnMetadata), - header: JSON.stringify(Object.fromEntries( - Object.entries(turnMetadata).filter(([key]) => !HEADER_OMITTED_TURN_METADATA_KEYS.has(key)), - )), - }; -}; - -const buildCodexClientMetadata = (identity: CodexRequestIdentity, turnMetadataJson: string): Record => ({ - 'x-codex-installation-id': identity.installationId, - session_id: identity.sessionId, - thread_id: identity.threadId, - 'x-codex-window-id': identity.windowId, - turn_id: identity.turnId, - 'x-codex-turn-metadata': turnMetadataJson, -}); - -interface PreparedCodexResponsesRequest { - identity: CodexRequestIdentity; - turnMetadataJson: CodexTurnMetadataJson; - lite?: CodexResponsesLiteRequest; - body: ReplayableBody; -} - -const prepareCodexResponsesRequest = ( - opts: CallCodexOpenAIResponsesOptions | CallCodexOpenAIResponsesCompactOptions, - action: 'generate' | 'compact', -): PreparedCodexResponsesRequest => { - const clientMetadata = { ...clientCodexClientMetadata(opts.body) }; - delete clientMetadata[CODEX_RESPONSES_LITE_CLIENT_METADATA_KEY]; - const clientTurnMetadata = callerTurnMetadata(opts, clientMetadata); - const identity = buildCodexRequestIdentity(opts, opts.body, clientMetadata, clientTurnMetadata); - const metadata: CodexTurnMetadataOptions = action === 'compact' - ? { requestKind: 'compaction' } - : opts.body.input.some(item => item.type === 'compaction_trigger') - ? CODEX_OPENAI_RESPONSES_COMPACTION_V2_TURN_METADATA - : { requestKind: 'turn' }; - const turnMetadataJson = buildCodexTurnMetadataJson(identity, metadata, clientTurnMetadata); - const standard = { ...opts.body, client_metadata: clientMetadata }; - const lite = codexModelUsesResponsesLite(opts.model) - ? encodeCodexResponsesLiteRequest(standard, identity.threadId) - : undefined; - const wire = lite?.body ?? standard; - return { - identity, - turnMetadataJson, - lite, - body: jsonRequestBody(action === 'compact' - ? buildCodexOpenAIResponsesCompactBody(wire, opts.model.id) - : buildCodexOpenAIResponsesBody(wire, opts.model.id, identity, turnMetadataJson.body)), - }; -}; - -const buildCodexOpenAIResponsesBody = ( - wire: CodexResponsesBody, - modelKey: string, - identity: CodexRequestIdentity, - turnMetadataJson: string, -): Record => { - const callerExtras: Record = {}; - for (const [k, v] of Object.entries(clientCodexClientMetadata(wire))) { - if (!IDENTITY_MIRRORED_CLIENT_METADATA_KEYS.has(k)) callerExtras[k] = v; - } - const body: Record = { - ...wire, - model: modelKey, - store: false, - stream: true, - client_metadata: { - ...buildCodexClientMetadata(identity, turnMetadataJson), - ...callerExtras, - }, - }; - if (body.prompt_cache_key === undefined) body.prompt_cache_key = identity.threadId; - return body; -}; - -// Codex's private compact endpoint accepts these create-only fields in addition -// to the public compact payload. Encode before projecting so relocated tool -// and instruction carriers reach the wire. -// https://github.com/openai/codex/blob/3d2ee51ca2d5db578f328aa75e20aa22c0197c9a/codex-rs/core/src/client.rs#L651-L678 -const CODEX_COMPACT_EXTENSION_FIELDS = ['tools', 'parallel_tool_calls', 'reasoning', 'text'] as const; - -const buildCodexOpenAIResponsesCompactBody = ( - wire: CodexResponsesBody, - modelKey: string, -): Record => { - const body: Record = { ...toCompactPayloadShape(wire), model: modelKey }; - for (const field of CODEX_COMPACT_EXTENSION_FIELDS) { - const value = wire[field]; - if (value !== undefined) body[field] = value; - } - return body; -}; - -interface CodexHttpCallRequest { - accessToken: string; - path: string; - accept: string; - body: ReplayableBody; - identity: CodexRequestIdentity; - turnMetadataJson: string | null; - responsesLite: boolean; -} - -// One upstream round-trip with quota-header persistence and terminal-401 -// classification. The returned Response is what the caller relays: -// - 2xx: caller decodes the body (SSE for /responses, JSON for /responses/compact) -// - 429: quota is already snapshotted; return verbatim -// - 401: an access-only credential preserves the upstream response verbatim -// and flips the row terminal, because it has nothing to retry with; on a -// renewable credential `token_invalidated` maps to a synthetic 503 and any -// other 401 is rebuilt with a re-readable body so the caller can decide to -// retry with a fresh access token -// - other: returned verbatim -const postCodexJson = async ( - opts: CodexBackendCallBase, - request: { - path: string; - accessToken: string; - body: Record; - headers: Headers; - quotaPolicy: 'always' | 'when-present'; - }, -): Promise => { - const { path, accessToken, body, headers, quotaPolicy } = request; - headers.set('authorization', `Bearer ${accessToken}`); - // A null account id omits the header rather than sending an empty one: the - // upstream reads absence as "whichever account this bearer belongs to". - if (opts.account.chatgptAccountId !== null) { - headers.set('chatgpt-account-id', opts.account.chatgptAccountId); - } - const response = await opts.call.wrapUpstreamCall(() => opts.call.fetcher(`${CODEX_BACKEND_BASE}${path}`, { - method: 'POST', - headers, - body: jsonRequestBody(body), - signal: opts.signal, - })); - return await classifyCodexHttpResponse(opts, response, quotaPolicy); -}; - -const dispatchCodexHttpCall = async ( - opts: CodexBackendCallBase, - request: CodexHttpCallRequest, -): Promise => { - const { accessToken, path, accept, body, identity, turnMetadataJson, responsesLite } = request; - const headers = new Headers(); - headers.set('authorization', `Bearer ${accessToken}`); - if (opts.account.chatgptAccountId !== null) { - headers.set('chatgpt-account-id', opts.account.chatgptAccountId); - } - headers.set('originator', CODEX_ORIGINATOR); - headers.set('user-agent', CODEX_USER_AGENT); - headers.set('version', CODEX_CLI_VERSION); - headers.set('accept', accept); - headers.set('content-type', 'application/json'); - headers.set('session-id', identity.sessionId); - headers.set('thread-id', identity.threadId); - headers.set('x-client-request-id', identity.clientRequestId); - headers.set('x-codex-window-id', identity.windowId); - if (turnMetadataJson !== null) headers.set('x-codex-turn-metadata', turnMetadataJson); - if (responsesLite) headers.set(CODEX_RESPONSES_LITE_HEADER, 'true'); - - const response = await opts.call.wrapUpstreamCall(() => opts.call.fetcher(`${CODEX_BACKEND_BASE}${path}`, { - method: 'POST', - headers, - body, - signal: opts.signal, - })); - - return await classifyCodexHttpResponse(opts, response); -}; - -export const classifyCodexHttpResponse = async ( - opts: CodexBackendCallBase, - response: Response, - quotaPolicy: 'always' | 'when-present' = 'always', -): Promise => { - if (response.ok) { - persistCodexQuotaObservation(opts, response, false, quotaPolicy); - return response; - } - - if (response.status === 429) { - persistCodexQuotaObservation(opts, response, true, quotaPolicy); - return response; - } - - if (response.status === 401) { - return await classifyCodexUnauthorizedResponse(opts, response, decodeCodexUpstreamError(await response.text())); - } - - return response; -}; - -const persistCodexQuotaObservation = ( - opts: CodexBackendCallBase, - response: Response, - isRateLimited: boolean, - policy: 'always' | 'when-present', -): void => { - const snapshot = parseCodexQuotaHeaders(response.headers, { now: new Date(), isRateLimited }); - if (policy === 'when-present' && !hasCodexQuotaReading(snapshot)) return; - registerBackgroundWrite(opts, putCodexQuota(opts.upstreamId, opts.account.chatgptAccountId, snapshot)); -}; - -const dispatchCodexImageCall = async ( - opts: CodexBackendCallBase, - request: CodexImageCallDispatchRequest, -): Promise => { - const { accessToken, path, body, turnId } = request; - const headers = new Headers({ - originator: trimHeader(opts.headers, 'originator') ?? CODEX_ORIGINATOR, - 'user-agent': CODEX_USER_AGENT, - version: CODEX_CLI_VERSION, - accept: 'application/json', - 'content-type': 'application/json', - 'x-codex-image-turn-id': turnId, - }); - return await postCodexJson(opts, { path, accessToken, body, headers, quotaPolicy: 'when-present' }); -}; - -// Recover from a 401 without deleting a sibling's newer credential: invalidate -// only the exact token that failed, reuse a winner already stored by another -// request, otherwise force a fresh coalesced mint. The resulting CAS write is -// awaited because it also resolves the latest plan observation for the retry. -// -// Every call site gates this behind `refresh_token !== null`: an access-only -// credential has nothing to re-mint from, its 401 was already classified as -// terminal in `classifyCodexHttpResponse`, and the verbatim upstream response -// is what reaches the client. -export const refreshAccessTokenForRetry = async ( - opts: CodexBackendCallBase, - failedEntry: CodexAccessTokenEntry, - fallbackPlan?: CodexPlanObservation, -): Promise<{ ok: true; accessToken: CodexAccessTokenEntry } | { ok: false; response: Response }> => { - try { - const retained = await invalidateCodexAccessToken( - opts.upstreamId, - opts.account.chatgptAccountId, - failedEntry.token, - ); - if (retained !== null) return { ok: true, accessToken: retained }; - const effective = await ensureCodexAccessToken( - opts.upstreamId, - opts.account.chatgptAccountId, - async refreshToken => { - const minted = await mintAccessToken(opts, refreshToken); - return mergeRetryPlan(minted, fallbackPlan ?? codexPlanObservation(failedEntry) ?? undefined); - }, - true, - ); - return { ok: true, accessToken: effective }; - } catch (err) { - if (err instanceof CodexOAuthSessionTerminatedError) return await codexRefreshFailed(opts, err); - throw err; - } -}; - -// The 401-retry gate's per-call inputs. The dispatchers bundle their access -// token, the response they saw, the recursion switch, and the retry / failure -// callbacks in one object rather than threading six positional arguments. -interface RetryCodexAccess401Options { - accessToken: CodexAccessTokenEntry; - response: Response | null; - alreadyRetried: boolean; - run: (fresh: CodexAccessTokenEntry) => Promise; - onRefreshFailure: (response: Response) => T; - fallbackPlan?: CodexPlanObservation; -} - -// The 401 retry gate every call dispatcher shares: on a renewable credential -// that has not already been retried, rotate the failed access token and re-run -// the operation once; if the refresh fails, relay the dispatcher's own failure -// result. The gate — a 401 on a credential with a refresh token, not already -// retried — is folded in here, so callers pass the response they saw and only -// branch on the retried outcome. An access-only credential's 401 is already -// classified terminal in `classifyCodexHttpResponse`, and the verbatim upstream -// response reaches the client. -const retryCodexAccess401 = async ( - opts: CodexBackendCallBase, - options: RetryCodexAccess401Options, -): Promise<{ retried: true; value: T } | { retried: false }> => { - const { accessToken, response, alreadyRetried, run, onRefreshFailure, fallbackPlan } = options; - if (response?.status !== 401 || opts.account.refresh_token === null || alreadyRetried) return { retried: false }; - const fresh = await refreshAccessTokenForRetry(opts, accessToken, fallbackPlan); - if (!fresh.ok) return { retried: true, value: onRefreshFailure(fresh.response) }; - return { retried: true, value: await run(fresh.accessToken) }; -}; - -const mergeRetryPlan = ( - entry: CodexAccessTokenEntry, - fallback: CodexPlanObservation | undefined, -): CodexAccessTokenEntry => { - if (entry.planType !== undefined || fallback === undefined) return entry; - return { - ...entry, - planType: fallback.planType, - ...(fallback.observedAt === undefined ? {} : { planObservedAt: fallback.observedAt }), - }; -}; - -const performStreamingOpenAIResponsesCall = async ( - opts: CallCodexOpenAIResponsesOptions, - prepared: PreparedCodexResponsesRequest, - accessToken: CodexAccessTokenEntry, - alreadyRetried: boolean, -): Promise> => { - const upstreamFetch = dispatchCodexHttpCall(opts, { - accessToken: accessToken.token, - path: CODEX_OPENAI_RESPONSES_PATH, - accept: 'text/event-stream', - body: prepared.body, - identity: prepared.identity, - turnMetadataJson: prepared.turnMetadataJson.header, - responsesLite: prepared.lite !== undefined, - }).then(ensureSseContentType); - - const lite = prepared.lite; - const result = await streamingProviderCall( - upstreamFetch, - (stream, parserOpts) => { - const frames = parseOpenAIResponsesStream(stream, parserOpts); - return restoreCodexResponsesOutput(lite === undefined - ? frames - : restoreCodexResponsesFrames(frames, lite.callableIdentities, lite.requestEchoes)); - }, - opts.model.id, - opts.signal, - ); - - const attempt = await retryCodexAccess401( - opts, - { - accessToken, - response: result.ok ? null : result.response, - alreadyRetried, - run: fresh => performStreamingOpenAIResponsesCall(opts, prepared, fresh, true), - onRefreshFailure: resp => ({ ok: false as const, modelKey: opts.model.id, response: resp }), - }, - ); - if (attempt.retried) return attempt.value; - - return result; -}; - -const performUnaryCompactCall = async ( - opts: CallCodexOpenAIResponsesCompactOptions, - prepared: PreparedCodexResponsesRequest, - accessToken: CodexAccessTokenEntry, - alreadyRetried: boolean, -): Promise => { - const response = await dispatchCodexHttpCall(opts, { - accessToken: accessToken.token, - path: CODEX_OPENAI_RESPONSES_COMPACT_PATH, - accept: 'application/json', - body: prepared.body, - identity: prepared.identity, - turnMetadataJson: prepared.turnMetadataJson.header, - responsesLite: prepared.lite !== undefined, - }); - - const attempt = await retryCodexAccess401( - opts, - { - accessToken, - response, - alreadyRetried, - run: fresh => performUnaryCompactCall(opts, prepared, fresh, true), - onRefreshFailure: resp => ({ ok: false as const, modelKey: opts.model.id, response: resp }), - }, - ); - if (attempt.retried) return attempt.value; - - if (!response.ok) return { ok: false, modelKey: opts.model.id, response }; - - const result = await response.json() as OpenAIResponsesCompactionResult; - return { - ok: true, - modelKey: opts.model.id, - result: prepared.lite === undefined - ? result - : restoreCodexResponsesCompactionResult(result, prepared.lite.callableIdentities, prepared.lite.generatedPrefix), - }; -}; - -const performAlphaSearchCall = async ( - opts: CallCodexAlphaSearchOptions, - accessToken: CodexAccessTokenEntry, - alreadyRetried: boolean, -): Promise => { - const requestId = stringField(opts.body, 'id'); - if (requestId === null) throw new Error('Normalized Codex alpha search request is missing id'); - const identity: CodexRequestIdentity = { - installationId: opts.account.openaiDeviceId, - sessionId: requestId, - threadId: requestId, - clientRequestId: requestId, - turnId: uuidV7(), - windowId: `${requestId}:0`, - }; - const turnMetadataJson = trimHeader(opts.headers, 'x-codex-turn-metadata'); - const response = await dispatchCodexHttpCall(opts, { - accessToken: accessToken.token, - path: CODEX_ALPHA_SEARCH_PATH, - accept: 'application/json', - body: jsonRequestBody({ ...opts.body, model: opts.model.id }), - identity, - turnMetadataJson, - responsesLite: false, - }); - - const attempt = await retryCodexAccess401( - opts, - { - accessToken, - response, - alreadyRetried, - run: fresh => performAlphaSearchCall(opts, fresh, true), - onRefreshFailure: resp => ({ modelKey: opts.model.id, response: resp }), - }, - ); - if (attempt.retried) return attempt.value; - return { modelKey: opts.model.id, response }; -}; - -// The fixed per-request inputs for one Codex image call. The path, body, and -// turn id are stable for the whole call, so the entry sites bundle them once -// and retries re-pass the same bundle rather than threading three positional -// arguments on every recursion. -interface CodexImageCallRequest { - path: string; - body: Record; - turnId: string; -} - -// The image-call bundle plus the resolved bearer, mirroring CodexHttpCallRequest -// for the HTTP call path. The entry sites bundle the stable path/body/turnId -// once; retries re-pass the same bundle with a fresh access token rather than -// threading four positional arguments on every recursion. -type CodexImageCallDispatchRequest = CodexImageCallRequest & { accessToken: string }; - -const performImageCall = async ( - opts: CodexBackendCallBase, - request: CodexImageCallRequest, - accessToken: CodexAccessTokenEntry, - effectivePlan: CodexPlanObservation, - alreadyRetried: boolean, -): Promise => { - const response = await dispatchCodexImageCall(opts, { accessToken: accessToken.token, ...request }); - const attempt = await retryCodexAccess401( - opts, - { - accessToken, - response, - alreadyRetried, - run: async entry => { - const refreshedPlan = codexPlanObservation(entry) ?? effectivePlan; - if (!codexPlanSupportsImages(refreshedPlan.planType)) return imageUnavailableResult(opts.model.id); - return await performImageCall(opts, request, entry, refreshedPlan, true); - }, - onRefreshFailure: resp => ({ modelKey: opts.model.id, response: resp }), - fallbackPlan: effectivePlan, - }, - ); - if (attempt.retried) return attempt.value; - return { modelKey: opts.model.id, response }; -}; - -export interface CodexUpstreamError { - readonly rawText: string; - readonly body: unknown; - readonly code: string | null; - readonly message: string; -} - -export const decodeCodexUpstreamError = (rawText: string): CodexUpstreamError => { - let body: unknown; - try { - body = JSON.parse(rawText); - } catch { - return { rawText, body: rawText, code: null, message: rawText.slice(0, 256) }; - } - const obj = typeof body === 'object' && body !== null ? body as { error?: { code?: unknown; message?: unknown }; detail?: unknown } : undefined; - const code = obj?.error && typeof obj.error === 'object' && typeof obj.error.code === 'string' ? obj.error.code : null; - const message = obj?.error && typeof obj.error === 'object' && typeof obj.error.message === 'string' - ? obj.error.message - : typeof obj?.detail === 'string' ? obj.detail : rawText.slice(0, 256); - return { rawText, body, code, message }; -}; - -export const classifyCodexUnauthorizedResponse = async (opts: CodexBackendCallBase, response: Response, parsed: CodexUpstreamError): Promise => { - const { rawText, code, message } = parsed; - if (opts.account.refresh_token === null) { - // Access-only credentials retain the authoritative upstream reply even when - // the terminal-state write fails. - try { - await opts.effects.persistTerminalState('session_terminated', message); - } catch { - // The upstream response remains authoritative. - } - return new Response(rawText, { status: 401, statusText: response.statusText, headers: response.headers }); - } - if (code === 'token_invalidated') { - await opts.effects.persistTerminalState('session_terminated', message); - return synthetic503(`Codex session terminated: ${message}`); - } - return new Response(rawText, { status: 401, statusText: response.statusText, headers: response.headers }); -}; - -const imageUnavailableResult = (modelKey: string): ProviderCallResult => ({ - modelKey, - response: new Response(JSON.stringify({ - error: { - type: 'image_tools_unavailable', - message: 'ChatGPT Free accounts do not provide Codex image tools.', - }, - }), { status: 403, headers: { 'content-type': 'application/json' } }), -}); - -const synthetic503 = (message: string): Response => new Response(JSON.stringify({ error: { type: 'codex_upstream_unavailable', message } }), { - status: 503, - headers: { 'content-type': 'application/json' }, -}); - -const codexRefreshFailed = async (opts: CodexBackendCallBase, err: CodexOAuthSessionTerminatedError): Promise<{ ok: false; response: Response }> => { - await opts.effects.persistTerminalState('refresh_failed', err.upstreamMessage); - return { ok: false, response: synthetic503(`Codex refresh failed: ${err.upstreamMessage}`) }; -}; - -// Codex backend serves SSE without setting `content-type: text/event-stream` -// (observed in production: only x-codex-* + standard CDN headers come back). -// The shared `streamingProviderCall` rejects 2xx responses lacking the SSE -// content-type as a contract violation, so we synthesize the header on the -// way through. Body stream is preserved verbatim. -const ensureSseContentType = (response: Response): Response => { - if (!response.ok || isEventStreamMediaType(response.headers.get('content-type'))) return response; - const headers = new Headers(response.headers); - headers.set('content-type', 'text/event-stream'); - return new Response(response.body, { status: response.status, statusText: response.statusText, headers }); -}; - -// Hand best-effort writes to waitUntil so workerd does not cancel them when -// the streaming response returns; the swallow guards against recoverable -// noise (transient storage errors, a state_json write that lost every one of -// its retries) tripping the request. -const registerBackgroundWrite = (opts: CodexBackendCallBase, write: Promise): void => { - opts.call.waitUntil(write.catch(() => {})); -}; diff --git a/packages/provider-codex/src/interceptors/openai-responses/index.ts b/packages/provider-codex/src/interceptors/openai-responses/index.ts deleted file mode 100644 index 873e279d15..0000000000 --- a/packages/provider-codex/src/interceptors/openai-responses/index.ts +++ /dev/null @@ -1,21 +0,0 @@ -// Codex-only OpenAI Responses workarounds. The chain is a boundary the Codex provider -// runs inside its own call methods, so the gateway main flow never knows that -// Codex has OpenAI Responses interceptors at all. - -import { injectDefaultInstructions } from './inject-default-instructions.ts'; -import { stripUnsupportedFields } from './strip-unsupported-fields.ts'; -import type { OpenAIResponsesBoundaryCtx } from './types.ts'; -import type { Interceptor } from '@floway-dev/interceptor'; -import type { ProviderOpenAIResponsesResult } from '@floway-dev/provider'; - -// Order rationale: neither interceptor below reads or writes a field the -// other touches, so order is positional only. -// -// Codex interceptors are pure payload/header mutators, so the chain's only -// terminal — the streaming `generate` + non-streaming `compact` dispatch — -// returns its `ProviderOpenAIResponsesResult` directly without any per-frame -// lift/lower step. -export const CODEX_OPENAI_RESPONSES_BOUNDARY: readonly Interceptor[] = [ - injectDefaultInstructions, - stripUnsupportedFields, -]; diff --git a/packages/provider-codex/src/interceptors/openai-responses/inject-default-instructions.ts b/packages/provider-codex/src/interceptors/openai-responses/inject-default-instructions.ts deleted file mode 100644 index eaa53fe37c..0000000000 --- a/packages/provider-codex/src/interceptors/openai-responses/inject-default-instructions.ts +++ /dev/null @@ -1,17 +0,0 @@ -import type { OpenAIResponsesBoundaryCtx } from './types.ts'; - -// ChatGPT-subscription catalog models reject missing or empty `instructions`. -// Native and translated callers may omit the field, so the provider supplies a -// neutral value at its boundary. Other values remain upstream-owned validation. -// https://github.com/im4codes/imcodes/blob/5f769d933dfd679e3a4d670183b0384a1baf62cd/src/agent/providers/codex-sdk.ts#L560-L579 -export const injectDefaultInstructions = async ( - ctx: OpenAIResponsesBoundaryCtx, - _env: object, - run: () => Promise, -): Promise => { - const instructions = ctx.payload.instructions; - if (instructions === undefined || instructions === null || instructions === '') { - ctx.payload = { ...ctx.payload, instructions: "You're a helpful assistant." }; - } - return await run(); -}; diff --git a/packages/provider-codex/src/interceptors/openai-responses/strip-unsupported-fields.ts b/packages/provider-codex/src/interceptors/openai-responses/strip-unsupported-fields.ts deleted file mode 100644 index a6d8e863bb..0000000000 --- a/packages/provider-codex/src/interceptors/openai-responses/strip-unsupported-fields.ts +++ /dev/null @@ -1,32 +0,0 @@ -import type { OpenAIResponsesBoundaryCtx } from './types.ts'; - -// Codex backend rejects requests carrying any of these fields with a -// `Unsupported parameter: ` 4xx. They are regular OpenAI Responses API -// fields that the ChatGPT-subscription path does not honor. Source-protocol -// translators legitimately set max_output_tokens / temperature / top_p (the -// caller's request might carry them), so we strip them at the Codex target -// boundary rather than at translation time, where they remain valid for -// other providers. -const CODEX_UNSUPPORTED_BODY_FIELDS = [ - 'max_output_tokens', - 'temperature', - 'top_p', - 'frequency_penalty', - 'presence_penalty', - 'user', - 'metadata', - 'prompt_cache_retention', - 'safety_identifier', - 'stream_options', -] as const; - -export const stripUnsupportedFields = async ( - ctx: OpenAIResponsesBoundaryCtx, - _env: object, - run: () => Promise, -): Promise => { - const next: Record = { ...(ctx.payload as unknown as Record) }; - for (const key of CODEX_UNSUPPORTED_BODY_FIELDS) delete next[key]; - ctx.payload = next as unknown as typeof ctx.payload; - return await run(); -}; diff --git a/packages/provider-codex/src/interceptors/openai-responses/types.ts b/packages/provider-codex/src/interceptors/openai-responses/types.ts deleted file mode 100644 index a68b1771b7..0000000000 --- a/packages/provider-codex/src/interceptors/openai-responses/types.ts +++ /dev/null @@ -1,17 +0,0 @@ -import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; -import type { ProviderModel, OpenAIResponsesAction } from '@floway-dev/provider'; - -// Boundary ctx for Codex OpenAI Responses interceptors. The same ctx feeds both the -// streaming `/responses` (action='generate') and the non-streaming compaction -// (action='compact') chains; the terminal switches on `action` to pick the -// wire shape (see provider.ts callOpenAIResponses). -export interface OpenAIResponsesBoundaryCtx { - payload: CanonicalOpenAIResponsesPayload; - headers: Headers; - readonly model: ProviderModel; - // Mirrors the gateway-side OpenAIResponsesInvocation.action. Interceptors MAY - // mutate it during the chain to re-route dispatch in the terminal - // handler — the terminal reads `ctx.action`, not the parameter the - // provider was originally called with. - action: OpenAIResponsesAction; -} diff --git a/packages/provider-codex/src/models.ts b/packages/provider-codex/src/models.ts index d844249fe8..0d7ebebdf4 100644 --- a/packages/provider-codex/src/models.ts +++ b/packages/provider-codex/src/models.ts @@ -163,7 +163,7 @@ export const codexModelUsesResponsesLite = (model: Pick): ProviderModel => { const contextWindow = assertContextWindow(raw.context_window, raw.id, 'context_window'); const maxContextWindow = raw.max_context_window === undefined diff --git a/packages/provider-codex/src/pipeline-facts.ts b/packages/provider-codex/src/pipeline-facts.ts index 7edaa35088..b21860a49c 100644 --- a/packages/provider-codex/src/pipeline-facts.ts +++ b/packages/provider-codex/src/pipeline-facts.ts @@ -1,11 +1,11 @@ import type { CodexPlanObservation } from './access-token.ts'; -import type { CodexCallEffects, CodexBackendCallBase } from './fetch.ts'; +import type { CodexCallEffects, CodexBackendCallBase } from './backend.ts'; import type { CodexAccountCredential, CodexAccessTokenEntry } from './state.ts'; import type { HttpRequestFacts } from '@floway-dev/http/pipeline'; import { secret, type Secret } from '@floway-dev/pipeline'; import type { ProviderModelFacts, ProviderOperationPayloads, ProviderRequest, ProviderServices } from '@floway-dev/provider'; -export type CodexOperation = 'alphaSearch' | 'openaiImagesGenerations' | 'openaiImagesEdits'; +export type CodexOperation = 'alphaSearch' | 'openaiImagesGenerations' | 'openaiImagesEdits' | 'openaiResponses' | 'openaiResponsesCompact'; export type CodexRequest = ProviderRequest; export type CodexToken = Omit & { readonly token: Secret }; export type CodexAccount = Omit & { @@ -25,6 +25,7 @@ export interface CodexHttpFacts extends HttpRequestFacts { 'request.codex.accessToken': CodexToken; 'request.codex.plan': CodexPlanObservation | null; 'request.codex.modelKey': string; + 'request.provider.modelKey': string; } export interface CodexPipelineConfig { diff --git a/packages/provider-codex/src/pipelines.ts b/packages/provider-codex/src/pipelines.ts index 330a78d19f..08cf17a15f 100644 --- a/packages/provider-codex/src/pipelines.ts +++ b/packages/provider-codex/src/pipelines.ts @@ -1,16 +1,25 @@ import type { CodexPipelineConfig, CodexRequest } from './pipeline-facts.ts'; import { authenticateCodex } from './stages/authenticate.ts'; import { authorizeCodexHeaders } from './stages/authorize-headers.ts'; +import { injectCodexDefaultInstructions } from './stages/inject-default-instructions.ts'; +import { normalizeCodexResponsesHeaders } from './stages/normalize-responses-headers.ts'; import { observeCodexResponse } from './stages/observe-response.ts'; import { prepareCodexRequest } from './stages/prepare-request.ts'; +import { prepareCodexResponsesPayload } from './stages/prepare-responses-content.ts'; +import { prepareCodexResponsesHttp } from './stages/prepare-responses.ts'; import { readCodexAccount } from './stages/read-account.ts'; +import { restoreCodexResponsesLite } from './stages/restore-responses-lite.ts'; +import { restoreCodexResponsesOutput } from './stages/restore-responses-output.ts'; import { retryCodexAccess } from './stages/retry-access.ts'; +import { stripCodexUnsupportedFields } from './stages/strip-unsupported-response-fields.ts'; import { http } from '@floway-dev/http/pipeline'; import { compose } from '@floway-dev/pipeline'; -import type { ProviderPipelines, ProviderResponse } from '@floway-dev/provider'; +import { decodeProviderResponse, observeProviderCall, observeProviderResponsesCall, selectProviderResponsesAction, type ProviderChatResponse, type ProviderPipelines, type ProviderResponse } from '@floway-dev/provider'; export const createCodexPipelines = (config: CodexPipelineConfig): ProviderPipelines => ({ - alphaSearch: compose, ProviderResponse>('codex.alphaSearch', [readCodexAccount<'alphaSearch'>(config), authenticateCodex(config, 'alphaSearch'), prepareCodexRequest('alphaSearch'), retryCodexAccess(config, 'alphaSearch'), authorizeCodexHeaders, observeCodexResponse(config, 'alphaSearch'), http]), - openaiImagesGenerations: compose, ProviderResponse>('codex.openaiImagesGenerations', [readCodexAccount<'openaiImagesGenerations'>(config), authenticateCodex(config, 'openaiImagesGenerations'), prepareCodexRequest('openaiImagesGenerations'), retryCodexAccess(config, 'openaiImagesGenerations'), authorizeCodexHeaders, observeCodexResponse(config, 'openaiImagesGenerations'), http]), - openaiImagesEdits: compose, ProviderResponse>('codex.openaiImagesEdits', [readCodexAccount<'openaiImagesEdits'>(config), authenticateCodex(config, 'openaiImagesEdits'), prepareCodexRequest('openaiImagesEdits'), retryCodexAccess(config, 'openaiImagesEdits'), authorizeCodexHeaders, observeCodexResponse(config, 'openaiImagesEdits'), http]), + openaiResponses: compose, ProviderChatResponse<'openaiResponses'>>('codex.openaiResponses', [selectProviderResponsesAction('generate'), readCodexAccount<'openaiResponses'>(config), injectCodexDefaultInstructions, stripCodexUnsupportedFields, prepareCodexResponsesPayload<'openaiResponses'>(), authenticateCodex(config, 'openaiResponses'), prepareCodexResponsesHttp<'openaiResponses'>(), restoreCodexResponsesOutput, restoreCodexResponsesLite, decodeProviderResponse('openaiResponses'), retryCodexAccess(config, 'openaiResponses'), authorizeCodexHeaders, normalizeCodexResponsesHeaders, observeCodexResponse(config, 'openaiResponses'), observeProviderResponsesCall, http]), + openaiResponsesCompact: compose, ProviderChatResponse<'openaiResponsesCompact'>>('codex.openaiResponsesCompact', [selectProviderResponsesAction('compact'), readCodexAccount<'openaiResponsesCompact'>(config), injectCodexDefaultInstructions, stripCodexUnsupportedFields, prepareCodexResponsesPayload<'openaiResponsesCompact'>(), authenticateCodex(config, 'openaiResponsesCompact'), prepareCodexResponsesHttp<'openaiResponsesCompact'>(), restoreCodexResponsesLite, decodeProviderResponse('openaiResponsesCompact'), retryCodexAccess(config, 'openaiResponsesCompact'), authorizeCodexHeaders, observeCodexResponse(config, 'openaiResponsesCompact'), observeProviderResponsesCall, http]), + alphaSearch: compose, ProviderResponse>('codex.alphaSearch', [readCodexAccount<'alphaSearch'>(config), authenticateCodex(config, 'alphaSearch'), prepareCodexRequest('alphaSearch'), retryCodexAccess(config, 'alphaSearch'), authorizeCodexHeaders, observeCodexResponse(config, 'alphaSearch'), observeProviderCall, http]), + openaiImagesGenerations: compose, ProviderResponse>('codex.openaiImagesGenerations', [readCodexAccount<'openaiImagesGenerations'>(config), authenticateCodex(config, 'openaiImagesGenerations'), prepareCodexRequest('openaiImagesGenerations'), retryCodexAccess(config, 'openaiImagesGenerations'), authorizeCodexHeaders, observeCodexResponse(config, 'openaiImagesGenerations'), observeProviderCall, http]), + openaiImagesEdits: compose, ProviderResponse>('codex.openaiImagesEdits', [readCodexAccount<'openaiImagesEdits'>(config), authenticateCodex(config, 'openaiImagesEdits'), prepareCodexRequest('openaiImagesEdits'), retryCodexAccess(config, 'openaiImagesEdits'), authorizeCodexHeaders, observeCodexResponse(config, 'openaiImagesEdits'), observeProviderCall, http]), }); diff --git a/packages/provider-codex/src/provider.ts b/packages/provider-codex/src/provider.ts index d48a409afa..73fb81e9d4 100644 --- a/packages/provider-codex/src/provider.ts +++ b/packages/provider-codex/src/provider.ts @@ -1,15 +1,12 @@ import { ensureCodexAccessToken, mintCodexAccessToken } from './access-token.ts'; import { CodexOAuthSessionTerminatedError } from './auth/oauth.ts'; +import type { CodexCallEffects } from './backend.ts'; import { assertCodexUpstreamRecord, type CodexUpstreamConfig } from './config.ts'; import { CODEX_DEFAULT_FLAGS } from './defaults.ts'; -import { callCodexAlphaSearch, callCodexOpenAIImagesEdits, callCodexOpenAIImagesGenerations, callCodexOpenAIResponses, callCodexOpenAIResponsesCompact, type CodexCallEffects } from './fetch.ts'; -import { CODEX_OPENAI_RESPONSES_BOUNDARY } from './interceptors/openai-responses/index.ts'; -import type { OpenAIResponsesBoundaryCtx } from './interceptors/openai-responses/types.ts'; import { codexImageProviderModel, codexPlanSupportsImages, codexRawToProviderModel, fetchCodexCatalog } from './models.ts'; import { createCodexPipelines } from './pipelines.ts'; import { assertCodexUpstreamState, findCodexAccountIndex, persistCodexRefreshTokenRotation, persistCodexTerminalState } from './state.ts'; -import { runInterceptors } from '@floway-dev/interceptor'; -import { getProviderRepo, resolveEffectiveFlags, type ProviderInstance, type Provider, type ProviderCallResult, type ProviderOpenAIResponsesResult, type ProviderStreamResult, type UpstreamRecord } from '@floway-dev/provider'; +import { getProviderRepo, resolveEffectiveFlags, type ProviderInstance, type Provider, type UpstreamRecord } from '@floway-dev/provider'; // https://github.com/openai/codex/blob/c607da9f371bb66a41cc772c6ddf1989d28137d3/codex-rs/codex-api/src/requests/headers.rs#L5-L12 // https://github.com/openai/codex/blob/c607da9f371bb66a41cc772c6ddf1989d28137d3/codex-rs/codex-api/src/endpoint/responses.rs#L87-L96 @@ -38,7 +35,7 @@ export const createCodexProvider = (record: UpstreamRecord): Provider => { // Computed once per provider instance: only the upstream layer applies // (no per-model override layer). Threaded into every ProviderModel emitted - // by getProvidedModels so interceptors can read the effective flag set + // by getProvidedModels so stages can read the effective flag set // without re-resolving. const enabledFlags = resolveEffectiveFlags([CODEX_DEFAULT_FLAGS, record.flagOverrides]); @@ -100,66 +97,6 @@ export const createCodexProvider = (record: UpstreamRecord): Provider => { if (codexPlanSupportsImages(access.planType ?? accountIdentity.planType ?? undefined)) models.push(codexImageProviderModel(enabledFlags)); return models; }, - - callAlphaSearch: async (model, body, signal, opts) => { - const { account } = await readActiveAccount(); - return await callCodexAlphaSearch({ - upstreamId: record.id, - account, - model, - headers: new Headers(opts.headers), - signal, - effects, - call: opts, - body, - }); - }, - - callOpenAIResponses: async (model, body, action, signal, opts) => { - const ctx: OpenAIResponsesBoundaryCtx = { - payload: { ...body, model: model.id }, - headers: new Headers(opts.headers), - model, - action, - }; - return await runInterceptors( - ctx, {}, CODEX_OPENAI_RESPONSES_BOUNDARY, async () => { - const { account } = await readActiveAccount(); - const { model: _ignored, ...wireBody } = ctx.payload; - const backendCallBase = { upstreamId: record.id, account, model, headers: ctx.headers, signal, effects, call: opts }; - switch (ctx.action) { - case 'compact': - // The fetch boundary selects the private wire format before compact - // projection so interceptor-provided tools and instructions survive. - return { action: 'compact', ...(await callCodexOpenAIResponsesCompact({ ...backendCallBase, body: wireBody })) }; - case 'generate': - return { action: 'generate', ...(await callCodexOpenAIResponses({ ...backendCallBase, body: wireBody })) }; - default: - ctx.action satisfies never; - throw new Error(`Unhandled OpenAIResponsesAction: ${ctx.action as string}`); - } - }, - ); - }, - - // Codex exposes OpenAI Responses and its provider-owned image endpoints. The - // remaining surfaces are unreachable through the advertised catalog, but - // a stray dispatch must still surface as a structured 405. - callAnthropicMessages: () => unsupportedStreamResult(), - callAnthropicMessagesCountTokens: () => unsupportedCallResult(), - callOpenAICompletions: () => unsupportedCallResult(), - callOpenAIChatCompletions: () => unsupportedStreamResult(), - callOpenAIEmbeddings: () => unsupportedCallResult(), - callOpenAIImagesGenerations: async (model, body, signal, opts) => { - const { account } = await readActiveAccount(); - return await callCodexOpenAIImagesGenerations({ upstreamId: record.id, account, model, headers: opts.headers, signal, effects, call: opts, body, fallbackPlanType: accountIdentity.planType ?? undefined }); - }, - callOpenAIImagesEdits: async (model, request, signal, opts) => { - const { account } = await readActiveAccount(); - return await callCodexOpenAIImagesEdits({ upstreamId: record.id, account, model, headers: opts.headers, signal, effects, call: opts, request, fallbackPlanType: accountIdentity.planType ?? undefined }); - }, - callOpenAIAudioTranscriptions: () => unsupportedCallResult(), - callRerank: () => Promise.reject(new Error('Codex provider does not support callRerank')), }; return { @@ -174,14 +111,3 @@ export const createCodexProvider = (record: UpstreamRecord): Provider => { instance, }; }; - -const synthetic405 = (): Response => new Response( - JSON.stringify({ error: { type: 'method_not_allowed', message: 'Endpoint not supported by codex provider' } }), - { status: 405, headers: { 'content-type': 'application/json' } }, -); - -const unsupportedStreamResult = (): Promise> => - Promise.resolve({ ok: false, modelKey: '', response: synthetic405() }); - -const unsupportedCallResult = (): Promise => - Promise.resolve({ modelKey: '', response: synthetic405() }); diff --git a/packages/provider-codex/src/stages/authenticate.ts b/packages/provider-codex/src/stages/authenticate.ts index 0adb1dbe20..c47fdb6db3 100644 --- a/packages/provider-codex/src/stages/authenticate.ts +++ b/packages/provider-codex/src/stages/authenticate.ts @@ -1,29 +1,29 @@ import { codexPlanObservation } from '../access-token.ts'; -import { prepareCodexCall } from '../fetch.ts'; +import { prepareCodexCall } from '../backend.ts'; import { codexPlanSupportsImages } from '../models.ts'; import { codexCall, tokenFacts, type CodexAccountFacts, type CodexAuthenticatedFacts, type CodexOperation, type CodexPipelineConfig } from '../pipeline-facts.ts'; import { takeHttpResponse } from '@floway-dev/http/pipeline'; import { defineStage, move, type Handed } from '@floway-dev/pipeline'; import type { ProviderResponse, ProviderServices } from '@floway-dev/provider'; -export const authenticateCodex = (config: CodexPipelineConfig, operation: O) => defineStage, CodexAuthenticatedFacts, ProviderResponse, ProviderResponse, ProviderResponse, ProviderServices>({ +export const authenticateCodex = (config: CodexPipelineConfig, operation: O) => defineStage, CodexAuthenticatedFacts, ProviderResponse, ProviderResponse, ProviderResponse & { 'response.provider.output': null }, ProviderServices>({ name: 'authenticateCodex', through: { request: { needs: ['request.codex.account', 'request.provider.model', 'request.http.callId'], consumes: [], provides: ['request.codex.accessToken', 'request.codex.plan'] }, response: { needs: [], consumes: [], provides: [] }, }, - return: { provides: ['response.http.exchange', 'response.http.body', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'] }, + return: { provides: ['response.http.exchange', 'response.http.body', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.provider.output'] }, execute: async (facts, next, use) => { const model = facts['request.provider.model']; const ready = await prepareCodexCall(codexCall(config, facts, use, model)); const failed = (response: Response) => { const exchange = takeHttpResponse(response); - return move({ ...facts, 'response.http.exchange': exchange, 'response.http.body': exchange.body, 'response.provider.modelKey': model.id, 'response.provider.called': false, 'response.provider.previousCalls': [] }); + return move({ ...facts, 'response.http.exchange': exchange, 'response.http.body': exchange.body, 'response.provider.modelKey': model.id, 'response.provider.called': false, 'response.provider.previousCalls': [], 'response.provider.output': null }); }; if (!ready.ok) return failed(ready.response); const observedPlan = codexPlanObservation(ready.accessToken); const plan = observedPlan ?? (config.fallbackPlanType === undefined ? null : { planType: config.fallbackPlanType }); - if (operation !== 'alphaSearch' && plan !== null && !codexPlanSupportsImages(plan.planType)) { + if ((operation === 'openaiImagesGenerations' || operation === 'openaiImagesEdits') && plan !== null && !codexPlanSupportsImages(plan.planType)) { return failed(Response.json({ error: { type: 'image_tools_unavailable', message: 'ChatGPT Free accounts do not provide Codex image tools.' } }, { status: 403 })); } return await next(move({ ...facts, 'request.codex.accessToken': tokenFacts(ready.accessToken), 'request.codex.plan': plan })) as Handed; diff --git a/packages/provider-codex/src/stages/inject-default-instructions.ts b/packages/provider-codex/src/stages/inject-default-instructions.ts new file mode 100644 index 0000000000..b80c415c6e --- /dev/null +++ b/packages/provider-codex/src/stages/inject-default-instructions.ts @@ -0,0 +1,28 @@ +import type { CodexAccountFacts } from '../pipeline-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; +import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; + +type Request = CodexAccountFacts<'openaiResponses' | 'openaiResponsesCompact'>; + +// ChatGPT-subscription catalog models reject missing or empty `instructions`. +// Native and translated callers may omit the field, so the provider supplies a +// neutral value at its boundary. Other values remain upstream-owned validation. +// https://github.com/im4codes/imcodes/blob/5f769d933dfd679e3a4d670183b0384a1baf62cd/src/agent/providers/codex-sdk.ts#L560-L579 +export const defaultInstructionsPayload =

>(payload: P): P => { + const instructions = payload.instructions; + return instructions === undefined || instructions === null || instructions === '' + ? { ...payload, instructions: "You're a helpful assistant." } + : payload; +}; + +export const injectCodexDefaultInstructions = defineStage, ProviderChatResponse<'openaiResponses'>, ProviderChatServices>({ + name: 'injectCodexDefaultInstructions', + through: { + request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => move({ + ...await next(move({ ...facts, 'request.provider.payload': defaultInstructionsPayload(facts['request.provider.payload']) })), + }), +}); diff --git a/packages/provider-codex/src/stages/normalize-responses-headers.ts b/packages/provider-codex/src/stages/normalize-responses-headers.ts new file mode 100644 index 0000000000..f901c40fe4 --- /dev/null +++ b/packages/provider-codex/src/stages/normalize-responses-headers.ts @@ -0,0 +1,28 @@ +import { CODEX_RESPONSES_LITE_HEADER } from '../constants.ts'; +import type { CodexResponsesHttpFacts } from './prepare-responses.ts'; +import type { HttpResponseFacts } from '@floway-dev/http/pipeline'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { isEventStreamMediaType } from '@floway-dev/protocols/common'; + +// The subscription endpoint can return valid SSE without the media-type header. +// Preserve the body lease and normalize the metadata before protocol decoding. +export const normalizeCodexResponsesHeaders = defineStage({ + name: 'normalizeCodexResponsesHeaders', + through: { + request: { needs: ['request.provider.responsesAction'], consumes: [], provides: [] }, + response: { needs: ['response.http.exchange', 'response.provider.responsesAction'], consumes: ['response.http.exchange'], provides: ['response.http.exchange'] }, + }, + execute: async (facts, next) => { + const back = await next(move({ ...facts })); + const exchange = back['response.http.exchange']; + if (back['response.provider.responsesAction'] === 'compact' || exchange.type !== 'response' || exchange.status < 200 || exchange.status >= 300) return move({ ...back }); + const contentType = exchange.headers.find(([name]) => name.toLowerCase() === 'content-type')?.[1]; + const headers = exchange.headers.filter(([name]) => name.toLowerCase() !== CODEX_RESPONSES_LITE_HEADER); + if (!isEventStreamMediaType(contentType)) { + const normalized = headers.filter(([name]) => name.toLowerCase() !== 'content-type'); + normalized.push(['content-type', 'text/event-stream']); + return move({ ...back, 'response.http.exchange': { ...exchange, headers: normalized } }); + } + return move({ ...back, 'response.http.exchange': { ...exchange, headers } }); + }, +}); diff --git a/packages/provider-codex/src/stages/observe-response.ts b/packages/provider-codex/src/stages/observe-response.ts index 72078ff26a..799d3a6201 100644 --- a/packages/provider-codex/src/stages/observe-response.ts +++ b/packages/provider-codex/src/stages/observe-response.ts @@ -1,4 +1,4 @@ -import { classifyCodexHttpResponse, classifyCodexUnauthorizedResponse, decodeCodexUpstreamError } from '../fetch.ts'; +import { classifyCodexUnauthorizedResponse, decodeCodexUpstreamError, writeCodexQuotaObservation } from '../backend.ts'; import { codexCall, type CodexHttpFacts, type CodexOperation, type CodexPipelineConfig } from '../pipeline-facts.ts'; import { exchangeResponse, takeHttpResponse, type HttpResponseFacts } from '@floway-dev/http/pipeline'; import { defer, defineStage, move, setRelease, type Deferred } from '@floway-dev/pipeline'; @@ -22,12 +22,18 @@ export const observeCodexResponse = (config: CodexPipelineConfig, operation: Cod let classified: Response; let failureBody: unknown = null; if (exchange.status === 401) { - const parsed = decodeCodexUpstreamError(await response.text()); if (exchange.body !== null) setRelease(exchange.body, async () => {}); + const parsed = decodeCodexUpstreamError(await response.text()); failureBody = parsed.body; + if (call.account.refresh_token === null) pending.push(call.effects.persistTerminalState('session_terminated', parsed.message)); classified = await classifyCodexUnauthorizedResponse(call, response, parsed); } else { - classified = await classifyCodexHttpResponse({ ...call, call: { ...call.call, waitUntil: work => { pending.push(work); } } }, response, operation === 'alphaSearch' ? 'always' : 'when-present'); + classified = response; + if (response.ok || response.status === 429) { + const policy = operation === 'alphaSearch' || operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? 'always' : 'when-present'; + const work = writeCodexQuotaObservation(call, response, response.status === 429, policy); + if (work !== null) pending.push(work); + } } const retained = classified === response ? exchange : takeHttpResponse(classified); return move({ ...back, 'response.http.exchange': retained, 'response.http.body': retained.body, 'response.codex.background': defer(Promise.all(pending)), 'response.codex.failureBody': failureBody }); diff --git a/packages/provider-codex/src/stages/prepare-request.ts b/packages/provider-codex/src/stages/prepare-request.ts index 993fb4c0b6..9518900c29 100644 --- a/packages/provider-codex/src/stages/prepare-request.ts +++ b/packages/provider-codex/src/stages/prepare-request.ts @@ -16,7 +16,7 @@ export const prepareCodexRequest = (operation: O) => d request: { needs: ['request.provider.model', 'request.provider.payload', 'request.http.headers', 'request.http.callId', 'request.codex.account', 'request.codex.accessToken', 'request.codex.plan'], consumes: ['request.provider.model', 'request.provider.payload', 'request.http.headers'], - provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.codex.modelKey', 'request.codex.model'], + provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.codex.modelKey', 'request.codex.model', 'request.provider.modelKey'], }, response: { needs: ['response.http.exchange', 'response.provider.called', 'response.provider.previousCalls'], consumes: [], provides: ['response.provider.modelKey'] }, }, @@ -50,6 +50,7 @@ export const prepareCodexRequest = (operation: O) => d ...rest, 'request.codex.model': model, 'request.codex.modelKey': model.id, + 'request.provider.modelKey': model.id, 'request.http.url': `${CODEX_BACKEND_BASE}${path}`, 'request.http.method': 'POST', 'request.http.headers': headers, diff --git a/packages/provider-codex/src/stages/prepare-responses-content.ts b/packages/provider-codex/src/stages/prepare-responses-content.ts new file mode 100644 index 0000000000..9bcc4d18fe --- /dev/null +++ b/packages/provider-codex/src/stages/prepare-responses-content.ts @@ -0,0 +1,15 @@ +import { prepareCodexResponsesContent, type PreparedCodexResponsesRequest } from '../backend.ts'; +import { accountValue, type CodexAccountFacts } from '../pipeline-facts.ts'; +import { defineStage, isSecret, move } from '@floway-dev/pipeline'; +import type { ProviderChatResponse } from '@floway-dev/provider'; + +export interface CodexPreparedResponsesFacts { 'request.codex.responsesPrepared': PreparedCodexResponsesRequest } +export const prepareCodexResponsesPayload = () => defineStage & { 'request.provider.responsesAction': 'generate' | 'compact' }, CodexAccountFacts & CodexPreparedResponsesFacts, ProviderChatResponse, ProviderChatResponse>({ + name: 'prepareCodexResponsesPayload', + through: { request: { needs: ['request.provider.model', 'request.provider.payload', 'request.http.headers', 'request.codex.account', 'request.provider.responsesAction'], consumes: [], provides: ['request.codex.responsesPrepared'] }, response: { needs: [], consumes: [], provides: [] } }, + execute: async (facts, next) => { + const headers = new Headers(facts['request.http.headers'].map(([name, value]): [string, string] => [name, isSecret(value) ? value.reveal() : value])); + const prepared = prepareCodexResponsesContent({ account: accountValue(facts['request.codex.account']), model: facts['request.provider.model'], body: facts['request.provider.payload'], headers }, facts['request.provider.responsesAction']); + return move({ ...await next(move({ ...facts, 'request.codex.responsesPrepared': prepared })) }); + }, +}); diff --git a/packages/provider-codex/src/stages/prepare-responses.ts b/packages/provider-codex/src/stages/prepare-responses.ts new file mode 100644 index 0000000000..5875841c08 --- /dev/null +++ b/packages/provider-codex/src/stages/prepare-responses.ts @@ -0,0 +1,43 @@ +import type { CodexPreparedResponsesFacts } from './prepare-responses-content.ts'; +import { CODEX_BACKEND_BASE, CODEX_CLI_VERSION, CODEX_OPENAI_RESPONSES_COMPACT_PATH, CODEX_OPENAI_RESPONSES_PATH, CODEX_ORIGINATOR, CODEX_RESPONSES_LITE_HEADER, CODEX_USER_AGENT } from '../constants.ts'; +import { type CodexAuthenticatedFacts, type CodexHttpFacts } from '../pipeline-facts.ts'; +import type { CodexResponsesLiteRequest } from '../responses-lite.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; + +export interface CodexResponsesHttpFacts extends CodexHttpFacts { + 'request.provider.responsesAction': 'generate' | 'compact'; + 'request.codex.responsesLite': CodexResponsesLiteRequest | null; +} + +export const prepareCodexResponsesHttp = () => defineStage & Pick & CodexPreparedResponsesFacts, CodexResponsesHttpFacts, ProviderChatResponse, ProviderChatResponse, ProviderChatServices>({ + name: 'prepareCodexResponsesHttp', + through: { + request: { + needs: ['request.provider.model', 'request.provider.payload', 'request.http.headers', 'request.http.callId', 'request.codex.account', 'request.codex.accessToken', 'request.codex.plan', 'request.provider.responsesAction', 'request.codex.responsesPrepared'], + consumes: ['request.provider.model', 'request.provider.payload', 'request.http.headers', 'request.codex.responsesPrepared'], + provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.codex.model', 'request.codex.modelKey', 'request.provider.modelKey', 'request.provider.responsesAction', 'request.codex.responsesLite'], + }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const { 'request.provider.model': model, 'request.provider.payload': _body, 'request.codex.responsesPrepared': prepared, ...rest } = facts; + const action = facts['request.provider.responsesAction']; + const { identity } = prepared; + const headers: [string, string][] = [ + ['originator', CODEX_ORIGINATOR], ['user-agent', CODEX_USER_AGENT], ['version', CODEX_CLI_VERSION], + ['accept', action === 'compact' ? 'application/json' : 'text/event-stream'], ['content-type', 'application/json'], + ['session-id', identity.sessionId], ['thread-id', identity.threadId], ['x-client-request-id', identity.clientRequestId], + ['x-codex-window-id', identity.windowId], ['x-codex-turn-metadata', prepared.turnMetadataJson.header], + ]; + if (prepared.lite !== undefined) headers.push([CODEX_RESPONSES_LITE_HEADER, 'true']); + return move({ + ...await next(move({ + ...rest, 'request.codex.model': model, 'request.codex.modelKey': model.id, 'request.provider.modelKey': model.id, + 'request.provider.responsesAction': action, 'request.codex.responsesLite': prepared.lite ?? null, + 'request.http.url': `${CODEX_BACKEND_BASE}${action === 'compact' ? CODEX_OPENAI_RESPONSES_COMPACT_PATH : CODEX_OPENAI_RESPONSES_PATH}`, + 'request.http.method': 'POST', 'request.http.headers': headers, 'request.http.body': prepared.body, 'request.http.encoding': 'json' as const, + })), + }); + }, +}); diff --git a/packages/provider-codex/src/stages/restore-responses-lite.ts b/packages/provider-codex/src/stages/restore-responses-lite.ts new file mode 100644 index 0000000000..df00963c45 --- /dev/null +++ b/packages/provider-codex/src/stages/restore-responses-lite.ts @@ -0,0 +1,24 @@ +import { restoreCodexResponsesCompactionResult, restoreCodexResponsesFrames } from '../responses-lite.ts'; +import type { CodexResponsesHttpFacts } from './prepare-responses.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { OpenAIResponsesCompactionResult } from '@floway-dev/protocols/openai-responses'; +import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; + +export const restoreCodexResponsesLite = defineStage, ProviderChatResponse<'openaiResponses'>, ProviderChatServices>({ + name: 'restoreCodexResponsesLite', + through: { + request: { needs: ['request.codex.responsesLite'], consumes: [], provides: [] }, + response: { needs: ['response.provider.output'], consumes: ['response.provider.output'], provides: ['response.provider.output'] }, + }, + execute: async (facts, next, use) => { + const back = await next(move({ ...facts })); + const lite = facts['request.codex.responsesLite']; + const output = back['response.provider.output']; + if (lite === null || output === null || !('kind' in output)) return move({ ...back }); + return move({ + ...back, 'response.provider.output': output.kind === 'value' + ? { kind: 'value' as const, body: restoreCodexResponsesCompactionResult(output.body as OpenAIResponsesCompactionResult, lite.callableIdentities, lite.generatedPrefix) } + : { kind: 'stream' as const, frames: use.recordProtocolFrames(restoreCodexResponsesFrames(output.frames, lite.callableIdentities, lite.requestEchoes)) }, + }); + }, +}); diff --git a/packages/provider-codex/src/stages/restore-responses-output.ts b/packages/provider-codex/src/stages/restore-responses-output.ts new file mode 100644 index 0000000000..f4391918a2 --- /dev/null +++ b/packages/provider-codex/src/stages/restore-responses-output.ts @@ -0,0 +1,19 @@ +import { restoreCodexResponsesOutput as restoreFrames } from '../responses-output.ts'; +import type { CodexResponsesHttpFacts } from './prepare-responses.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; + +export const restoreCodexResponsesOutput = defineStage, ProviderChatResponse<'openaiResponses'>, ProviderChatServices>({ + name: 'restoreCodexResponsesOutput', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { needs: ['response.provider.output'], consumes: ['response.provider.output'], provides: ['response.provider.output'] }, + }, + execute: async (facts, next, use) => { + const back = await next(move({ ...facts })); + const output = back['response.provider.output']; + return output !== null && 'kind' in output && output.kind === 'stream' + ? move({ ...back, 'response.provider.output': { kind: 'stream' as const, frames: use.recordProtocolFrames(restoreFrames(output.frames)) } }) + : move({ ...back }); + }, +}); diff --git a/packages/provider-codex/src/stages/retry-access.ts b/packages/provider-codex/src/stages/retry-access.ts index da73e6a0fc..137e34f642 100644 --- a/packages/provider-codex/src/stages/retry-access.ts +++ b/packages/provider-codex/src/stages/retry-access.ts @@ -1,16 +1,16 @@ import { codexPlanObservation } from '../access-token.ts'; -import { refreshAccessTokenForRetry } from '../fetch.ts'; +import { refreshAccessTokenForRetry } from '../backend.ts'; import { codexPlanSupportsImages } from '../models.ts'; import { codexCall, tokenFacts, type CodexHttpFacts, type CodexOperation, type CodexPipelineConfig } from '../pipeline-facts.ts'; import { takeHttpResponse, type HttpResponseFacts } from '@floway-dev/http/pipeline'; import { defineStage, move } from '@floway-dev/pipeline'; import type { ProviderCallResponse, ProviderServices } from '@floway-dev/provider'; -export const retryCodexAccess = (config: CodexPipelineConfig, operation: CodexOperation) => defineStage({ +export const retryCodexAccess = (config: CodexPipelineConfig, operation: CodexOperation) => defineStage({ name: 'retryCodexAccess', through: { request: { needs: ['request.codex.account', 'request.codex.accessToken', 'request.codex.plan', 'request.codex.model', 'request.codex.modelKey', 'request.http.callId'], consumes: ['request.codex.accessToken', 'request.codex.plan'], provides: ['request.codex.accessToken', 'request.codex.plan'] }, - response: { needs: ['response.http.exchange', 'response.http.body'], consumes: ['response.http.exchange', 'response.http.body'], provides: ['response.http.exchange', 'response.http.body', 'response.provider.called', 'response.provider.previousCalls'] }, + response: { needs: ['response.http.exchange', 'response.http.body'], consumes: ['response.http.exchange', 'response.http.body'], provides: ['response.http.exchange', 'response.http.body', 'response.provider.called', 'response.provider.previousCalls', ...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? ['response.provider.responsesAction' as const] : [])] }, }, execute: async (facts, next, use) => { const first = await next(move({ ...facts })); @@ -21,12 +21,12 @@ export const retryCodexAccess = (config: CodexPipelineConfig, operation: CodexOp const fresh = await refreshAccessTokenForRetry(codexCall(config, facts, use, facts['request.codex.model']), { ...failedToken, token: failedToken.token.reveal() }, facts['request.codex.plan'] ?? undefined); const declined = (response: Response) => { const nextExchange = takeHttpResponse(response); - return move({ ...first, 'response.http.exchange': nextExchange, 'response.http.body': nextExchange.body, 'response.provider.called': false, 'response.provider.previousCalls': [{ modelKey: facts['request.codex.modelKey'] }] }); + return move({ ...first, 'response.http.exchange': nextExchange, 'response.http.body': nextExchange.body, 'response.provider.called': false, 'response.provider.previousCalls': [{ modelKey: facts['request.codex.modelKey'] }], ...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? { 'response.provider.responsesAction': facts['request.provider.responsesAction']! } : {}) }); }; if (!fresh.ok) return declined(fresh.response); const plan = codexPlanObservation(fresh.accessToken) ?? facts['request.codex.plan']; - if (operation !== 'alphaSearch' && plan !== null && !codexPlanSupportsImages(plan.planType)) return declined(Response.json({ error: { type: 'image_tools_unavailable', message: 'ChatGPT Free accounts do not provide Codex image tools.' } }, { status: 403 })); + if ((operation === 'openaiImagesGenerations' || operation === 'openaiImagesEdits') && plan !== null && !codexPlanSupportsImages(plan.planType)) return declined(Response.json({ error: { type: 'image_tools_unavailable', message: 'ChatGPT Free accounts do not provide Codex image tools.' } }, { status: 403 })); const retried = await next(move({ ...facts, 'request.codex.accessToken': tokenFacts(fresh.accessToken), 'request.codex.plan': plan })); - return move({ ...retried, 'response.provider.called': retried['response.http.exchange'].type === 'response', 'response.provider.previousCalls': [{ modelKey: facts['request.codex.modelKey'] }] }); + return move({ ...retried, 'response.provider.called': retried['response.http.exchange'].type === 'response', 'response.provider.previousCalls': [{ modelKey: facts['request.codex.modelKey'] }], ...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? { 'response.provider.responsesAction': facts['request.provider.responsesAction']! } : {}) }); }, }); diff --git a/packages/provider-codex/src/stages/strip-unsupported-response-fields.ts b/packages/provider-codex/src/stages/strip-unsupported-response-fields.ts new file mode 100644 index 0000000000..95bb65ee80 --- /dev/null +++ b/packages/provider-codex/src/stages/strip-unsupported-response-fields.ts @@ -0,0 +1,40 @@ +import type { CodexAccountFacts } from '../pipeline-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; +import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; + +type Request = CodexAccountFacts<'openaiResponses' | 'openaiResponsesCompact'>; + +// Codex backend rejects requests carrying any of these fields with a +// `Unsupported parameter: ` 4xx. They are regular OpenAI Responses API +// fields that the ChatGPT-subscription path does not honor. Source-protocol +// translators legitimately set max_output_tokens / temperature / top_p (the +// caller's request might carry them), so we strip them at the Codex target +// boundary rather than at translation time, where they remain valid for +// other providers. +const CODEX_UNSUPPORTED_BODY_FIELDS = [ + 'max_output_tokens', + 'temperature', + 'top_p', + 'frequency_penalty', + 'presence_penalty', + 'user', + 'metadata', + 'prompt_cache_retention', + 'safety_identifier', + 'stream_options', +] as const; + +export const supportedResponsesPayload =

>(payload: P): P => + Object.fromEntries(Object.entries(payload).filter(([key]) => !(CODEX_UNSUPPORTED_BODY_FIELDS as readonly string[]).includes(key))) as P; + +export const stripCodexUnsupportedFields = defineStage, ProviderChatResponse<'openaiResponses'>, ProviderChatServices>({ + name: 'stripCodexUnsupportedFields', + through: { + request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => move({ + ...await next(move({ ...facts, 'request.provider.payload': supportedResponsesPayload(facts['request.provider.payload']) })), + }), +}); diff --git a/packages/provider-copilot/__tests__/auth_test.ts b/packages/provider-copilot/__tests__/auth_test.ts index f0fce4bf6e..8ae3d02f1d 100644 --- a/packages/provider-copilot/__tests__/auth_test.ts +++ b/packages/provider-copilot/__tests__/auth_test.ts @@ -251,7 +251,7 @@ describe('Copilot token exchange retries', () => { }); }); -test('copilotAuthedFetch overlays interceptor headers on the pinned base set', async () => { +test('copilotAuthedFetch overlays request headers on the pinned base set', async () => { await mockTokenAndCapture([['x-initiator', 'agent'], ['copilot-vision-request', 'true']], headers => { assertEquals(headers.get('x-initiator'), 'agent'); assertEquals(headers.get('copilot-vision-request'), 'true'); @@ -261,7 +261,7 @@ test('copilotAuthedFetch overlays interceptor headers on the pinned base set', a }); }); -test('copilotAuthedFetch deletes a base header when the interceptor passes an empty-string value', async () => { +test('copilotAuthedFetch deletes a base header when the caller passes an empty-string value', async () => { // Sentinel contract: empty string means drop this base header from the pinned set. await mockTokenAndCapture([['copilot-integration-id', '']], headers => { assertEquals(headers.has('copilot-integration-id'), false); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/detect-claude-code-metadata_test.ts b/packages/provider-copilot/__tests__/claude-code-metadata_test.ts similarity index 94% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/detect-claude-code-metadata_test.ts rename to packages/provider-copilot/__tests__/claude-code-metadata_test.ts index 5cc0be8dd1..856ff19f48 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/detect-claude-code-metadata_test.ts +++ b/packages/provider-copilot/__tests__/claude-code-metadata_test.ts @@ -1,6 +1,6 @@ import { test } from 'vitest'; -import { parseUserIdMetadata } from '../../../src/interceptors/anthropic-messages/detect-claude-code-metadata.ts'; +import { parseUserIdMetadata } from '../src/claude-code-metadata.ts'; import { assertEquals } from '@floway-dev/test-utils'; test('parseUserIdMetadata returns nulls for undefined input', () => { diff --git a/packages/provider-copilot/__tests__/interceptors/image-size_test.ts b/packages/provider-copilot/__tests__/image-size_test.ts similarity index 93% rename from packages/provider-copilot/__tests__/interceptors/image-size_test.ts rename to packages/provider-copilot/__tests__/image-size_test.ts index 28276fa1f7..4f1e9f19e6 100644 --- a/packages/provider-copilot/__tests__/interceptors/image-size_test.ts +++ b/packages/provider-copilot/__tests__/image-size_test.ts @@ -1,6 +1,6 @@ import { test } from 'vitest'; -import { targetSizeForOpenAIResponsesChat } from '../../src/interceptors/image-size.ts'; +import { targetSizeForOpenAIResponsesChat } from '../src/image-size.ts'; import { assertEquals } from '@floway-dev/test-utils'; test('gpt-4o / gpt-4.1 use the 2048 box + 768 short edge', () => { diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/boundary-chain_test.ts b/packages/provider-copilot/__tests__/interceptors/anthropic-messages/boundary-chain_test.ts deleted file mode 100644 index dc3742bb29..0000000000 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/boundary-chain_test.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { test } from 'vitest'; - -import { CLAUDE_AGENT_USER_AGENT } from '../../../src/auth.ts'; -import { COPILOT_ANTHROPIC_MESSAGES_BOUNDARY } from '../../../src/interceptors/anthropic-messages/index.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; -import { runInterceptors } from '@floway-dev/interceptor'; -import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; -import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; - -const stubRequest = {}; - -const okEvents = (): Promise>> => - Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); - -const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesBoundaryCtx => ({ - payload, - headers: new Headers(), - anthropicBeta: [], - model: stubProviderModel({ endpoints: { anthropicMessages: {} } }), -}); - -const COMPACT_LAST_MESSAGE_TEXT = - 'Your task is to create a detailed summary of the conversation so far.\n\n' + - 'CRITICAL: Respond with TEXT ONLY. Do NOT call any tools.\n\n' + - 'Pending Tasks:\n- finish refactor\n\nCurrent Work:\n- reviewing diff'; - -test('Claude Code SDK compact request: Claude-agent overrides compact intent, both halves of metadata threaded through', async () => { - // This is the realistic ordering case: a Claude Code compact summary call - // ALSO carries the Claude Code SDK fingerprint. We expect the final wire - // headers to be `messages-proxy` (Claude-agent wins over compact's - // `conversation-compaction`), the user-agent and integration-id deletion - // from Claude-agent, and an `x-interaction-id` from the interaction-id - // interceptor — matching what VSCode Copilot Chat sends for the same call. - const ctx = invocation({ - model: 'claude-test', - max_tokens: 10, - metadata: { user_id: JSON.stringify({ device_id: 'dev-1', session_id: 'sess-1' }) }, - messages: [{ role: 'user', content: COMPACT_LAST_MESSAGE_TEXT }], - }); - - await runInterceptors>>( - ctx, - stubRequest, - COPILOT_ANTHROPIC_MESSAGES_BOUNDARY, - okEvents, - ); - - // Compact set `x-initiator: agent` early; `withInitiatorHeaderSet` runs - // later in the merged boundary chain and re-derives x-initiator from the - // last-message structure, so the final wire value reflects the wire-shape - // pass. (That is the same value the pre-merge production code shipped, - // because the target chain always overrode the source-side tag.) - assertEquals(ctx.headers.get('x-initiator'), 'user'); - // Compact set `conversation-compaction`; Claude-agent's `messages-proxy` - // runs after and overrides it. This mirrors caozhiyuan/copilot-api's - // prepareForCompact → prepareMessageProxyHeaders order. - assertEquals(ctx.headers.get('x-interaction-type'), 'messages-proxy'); - assertEquals(ctx.headers.get('openai-intent'), 'messages-proxy'); - assertEquals(ctx.headers.get('user-agent'), CLAUDE_AGENT_USER_AGENT); - // Empty-string sentinel: `copilotAuthedFetch` in - // `packages/provider-copilot/src/auth.ts` deletes the base header. - assertEquals(ctx.headers.get('copilot-integration-id'), ''); - // SHA-256-then-UUIDv4 of 'sess-1' (matches caozhiyuan's getUUID). - assertEquals(ctx.headers.get('x-interaction-id'), 'abe633f3-a47a-4758-974e-abe9160daf36'); -}); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/item-id-carrier_test.ts b/packages/provider-copilot/__tests__/opaque-item-id_test.ts similarity index 95% rename from packages/provider-copilot/__tests__/interceptors/openai-responses/item-id-carrier_test.ts rename to packages/provider-copilot/__tests__/opaque-item-id_test.ts index 006d304875..c48c475951 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/item-id-carrier_test.ts +++ b/packages/provider-copilot/__tests__/opaque-item-id_test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { unwrapCopilotItemId, wrapCopilotItemId } from '../../../src/interceptors/openai-responses/item-id-carrier.ts'; +import { unwrapCopilotItemId, wrapCopilotItemId } from '../src/opaque-item-id.ts'; describe('Copilot item id carrier', () => { test.each([ diff --git a/packages/provider-copilot/__tests__/provider_test.ts b/packages/provider-copilot/__tests__/provider_test.ts index 616a9a6000..477079b0ba 100644 --- a/packages/provider-copilot/__tests__/provider_test.ts +++ b/packages/provider-copilot/__tests__/provider_test.ts @@ -10,7 +10,7 @@ import { createInMemoryImageProcessor, initImageProcessor } from '@floway-dev/pl import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; import type { UpstreamRecord } from '@floway-dev/provider'; import { directFetcher, initProviderRepo } from '@floway-dev/provider'; -import { callProviderPipeline, assertEquals, assertRejects, assertThrows, jsonResponse, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, sseResponse, withMockedFetch } from '@floway-dev/test-utils'; +import { collectChatProviderPipeline, callProviderPipeline, assertEquals, assertRejects, assertThrows, jsonResponse, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, sseResponse, withMockedFetch } from '@floway-dev/test-utils'; const mergeVariantsControl = vi.hoisted<{ override: ((merged: CopilotRawModel[]) => CopilotRawModel[]) | null; @@ -273,7 +273,7 @@ test('Copilot provider owns the claude-* Anthropic Messages capability workaroun assertEquals(providerModel.id, 'claude-haiku-chat-listed'); assertEquals(providerModel.endpoints, { anthropicMessages: {} }); - await provider.callAnthropicMessages(providerModel, { + await collectChatProviderPipeline(instance, 'anthropicMessages', providerModel, { max_tokens: 100, messages: [{ role: 'user', content: 'hello' }], }, undefined, noopAnthropicMessagesUpstreamCallOptions()); @@ -329,7 +329,7 @@ test('Copilot provider selects raw variants that support the target endpoint', a }, async () => { const [providerModel] = await provider.getProvidedModels(directFetcher); - await provider.callOpenAIResponses(providerModel, { + await collectChatProviderPipeline(instance, 'openaiResponses', providerModel, { input: [{ type: 'additional_tools', role: 'developer', @@ -341,7 +341,7 @@ test('Copilot provider selects raw variants that support the target endpoint', a }], }], reasoning: { effort: 'xhigh' }, - }, 'generate', undefined, noopUpstreamCallOptions()); + }, undefined, noopUpstreamCallOptions()); }, ); @@ -400,7 +400,7 @@ test('Copilot provider runs the OpenAI Responses boundary chain on the compact p // service_tier is set so withServiceTierStripped has something to strip; // an input_image is included so withVisionHeaderSet fires; the last // input item is a user message so withInitiatorHeaderSet picks 'user'. - const result = await provider.callOpenAIResponses(providerModel, { + const result = await collectChatProviderPipeline(instance, 'openaiResponsesCompact', providerModel, { input: [ { type: 'additional_tools', @@ -422,11 +422,11 @@ test('Copilot provider runs the OpenAI Responses boundary chain on the compact p }, ], service_tier: 'priority', - }, 'compact', undefined, noopUpstreamCallOptions()); + }, undefined, noopUpstreamCallOptions()); - if (!result.ok) throw new Error('expected ok compaction result'); - if (result.action !== 'compact') throw new Error(`expected compact action tag, got ${result.action}`); - assertEquals(result.result.object, 'response.compaction'); + if (!(result.output !== null && 'kind' in result.output)) throw new Error('expected ok compaction result'); + if (result.output === null || !('kind' in result.output) || result.output.kind !== 'value') throw new Error('expected compact value'); + assertEquals(result.output.body.object, 'response.compaction'); }, ); @@ -590,10 +590,10 @@ test('Copilot provider forces stream=true for streaming endpoints and leaves cou const opts = noopUpstreamCallOptions(); const anthropicMessagesOpts = noopAnthropicMessagesUpstreamCallOptions(); - await provider.callOpenAIChatCompletions(byId.get('gpt-chat')!, { messages: [{ role: 'user', content: 'hi' }] }, undefined, opts); - await provider.callOpenAIResponses(byId.get('gpt-resp')!, { input: [] }, 'generate', undefined, opts); - await provider.callAnthropicMessages(byId.get('claude-msg')!, { max_tokens: 10, messages: [{ role: 'user', content: 'hi' }] }, undefined, anthropicMessagesOpts); - await provider.callAnthropicMessagesCountTokens(byId.get('claude-msg')!, { max_tokens: 10, messages: [{ role: 'user', content: 'hi' }] }, undefined, anthropicMessagesOpts); + await collectChatProviderPipeline(instance, 'openaiChatCompletions', byId.get('gpt-chat')!, { messages: [{ role: 'user', content: 'hi' }] }, undefined, opts); + await collectChatProviderPipeline(instance, 'openaiResponses', byId.get('gpt-resp')!, { input: [] }, undefined, opts); + await collectChatProviderPipeline(instance, 'anthropicMessages', byId.get('claude-msg')!, { max_tokens: 10, messages: [{ role: 'user', content: 'hi' }] }, undefined, anthropicMessagesOpts); + await collectChatProviderPipeline(instance, 'anthropicMessagesCountTokens', byId.get('claude-msg')!, { max_tokens: 10, messages: [{ role: 'user', content: 'hi' }] }, undefined, anthropicMessagesOpts); await callProviderPipeline(instance, 'openaiEmbeddings', byId.get('emb-mini')!, { input: 'hi' }, undefined, opts); }, ); @@ -611,10 +611,8 @@ test('Copilot provider sets copilot-vision-request when an image is nested insid const provider = instance.instance; const visionHeaders: (string | null)[] = []; - // The vision-detection interceptor runs inside `provider.callAnthropicMessages`, so - // it must walk into nested `tool_result.content` to find the image. const driveAnthropicMessages = async (providerModel: Awaited>[number], body: Omit): Promise => { - await provider.callAnthropicMessages(providerModel, body, undefined, noopAnthropicMessagesUpstreamCallOptions()); + await collectChatProviderPipeline(instance, 'anthropicMessages', providerModel, body, undefined, noopAnthropicMessagesUpstreamCallOptions()); }; await withMockedFetch( @@ -689,12 +687,8 @@ test('Copilot provider sets copilot-vision-request when an image is nested insid }); test('Copilot Anthropic Messages boundary chain does NOT fire on the OpenAI Chat Completions wire (translated path)', async () => { - // Boundary isolation: each provider call method runs only its own protocol - // boundary chain. The Anthropic-Messages-only `withClaudeAgentHeadersSet` interceptor - // would set x-interaction-type to 'messages-proxy' for Claude Code SDK - // metadata, but it MUST NOT run when the translated path calls Copilot's - // openai-chat-completions wire — that path runs `COPILOT_OPENAI_CHAT_COMPLETIONS_BOUNDARY`, - // which has no Anthropic-Messages-source headers in it. + // The operation chain selects wire-specific stages. Claude Code metadata must + // not install Anthropic Messages headers on a Chat Completions request. const { copilotUpstream } = await setupCopilotTest(); const instance = createCopilotProvider(copilotUpstream); const provider = instance.instance; @@ -719,21 +713,15 @@ test('Copilot Anthropic Messages boundary chain does NOT fire on the OpenAI Chat }, async () => { const [providerModel] = await provider.getProvidedModels(directFetcher); - // Even with a Claude-Code-shaped metadata blob, the openai-chat-completions - // boundary chain has no Anthropic-Messages-source interceptor, so the - // messages-proxy intent must not appear on the wire. - await provider.callOpenAIChatCompletions(providerModel, { + await collectChatProviderPipeline(instance, 'openaiChatCompletions', providerModel, { messages: [{ role: 'user', content: 'hi' }], metadata: { user_id: JSON.stringify({ device_id: 'dev-1', session_id: 'sess-1' }) }, }, undefined, noopUpstreamCallOptions()); }, ); - // The openai-chat-completions wire defaults to `conversation-agent` (set by - // `copilotAuthedFetch` in `packages/provider-copilot/src/auth.ts`). The - // Anthropic-Messages-boundary `withClaudeAgentHeadersSet` would overwrite it to - // `messages-proxy` if it had run — its absence is the - // proof that the Anthropic Messages boundary chain did NOT fire on this wire. + // Authentication sets conversation-agent; the Anthropic-only header stage + // would overwrite it to messages-proxy if it ran on this operation. assertEquals(observedInteractionType, ['conversation-agent']); }); @@ -1013,7 +1001,8 @@ const betaIntent = ['context-1m-2025-08-07', 'advanced-tool-use-2025-11-20', 'un test('Copilot consumes Anthropic Messages beta intent before serializing its supported wire subset', async () => { const { copilotUpstream } = await setupCopilotTest(); - const provider = createCopilotProvider(copilotUpstream).instance; + const instance = createCopilotProvider(copilotUpstream); + const provider = instance.instance; let upstreamModel: unknown; let upstreamBeta: string | null = null; @@ -1036,13 +1025,8 @@ test('Copilot consumes Anthropic Messages beta intent before serializing its sup }, async () => { const [providerModel] = await provider.getProvidedModels(directFetcher); - const result = await provider.callAnthropicMessages( - providerModel, - { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }] }, - undefined, - noopAnthropicMessagesUpstreamCallOptions({ anthropicBeta: betaIntent }), - ); - assertEquals(result.modelKey, 'claude-opus-4.6-1m'); + const result = await collectChatProviderPipeline(instance, 'anthropicMessages', providerModel, { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }] }, undefined, noopAnthropicMessagesUpstreamCallOptions({ anthropicBeta: betaIntent })); + assertEquals(result.facts['response.provider.modelKey'], 'claude-opus-4.6-1m'); }, ); @@ -1052,7 +1036,8 @@ test('Copilot consumes Anthropic Messages beta intent before serializing its sup test('Copilot count_tokens consumes and serializes the same Anthropic Messages beta intent', async () => { const { copilotUpstream } = await setupCopilotTest(); - const provider = createCopilotProvider(copilotUpstream).instance; + const instance = createCopilotProvider(copilotUpstream); + const provider = instance.instance; let upstreamModel: unknown; let upstreamBeta: string | null = null; @@ -1073,13 +1058,8 @@ test('Copilot count_tokens consumes and serializes the same Anthropic Messages b }, async () => { const [providerModel] = await provider.getProvidedModels(directFetcher); - const result = await provider.callAnthropicMessagesCountTokens( - providerModel, - { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }] }, - undefined, - noopAnthropicMessagesUpstreamCallOptions({ anthropicBeta: betaIntent }), - ); - assertEquals(result.modelKey, 'claude-opus-4.6-1m'); + const result = await collectChatProviderPipeline(instance, 'anthropicMessagesCountTokens', providerModel, { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }] }, undefined, noopAnthropicMessagesUpstreamCallOptions({ anthropicBeta: betaIntent })); + assertEquals(result.facts['response.provider.modelKey'], 'claude-opus-4.6-1m'); }, ); @@ -1111,17 +1091,12 @@ test('Copilot provider routes speed=fast to the -fast raw variant and stamps usa const [providerModel] = await provider.getProvidedModels(directFetcher); assertEquals(providerModel.id, 'claude-opus-4-6'); - const result = await provider.callAnthropicMessages( - providerModel, - { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }], speed: 'fast' }, - undefined, - noopAnthropicMessagesUpstreamCallOptions(), - ); + const result = await collectChatProviderPipeline(instance, 'anthropicMessages', providerModel, { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }], speed: 'fast' }, undefined, noopAnthropicMessagesUpstreamCallOptions()); - if (!result.ok) throw new Error(`expected ok stream, got ${JSON.stringify(result.response)}`); + if (!(result.output !== null && 'kind' in result.output)) throw new Error(`expected ok stream, got ${JSON.stringify(result.response)}`); const frames = []; - for await (const frame of result.events) frames.push(frame); + for (const frame of result.frames) frames.push(frame); const messageStart = frames.find(f => f.type === 'event' && f.event.type === 'message_start'); if (messageStart?.type !== 'event' || messageStart.event.type !== 'message_start') { @@ -1135,7 +1110,7 @@ test('Copilot provider routes speed=fast to the -fast raw variant and stamps usa } assertEquals(messageDelta.event.usage?.speed, 'fast'); - assertEquals(result.modelKey, 'claude-opus-4.6-fast'); + assertEquals(result.facts['response.provider.modelKey'], 'claude-opus-4.6-fast'); }, ); @@ -1169,23 +1144,18 @@ test('Copilot provider returns HTTP 400 invalid_request_error when speed=fast hi async () => { const [providerModel] = await provider.getProvidedModels(directFetcher); - const result = await provider.callAnthropicMessages( - providerModel, - { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }], speed: 'fast' }, - undefined, - noopAnthropicMessagesUpstreamCallOptions(), - ); + const result = await collectChatProviderPipeline(instance, 'anthropicMessages', providerModel, { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }], speed: 'fast' }, undefined, noopAnthropicMessagesUpstreamCallOptions()); - if (result.ok) throw new Error('expected 400 error, got ok stream'); - assertEquals(result.response.status, 400); - const body = (await result.response.json()) as { type: string; error: { type: string; message: string } }; + if ((result.output !== null && 'kind' in result.output)) throw new Error('expected 400 error, got ok stream'); + assertEquals(result.response!.status, 400); + const body = (await result.response!.json()) as { type: string; error: { type: string; message: string } }; // Byte-identical to the wire string Anthropic emits on real api.anthropic.com // for the same failure mode — recorded verbatim from a live response in // https://github.com/Yeachan-Heo/gajae-code/blob/main/packages/ai/test/anthropic-fast-mode.test.ts assertEquals(body.type, 'error'); assertEquals(body.error.type, 'invalid_request_error'); assertEquals(body.error.message, "'claude-haiku-4-5' does not support the `speed` parameter."); - assertEquals(result.modelKey, providerModel.id); + assertEquals(result.facts['response.provider.modelKey'], providerModel.id); }, ); @@ -1218,12 +1188,7 @@ test('Copilot provider passes unknown speed values to the upstream verbatim so t // does not own rejecting it and must not strip it either — let // Copilot surface whatever error its strict validator returns. const speedValue = 'priority' as AnthropicMessagesPayload['speed']; - await provider.callAnthropicMessages( - providerModel, - { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }], speed: speedValue }, - undefined, - noopAnthropicMessagesUpstreamCallOptions(), - ); + await collectChatProviderPipeline(instance, 'anthropicMessages', providerModel, { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }], speed: speedValue }, undefined, noopAnthropicMessagesUpstreamCallOptions()); }, ); @@ -1376,7 +1341,8 @@ test('Copilot chat field: no capabilities → no chat field', async () => { // the SSE body, so a streaming call snapshots at stream start. test('Copilot provider persists the quota snapshot a data-plane response carries', async () => { const harness = await setupCopilotTest(); - const provider = createCopilotProvider(harness.copilotUpstream).instance; + const instance = createCopilotProvider(harness.copilotUpstream); + const provider = instance.instance; await withMockedFetch( request => { @@ -1402,7 +1368,7 @@ test('Copilot provider persists the quota snapshot a data-plane response carries }, async () => { const [model] = await provider.getProvidedModels(directFetcher); - await provider.callOpenAIChatCompletions(model, { messages: [{ role: 'user', content: 'hi' }] }, undefined, noopUpstreamCallOptions()); + await collectChatProviderPipeline(instance, 'openaiChatCompletions', model, { messages: [{ role: 'user', content: 'hi' }] }, undefined, noopUpstreamCallOptions()); // The persist is fire-and-forget: on workerd `waitUntil` keeps it alive // past the response, and here it settles on the host event loop. await vi.waitFor(() => { @@ -1424,7 +1390,8 @@ test('Copilot provider persists the quota snapshot a data-plane response carries // no upside, so the slot is left exactly as it was. test('Copilot provider leaves the persisted snapshot alone when a response carries no quota headers', async () => { const harness = await setupCopilotTest(); - const provider = createCopilotProvider(harness.copilotUpstream).instance; + const instance = createCopilotProvider(harness.copilotUpstream); + const provider = instance.instance; await withMockedFetch( request => { @@ -1445,7 +1412,7 @@ test('Copilot provider leaves the persisted snapshot alone when a response carri }, async () => { const [model] = await provider.getProvidedModels(directFetcher); - await provider.callOpenAIChatCompletions(model, { messages: [{ role: 'user', content: 'hi' }] }, undefined, noopUpstreamCallOptions()); + await collectChatProviderPipeline(instance, 'openaiChatCompletions', model, { messages: [{ role: 'user', content: 'hi' }] }, undefined, noopUpstreamCallOptions()); }, ); @@ -1501,10 +1468,10 @@ test('Copilot provider merges the -fast raw variant and reaches it through servi const sol = models.find(model => model.id === 'gpt-5.6-sol')!; const opts = noopUpstreamCallOptions(); - await provider.callOpenAIResponses(sol, { input: [] }, 'generate', undefined, opts); - await provider.callOpenAIResponses(sol, { input: [], service_tier: 'priority' }, 'generate', undefined, opts); - await provider.callOpenAIResponses(sol, { input: [], service_tier: 'fast' }, 'generate', undefined, opts); - await provider.callOpenAIResponses(sol, { input: [], service_tier: 'flex' }, 'generate', undefined, opts); + await collectChatProviderPipeline(instance, 'openaiResponses', sol, { input: [] }, undefined, opts); + await collectChatProviderPipeline(instance, 'openaiResponses', sol, { input: [], service_tier: 'priority' }, undefined, opts); + await collectChatProviderPipeline(instance, 'openaiResponses', sol, { input: [], service_tier: 'fast' }, undefined, opts); + await collectChatProviderPipeline(instance, 'openaiResponses', sol, { input: [], service_tier: 'flex' }, undefined, opts); }, ); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/apply-top-level-cache-control_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/apply-top-level-cache-control_test.ts similarity index 82% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/apply-top-level-cache-control_test.ts rename to packages/provider-copilot/__tests__/stages/anthropic-messages/apply-top-level-cache-control_test.ts index 99da6bf30a..dcd4e92386 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/apply-top-level-cache-control_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/apply-top-level-cache-control_test.ts @@ -1,19 +1,15 @@ import { test } from 'vitest'; -import { withTopLevelCacheControlApplied } from '../../../src/interceptors/anthropic-messages/apply-top-level-cache-control.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { copilotAnthropicMessagesApplyTopLevelCacheControl } from '../../../src/stages/anthropic-messages/apply-top-level-cache-control.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type AnthropicMessagesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: AnthropicMessagesPayload & { cache_control?: { type: 'ephemeral' } }): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload & { cache_control?: { type: 'ephemeral' } }): AnthropicMessagesProbe => ({ payload, headers: new Headers(), anthropicBeta: [], @@ -32,7 +28,7 @@ test('ports top-level cache_control onto the last cacheable block and drops the ], }); - await withTopLevelCacheControlApplied(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesApplyTopLevelCacheControl, ctx, okEvents); assertEquals((ctx.payload as { cache_control?: unknown }).cache_control, undefined); assertEquals(ctx.payload.messages[2].content, [{ type: 'text', text: 'second', cache_control: { type: 'ephemeral' } }]); @@ -48,7 +44,7 @@ test('respects an explicit cache_control on the last cacheable block', async () messages: [{ role: 'user', content: [{ type: 'text', text: 'hi', cache_control: explicit }] }], }); - await withTopLevelCacheControlApplied(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesApplyTopLevelCacheControl, ctx, okEvents); assertEquals((ctx.payload as { cache_control?: unknown }).cache_control, undefined); assertEquals(ctx.payload.messages[0].content, [{ type: 'text', text: 'hi', cache_control: explicit }]); @@ -62,7 +58,7 @@ test('promotes string message content so the auto-marker has a block to land on' messages: [{ role: 'user', content: 'plain' }], }); - await withTopLevelCacheControlApplied(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesApplyTopLevelCacheControl, ctx, okEvents); assertEquals((ctx.payload as { cache_control?: unknown }).cache_control, undefined); assertEquals(ctx.payload.messages[0].content, [{ type: 'text', text: 'plain', cache_control: { type: 'ephemeral' } }]); @@ -79,7 +75,7 @@ test('walks past trailing non-cacheable blocks to find the last cacheable one', ], }); - await withTopLevelCacheControlApplied(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesApplyTopLevelCacheControl, ctx, okEvents); assertEquals((ctx.payload as { cache_control?: unknown }).cache_control, undefined); assertEquals(ctx.payload.messages[1].content, [{ type: 'thinking', thinking: 'pondering' }]); @@ -94,7 +90,7 @@ test('drops top-level cache_control when no cacheable block exists', async () => messages: [{ role: 'assistant', content: [{ type: 'thinking', thinking: 'no text yet' }] }], }); - await withTopLevelCacheControlApplied(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesApplyTopLevelCacheControl, ctx, okEvents); assertEquals((ctx.payload as { cache_control?: unknown }).cache_control, undefined); assertEquals(ctx.payload.messages[0].content, [{ type: 'thinking', thinking: 'no text yet' }]); @@ -107,7 +103,7 @@ test('no-op when payload has no top-level cache_control', async () => { messages: [{ role: 'user', content: [{ type: 'text', text: 'hi' }] }], }); - await withTopLevelCacheControlApplied(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesApplyTopLevelCacheControl, ctx, okEvents); assertEquals((ctx.payload as { cache_control?: unknown }).cache_control, undefined); assertEquals(ctx.payload.messages[0].content, [{ type: 'text', text: 'hi' }]); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/compress-images_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/compress-images_test.ts similarity index 91% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/compress-images_test.ts rename to packages/provider-copilot/__tests__/stages/anthropic-messages/compress-images_test.ts index 8835ddccd8..9f53ffc937 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/compress-images_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/compress-images_test.ts @@ -1,16 +1,12 @@ import { test } from 'vitest'; -import { withInlineImagesCompressed } from '../../../src/interceptors/anthropic-messages/compress-images.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { copilotAnthropicMessagesCompressImages } from '../../../src/stages/anthropic-messages/compress-images.ts'; import { type ImageDimensions, type ImageProcessor, initImageProcessor } from '@floway-dev/platform'; import type { AnthropicMessagesImageBlock, AnthropicMessagesPayload, AnthropicMessagesStreamEvent, AnthropicMessagesToolResultBlock } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type AnthropicMessagesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assert, assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); @@ -29,7 +25,7 @@ const spyProcessor = (): { processor: ImageProcessor; inputs: Uint8Array[]; targ return { processor, inputs, targets }; }; -const invocation = (payload: AnthropicMessagesPayload, upstreamModelId = 'claude-test'): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload, upstreamModelId = 'claude-test'): AnthropicMessagesProbe => ({ payload, headers: new Headers(), anthropicBeta: [], @@ -59,7 +55,7 @@ test('compresses a top-level image block to WebP', async () => { }; const ctx = invocation(payload); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesCompressImages, ctx, okEvents); const block = (ctx.payload.messages[0].content as Array<{ type: string; source?: { media_type: string; data: string } }>)[1]; assertEquals(block.source?.media_type, 'image/webp'); @@ -110,7 +106,7 @@ test('compresses an image nested inside tool_result content', async () => { }; const ctx = invocation(payload); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesCompressImages, ctx, okEvents); const rewrittenContent = ctx.payload.messages[0].content; if (!Array.isArray(rewrittenContent)) throw new Error('expected rewritten user content'); @@ -144,7 +140,7 @@ test('leaves image-free payloads untouched and does not invoke the processor', a }; const ctx = invocation(payload); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesCompressImages, ctx, okEvents); assertEquals(inputs.length, 0); assert(ctx.payload === payload); @@ -174,7 +170,7 @@ test('compresses each unique inline image only once when the same base64 data re ], }); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesCompressImages, ctx, okEvents); assertEquals(inputs.length, 2); }); @@ -197,7 +193,7 @@ test('high-res Opus clamps the long edge to 2576 on very wide images', async () assertEquals(await capTargetFor('claude-opus-4.7', { width: 8000, height: 1000 }), { width: 2576, height: 322 }); }); -// Drives the interceptor end-to-end with a header-only PNG of the requested +// Drives the stage end-to-end with a header-only PNG of the requested // dimensions and reads back the target the per-model cap resolved to. The // PNG carries no pixel payload — image-size reads dimensions from IHDR and // does not validate the rest of the file. @@ -214,7 +210,7 @@ const capTargetFor = async (upstreamModelId: string, source: ImageDimensions): P }, ], }, upstreamModelId); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesCompressImages, ctx, okEvents); return targets[0]; }; diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/handle-speed-fast_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/handle-speed-fast_test.ts similarity index 68% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/handle-speed-fast_test.ts rename to packages/provider-copilot/__tests__/stages/anthropic-messages/handle-speed-fast_test.ts index 1d03f5c413..f764bcd3f7 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/handle-speed-fast_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/handle-speed-fast_test.ts @@ -1,11 +1,9 @@ import { test } from 'vitest'; -import { withSpeedFast } from '../../../src/interceptors/anthropic-messages/handle-speed-fast.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { copilotAnthropicMessagesSpeedFast } from '../../../src/stages/anthropic-messages/handle-speed-fast.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent, AnthropicMessagesUsage } from '@floway-dev/protocols/anthropic-messages'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type AnthropicMessagesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const collect = async (events: AsyncIterable): Promise => { @@ -16,7 +14,7 @@ const collect = async (events: AsyncIterable): Promise => { const baseUsage: AnthropicMessagesUsage = { input_tokens: 10, output_tokens: 0 }; -const makeCtx = (speed?: unknown): AnthropicMessagesBoundaryCtx => ({ +const makeCtx = (speed?: unknown): AnthropicMessagesProbe => ({ payload: { model: 'claude-opus-4.6-fast', messages: [{ role: 'user', content: 'hi' }], @@ -28,8 +26,6 @@ const makeCtx = (speed?: unknown): AnthropicMessagesBoundaryCtx => ({ model: stubProviderModel({ endpoints: { anthropicMessages: {} } }), }); -const stubRequest = {}; - const messageStart = (usage: AnthropicMessagesUsage): ProtocolFrame => eventFrame({ type: 'message_start', message: { @@ -60,38 +56,38 @@ const streamResult = (frames: ProtocolFrame[]): Ex testTelemetryModelIdentity, ); -test('withSpeedFast strips speed=fast from the outbound payload', async () => { +test('copilotAnthropicMessagesSpeedFast strips speed=fast from the outbound payload', async () => { const ctx = makeCtx('fast'); - await withSpeedFast(ctx, stubRequest, () => Promise.resolve(streamResult([]))); + await applyProviderStage(copilotAnthropicMessagesSpeedFast, ctx, () => Promise.resolve(streamResult([]))); assertEquals('speed' in ctx.payload, false); }); -test('withSpeedFast strips speed=standard (semantically equivalent to omitted)', async () => { +test('copilotAnthropicMessagesSpeedFast strips speed=standard (semantically equivalent to omitted)', async () => { const ctx = makeCtx('standard'); - await withSpeedFast(ctx, stubRequest, () => Promise.resolve(streamResult([]))); + await applyProviderStage(copilotAnthropicMessagesSpeedFast, ctx, () => Promise.resolve(streamResult([]))); assertEquals('speed' in ctx.payload, false); }); -test('withSpeedFast leaves unknown speed values untouched so the upstream rejects them', async () => { +test('copilotAnthropicMessagesSpeedFast leaves unknown speed values untouched so the upstream rejects them', async () => { // Anthropic returns 400 invalid_request_error on unknown enum values rather // than silently downgrading; the gateway mirrors that by passing the // unknown value through to Copilot, which will surface the same shape of // error. We never invent a fall-through here. const ctx = makeCtx('priority'); - await withSpeedFast(ctx, stubRequest, () => Promise.resolve(streamResult([]))); + await applyProviderStage(copilotAnthropicMessagesSpeedFast, ctx, () => Promise.resolve(streamResult([]))); assertEquals(ctx.payload.speed, 'priority'); }); -test('withSpeedFast stamps usage.speed=fast on message_start when fast was requested', async () => { +test('copilotAnthropicMessagesSpeedFast stamps usage.speed=fast on message_start when fast was requested', async () => { const ctx = makeCtx('fast'); - const result = await withSpeedFast(ctx, stubRequest, () => + const result = await applyProviderStage(copilotAnthropicMessagesSpeedFast, ctx, () => Promise.resolve(streamResult([messageStart(baseUsage), messageStop(), doneFrame()]))); assertEquals(result.type, 'events'); @@ -104,10 +100,10 @@ test('withSpeedFast stamps usage.speed=fast on message_start when fast was reque ]); }); -test('withSpeedFast stamps usage.speed=fast on every message_delta carrying usage', async () => { +test('copilotAnthropicMessagesSpeedFast stamps usage.speed=fast on every message_delta carrying usage', async () => { const ctx = makeCtx('fast'); - const result = await withSpeedFast(ctx, stubRequest, () => + const result = await applyProviderStage(copilotAnthropicMessagesSpeedFast, ctx, () => Promise.resolve(streamResult([messageStart(baseUsage), messageDelta(7), messageStop()]))); assertEquals(result.type, 'events'); @@ -124,11 +120,11 @@ test('withSpeedFast stamps usage.speed=fast on every message_delta carrying usag ]); }); -test('withSpeedFast leaves the stream untouched when speed is absent', async () => { +test('copilotAnthropicMessagesSpeedFast leaves the stream untouched when speed is absent', async () => { const ctx = makeCtx(); const frames = [messageStart(baseUsage), messageDelta(3), messageStop()]; - const result = await withSpeedFast(ctx, stubRequest, () => Promise.resolve(streamResult(frames))); + const result = await applyProviderStage(copilotAnthropicMessagesSpeedFast, ctx, () => Promise.resolve(streamResult(frames))); assertEquals(result.type, 'events'); if (result.type !== 'events') throw new Error('expected events'); @@ -136,21 +132,21 @@ test('withSpeedFast leaves the stream untouched when speed is absent', async () assertEquals(await collect(result.events), frames); }); -test('withSpeedFast leaves the stream untouched when speed=standard (no fast intent to stamp)', async () => { +test('copilotAnthropicMessagesSpeedFast leaves the stream untouched when speed=standard (no fast intent to stamp)', async () => { const ctx = makeCtx('standard'); const frames = [messageStart(baseUsage), messageStop()]; - const result = await withSpeedFast(ctx, stubRequest, () => Promise.resolve(streamResult(frames))); + const result = await applyProviderStage(copilotAnthropicMessagesSpeedFast, ctx, () => Promise.resolve(streamResult(frames))); assertEquals(result.type, 'events'); if (result.type !== 'events') throw new Error('expected events'); assertEquals(await collect(result.events), frames); }); -test('withSpeedFast surfaces non-events results (api-error / internal-error) verbatim', async () => { +test('copilotAnthropicMessagesSpeedFast surfaces non-events results (api-error / internal-error) verbatim', async () => { const ctx = makeCtx('fast'); - const result = await withSpeedFast(ctx, stubRequest, () => + const result = await applyProviderStage(copilotAnthropicMessagesSpeedFast, ctx, () => Promise.resolve({ type: 'internal-error', status: 502, diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/normalize-anthropic-beta_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/normalize-anthropic-beta_test.ts similarity index 75% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/normalize-anthropic-beta_test.ts rename to packages/provider-copilot/__tests__/stages/anthropic-messages/normalize-anthropic-beta_test.ts index 560d6bffdf..ea1ed2a772 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/normalize-anthropic-beta_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/normalize-anthropic-beta_test.ts @@ -1,20 +1,18 @@ import { test } from 'vitest'; -import { withAnthropicBetaNormalized } from '../../../src/interceptors/anthropic-messages/normalize-anthropic-beta.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { copilotAnthropicMessagesNormalizeAnthropicBeta } from '../../../src/stages/anthropic-messages/normalize-anthropic-beta.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import { eventResult, type ExecuteResult } from '@floway-dev/provider'; +import { applyProviderStage, type AnthropicMessagesProbe, eventResult, type ExecuteResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); const invocation = ( payload: AnthropicMessagesPayload & { context_management?: unknown }, anthropicBeta: string[] = [], -): AnthropicMessagesBoundaryCtx => ({ +): AnthropicMessagesProbe => ({ payload: payload as AnthropicMessagesPayload, headers: new Headers(), anthropicBeta, @@ -34,7 +32,7 @@ test('keeps only supported caller beta values in first-seen order', async () => 'advanced-tool-use-2025-11-20', 'context-1m-2025-08-07', ]); - await withAnthropicBetaNormalized(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesNormalizeAnthropicBeta, ctx, okEvents); assertEquals(ctx.anthropicBeta, ['advanced-tool-use-2025-11-20']); }); @@ -42,15 +40,15 @@ test('synthesizes interleaved thinking only when the caller supplied no beta int const payload = { ...baseBody, thinking: { type: 'enabled' as const, budget_tokens: 1024 } }; const silent = invocation(payload); const explicit = invocation(payload, ['context-1m-2025-08-07']); - await withAnthropicBetaNormalized(silent, stubRequest, okEvents); - await withAnthropicBetaNormalized(explicit, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesNormalizeAnthropicBeta, silent, okEvents); + await applyProviderStage(copilotAnthropicMessagesNormalizeAnthropicBeta, explicit, okEvents); assertEquals(silent.anthropicBeta, ['interleaved-thinking-2025-05-14']); assertEquals(explicit.anthropicBeta, []); }); test('does not synthesize interleaved thinking for adaptive thinking', async () => { const ctx = invocation({ ...baseBody, thinking: { type: 'adaptive', budget_tokens: 1024 } }); - await withAnthropicBetaNormalized(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesNormalizeAnthropicBeta, ctx, okEvents); assertEquals(ctx.anthropicBeta, []); }); @@ -59,7 +57,7 @@ test('pairs context management with its required beta token', async () => { { ...baseBody, context_management: { edits: [] } }, ['interleaved-thinking-2025-05-14'], ); - await withAnthropicBetaNormalized(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesNormalizeAnthropicBeta, ctx, okEvents); assertEquals(ctx.anthropicBeta, [ 'interleaved-thinking-2025-05-14', 'context-management-2025-06-27', diff --git a/packages/provider-copilot/__tests__/stages/anthropic-messages/pipeline-headers_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/pipeline-headers_test.ts new file mode 100644 index 0000000000..c52c30840d --- /dev/null +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/pipeline-headers_test.ts @@ -0,0 +1,31 @@ +import { test } from 'vitest'; + +import { CLAUDE_AGENT_USER_AGENT, clearInProcessCopilotTokenCache } from '../../../src/auth.ts'; +import { createCopilotProvider } from '../../../src/provider.ts'; +import { emptyCopilotUpstreamState } from '../../../src/state.ts'; +import { initProviderRepo, type UpstreamRecord } from '@floway-dev/provider'; +import { assertEquals, collectChatProviderPipeline, noopAnthropicMessagesUpstreamCallOptions, sseResponse, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; + +const compactText = 'Your task is to create a detailed summary of the conversation so far.\n\nCRITICAL: Respond with TEXT ONLY. Do NOT call any tools.\n\nPending Tasks:\n- finish refactor\n\nCurrent Work:\n- reviewing diff'; + +test('Claude Code compact intent, identity and interaction metadata reach the final Copilot wire in stage order', async () => { + clearInProcessCopilotTokenCache(); + const record: UpstreamRecord = { id: 'up_header_stages', kind: 'copilot', name: 'Test', enabled: true, sortOrder: 0, createdAt: '', updatedAt: '', config: { githubHost: 'github.com', githubToken: 'test-token', user: { id: 1, login: 'test', name: null, avatar_url: '' } }, state: { ...emptyCopilotUpstreamState(), copilotToken: { token: 'access-token', expiresAt: 4102444800, baseUrl: 'https://copilot.example' } }, flagOverrides: {}, disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, modelsCache: null, hue: 210 }; + initProviderRepo(() => ({ upstreams: { getById: async () => record, saveState: async () => {} } })); + const provider = createCopilotProvider(record); + const model = stubProviderModel({ id: 'claude-test', providerData: { rawModels: [{ id: 'claude-test', supported_endpoints: ['/v1/messages'] }] } }); + let observed: Headers | undefined; + await withMockedFetch(request => { + observed = request.headers; + return sseResponse('event: message_stop\ndata: {"type":"message_stop"}\n\n'); + }, async () => { + await collectChatProviderPipeline(provider, 'anthropicMessages', model, { max_tokens: 10, metadata: { user_id: JSON.stringify({ device_id: 'dev-1', session_id: 'sess-1' }) }, messages: [{ role: 'user', content: compactText }] }, undefined, noopAnthropicMessagesUpstreamCallOptions()); + }); + if (observed === undefined) throw new Error('Expected model endpoint dispatch'); + assertEquals(observed.get('x-initiator'), 'user'); + assertEquals(observed.get('x-interaction-type'), 'messages-proxy'); + assertEquals(observed.get('openai-intent'), 'messages-proxy'); + assertEquals(observed.get('user-agent'), CLAUDE_AGENT_USER_AGENT); + assertEquals(observed.get('copilot-integration-id'), null); + assertEquals(observed.get('x-interaction-id'), 'abe633f3-a47a-4758-974e-abe9160daf36'); +}); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/promote-thinking-display_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/promote-thinking-display_test.ts similarity index 78% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/promote-thinking-display_test.ts rename to packages/provider-copilot/__tests__/stages/anthropic-messages/promote-thinking-display_test.ts index 0af6c7c173..852978f32d 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/promote-thinking-display_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/promote-thinking-display_test.ts @@ -1,11 +1,9 @@ import { test } from 'vitest'; -import { resolveAnthropicMessagesDownstreamThinkingDisplay, withThinkingDisplayPromoted } from '../../../src/interceptors/anthropic-messages/promote-thinking-display.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { resolveAnthropicMessagesDownstreamThinkingDisplay, copilotAnthropicMessagesThinkingDisplay } from '../../../src/stages/anthropic-messages/promote-thinking-display.ts'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type AnthropicMessagesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; const collect = async (events: AsyncIterable): Promise => { @@ -15,11 +13,11 @@ const collect = async (events: AsyncIterable): Promise => { }; const makeCtx = ( - thinking: AnthropicMessagesBoundaryCtx['payload']['thinking'], + thinking: AnthropicMessagesProbe['payload']['thinking'], overrides: { model?: string; } = {}, -): AnthropicMessagesBoundaryCtx => ({ +): AnthropicMessagesProbe => ({ payload: { model: overrides.model ?? 'claude-opus-4.7-1m-internal', messages: [{ role: 'user', content: 'hi' }], @@ -31,8 +29,6 @@ const makeCtx = ( model: stubProviderModel({ endpoints: { anthropicMessages: {} } }), }); -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); test('resolveAnthropicMessagesDownstreamThinkingDisplay exposes 4.7+ omitted by default and older Claude as summarized', () => { @@ -82,10 +78,10 @@ test('resolveAnthropicMessagesDownstreamThinkingDisplay ignores unknown explicit assertEquals(resolveAnthropicMessagesDownstreamThinkingDisplay(ctx), undefined); }); -test('withThinkingDisplayPromoted sends summarized upstream when thinking display is omitted', async () => { +test('copilotAnthropicMessagesThinkingDisplay sends summarized upstream when thinking display is omitted', async () => { const ctx = makeCtx({ type: 'adaptive' }); - await withThinkingDisplayPromoted(ctx, stubRequest, () => + await applyProviderStage(copilotAnthropicMessagesThinkingDisplay, ctx, () => Promise.resolve({ type: 'internal-error', status: 418, @@ -101,41 +97,41 @@ test('withThinkingDisplayPromoted sends summarized upstream when thinking displa assertEquals(ctx.payload.thinking?.display, 'summarized'); }); -test('withThinkingDisplayPromoted overrides omitted but preserves full', async () => { +test('copilotAnthropicMessagesThinkingDisplay overrides omitted but preserves full', async () => { const omittedCtx = makeCtx({ type: 'adaptive', display: 'omitted' }); const fullCtx = makeCtx({ type: 'adaptive', display: 'full' }); - await withThinkingDisplayPromoted(omittedCtx, stubRequest, okEvents); - await withThinkingDisplayPromoted(fullCtx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesThinkingDisplay, omittedCtx, okEvents); + await applyProviderStage(copilotAnthropicMessagesThinkingDisplay, fullCtx, okEvents); assertEquals(omittedCtx.payload.thinking?.display, 'summarized'); assertEquals(fullCtx.payload.thinking?.display, 'full'); }); -test('withThinkingDisplayPromoted leaves disabled or absent thinking untouched', async () => { +test('copilotAnthropicMessagesThinkingDisplay leaves disabled or absent thinking untouched', async () => { const disabledCtx = makeCtx({ type: 'disabled' }); const absentCtx = makeCtx(undefined); - await withThinkingDisplayPromoted(disabledCtx, stubRequest, okEvents); - await withThinkingDisplayPromoted(absentCtx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesThinkingDisplay, disabledCtx, okEvents); + await applyProviderStage(copilotAnthropicMessagesThinkingDisplay, absentCtx, okEvents); assertEquals(disabledCtx.payload.thinking, { type: 'disabled' }); assertEquals(absentCtx.payload.thinking, undefined); }); -test('withThinkingDisplayPromoted leaves unknown display values for upstream validation', async () => { +test('copilotAnthropicMessagesThinkingDisplay leaves unknown display values for upstream validation', async () => { const ctx = makeCtx({ type: 'adaptive' }); (ctx.payload.thinking as { display?: unknown }).display = 'omit'; - await withThinkingDisplayPromoted(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesThinkingDisplay, ctx, okEvents); assertEquals((ctx.payload.thinking as { display?: unknown }).display, 'omit'); }); -test('withThinkingDisplayPromoted simulates omitted display on protocol events', async () => { +test('copilotAnthropicMessagesThinkingDisplay simulates omitted display on protocol events', async () => { const ctx = makeCtx({ type: 'adaptive' }); - const result = await withThinkingDisplayPromoted(ctx, stubRequest, () => + const result = await applyProviderStage(copilotAnthropicMessagesThinkingDisplay, ctx, () => Promise.resolve( eventResult( (async function* (): AsyncGenerator> { diff --git a/packages/provider-copilot/__tests__/stages/anthropic-messages/rewrite-context-window-error_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/rewrite-context-window-error_test.ts new file mode 100644 index 0000000000..20f7926adb --- /dev/null +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/rewrite-context-window-error_test.ts @@ -0,0 +1,28 @@ +import { expect, test } from 'vitest'; + +import { rewriteCopilotContextWindowError } from '../../../src/stages/anthropic-messages/rewrite-context-window-error.ts'; +import { http } from '@floway-dev/http/pipeline'; +import { compose, getFailureFacts, move, run } from '@floway-dev/pipeline'; +import { decodeProviderResponse, observeProviderCall, type ProviderChatServices } from '@floway-dev/provider'; +import { noopUpstreamCallOptions } from '@floway-dev/test-utils'; + +test('failed finite error-body reading preserves its original failure and the actual call facts', async () => { + const error = new Error('error response body failed'); + const body = new ReadableStream({ start(controller) { controller.error(error); } }); + const services: ProviderChatServices = { + httpCall: () => ({ ...noopUpstreamCallOptions(), signal: undefined, fetcher: async () => new Response(body, { status: 400 }) }), + recordProtocolFrames: frames => frames, + }; + const entry = move({ + 'request.provider.modelKey': 'claude-test', + 'request.http.callId': 1, + 'request.http.url': 'https://copilot.example/v1/messages', + 'request.http.method': 'POST', + 'request.http.headers': [], + 'request.http.body': {}, + 'request.http.encoding': 'json' as const, + }); + const pipeline = compose('copilotErrorBody', [rewriteCopilotContextWindowError, decodeProviderResponse('anthropicMessages'), observeProviderCall, http]); + await expect(run(pipeline, entry, services)).rejects.toBe(error); + expect(getFailureFacts(error)).toMatchObject({ 'response.provider.called': true, 'response.provider.modelKey': 'claude-test' }); +}); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-claude-agent-headers_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-claude-agent-headers_test.ts similarity index 80% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-claude-agent-headers_test.ts rename to packages/provider-copilot/__tests__/stages/anthropic-messages/set-claude-agent-headers_test.ts index dcd035ba0a..2adfb96e7f 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-claude-agent-headers_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-claude-agent-headers_test.ts @@ -1,20 +1,16 @@ import { test } from 'vitest'; import { CLAUDE_AGENT_USER_AGENT } from '../../../src/auth.ts'; -import { withClaudeAgentHeadersSet } from '../../../src/interceptors/anthropic-messages/set-claude-agent-headers.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { copilotAnthropicMessagesSetClaudeAgentHeaders } from '../../../src/stages/anthropic-messages/set-claude-agent-headers.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type AnthropicMessagesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesProbe => ({ payload, headers: new Headers(), anthropicBeta: [], @@ -31,7 +27,7 @@ const basePayload = (userId: string | undefined): AnthropicMessagesPayload => ({ test('Claude agent headers set for the legacy fingerprint with both halves', async () => { const ctx = invocation(basePayload('user_acct-1_account__session_sess-1')); - await withClaudeAgentHeadersSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetClaudeAgentHeaders, ctx, okEvents); assertEquals(ctx.headers.get('x-interaction-type'), 'messages-proxy'); assertEquals(ctx.headers.get('openai-intent'), 'messages-proxy'); @@ -44,7 +40,7 @@ test('Claude agent headers set for the legacy fingerprint with both halves', asy test('Claude agent headers set for the JSON fingerprint with device_id + session_id', async () => { const ctx = invocation(basePayload(JSON.stringify({ device_id: 'dev-1', session_id: 'sess-1' }))); - await withClaudeAgentHeadersSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetClaudeAgentHeaders, ctx, okEvents); assertEquals(ctx.headers.get('user-agent'), CLAUDE_AGENT_USER_AGENT); assertEquals(ctx.headers.get('copilot-integration-id'), ''); @@ -53,7 +49,7 @@ test('Claude agent headers set for the JSON fingerprint with device_id + session test('Claude agent headers absent when session_id is missing', async () => { const ctx = invocation(basePayload(JSON.stringify({ device_id: 'dev-1' }))); - await withClaudeAgentHeadersSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetClaudeAgentHeaders, ctx, okEvents); assertEquals(ctx.headers.has('user-agent'), false); assertEquals(ctx.headers.has('copilot-integration-id'), false); @@ -62,7 +58,7 @@ test('Claude agent headers absent when session_id is missing', async () => { test('Claude agent headers absent when safety identifier is missing', async () => { const ctx = invocation(basePayload(JSON.stringify({ session_id: 'sess-only' }))); - await withClaudeAgentHeadersSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetClaudeAgentHeaders, ctx, okEvents); assertEquals(ctx.headers.has('user-agent'), false); assertEquals(ctx.headers.has('copilot-integration-id'), false); @@ -71,7 +67,7 @@ test('Claude agent headers absent when safety identifier is missing', async () = test('Claude agent headers absent when metadata is not provided', async () => { const ctx = invocation(basePayload(undefined)); - await withClaudeAgentHeadersSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetClaudeAgentHeaders, ctx, okEvents); assertEquals(ctx.headers.has('user-agent'), false); assertEquals(ctx.headers.has('copilot-integration-id'), false); @@ -80,7 +76,7 @@ test('Claude agent headers absent when metadata is not provided', async () => { test('Claude agent headers skipped on claude-opus-4-8 even with full fingerprint', async () => { const ctx = invocation({ ...basePayload(JSON.stringify({ device_id: 'dev-1', session_id: 'sess-1' })), model: 'claude-opus-4-8' }); - await withClaudeAgentHeadersSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetClaudeAgentHeaders, ctx, okEvents); assertEquals(ctx.headers.has('user-agent'), false); assertEquals(ctx.headers.has('openai-intent'), false); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-compact-headers_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-compact-headers_test.ts similarity index 85% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-compact-headers_test.ts rename to packages/provider-copilot/__tests__/stages/anthropic-messages/set-compact-headers_test.ts index 308b0d01e9..78c22dacbc 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-compact-headers_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-compact-headers_test.ts @@ -1,19 +1,15 @@ import { test } from 'vitest'; -import { withCompactHeadersSet } from '../../../src/interceptors/anthropic-messages/set-compact-headers.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { copilotAnthropicMessagesSetCompactHeaders } from '../../../src/stages/anthropic-messages/set-compact-headers.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type AnthropicMessagesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesProbe => ({ payload, headers: new Headers(), anthropicBeta: [], @@ -32,7 +28,7 @@ test('Compact headers set when the last user message carries all three markers', messages: [{ role: 'user', content: COMPACT_LAST_MESSAGE_TEXT }], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); assertEquals(ctx.headers.get('x-interaction-type'), 'conversation-compaction'); @@ -54,7 +50,7 @@ test('Compact headers set from a multi-block last message that joins to the full ], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.get('x-interaction-type'), 'conversation-compaction'); }); @@ -67,7 +63,7 @@ test('Compact headers set when the system prompt starts with a compact summariza messages: [{ role: 'user', content: 'go ahead' }], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); assertEquals(ctx.headers.get('x-interaction-type'), 'conversation-compaction'); @@ -84,7 +80,7 @@ test('Compact headers set when an array system prompt contains a compact prefix messages: [{ role: 'user', content: 'hi' }], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.get('x-interaction-type'), 'conversation-compaction'); }); @@ -96,7 +92,7 @@ test('Compact headers absent when only the text-only guard is present (other mar messages: [{ role: 'user', content: 'CRITICAL: Respond with TEXT ONLY. Do NOT call any tools.' }], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.has('x-initiator'), false); assertEquals(ctx.headers.has('x-interaction-type'), false); @@ -109,7 +105,7 @@ test('Compact headers absent for an ordinary user turn', async () => { messages: [{ role: 'user', content: 'hello there' }], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.has('x-initiator'), false); assertEquals(ctx.headers.has('x-interaction-type'), false); @@ -130,7 +126,7 @@ test('Compact headers absent when the last message is assistant-role with compac ], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.has('x-initiator'), false); assertEquals(ctx.headers.has('x-interaction-type'), false); @@ -152,7 +148,7 @@ test('Compact headers absent when the last assistant message is a multi-block co ], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.has('x-initiator'), false); assertEquals(ctx.headers.has('x-interaction-type'), false); @@ -171,7 +167,7 @@ test('Auto-continue absent when the assistant role carries the resume prompt ver ], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.has('x-initiator'), false); assertEquals(ctx.headers.has('x-interaction-type'), false); @@ -189,7 +185,7 @@ test('Auto-continue marks Claude Code resume prompts with x-initiator: agent onl ], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); assertEquals(ctx.headers.has('x-interaction-type'), false); @@ -207,7 +203,7 @@ test('Auto-continue marks OpenCode primary continuation prompts with x-initiator ], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); assertEquals(ctx.headers.has('x-interaction-type'), false); @@ -225,7 +221,7 @@ test('Auto-continue marks OpenCode media-eviction continuation prompts with x-in ], }); - await withCompactHeadersSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetCompactHeaders, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); assertEquals(ctx.headers.has('x-interaction-type'), false); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-initiator-header_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-initiator-header_test.ts similarity index 79% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-initiator-header_test.ts rename to packages/provider-copilot/__tests__/stages/anthropic-messages/set-initiator-header_test.ts index 86e88414cd..e1e9903b97 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-initiator-header_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-initiator-header_test.ts @@ -1,19 +1,15 @@ import { test } from 'vitest'; -import { withInitiatorHeaderSet } from '../../../src/interceptors/anthropic-messages/set-initiator-header.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { copilotAnthropicMessagesSetInitiatorHeader } from '../../../src/stages/anthropic-messages/set-initiator-header.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type AnthropicMessagesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesProbe => ({ payload, headers: new Headers(), anthropicBeta: [], @@ -27,7 +23,7 @@ test('Anthropic Messages initiator is user when the last message is a plain user messages: [{ role: 'user', content: 'hello' }], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'user'); }); @@ -47,7 +43,7 @@ test('Anthropic Messages initiator is user when the last user turn mixes text an ], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'user'); }); @@ -66,7 +62,7 @@ test('Anthropic Messages initiator is agent when the last user turn is entirely ], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); @@ -81,7 +77,7 @@ test('Anthropic Messages initiator is agent when the final message is from the a ], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-interaction-id-header_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-interaction-id-header_test.ts similarity index 78% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-interaction-id-header_test.ts rename to packages/provider-copilot/__tests__/stages/anthropic-messages/set-interaction-id-header_test.ts index c042cfb3b0..8fd6793abd 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-interaction-id-header_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-interaction-id-header_test.ts @@ -1,20 +1,16 @@ import { parse, validate, version } from 'uuid'; import { test } from 'vitest'; -import { withInteractionIdHeaderSet } from '../../../src/interceptors/anthropic-messages/set-interaction-id-header.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { copilotAnthropicMessagesSetInteractionIdHeader } from '../../../src/stages/anthropic-messages/set-interaction-id-header.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type AnthropicMessagesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assert, assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesProbe => ({ payload, headers: new Headers(), anthropicBeta: [], @@ -39,7 +35,7 @@ const SESS_ALONE_UUID = '19be0c9e-9a65-43eb-be80-efb2770d4510'; test('Interaction-id forwarded as a SHA-256 UUID for the legacy fingerprint with both halves', async () => { const ctx = invocation(payloadWith('user_acct-1_account__session_sess-legacy')); - await withInteractionIdHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInteractionIdHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-interaction-id'), SESS_LEGACY_UUID); }); @@ -47,7 +43,7 @@ test('Interaction-id forwarded as a SHA-256 UUID for the legacy fingerprint with test('Interaction-id forwarded as a SHA-256 UUID for the JSON fingerprint carrying session_id', async () => { const ctx = invocation(payloadWith(JSON.stringify({ device_id: 'dev', session_id: 'sess-json' }))); - await withInteractionIdHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInteractionIdHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-interaction-id'), SESS_JSON_UUID); }); @@ -57,7 +53,7 @@ test('Interaction-id forwarded as a SHA-256 UUID even when only the sessionId ha // device_id / account_uuid; we still want trace correlation in that case. const ctx = invocation(payloadWith(JSON.stringify({ session_id: 'sess-alone' }))); - await withInteractionIdHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInteractionIdHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-interaction-id'), SESS_ALONE_UUID); }); @@ -66,8 +62,8 @@ test('Interaction-id hashing is deterministic across repeated invocations', asyn const first = invocation(payloadWith(JSON.stringify({ session_id: 'sess-alone' }))); const second = invocation(payloadWith(JSON.stringify({ session_id: 'sess-alone' }))); - await withInteractionIdHeaderSet(first, stubRequest, okEvents); - await withInteractionIdHeaderSet(second, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInteractionIdHeader, first, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInteractionIdHeader, second, okEvents); assertEquals(first.headers.get('x-interaction-id'), second.headers.get('x-interaction-id')); }); @@ -75,7 +71,7 @@ test('Interaction-id hashing is deterministic across repeated invocations', asyn test('Interaction-id has RFC UUIDv4 version and variant bits', async () => { const ctx = invocation(payloadWith(JSON.stringify({ session_id: 'sess-shape-check' }))); - await withInteractionIdHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInteractionIdHeader, ctx, okEvents); const value = ctx.headers.get('x-interaction-id'); assert(value !== null && validate(value)); @@ -87,7 +83,7 @@ test('Interaction-id has RFC UUIDv4 version and variant bits', async () => { test('Interaction-id absent when metadata.user_id is missing', async () => { const ctx = invocation(payloadWith(undefined)); - await withInteractionIdHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInteractionIdHeader, ctx, okEvents); assertEquals(ctx.headers.has('x-interaction-id'), false); }); @@ -95,7 +91,7 @@ test('Interaction-id absent when metadata.user_id is missing', async () => { test('Interaction-id absent when metadata.user_id carries no session marker', async () => { const ctx = invocation(payloadWith('user_acct-1_account')); - await withInteractionIdHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetInteractionIdHeader, ctx, okEvents); assertEquals(ctx.headers.has('x-interaction-id'), false); }); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-vision-header_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-vision-header_test.ts similarity index 81% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-vision-header_test.ts rename to packages/provider-copilot/__tests__/stages/anthropic-messages/set-vision-header_test.ts index 6e6677b542..50f4bd1fcd 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/set-vision-header_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/set-vision-header_test.ts @@ -1,19 +1,15 @@ import { test } from 'vitest'; -import { withVisionHeaderSet } from '../../../src/interceptors/anthropic-messages/set-vision-header.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { copilotAnthropicMessagesSetVisionHeader } from '../../../src/stages/anthropic-messages/set-vision-header.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type AnthropicMessagesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesProbe => ({ payload, headers: new Headers(), anthropicBeta: [], @@ -35,7 +31,7 @@ test('Anthropic Messages vision header set when a top-level image block is prese ], }); - await withVisionHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetVisionHeader, ctx, okEvents); assertEquals(ctx.headers.get('copilot-vision-request'), 'true'); }); @@ -60,7 +56,7 @@ test('Anthropic Messages vision header set when an image is nested inside tool_r ], }); - await withVisionHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetVisionHeader, ctx, okEvents); assertEquals(ctx.headers.get('copilot-vision-request'), 'true'); }); @@ -83,7 +79,7 @@ test('Anthropic Messages vision header absent when no image is present', async ( ], }); - await withVisionHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesSetVisionHeader, ctx, okEvents); assertEquals(ctx.headers.has('copilot-vision-request'), false); }); diff --git a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/strip-cache-control-extensions_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/strip-cache-control-extensions_test.ts similarity index 85% rename from packages/provider-copilot/__tests__/interceptors/anthropic-messages/strip-cache-control-extensions_test.ts rename to packages/provider-copilot/__tests__/stages/anthropic-messages/strip-cache-control-extensions_test.ts index 68e7f027e8..c2a72743dc 100644 --- a/packages/provider-copilot/__tests__/interceptors/anthropic-messages/strip-cache-control-extensions_test.ts +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/strip-cache-control-extensions_test.ts @@ -1,19 +1,15 @@ import { test } from 'vitest'; -import { withCacheControlExtensionsStripped } from '../../../src/interceptors/anthropic-messages/strip-cache-control-extensions.ts'; -import type { AnthropicMessagesBoundaryCtx } from '../../../src/interceptors/anthropic-messages/types.ts'; +import { copilotAnthropicMessagesStripCacheControlExtensions } from '../../../src/stages/anthropic-messages/strip-cache-control-extensions.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type AnthropicMessagesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesBoundaryCtx => ({ +const invocation = (payload: AnthropicMessagesPayload): AnthropicMessagesProbe => ({ payload, headers: new Headers(), anthropicBeta: [], @@ -52,7 +48,7 @@ test('strips scope and ttl from system, tools, and message content blocks while ], }); - await withCacheControlExtensionsStripped(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesStripCacheControlExtensions, ctx, okEvents); assertEquals(ctx.payload.system, [{ type: 'text', text: 'sys', cache_control: { type: 'ephemeral' } }]); assertEquals(ctx.payload.tools, [{ name: 't', input_schema: { type: 'object' }, cache_control: { type: 'ephemeral' } }]); @@ -74,7 +70,7 @@ test('deletes cache_control entirely if no recognised field survives the strip', messages: [{ role: 'user', content: 'hi' }], }); - await withCacheControlExtensionsStripped(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesStripCacheControlExtensions, ctx, okEvents); assertEquals(ctx.payload.system, [{ type: 'text', text: 'sys' }]); }); @@ -88,7 +84,7 @@ test('no-op when no cache_control is present anywhere', async () => { tools: [{ name: 't', input_schema: { type: 'object' } }], }); - await withCacheControlExtensionsStripped(ctx, stubRequest, okEvents); + await applyProviderStage(copilotAnthropicMessagesStripCacheControlExtensions, ctx, okEvents); assertEquals(ctx.payload.system, 'plain system'); assertEquals(ctx.payload.tools, [{ name: 't', input_schema: { type: 'object' } }]); diff --git a/packages/provider-copilot/__tests__/stages/anthropic-messages/strip-eager-input-streaming_test.ts b/packages/provider-copilot/__tests__/stages/anthropic-messages/strip-eager-input-streaming_test.ts new file mode 100644 index 0000000000..e00c540901 --- /dev/null +++ b/packages/provider-copilot/__tests__/stages/anthropic-messages/strip-eager-input-streaming_test.ts @@ -0,0 +1,17 @@ +import { expect, test } from 'vitest'; + +import { copilotAnthropicMessagesStripEagerInputStreaming } from '../../../src/stages/anthropic-messages/strip-eager-input-streaming.ts'; +import { applyProviderStage, stubProviderModel } from '@floway-dev/test-utils'; + +test('removes eager_input_streaming while preserving unaffected tool identity and the immutable input', async () => { + const kept = { name: 'kept', input_schema: { type: 'object' as const, properties: {} } }; + const removed = { name: 'changed', input_schema: { type: 'object' as const, properties: {} }, eager_input_streaming: true }; + const payload = { model: 'claude', max_tokens: 16, messages: [], tools: [kept, removed] }; + const probe = { payload, headers: new Headers(), anthropicBeta: [], model: stubProviderModel() }; + await applyProviderStage(copilotAnthropicMessagesStripEagerInputStreaming, probe, async () => {}); + expect(probe.payload.tools[0]).toBe(kept); + expect(probe.payload.tools[1]).not.toBe(removed); + expect(probe.payload.tools[1]).not.toHaveProperty('eager_input_streaming'); + expect(payload.tools[1]).toBe(removed); + expect(removed.eager_input_streaming).toBe(true); +}); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts b/packages/provider-copilot/__tests__/stages/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts similarity index 88% rename from packages/provider-copilot/__tests__/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts rename to packages/provider-copilot/__tests__/stages/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts index 44ae3ee9de..3ae90bf500 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-chat-completions/abort-on-tool-argument-whitespace_test.ts @@ -1,22 +1,18 @@ import { test } from 'vitest'; -import { withToolArgumentWhitespaceAborted } from '../../../src/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace.ts'; -import type { OpenAIChatCompletionsBoundaryCtx } from '../../../src/interceptors/openai-chat-completions/types.ts'; -import { MAX_CONSECUTIVE_WHITESPACE } from '../../../src/interceptors/shared/whitespace-overflow.ts'; +import { copilotOpenAIChatCompletionsAbortToolWhitespace } from '../../../src/stages/openai-chat-completions/abort-on-tool-argument-whitespace.ts'; +import { MAX_CONSECUTIVE_WHITESPACE } from '../../../src/tool-argument-whitespace.ts'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type OpenAIChatCompletionsProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assert, assertEquals, assertStringIncludes, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const invocation = (): OpenAIChatCompletionsBoundaryCtx => ({ +const invocation = (): OpenAIChatCompletionsProbe => ({ payload: { model: 'test-model', messages: [] }, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiChatCompletions: {} } }), }); -const stubRequest = {}; - const baseChunk = (overrides: Partial): OpenAIChatCompletionsStreamEvent => ({ id: 'chatcmpl_1', object: 'chat.completion.chunk', @@ -34,7 +30,7 @@ const collect = async (result: ExecuteResult[]): Promise[]> => { - const result = await withToolArgumentWhitespaceAborted(invocation(), stubRequest, () => + const result = await applyProviderStage(copilotOpenAIChatCompletionsAbortToolWhitespace, invocation(), () => Promise.resolve( eventResult( (async function* () { @@ -53,7 +49,7 @@ const runExpectingThrow = async (frames: ProtocolFrame { diff --git a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/attach-cache-control-markers_test.ts b/packages/provider-copilot/__tests__/stages/openai-chat-completions/attach-cache-control-markers_test.ts similarity index 81% rename from packages/provider-copilot/__tests__/interceptors/openai-chat-completions/attach-cache-control-markers_test.ts rename to packages/provider-copilot/__tests__/stages/openai-chat-completions/attach-cache-control-markers_test.ts index bb51f13ce0..a03ab56431 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/attach-cache-control-markers_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-chat-completions/attach-cache-control-markers_test.ts @@ -1,19 +1,15 @@ import { test } from 'vitest'; -import { type CopilotCacheableMessage, withCacheControlMarkersAttached } from '../../../src/interceptors/openai-chat-completions/attach-cache-control-markers.ts'; -import type { OpenAIChatCompletionsBoundaryCtx } from '../../../src/interceptors/openai-chat-completions/types.ts'; +import { type CopilotCacheableMessage, copilotOpenAIChatCompletionsAttachCacheControlMarkers } from '../../../src/stages/openai-chat-completions/attach-cache-control-markers.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent, OpenAIChatCompletionsMessage } from '@floway-dev/protocols/openai-chat-completions'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type OpenAIChatCompletionsProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assert, assertEquals, assertFalse, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (messages: OpenAIChatCompletionsMessage[]): OpenAIChatCompletionsBoundaryCtx => ({ +const invocation = (messages: OpenAIChatCompletionsMessage[]): OpenAIChatCompletionsProbe => ({ payload: { model: 'gpt-test', messages }, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiChatCompletions: {} } }), @@ -32,7 +28,7 @@ test('OpenAI Chat Completions cache markers attach to first two systems and last { role: 'assistant', content: 'assistant 2' }, ]); - await withCacheControlMarkersAttached(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsAttachCacheControlMarkers, ctx, okEvents); assertEquals(markedIndexes(ctx.payload.messages), [0, 1, 4, 5]); for (const index of [0, 1, 4, 5]) { @@ -45,7 +41,7 @@ test('OpenAI Chat Completions cache markers handle a single system message', asy { role: 'system', content: 'only system' }, ]); - await withCacheControlMarkersAttached(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsAttachCacheControlMarkers, ctx, okEvents); assertEquals(markedIndexes(ctx.payload.messages), [0]); }); @@ -58,7 +54,7 @@ test('OpenAI Chat Completions cache markers attach to last two non-systems when { role: 'assistant', content: 'assistant 2' }, ]); - await withCacheControlMarkersAttached(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsAttachCacheControlMarkers, ctx, okEvents); assertEquals(markedIndexes(ctx.payload.messages), [2, 3]); }); @@ -69,7 +65,7 @@ test('OpenAI Chat Completions cache markers skip empty string content', async () { role: 'user', content: '' }, ]); - await withCacheControlMarkersAttached(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsAttachCacheControlMarkers, ctx, okEvents); assertEquals(markedIndexes(ctx.payload.messages), []); }); @@ -80,7 +76,7 @@ test('OpenAI Chat Completions cache markers skip empty array content', async () { role: 'user', content: [] }, ]); - await withCacheControlMarkersAttached(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsAttachCacheControlMarkers, ctx, okEvents); assertEquals(markedIndexes(ctx.payload.messages), []); }); @@ -95,7 +91,7 @@ test('OpenAI Chat Completions cache markers only mark first two of five systems' { role: 'user', content: 'user 1' }, ]); - await withCacheControlMarkersAttached(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsAttachCacheControlMarkers, ctx, okEvents); // First two systems + last (and only) non-system. assertEquals(markedIndexes(ctx.payload.messages), [0, 1, 5]); @@ -111,7 +107,7 @@ test('OpenAI Chat Completions cache markers are independent object instances per { role: 'user', content: 'user 1' }, ]); - await withCacheControlMarkersAttached(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsAttachCacheControlMarkers, ctx, okEvents); const a = (ctx.payload.messages[0] as CopilotCacheableMessage).copilot_cache_control; const b = (ctx.payload.messages[1] as CopilotCacheableMessage).copilot_cache_control; diff --git a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/compress-images_test.ts b/packages/provider-copilot/__tests__/stages/openai-chat-completions/compress-images_test.ts similarity index 87% rename from packages/provider-copilot/__tests__/interceptors/openai-chat-completions/compress-images_test.ts rename to packages/provider-copilot/__tests__/stages/openai-chat-completions/compress-images_test.ts index 1da921fbfa..9bf8687b2b 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/compress-images_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-chat-completions/compress-images_test.ts @@ -1,16 +1,12 @@ import { test } from 'vitest'; -import { withInlineImagesCompressed } from '../../../src/interceptors/openai-chat-completions/compress-images.ts'; -import type { OpenAIChatCompletionsBoundaryCtx } from '../../../src/interceptors/openai-chat-completions/types.ts'; +import { copilotOpenAIChatCompletionsCompressImages } from '../../../src/stages/openai-chat-completions/compress-images.ts'; import { type ImageProcessor, initImageProcessor } from '@floway-dev/platform'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent, OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type OpenAIChatCompletionsProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assert, assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); @@ -18,7 +14,7 @@ const fixedProcessor: ImageProcessor = { compressToWebp: () => Promise.resolve(new Uint8Array([1, 2, 3])), }; -const invocation = (payload: OpenAIChatCompletionsPayload): OpenAIChatCompletionsBoundaryCtx => ({ +const invocation = (payload: OpenAIChatCompletionsPayload): OpenAIChatCompletionsProbe => ({ payload, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiChatCompletions: {} } }), @@ -47,7 +43,7 @@ test('rewrites a base64 image_url data URL to a WebP data URL', async () => { }; const ctx = invocation(payload); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsCompressImages, ctx, okEvents); assertEquals(imageUrl(ctx.payload), 'data:image/webp;base64,AQID'); assertEquals(imageUrl(payload), 'data:image/png;base64,AAAA'); @@ -79,7 +75,7 @@ test('leaves remote https image references untouched', async () => { }; const ctx = invocation(payload); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsCompressImages, ctx, okEvents); assertEquals(imageUrl(ctx.payload), 'https://example.com/cat.png'); assert(ctx.payload === payload); @@ -114,7 +110,7 @@ test('compresses each unique inline image only once when the same data URL repea ], }); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsCompressImages, ctx, okEvents); assertEquals(calls, 2); }); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/set-initiator-header_test.ts b/packages/provider-copilot/__tests__/stages/openai-chat-completions/set-initiator-header_test.ts similarity index 79% rename from packages/provider-copilot/__tests__/interceptors/openai-chat-completions/set-initiator-header_test.ts rename to packages/provider-copilot/__tests__/stages/openai-chat-completions/set-initiator-header_test.ts index 3ed8f66085..923844ab6a 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/set-initiator-header_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-chat-completions/set-initiator-header_test.ts @@ -1,19 +1,15 @@ import { test } from 'vitest'; -import { withInitiatorHeaderSet } from '../../../src/interceptors/openai-chat-completions/set-initiator-header.ts'; -import type { OpenAIChatCompletionsBoundaryCtx } from '../../../src/interceptors/openai-chat-completions/types.ts'; +import { copilotOpenAIChatCompletionsSetInitiatorHeader } from '../../../src/stages/openai-chat-completions/set-initiator-header.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent, OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type OpenAIChatCompletionsProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: OpenAIChatCompletionsPayload): OpenAIChatCompletionsBoundaryCtx => ({ +const invocation = (payload: OpenAIChatCompletionsPayload): OpenAIChatCompletionsProbe => ({ payload, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiChatCompletions: {} } }), @@ -25,7 +21,7 @@ test('OpenAI Chat Completions initiator is user when the last message is from th messages: [{ role: 'user', content: 'hello' }], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'user'); }); @@ -39,7 +35,7 @@ test('OpenAI Chat Completions initiator is agent when the last message is an ass ], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); @@ -58,7 +54,7 @@ test('OpenAI Chat Completions initiator is agent when the last message is a tool ], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); @@ -78,7 +74,7 @@ test('OpenAI Chat Completions initiator follows the final user role for a lifted ], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'user'); }); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/set-vision-header_test.ts b/packages/provider-copilot/__tests__/stages/openai-chat-completions/set-vision-header_test.ts similarity index 75% rename from packages/provider-copilot/__tests__/interceptors/openai-chat-completions/set-vision-header_test.ts rename to packages/provider-copilot/__tests__/stages/openai-chat-completions/set-vision-header_test.ts index 63605f9452..5f540136d9 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-chat-completions/set-vision-header_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-chat-completions/set-vision-header_test.ts @@ -1,19 +1,15 @@ import { test } from 'vitest'; -import { withVisionHeaderSet } from '../../../src/interceptors/openai-chat-completions/set-vision-header.ts'; -import type { OpenAIChatCompletionsBoundaryCtx } from '../../../src/interceptors/openai-chat-completions/types.ts'; +import { copilotOpenAIChatCompletionsSetVisionHeader } from '../../../src/stages/openai-chat-completions/set-vision-header.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent, OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type OpenAIChatCompletionsProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: OpenAIChatCompletionsPayload): OpenAIChatCompletionsBoundaryCtx => ({ +const invocation = (payload: OpenAIChatCompletionsPayload): OpenAIChatCompletionsProbe => ({ payload, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiChatCompletions: {} } }), @@ -33,7 +29,7 @@ test('OpenAI Chat Completions vision header set when an image_url content part i ], }); - await withVisionHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsSetVisionHeader, ctx, okEvents); assertEquals(ctx.headers.get('copilot-vision-request'), 'true'); }); @@ -47,7 +43,7 @@ test('OpenAI Chat Completions vision header absent when content is pure text', a ], }); - await withVisionHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIChatCompletionsSetVisionHeader, ctx, okEvents); assertEquals(ctx.headers.has('copilot-vision-request'), false); }); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/abort-on-tool-argument-whitespace_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/abort-on-tool-argument-whitespace_test.ts similarity index 84% rename from packages/provider-copilot/__tests__/interceptors/openai-responses/abort-on-tool-argument-whitespace_test.ts rename to packages/provider-copilot/__tests__/stages/openai-responses/abort-on-tool-argument-whitespace_test.ts index 0436fecb30..405fc04faa 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/abort-on-tool-argument-whitespace_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/abort-on-tool-argument-whitespace_test.ts @@ -1,14 +1,13 @@ import { test } from 'vitest'; -import { withToolArgumentWhitespaceAborted } from '../../../src/interceptors/openai-responses/abort-on-tool-argument-whitespace.ts'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; -import { MAX_CONSECUTIVE_WHITESPACE } from '../../../src/interceptors/shared/whitespace-overflow.ts'; +import { copilotOpenAIResponsesAbortToolWhitespace } from '../../../src/stages/openai-responses/abort-on-tool-argument-whitespace.ts'; +import { MAX_CONSECUTIVE_WHITESPACE } from '../../../src/tool-argument-whitespace.ts'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import type { ProviderOpenAIResponsesResult } from '@floway-dev/provider'; +import { applyProviderStage, type OpenAIResponsesProbe, type ProviderOpenAIResponsesResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; -const invocation = (): OpenAIResponsesBoundaryCtx => ({ +const invocation = (): OpenAIResponsesProbe => ({ payload: { model: 'test-model', input: [], @@ -28,8 +27,6 @@ const invocation = (): OpenAIResponsesBoundaryCtx => ({ action: 'generate', }); -const stubRequest = {}; - const argsDelta = (outputIndex: number, delta: string): OpenAIResponsesStreamEvent => ({ type: 'response.function_call_arguments.delta', @@ -46,7 +43,7 @@ const collect = async (result: ProviderOpenAIResponsesResult): Promise[]): Promise[]> => { - const result = await withToolArgumentWhitespaceAborted(invocation(), stubRequest, () => + const result = await applyProviderStage(copilotOpenAIResponsesAbortToolWhitespace, invocation(), () => Promise.resolve({ action: 'generate', ok: true, @@ -74,7 +71,7 @@ test('aborts and emits an error event + done when whitespace exceeds the thresho const wsDelta = '\n'.repeat(MAX_CONSECUTIVE_WHITESPACE + 1); const frames: ProtocolFrame[] = [ eventFrame(argsDelta(0, wsDelta)), - // Should not be observed: interceptor aborts on the first offending delta. + // The stage aborts before the second delta can reach the client. eventFrame(argsDelta(0, '\n\n\n')), doneFrame(), ]; diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/action-pivot_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/action-pivot_test.ts similarity index 56% rename from packages/provider-copilot/__tests__/interceptors/openai-responses/action-pivot_test.ts rename to packages/provider-copilot/__tests__/stages/openai-responses/action-pivot_test.ts index 0ed25c8408..55daa65fcb 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/action-pivot_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/action-pivot_test.ts @@ -1,42 +1,31 @@ import { test, vi } from 'vitest'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; -import type { Interceptor } from '@floway-dev/interceptor'; -import type { ProviderOpenAIResponsesResult } from '@floway-dev/provider'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; import { assertEquals } from '@floway-dev/test-utils'; -// `provider.callOpenAIResponses` runs the boundary chain and the terminal -// switches on `ctx.action`, not on the closure-captured `action` parameter -// — so an interceptor that flips `ctx.action` mid-chain reroutes dispatch. -// To prove that contract end-to-end, swap the boundary chain for one -// containing a pivot interceptor (compact → generate) via `vi.mock`, -// then drive `provider.callOpenAIResponses(model, body, 'compact', ...)`. The -// observed wire request must be the streaming /responses shape -// (stream:true, no `compaction_trigger`), and the typed result must -// surface as the `action: 'generate'` variant. -const pivotCompactToGenerate: Interceptor = async (ctx, _env, run) => { - ctx.action = 'generate'; - return await run(); -}; +type Facts = { 'request.provider.payload': CanonicalOpenAIResponsesPayload; 'request.provider.responsesAction': 'generate' | 'compact' }; -vi.mock('../../../src/interceptors/openai-responses/index.ts', async () => { - const original = await vi.importActual('../../../src/interceptors/openai-responses/index.ts'); - return { - ...original, - COPILOT_OPENAI_RESPONSES_BOUNDARY: [...original.COPILOT_OPENAI_RESPONSES_BOUNDARY, pivotCompactToGenerate], - }; -}); +vi.mock('../../../src/stages/openai-responses/force-store-false.ts', () => ({ + copilotOpenAIResponsesForceStoreFalse: defineStage({ + name: 'pivotCompactToGenerate', + through: { + request: { needs: ['request.provider.payload', 'request.provider.responsesAction'], consumes: ['request.provider.payload', 'request.provider.responsesAction'], provides: ['request.provider.payload', 'request.provider.responsesAction'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => move({ ...await next(move({ ...facts, 'request.provider.payload': { ...facts['request.provider.payload'], store: false }, 'request.provider.responsesAction': 'generate' })) }), + }), +})); -// Imports below MUST follow the vi.mock so the provider module resolves -// against the mocked chain on first import. +// Resolve the provider assembly after installing the request-stage fixture. const { clearInProcessCopilotTokenCache } = await import('../../../src/auth.ts'); const { createCopilotProvider } = await import('../../../src/provider.ts'); const { createInMemoryImageProcessor, initImageProcessor } = await import('@floway-dev/platform'); const { directFetcher, initProviderRepo } = await import('@floway-dev/provider'); -const { jsonResponse, noopUpstreamCallOptions, sseResponse, withMockedFetch } = await import('@floway-dev/test-utils'); +const { collectChatProviderPipeline, jsonResponse, noopUpstreamCallOptions, sseResponse, withMockedFetch } = await import('@floway-dev/test-utils'); type UpstreamRecord = import('@floway-dev/provider').UpstreamRecord; -test('Copilot provider terminal dispatches on post-chain ctx.action (interceptor flip compact→generate routes to the streaming generate path)', async () => { +test('Copilot dispatch and decoding follow the action changed by a stage (compact→generate)', async () => { const upstream: UpstreamRecord = { id: 'up_copilot_pivot', kind: 'copilot', @@ -98,17 +87,15 @@ test('Copilot provider terminal dispatches on post-chain ctx.action (interceptor }, async () => { const [providerModel] = await provider.getProvidedModels(directFetcher); - const result = await provider.callOpenAIResponses(providerModel, { + const result = await collectChatProviderPipeline(instance, 'openaiResponsesCompact', providerModel, { input: [{ type: 'message', role: 'user', content: 'hi' }], - }, 'compact', undefined, noopUpstreamCallOptions()); - if (!result.ok) throw new Error('expected ok result'); - if (result.action !== 'generate') throw new Error(`expected generate variant after pivot, got ${result.action}`); + }, undefined, noopUpstreamCallOptions()); + if (result.output === null || !('kind' in result.output) || result.output.kind !== 'stream') throw new Error('expected generate stream after action pivot'); + assertEquals(result.facts['response.provider.responsesAction'], 'generate'); }, ); if (!openaiResponsesBody) throw new Error('expected /responses to be hit'); - // Stream-true wire shape proves the terminal took the generate branch, - // not the synth-via-trigger compact branch. assertEquals(openaiResponsesBody.stream, true); const wireInput = openaiResponsesBody.input as Array<{ type: string }>; assertEquals(wireInput.some(item => item.type === 'compaction_trigger'), false); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/compress-images_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/compress-images_test.ts similarity index 81% rename from packages/provider-copilot/__tests__/interceptors/openai-responses/compress-images_test.ts rename to packages/provider-copilot/__tests__/stages/openai-responses/compress-images_test.ts index 33bec4785f..061d7beddb 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/compress-images_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/compress-images_test.ts @@ -1,16 +1,12 @@ -import { test } from 'vitest'; +import { expect, test } from 'vitest'; -import { withInlineImagesCompressed } from '../../../src/interceptors/openai-responses/compress-images.ts'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; +import { copilotOpenAIResponsesCompressImages } from '../../../src/stages/openai-responses/compress-images.ts'; import { type ImageProcessor, initImageProcessor } from '@floway-dev/platform'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesInputContent, OpenAIResponsesInputImage, OpenAIResponsesInputItem, OpenAIResponsesStreamEvent, OpenAIResponsesToolOutputContent } from '@floway-dev/protocols/openai-responses'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type OpenAIResponsesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assert, assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); @@ -18,7 +14,7 @@ const fixedProcessor: ImageProcessor = { compressToWebp: () => Promise.resolve(new Uint8Array([1, 2, 3])), }; -const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBoundaryCtx => ({ +const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesProbe => ({ payload, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiResponses: {} } }), @@ -55,7 +51,7 @@ test.each(Object.entries(contentContainers))('compresses inline images in %s', a }; const ctx = invocation(payload); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIResponsesCompressImages, ctx, okEvents); assertEquals(imageUrlOf(ctx.payload.input[0]), 'data:image/webp;base64,AQID'); assertEquals(imageUrlOf(sourceItem), 'data:image/png;base64,AAAA'); @@ -81,7 +77,7 @@ test.each(Object.entries(contentContainers))('leaves remote images in %s untouch }; const ctx = invocation(payload); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIResponsesCompressImages, ctx, okEvents); assertEquals(imageUrlOf(ctx.payload.input[0]), 'https://example.com/cat.png'); assert(ctx.payload === payload); @@ -127,7 +123,7 @@ test('compresses each unique inline image only once when the same data URL appea ], }); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIResponsesCompressImages, ctx, okEvents); // Two unique data URLs across five targets → exactly two compress calls. assertEquals(calls, 2); @@ -151,16 +147,24 @@ test('reuses its compressed image when the same attempt payload is retried', asy }], }); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIResponsesCompressImages, ctx, okEvents); + await applyProviderStage(copilotOpenAIResponsesCompressImages, ctx, okEvents); assertEquals(calls, 1); assertEquals(imageUrlOf(ctx.payload.input[0]), 'data:image/webp;base64,AQID'); const item = ctx.payload.input[0]; - if (item.type !== 'message' || !Array.isArray(item.content) || item.content[0]?.type !== 'input_image') throw new Error('expected image content'); - item.content[0].image_url = 'data:image/png;base64,BBBB'; - await withInlineImagesCompressed(ctx, stubRequest, okEvents); + if (item.type !== 'message' || !Array.isArray(item.content)) throw new Error('expected image message'); + const content = item.content; + const image = content[0]; + if (image?.type !== 'input_image') throw new Error('expected image content'); + expect(() => { image.image_url = 'data:image/png;base64,BBBB'; }).toThrow(); + ctx.payload = { + ...ctx.payload, input: ctx.payload.input.map((candidate, index) => index === 0 + ? { ...item, content: content.map((part, partIndex) => partIndex === 0 ? { ...part, image_url: 'data:image/png;base64,BBBB' } : part) } + : candidate), + }; + await applyProviderStage(copilotOpenAIResponsesCompressImages, ctx, okEvents); assertEquals(calls, 2); }); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/fill-empty-namespace-descriptions_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/fill-empty-namespace-descriptions_test.ts similarity index 80% rename from packages/provider-copilot/__tests__/interceptors/openai-responses/fill-empty-namespace-descriptions_test.ts rename to packages/provider-copilot/__tests__/stages/openai-responses/fill-empty-namespace-descriptions_test.ts index d50a82f583..ee7518a7e6 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/fill-empty-namespace-descriptions_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/fill-empty-namespace-descriptions_test.ts @@ -1,11 +1,11 @@ import { test } from 'vitest'; -import { withEmptyNamespaceDescriptionsFilled } from '../../../src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; +import { copilotOpenAIResponsesFillEmptyNamespaceDescriptions } from '../../../src/stages/openai-responses/fill-empty-namespace-descriptions.ts'; import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; +import { applyProviderStage, type OpenAIResponsesProbe } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; -const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBoundaryCtx => ({ +const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesProbe => ({ payload, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiResponses: {} } }), @@ -36,7 +36,7 @@ test('fills Codex OpenAI Responses Lite namespace descriptions before Copilot di }], }); - await withEmptyNamespaceDescriptionsFilled(ctx, {}, async () => {}); + await applyProviderStage(copilotOpenAIResponsesFillEmptyNamespaceDescriptions, ctx, async () => {}); const [item] = ctx.payload.input; if (item?.type !== 'additional_tools') throw new Error('expected additional_tools input'); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/force-store-false_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/force-store-false_test.ts similarity index 72% rename from packages/provider-copilot/__tests__/interceptors/openai-responses/force-store-false_test.ts rename to packages/provider-copilot/__tests__/stages/openai-responses/force-store-false_test.ts index 6e69cfca35..3009aad07b 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/force-store-false_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/force-store-false_test.ts @@ -1,19 +1,15 @@ import { test } from 'vitest'; -import { withStoreForcedFalse } from '../../../src/interceptors/openai-responses/force-store-false.ts'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; +import { copilotOpenAIResponsesForceStoreFalse } from '../../../src/stages/openai-responses/force-store-false.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type OpenAIResponsesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBoundaryCtx => ({ +const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesProbe => ({ payload, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiResponses: {} } }), @@ -23,7 +19,7 @@ const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBo test('forces store:false when the caller requested store:true', async () => { const ctx = invocation({ model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }], store: true }); - await withStoreForcedFalse(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIResponsesForceStoreFalse, ctx, okEvents); assertEquals(ctx.payload.store, false); }); @@ -31,7 +27,7 @@ test('forces store:false when the caller requested store:true', async () => { test('sets store:false when the caller omitted store', async () => { const ctx = invocation({ model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }] }); - await withStoreForcedFalse(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIResponsesForceStoreFalse, ctx, okEvents); assertEquals(ctx.payload.store, false); }); @@ -39,7 +35,7 @@ test('sets store:false when the caller omitted store', async () => { test('leaves an explicit store:false untouched', async () => { const ctx = invocation({ model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'hello' }], store: false }); - await withStoreForcedFalse(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIResponsesForceStoreFalse, ctx, okEvents); assertEquals(ctx.payload.store, false); }); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/item-id-membrane_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/item-id-membrane_test.ts similarity index 97% rename from packages/provider-copilot/__tests__/interceptors/openai-responses/item-id-membrane_test.ts rename to packages/provider-copilot/__tests__/stages/openai-responses/item-id-membrane_test.ts index 382895cbeb..7b04832154 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/item-id-membrane_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/item-id-membrane_test.ts @@ -1,14 +1,13 @@ import { expect, test } from 'vitest'; -import { unwrapCopilotItemId, wrapCopilotItemId } from '../../../src/interceptors/openai-responses/item-id-carrier.ts'; -import { withCopilotOpenAIResponsesItemIdMembrane } from '../../../src/interceptors/openai-responses/item-id-membrane.ts'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; +import { unwrapCopilotItemId, wrapCopilotItemId } from '../../../src/opaque-item-id.ts'; +import { copilotOpenAIResponsesItemIdMembrane } from '../../../src/stages/openai-responses/item-id-membrane.ts'; import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; import { openaiResponsesResultToEvents, type OpenAIResponsesInputItem, type OpenAIResponsesOutputItem, type OpenAIResponsesResult, type OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import type { ProviderOpenAIResponsesResult } from '@floway-dev/provider'; +import { applyProviderStage, type OpenAIResponsesProbe, type ProviderOpenAIResponsesResult } from '@floway-dev/test-utils'; import { stubProviderModel } from '@floway-dev/test-utils'; -const invocation = (input: OpenAIResponsesInputItem[] = []): OpenAIResponsesBoundaryCtx => ({ +const invocation = (input: OpenAIResponsesInputItem[] = []): OpenAIResponsesProbe => ({ payload: { model: 'test-model', input, @@ -49,13 +48,13 @@ const collect = async (result: ProviderOpenAIResponsesResult): Promise> | ProtocolFrame[], ctx = invocation(), -): Promise<{ result: ProviderOpenAIResponsesResult; ctx: OpenAIResponsesBoundaryCtx }> => { +): Promise<{ result: ProviderOpenAIResponsesResult; ctx: OpenAIResponsesProbe }> => { const iterable = Symbol.asyncIterator in frames ? frames as AsyncIterable> : (async function* () { yield* frames as ProtocolFrame[]; })(); - const result = await withCopilotOpenAIResponsesItemIdMembrane(ctx, {}, () => Promise.resolve({ - action: 'generate', - ok: true, + const result = await applyProviderStage(copilotOpenAIResponsesItemIdMembrane, ctx, () => Promise.resolve({ + action: 'generate' as const, + ok: true as const, events: iterable, modelKey: 'test-model', })); @@ -404,7 +403,7 @@ test('restores owned blob ids for Copilot input and leaves foreign items unchang ]; const ctx = invocation(input); let wireInput: OpenAIResponsesInputItem[] | undefined; - await withCopilotOpenAIResponsesItemIdMembrane(ctx, {}, () => { + await applyProviderStage(copilotOpenAIResponsesItemIdMembrane, ctx, () => { wireInput = structuredClone(ctx.payload.input); return Promise.resolve({ action: 'generate', @@ -445,7 +444,7 @@ test('rejects conflicting ids carried by one input item', async () => { ], }]); - await expect(withCopilotOpenAIResponsesItemIdMembrane(ctx, {}, () => { + await expect(applyProviderStage(copilotOpenAIResponsesItemIdMembrane, ctx, () => { throw new Error('must not reach upstream'); })).rejects.toThrow(/conflicting upstream ids/); }); @@ -455,7 +454,7 @@ test('normalizes the generated compaction item without touching retained compact { type: 'message', id: 'msg_retained', status: 'completed', role: 'assistant', content: [] }, { type: 'compaction', id: 'cmp_raw', encrypted_content: 'compact state' }, ]); - const result = await withCopilotOpenAIResponsesItemIdMembrane(invocation(), {}, () => Promise.resolve({ + const result = await applyProviderStage(copilotOpenAIResponsesItemIdMembrane, invocation(), () => Promise.resolve({ action: 'compact', ok: true, result: compactResult, diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/set-initiator-header_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/set-initiator-header_test.ts similarity index 82% rename from packages/provider-copilot/__tests__/interceptors/openai-responses/set-initiator-header_test.ts rename to packages/provider-copilot/__tests__/stages/openai-responses/set-initiator-header_test.ts index 4fa9098ad1..1f1ead9086 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/set-initiator-header_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/set-initiator-header_test.ts @@ -1,19 +1,15 @@ import { test } from 'vitest'; -import { withInitiatorHeaderSet } from '../../../src/interceptors/openai-responses/set-initiator-header.ts'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; +import { copilotOpenAIResponsesSetInitiatorHeader } from '../../../src/stages/openai-responses/set-initiator-header.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type OpenAIResponsesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBoundaryCtx => ({ +const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesProbe => ({ payload, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiResponses: {} } }), @@ -32,7 +28,7 @@ test.each(['user', 'system', 'developer'] as const)('OpenAI Responses initiator ], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIResponsesSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'user'); }); @@ -40,7 +36,7 @@ test.each(['user', 'system', 'developer'] as const)('OpenAI Responses initiator test('OpenAI Responses initiator is user when input is an empty array', async () => { const ctx = invocation({ model: 'gpt-test', input: [] }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIResponsesSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'user'); }); @@ -55,7 +51,7 @@ test('OpenAI Responses initiator is user for the role-bearing additional_tools i }], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIResponsesSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'user'); }); @@ -84,7 +80,7 @@ test('OpenAI Responses initiator is agent when the last input item is a function ], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIResponsesSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); @@ -112,7 +108,7 @@ test('OpenAI Responses initiator is agent when the last input item is a custom_t ], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIResponsesSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); @@ -130,7 +126,7 @@ test('OpenAI Responses initiator is agent when the last canonical item is reason ], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIResponsesSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); @@ -152,7 +148,7 @@ test('OpenAI Responses initiator is agent when the last input item is an assista ], }); - await withInitiatorHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIResponsesSetInitiatorHeader, ctx, okEvents); assertEquals(ctx.headers.get('x-initiator'), 'agent'); }); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/set-vision-header_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/set-vision-header_test.ts similarity index 80% rename from packages/provider-copilot/__tests__/interceptors/openai-responses/set-vision-header_test.ts rename to packages/provider-copilot/__tests__/stages/openai-responses/set-vision-header_test.ts index 54b1b1041e..28119e6db5 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/set-vision-header_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/set-vision-header_test.ts @@ -1,19 +1,15 @@ import { test } from 'vitest'; -import { withVisionHeaderSet } from '../../../src/interceptors/openai-responses/set-vision-header.ts'; -import type { OpenAIResponsesBoundaryCtx } from '../../../src/interceptors/openai-responses/types.ts'; +import { copilotOpenAIResponsesSetVisionHeader } from '../../../src/stages/openai-responses/set-vision-header.ts'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesInputContent, OpenAIResponsesInputItem, OpenAIResponsesStreamEvent, OpenAIResponsesToolOutputContent } from '@floway-dev/protocols/openai-responses'; -import type { ExecuteResult } from '@floway-dev/provider'; -import { eventResult } from '@floway-dev/provider'; +import { applyProviderStage, type OpenAIResponsesProbe, type ExecuteResult, eventResult } from '@floway-dev/test-utils'; import { assertEquals, stubProviderModel, testTelemetryModelIdentity } from '@floway-dev/test-utils'; -const stubRequest = {}; - const okEvents = (): Promise>> => Promise.resolve(eventResult((async function* (): AsyncGenerator> {})(), testTelemetryModelIdentity)); -const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesBoundaryCtx => ({ +const invocation = (payload: CanonicalOpenAIResponsesPayload): OpenAIResponsesProbe => ({ payload, headers: new Headers(), model: stubProviderModel({ endpoints: { openaiResponses: {} } }), @@ -35,7 +31,7 @@ test.each(Object.entries(contentContainers))('OpenAI Responses vision header det ])], }); - await withVisionHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIResponsesSetVisionHeader, ctx, okEvents); assertEquals(ctx.headers.get('copilot-vision-request'), 'true'); }); @@ -46,7 +42,7 @@ test.each(Object.entries(contentContainers))('OpenAI Responses vision header ign input: [wrap([{ type: 'input_text', text: 'plain text only' }])], }); - await withVisionHeaderSet(ctx, stubRequest, okEvents); + await applyProviderStage(copilotOpenAIResponsesSetVisionHeader, ctx, okEvents); assertEquals(ctx.headers.has('copilot-vision-request'), false); }); diff --git a/packages/provider-copilot/__tests__/interceptors/openai-responses/strip-image-generation_test.ts b/packages/provider-copilot/__tests__/stages/openai-responses/strip-image-generation_test.ts similarity index 74% rename from packages/provider-copilot/__tests__/interceptors/openai-responses/strip-image-generation_test.ts rename to packages/provider-copilot/__tests__/stages/openai-responses/strip-image-generation_test.ts index 3b8218a32f..9517e9bebb 100644 --- a/packages/provider-copilot/__tests__/interceptors/openai-responses/strip-image-generation_test.ts +++ b/packages/provider-copilot/__tests__/stages/openai-responses/strip-image-generation_test.ts @@ -1,11 +1,11 @@ -import { test } from 'vitest'; +import { expect, test } from 'vitest'; -import { stripImageGenerationFromPayload } from '../../../src/interceptors/openai-responses/strip-image-generation.ts'; +import { stripImageGenerationFromPayload } from '../../../src/stages/openai-responses/strip-image-generation.ts'; import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; import { assert, assertEquals, assertFalse } from '@floway-dev/test-utils'; test('stripImageGenerationFromPayload removes image_generation tools', () => { - const payload = { + let payload = { model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'draw this' }], tools: [ @@ -20,7 +20,7 @@ test('stripImageGenerationFromPayload removes image_generation tools', () => { tool_choice: 'auto', } as CanonicalOpenAIResponsesPayload; - stripImageGenerationFromPayload(payload); + payload = stripImageGenerationFromPayload(payload); assertEquals(payload.tools?.length, 1); assertEquals(payload.tools?.[0].type, 'function'); @@ -28,21 +28,21 @@ test('stripImageGenerationFromPayload removes image_generation tools', () => { }); test('stripImageGenerationFromPayload removes forced image_generation tool_choice', () => { - const payload = { + let payload = { model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'draw this' }], tools: [{ type: 'image_generation' }], tool_choice: { type: 'image_generation' }, } as CanonicalOpenAIResponsesPayload; - stripImageGenerationFromPayload(payload); + payload = stripImageGenerationFromPayload(payload); assertFalse('tools' in payload); assertFalse('tool_choice' in payload); }); test('forced image generation cannot expose a surviving input-carried client tool', () => { - const payload: CanonicalOpenAIResponsesPayload = { + let payload: CanonicalOpenAIResponsesPayload = { model: 'gpt-test', input: [{ type: 'additional_tools', role: 'developer', @@ -54,7 +54,7 @@ test('forced image generation cannot expose a surviving input-carried client too tool_choice: { type: 'image_generation' }, }; - stripImageGenerationFromPayload(payload); + payload = stripImageGenerationFromPayload(payload); assertEquals(payload.tool_choice, 'none'); const item = payload.input[0]; @@ -63,14 +63,14 @@ test('forced image generation cannot expose a surviving input-carried client too }); test('stripImageGenerationFromPayload removes required tool_choice when no tools remain', () => { - const payload = { + let payload = { model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'draw this' }], tools: [{ type: 'image_generation' }], tool_choice: 'required', } as CanonicalOpenAIResponsesPayload; - stripImageGenerationFromPayload(payload); + payload = stripImageGenerationFromPayload(payload); assertFalse('tools' in payload); assertFalse('tool_choice' in payload); @@ -80,7 +80,7 @@ test('stripImageGenerationFromPayload preserves Copilot-accepted hosted and defe // Codex uses `tool_search` and `namespace` for client-executed deferred tool // discovery and Copilot accepts `web_search`; the Copilot OpenAI Responses target // must still see those entries even after image_generation is dropped. - const payload = { + let payload = { model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'search the web' }], tools: [ @@ -98,7 +98,7 @@ test('stripImageGenerationFromPayload preserves Copilot-accepted hosted and defe tool_choice: 'auto', } as CanonicalOpenAIResponsesPayload; - stripImageGenerationFromPayload(payload); + payload = stripImageGenerationFromPayload(payload); assertEquals(payload.tools?.map(tool => tool.type), ['function', 'web_search', 'tool_search', 'namespace']); assertEquals(payload.tool_choice, 'auto'); @@ -106,14 +106,14 @@ test('stripImageGenerationFromPayload preserves Copilot-accepted hosted and defe test('stripImageGenerationFromPayload preserves forced non-image hosted and deferred tool_choices', () => { for (const type of ['web_search', 'tool_search', 'namespace'] as const) { - const payload = { + let payload = { model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'search' }], tools: [{ type }], tool_choice: { type }, } as CanonicalOpenAIResponsesPayload; - stripImageGenerationFromPayload(payload); + payload = stripImageGenerationFromPayload(payload); assertEquals(payload.tools, [{ type }]); assertEquals(payload.tool_choice, { type }); @@ -121,7 +121,7 @@ test('stripImageGenerationFromPayload preserves forced non-image hosted and defe }); test('stripImageGenerationFromPayload preserves custom Freeform tools for downstream wrapping', () => { - const payload = { + let payload = { model: 'gpt-test', input: [{ type: 'message', role: 'user', content: 'do x' }], tools: [ @@ -136,15 +136,15 @@ test('stripImageGenerationFromPayload preserves custom Freeform tools for downst tool_choice: { type: 'custom', name: 'freeform_other' }, } as CanonicalOpenAIResponsesPayload; - stripImageGenerationFromPayload(payload); + payload = stripImageGenerationFromPayload(payload); assertEquals(payload.tools?.length, 2); assertEquals(payload.tools?.[1].type, 'custom'); assertEquals(payload.tool_choice, { type: 'custom', name: 'freeform_other' }); }); -test('strips image generation from input tool carriers in place', () => { - const payload: CanonicalOpenAIResponsesPayload = { +test('strips image generation from input tool carriers without changing their positions', () => { + let payload: CanonicalOpenAIResponsesPayload = { model: 'gpt-test', tools: [], tool_choice: 'required', @@ -155,7 +155,7 @@ test('strips image generation from input tool carriers in place', () => { ], }; - stripImageGenerationFromPayload(payload); + payload = stripImageGenerationFromPayload(payload); assertEquals(payload.input, [ { type: 'additional_tools', role: 'developer', id: 'at_1', tools: [] }, @@ -166,20 +166,20 @@ test('strips image generation from input tool carriers in place', () => { }); test('drops required tool choice when input only supplied removed image generation', () => { - const payload: CanonicalOpenAIResponsesPayload = { + let payload: CanonicalOpenAIResponsesPayload = { model: 'gpt-test', input: [{ type: 'additional_tools', role: 'developer', tools: [{ type: 'image_generation' }] }], tool_choice: 'required', }; - stripImageGenerationFromPayload(payload); + payload = stripImageGenerationFromPayload(payload); assertEquals(payload.input, [{ type: 'additional_tools', role: 'developer', tools: [] }]); assertFalse('tool_choice' in payload); }); test('removes filtered hosted selectors from allowed_tools without dropping client tools', () => { - const payload: CanonicalOpenAIResponsesPayload = { + let payload: CanonicalOpenAIResponsesPayload = { model: 'gpt-test', input: [{ type: 'additional_tools', role: 'developer', @@ -194,25 +194,25 @@ test('removes filtered hosted selectors from allowed_tools without dropping clie }, }; - stripImageGenerationFromPayload(payload); + payload = stripImageGenerationFromPayload(payload); assertEquals(payload.tool_choice, { type: 'allowed_tools', mode: 'required', tools: [{ type: 'function', name: 'lookup' }] }); }); test('keeps an unsatisfiable required allowed_tools choice explicit', () => { - const payload: CanonicalOpenAIResponsesPayload = { + let payload: CanonicalOpenAIResponsesPayload = { model: 'gpt-test', input: [{ type: 'tool_search_output', tools: [{ type: 'image_generation' }] }], tool_choice: { type: 'allowed_tools', mode: 'required', tools: [{ type: 'image_generation' }] }, }; - stripImageGenerationFromPayload(payload); + payload = stripImageGenerationFromPayload(payload); assertEquals(payload.tool_choice, { type: 'allowed_tools', mode: 'required', tools: [] }); }); test('auto allowed_tools cannot expose tools excluded by the original selector', () => { - const payload: CanonicalOpenAIResponsesPayload = { + let payload: CanonicalOpenAIResponsesPayload = { model: 'gpt-test', input: [{ type: 'additional_tools', role: 'developer', @@ -224,7 +224,7 @@ test('auto allowed_tools cannot expose tools excluded by the original selector', tool_choice: { type: 'allowed_tools', mode: 'auto', tools: [{ type: 'image_generation' }] }, }; - stripImageGenerationFromPayload(payload); + payload = stripImageGenerationFromPayload(payload); assertEquals(payload.tool_choice, 'none'); const item = payload.input[0]; @@ -233,12 +233,30 @@ test('auto allowed_tools cannot expose tools excluded by the original selector', }); test('leaves unrelated allowed_tools choices unchanged', () => { - const payload: CanonicalOpenAIResponsesPayload = { + let payload: CanonicalOpenAIResponsesPayload = { model: 'gpt-test', input: [], tool_choice: { type: 'allowed_tools', mode: 'auto', tools: [] }, }; - stripImageGenerationFromPayload(payload); + payload = stripImageGenerationFromPayload(payload); assertEquals(payload.tool_choice, { type: 'allowed_tools', mode: 'auto', tools: [] }); }); + +test('filtering a frozen request shares unchanged carriers and schemas', () => { + const schema = Object.freeze({ type: 'object' }); + const callable = Object.freeze({ type: 'function' as const, name: 'lookup', parameters: schema }); + const kept = Object.freeze({ type: 'message' as const, role: 'user' as const, content: 'hello' }); + const tools = Object.freeze([{ type: 'image_generation' as const }, callable]); + const carrier = Object.freeze({ type: 'additional_tools' as const, role: 'developer' as const, tools }); + const source = Object.freeze({ model: 'test', input: Object.freeze([kept, carrier]) }) as unknown as CanonicalOpenAIResponsesPayload; + const filtered = stripImageGenerationFromPayload(source); + expect(filtered).not.toBe(source); + expect(filtered.input[0]).toBe(kept); + const changed = filtered.input[1]; + expect(changed).not.toBe(carrier); + assert(changed.type === 'additional_tools'); + expect(changed.tools[0]).toBe(callable); + expect(carrier.tools).toHaveLength(2); + expect(stripImageGenerationFromPayload(filtered)).toBe(filtered); +}); diff --git a/packages/provider-copilot/__tests__/stages/pipelines_test.ts b/packages/provider-copilot/__tests__/stages/pipelines_test.ts new file mode 100644 index 0000000000..7d7f6867f4 --- /dev/null +++ b/packages/provider-copilot/__tests__/stages/pipelines_test.ts @@ -0,0 +1,70 @@ +import { expect, test } from 'vitest'; + +import { clearInProcessCopilotTokenCache } from '../../src/auth.ts'; +import { createCopilotPipelines } from '../../src/pipelines.ts'; +import { emptyCopilotUpstreamState } from '../../src/state.ts'; +import { move, run } from '@floway-dev/pipeline'; +import { initProviderRepo, providerModelFacts, type ProviderChatServices, type ProviderOperationRequest, type UpstreamRecord } from '@floway-dev/provider'; +import { jsonResponse, noopUpstreamCallOptions, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; + +const auth = { id: 'up_pipeline', githubHost: 'github.com', githubToken: 'test-token' }; +const setup = (): void => { + clearInProcessCopilotTokenCache(); + const upstream: UpstreamRecord = { id: auth.id, kind: 'copilot', name: 'Test', enabled: true, sortOrder: 0, createdAt: '', updatedAt: '', config: {}, state: { ...emptyCopilotUpstreamState(), copilotToken: { token: 'access-token', expiresAt: 4102444800, baseUrl: 'https://copilot.example' } }, flagOverrides: {}, disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, modelsCache: null, hue: 210 }; + initProviderRepo(() => ({ upstreams: { getById: async () => upstream, saveState: async () => {} } })); +}; +const services: ProviderChatServices = { httpCall: () => ({ ...noopUpstreamCallOptions(), signal: undefined }), recordProtocolFrames: frames => frames }; + +test('Copilot compaction runs vendor request stages and normalizes the generated value in the same run', async () => { + setup(); + const pipeline = createCopilotPipelines(auth).openaiResponsesCompact; + if (pipeline === undefined) throw new Error('Missing compact chain'); + const model = stubProviderModel({ id: 'gpt-5.4', providerData: { rawModels: [{ id: 'gpt-5.4', supported_endpoints: ['/responses'] }] } }); + const input: ProviderOperationRequest<'openaiResponsesCompact'> = move({ + 'request.provider.model': providerModelFacts(model), + 'request.provider.payload': { input: [{ type: 'message', role: 'user', content: 'hello' }], store: true, service_tier: 'priority', tools: [{ type: 'namespace', name: 'tools', description: '', tools: [] }] }, + 'request.http.callId': 1, + 'request.http.headers': [], + }); + await withMockedFetch(async request => { + expect(request.url).toBe('https://copilot.example/responses'); + expect(request.headers.get('authorization')).toBe('Bearer access-token'); + const body = await request.json() as Record; + expect(body).toMatchObject({ model: 'gpt-5.4', store: false, stream: false, tools: [{ type: 'namespace', description: 'Tools in the tools namespace.' }] }); + expect(body).not.toHaveProperty('service_tier'); + expect(body.input).toEqual([{ type: 'message', role: 'user', content: 'hello' }, { type: 'compaction_trigger' }]); + return jsonResponse({ id: 'response', object: 'response', output: [{ type: 'compaction', id: 'upstream-compaction-id', encrypted_content: 'opaque' }] }); + }, async () => { + const executed = await run(pipeline, input, services); + const output = executed.facts['response.provider.output']; + if (output === null || !('kind' in output) || output.kind !== 'value') throw new Error('Expected compact value'); + expect(output.body.object).toBe('response.compaction'); + expect(output.body.output).toHaveLength(2); + expect(output.body.output[1]).toMatchObject({ type: 'compaction' }); + expect(output.body.output[1].id).not.toBe('upstream-compaction-id'); + expect(executed.facts['response.provider.called']).toBe(true); + expect(executed.facts['response.provider.modelKey']).toBe('gpt-5.4'); + expect(input['request.provider.payload'].store).toBe(true); + await executed.drain(); + }); +}); + +test('unsupported Anthropic Fast Mode is a preflight value with no model endpoint observation', async () => { + setup(); + const pipeline = createCopilotPipelines(auth).anthropicMessages; + if (pipeline === undefined) throw new Error('Missing Messages chain'); + const input: ProviderOperationRequest<'anthropicMessages'> = move({ + 'request.provider.model': providerModelFacts(stubProviderModel({ id: 'claude-sonnet-4.6', providerData: { rawModels: [{ id: 'claude-sonnet-4.6', supported_endpoints: ['/v1/messages'] }] } })), + 'request.provider.payload': { max_tokens: 16, messages: [], speed: 'fast' }, + 'request.provider.anthropicBeta': [], + 'request.http.callId': 1, + 'request.http.headers': [], + }); + await withMockedFetch(() => { throw new Error('No network request expected'); }, async () => { + const executed = await run(pipeline, input, services); + expect(executed.facts['response.provider.called']).toBe(false); + expect(executed.facts['response.provider.previousCalls']).toEqual([]); + expect(executed.facts['response.http.exchange']).toMatchObject({ type: 'response', status: 400 }); + await executed.drain(); + }); +}); diff --git a/packages/provider-copilot/package.json b/packages/provider-copilot/package.json index d813df5f67..aa5dd56db0 100644 --- a/packages/provider-copilot/package.json +++ b/packages/provider-copilot/package.json @@ -4,7 +4,10 @@ "version": "0.0.0", "type": "module", "exports": { - ".": { "import": "./src/index.ts", "types": "./src/index.ts" } + ".": { + "import": "./src/index.ts", + "types": "./src/index.ts" + } }, "scripts": { "typecheck": "tsc --noEmit" @@ -12,7 +15,6 @@ "dependencies": { "@floway-dev/http": "workspace:*", "@floway-dev/pipeline": "workspace:*", - "@floway-dev/interceptor": "workspace:*", "@floway-dev/platform": "workspace:*", "@floway-dev/protocols": "workspace:*", "@floway-dev/provider": "workspace:*", diff --git a/packages/provider-copilot/src/auth.ts b/packages/provider-copilot/src/auth.ts index 98a1e4b06c..341c2013c2 100644 --- a/packages/provider-copilot/src/auth.ts +++ b/packages/provider-copilot/src/auth.ts @@ -246,14 +246,14 @@ export const copilotRequestHeaders = (entry: CopilotTokenEntry, initHeaders: Hea headers.set('x-interaction-type', 'conversation-agent'); // Provider-attached invocation headers (vision, initiator, anthropic-beta, - // ...) flow through unchanged. The provider's boundary interceptors decide + // ...) flow through unchanged. The provider's boundary stages decide // which headers each upstream call needs; this layer only knows how to ship - // them. Setting them last lets workaround interceptors override the static + // them. Setting them last lets workaround stages override the static // VSCode identification block when a future workaround needs to. // - // Convention: an empty-string value from an interceptor means "delete this - // base header" — the interceptor wants Copilot to NOT see a default we'd - // otherwise pin. An interceptor that wants to clear an arbitrary downstream + // Convention: an empty-string value from an stage means "delete this + // base header" — the stage wants Copilot to NOT see a default we'd + // otherwise pin. An stage that wants to clear an arbitrary downstream // header value must do so by name through this sentinel; the layer does not // otherwise expose a per-header delete API. if (attached) { diff --git a/packages/provider-copilot/src/chat-facts.ts b/packages/provider-copilot/src/chat-facts.ts new file mode 100644 index 0000000000..e8c711b416 --- /dev/null +++ b/packages/provider-copilot/src/chat-facts.ts @@ -0,0 +1,22 @@ +import type { CopilotSession } from './pipeline-facts.ts'; +import type { HttpHeaders, HttpRequestFacts } from '@floway-dev/http/pipeline'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; +import type { ProviderModelFacts } from '@floway-dev/provider'; + +export interface ChatFacts

{ + 'request.provider.model': ProviderModelFacts; + 'request.provider.payload': P; + 'request.provider.modelKey': string; + 'request.http.callId': number; + 'request.http.headers': HttpHeaders; +} +export type ChatCompletionsFacts = ChatFacts; +export type ResponsesFacts = ChatFacts & { 'request.provider.responsesAction': 'generate' | 'compact' }; +export type MessagesFacts = ChatFacts & { 'request.provider.anthropicBeta': readonly string[] }; +export type CopilotChatFacts = ChatFacts & { 'request.provider.anthropicBeta'?: readonly string[]; 'request.provider.responsesAction'?: 'generate' | 'compact' }; +export type AuthenticatedChatFacts = CopilotChatFacts & { 'request.copilot.session': CopilotSession }; +export interface CopilotChatHttpFacts extends HttpRequestFacts { + 'request.provider.modelKey': string; +} diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/detect-claude-code-metadata.ts b/packages/provider-copilot/src/claude-code-metadata.ts similarity index 94% rename from packages/provider-copilot/src/interceptors/anthropic-messages/detect-claude-code-metadata.ts rename to packages/provider-copilot/src/claude-code-metadata.ts index 4762a6217c..3fb6f74e8b 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/detect-claude-code-metadata.ts +++ b/packages/provider-copilot/src/claude-code-metadata.ts @@ -11,8 +11,8 @@ * attempt JSON parsing — Claude Code never combines the two formats, and * skipping the parse avoids a thrown `SyntaxError` allocation on the hot path. * - * Used by withClaudeAgentHeadersSet (needs BOTH halves to fire) and by - * withInteractionIdHeaderSet (needs only sessionId). + * The Claude agent header stage needs BOTH halves to fire; the interaction ID + * stage needs only sessionId. * * References: * - https://github.com/caozhiyuan/copilot-api/blob/main/src/lib/utils.ts#L151 diff --git a/packages/provider-copilot/src/control-http.ts b/packages/provider-copilot/src/control-http.ts new file mode 100644 index 0000000000..630039b0d8 --- /dev/null +++ b/packages/provider-copilot/src/control-http.ts @@ -0,0 +1,45 @@ +import { copilotAuthedFetch, isCopilotTokenFetchError, type CopilotAuth } from './auth.ts'; +import { parseCopilotQuotaHeaders, putCopilotQuota } from './quota.ts'; +import type { FetchInit, UpstreamFetchOptions } from '@floway-dev/provider'; + +export type CopilotFetchConfig = CopilotAuth; + +export interface CopilotControlFetchOptions extends UpstreamFetchOptions { + /** Extends control-call persistence beyond the response where the runtime supports it. */ + waitUntil?: (promise: Promise) => void; +} + +// Keep any entitlement metadata returned by the catalog refresh without delaying its result. +const captureQuotaFireAndForget = ( + upstreamId: string, + headers: Headers, + waitUntil: ((promise: Promise) => void) | undefined, +): void => { + const snapshot = parseCopilotQuotaHeaders(headers, new Date()); + if (snapshot === null) return; + const persist = putCopilotQuota(upstreamId, snapshot).catch((error: unknown) => { + console.warn(`Failed to persist Copilot quota snapshot for ${upstreamId}:`, error); + }); + waitUntil?.(persist); +}; + +// A rejected credential exchange retains its HTTP reply for catalog error diagnostics. +export const copilotFetchModels = async ( + config: CopilotFetchConfig, + init: FetchInit, + options: CopilotControlFetchOptions, +): Promise => { + const response = await copilotAuthedFetch('/models', init, config, { + headers: options.extraHeaders, + fetcher: options.fetcher, + wrapUpstreamCall: options.wrapUpstreamCall, + }).catch(error => { + if (!isCopilotTokenFetchError(error)) throw error; + return new Response(error.body, { + status: error.status, + headers: new Headers(error.headers), + }); + }); + captureQuotaFireAndForget(config.id, response.headers, options.waitUntil); + return response; +}; diff --git a/packages/provider-copilot/src/fetch-models.ts b/packages/provider-copilot/src/fetch-models.ts index 6614b79b7a..04a65d9f01 100644 --- a/packages/provider-copilot/src/fetch-models.ts +++ b/packages/provider-copilot/src/fetch-models.ts @@ -1,4 +1,4 @@ -import { copilotFetchModels, type CopilotFetchConfig } from './fetch.ts'; +import { copilotFetchModels, type CopilotFetchConfig } from './control-http.ts'; import type { CopilotModelsResponse } from './types.ts'; import { fetchUpstreamModels, type Fetcher, type HttpHeaderLines, identityWrapUpstreamCall } from '@floway-dev/provider'; diff --git a/packages/provider-copilot/src/fetch.ts b/packages/provider-copilot/src/fetch.ts deleted file mode 100644 index a7d2449545..0000000000 --- a/packages/provider-copilot/src/fetch.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { copilotAuthedFetch, isCopilotTokenFetchError, type CopilotAuth } from './auth.ts'; -import { parseCopilotQuotaHeaders, putCopilotQuota } from './quota.ts'; -import type { FetchInit, UpstreamFetchOptions } from '@floway-dev/provider'; - -export type CopilotFetchConfig = CopilotAuth; - -export interface CopilotDataPlaneFetchOptions extends UpstreamFetchOptions { - /** Extends the runtime past the response so the quota persist below - * survives on workerd, which cancels orphan promises the moment the client - * response is sent. The catalog fetch threads none: a `/models` response - * carries no quota headers, so nothing is ever scheduled on that path. */ - waitUntil?: (promise: Promise) => void; -} - -// Every Copilot data-plane response carries the seat's entitlement in -// `x-quota-snapshot-*`, so this funnel — the one place all of them pass -// through — is where the snapshot gets harvested. Streaming included: the -// headers land ahead of the first SSE byte. -// -// Best-effort by construction. The response is already the caller's to relay, -// and a snapshot is a strictly better-than-nothing dashboard read: a CAS loss -// to a concurrent state write, or a storage blip, must not surface on the hot -// path. A response with no quota headers (every 4xx we have observed, plus -// `/models`) leaves the persisted snapshot alone rather than erasing it. -const captureQuotaFireAndForget = ( - upstreamId: string, - headers: Headers, - waitUntil: ((promise: Promise) => void) | undefined, -): void => { - const snapshot = parseCopilotQuotaHeaders(headers, new Date()); - if (snapshot === null) return; - const persist = putCopilotQuota(upstreamId, snapshot).catch((error: unknown) => { - console.warn(`Failed to persist Copilot quota snapshot for ${upstreamId}:`, error); - }); - waitUntil?.(persist); -}; - -// Token-exchange failures surface as regular OpenAI Responses so callers handle them via the same 4xx/5xx path. -const copilotFetchInternal = async ( - config: CopilotFetchConfig, - path: string, - init: FetchInit, - options: CopilotDataPlaneFetchOptions, -): Promise => { - const response = await copilotAuthedFetch(path, init, config, { - headers: options.extraHeaders, - fetcher: options.fetcher, - wrapUpstreamCall: options.wrapUpstreamCall, - }).catch(error => { - if (!isCopilotTokenFetchError(error)) throw error; - return new Response(error.body, { - status: error.status, - headers: new Headers(error.headers), - }); - }); - captureQuotaFireAndForget(config.id, response.headers, options.waitUntil); - return response; -}; - -export const copilotFetchOpenAIChatCompletions = (config: CopilotFetchConfig, init: FetchInit, options: CopilotDataPlaneFetchOptions): Promise => - copilotFetchInternal(config, '/chat/completions', init, options); -export const copilotFetchOpenAIResponses = (config: CopilotFetchConfig, init: FetchInit, options: CopilotDataPlaneFetchOptions): Promise => - copilotFetchInternal(config, '/responses', init, options); -export const copilotFetchAnthropicMessages = (config: CopilotFetchConfig, init: FetchInit, options: CopilotDataPlaneFetchOptions): Promise => - copilotFetchInternal(config, '/v1/messages', init, options); -export const copilotFetchAnthropicMessagesCountTokens = (config: CopilotFetchConfig, init: FetchInit, options: CopilotDataPlaneFetchOptions): Promise => - copilotFetchInternal(config, '/v1/messages/count_tokens', init, options); -export const copilotFetchOpenAIEmbeddings = (config: CopilotFetchConfig, init: FetchInit, options: CopilotDataPlaneFetchOptions): Promise => - copilotFetchInternal(config, '/embeddings', init, options); -export const copilotFetchModels = (config: CopilotFetchConfig, init: FetchInit, options: CopilotDataPlaneFetchOptions): Promise => - copilotFetchInternal(config, '/models', init, options); diff --git a/packages/provider-copilot/src/interceptors/image-compression.ts b/packages/provider-copilot/src/image-compression.ts similarity index 96% rename from packages/provider-copilot/src/interceptors/image-compression.ts rename to packages/provider-copilot/src/image-compression.ts index 927c4553bb..c1d2625a0b 100644 --- a/packages/provider-copilot/src/interceptors/image-compression.ts +++ b/packages/provider-copilot/src/image-compression.ts @@ -23,7 +23,7 @@ const compressImageDataUrlToWebp = async ( }; // A single agentic request often replays the same screenshot across many turns, -// so the boundary interceptors run `Promise.all` over dozens of inline images +// so the boundary stages run `Promise.all` over dozens of inline images // that hash to the same cache key. Without dedup, every duplicate races a // concurrent cache write on that one key, tripping Cloudflare KV's per-key // 1-write/sec limit and wasting work on the Node target. The returned function diff --git a/packages/provider-copilot/src/interceptors/image-size.ts b/packages/provider-copilot/src/image-size.ts similarity index 100% rename from packages/provider-copilot/src/interceptors/image-size.ts rename to packages/provider-copilot/src/image-size.ts diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/apply-top-level-cache-control.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/apply-top-level-cache-control.ts deleted file mode 100644 index f7459acf79..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/apply-top-level-cache-control.ts +++ /dev/null @@ -1,63 +0,0 @@ -import type { CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; -import type { - AnthropicMessagesAssistantInputContentBlock, - AnthropicMessagesTextBlock, - AnthropicMessagesUserContentBlock, -} from '@floway-dev/protocols/anthropic-messages'; - -/** - * Anthropic's Anthropic Messages API defines a top-level `cache_control` field that - * "automatically applies a cache_control marker to the last cacheable block - * in the request" (per `MessageCreateParamsBase` in the official SDK). - * Copilot's `/v1/messages` deployment validates against an older schema for - * several model slots (claude-haiku-4.5, claude-sonnet-4.5, claude-sonnet-4.6, - * and intermittently claude-opus-4.5) and rejects the top-level field with - * `cache_control: Extra inputs are not permitted`. Newer slots (opus 4.6/4.7) - * silently accept it. - * - * Port the marker onto the last cacheable content block (mirroring the - * documented semantics), then drop the top-level field. If the last cacheable - * block already carries its own `cache_control`, leave it alone — an explicit - * marker wins over the auto-apply. Sub-field extensions (`scope`, `ttl`) - * carried in the ported value are cleaned up by - * `withCacheControlExtensionsStripped`, which runs immediately after. - * - * References: - * - https://github.com/anthropics/anthropic-sdk-typescript/blob/a53f60d59ca904f3e79296586642aac3ce68ae02/src/resources/messages/messages.ts#L2909-L2913 - * - https://github.com/caozhiyuan/copilot-api/issues/269 - */ - -type CacheableBlock = Extract< - AnthropicMessagesUserContentBlock | AnthropicMessagesAssistantInputContentBlock, - { cache_control?: unknown } ->; - -const isCacheableBlock = (block: AnthropicMessagesUserContentBlock | AnthropicMessagesAssistantInputContentBlock): block is CacheableBlock => - block.type === 'text' || block.type === 'image' || block.type === 'tool_use' || block.type === 'tool_result'; - -export const withTopLevelCacheControlApplied: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, _env, run) => { - const payload = ctx.payload as typeof ctx.payload & { cache_control?: { type: 'ephemeral' } }; - const topLevel = payload.cache_control; - if (topLevel === undefined) return await run(); - - delete payload.cache_control; - - for (let m = payload.messages.length - 1; m >= 0; m--) { - const message = payload.messages[m]; - - if (typeof message.content === 'string') { - const block: AnthropicMessagesTextBlock = { type: 'text', text: message.content, cache_control: topLevel }; - message.content = [block] as AnthropicMessagesUserContentBlock[] | AnthropicMessagesAssistantInputContentBlock[]; - return await run(); - } - - for (let b = message.content.length - 1; b >= 0; b--) { - const block = message.content[b]; - if (!isCacheableBlock(block)) continue; - block.cache_control ??= topLevel; - return await run(); - } - } - - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/index.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/index.ts deleted file mode 100644 index 2ab0197f7d..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/index.ts +++ /dev/null @@ -1,85 +0,0 @@ -// Copilot-only Anthropic Messages workarounds. Each list is a boundary chain the -// Copilot provider runs inside its own `callX` methods, so the gateway main -// flow never knows that Copilot has interceptors at all. - -import { withTopLevelCacheControlApplied } from './apply-top-level-cache-control.ts'; -import { withInlineImagesCompressed } from './compress-images.ts'; -import { withSpeedFast } from './handle-speed-fast.ts'; -import { withAnthropicBetaNormalized } from './normalize-anthropic-beta.ts'; -import { withThinkingDisplayPromoted } from './promote-thinking-display.ts'; -import { rewriteContextWindowError } from './rewrite-context-window-error.ts'; -import { withClaudeAgentHeadersSet } from './set-claude-agent-headers.ts'; -import { withCompactHeadersSet } from './set-compact-headers.ts'; -import { withInitiatorHeaderSet } from './set-initiator-header.ts'; -import { withInteractionIdHeaderSet } from './set-interaction-id-header.ts'; -import { withVisionHeaderSet } from './set-vision-header.ts'; -import { withCacheControlExtensionsStripped } from './strip-cache-control-extensions.ts'; -import { withEagerInputStreamingStripped } from './strip-eager-input-streaming.ts'; -import type { CopilotAnthropicMessagesBoundaryInterceptor, CopilotAnthropicMessagesCountTokensBoundaryInterceptor } from './types.ts'; - -// Order rationale, split into two lanes that run back-to-back: -// -// Lane 1 — source-shape header derivation (must read the pre-mutation -// payload): -// - rewriteContextWindowError wraps the whole chain so any upstream context- -// window failure surfaced from the terminal is rewritten into a -// Anthropic-Messages-shaped invalid_request_error before later interceptors see it. -// - withCompactHeadersSet pins the compact/auto-continue intent first. -// - withClaudeAgentHeadersSet then overrides those intents (and the -// user-agent / copilot-integration-id) for Claude Code SDK proxy traffic. -// - withInteractionIdHeaderSet finally sets `x-interaction-id` from the -// same parsed metadata. -// -// Lane 2 — wire-shape mutators followed by header-from-wire derivation: -// Payload mutators run first so the header interceptors see the final -// outgoing payload; withTopLevelCacheControlApplied runs before -// withCacheControlExtensionsStripped so the ported marker on the last -// cacheable block is cleaned in the same pass. withSpeedFast strips the -// client's `speed` field (already consumed by callAnthropicMessages for raw-variant -// selection) and post-`run()` stamps `usage.speed='fast'` onto outbound -// message_start/message_delta events when Fast Mode was requested. The -// header lane closes by normalizing admitted caller beta intent, then adding -// tokens required by the final thinking and context-management shape. -// `withInitiatorHeaderSet` re-derives x-initiator from the final last-message -// structure and may overwrite the -// compact-tagged value above — that mirrors the pre-boundary target-side -// override. -// -// `withAnthropicMessagesWebSearchShim` is intentionally NOT registered here. It runs -// in the gateway's `anthropicMessagesInterceptors` (filtered by enabled flags); the -// Copilot provider opts in by listing `anthropic-messages-web-search-shim` in its -// default flag set (see COPILOT_DEFAULT_FLAGS in ../../defaults.ts). -export const COPILOT_ANTHROPIC_MESSAGES_BOUNDARY = [ - rewriteContextWindowError, - withCompactHeadersSet, - withClaudeAgentHeadersSet, - withInteractionIdHeaderSet, - withInlineImagesCompressed, - withSpeedFast, - withThinkingDisplayPromoted, - withTopLevelCacheControlApplied, - withCacheControlExtensionsStripped, - withEagerInputStreamingStripped, - withVisionHeaderSet, - withInitiatorHeaderSet, - withAnthropicBetaNormalized, -] as const satisfies readonly CopilotAnthropicMessagesBoundaryInterceptor[]; - -// /v1/messages/count_tokens is a one-shot HTTP exchange that returns the raw -// upstream Response. The Copilot provider applies vision detection, -// x-initiator classification and normalized caller/payload beta intent to both -// chat and count_tokens. -// -// withInlineImagesCompressed runs first so count_tokens sizes the same -// WebP-recompressed payload the chat path sends, keeping the estimate -// consistent with the real request. The chat boundary's other -// entries stay out: its post-`run()` inspectors cannot be expressed against a -// raw Response at all, and the remaining payload mutators and header setters -// each answer something we observed on the generation endpoint, with no -// equivalent need seen on count_tokens. -export const COPILOT_ANTHROPIC_MESSAGES_COUNT_TOKENS_BOUNDARY = [ - withInlineImagesCompressed, - withVisionHeaderSet, - withInitiatorHeaderSet, - withAnthropicBetaNormalized, -] as const satisfies readonly CopilotAnthropicMessagesCountTokensBoundaryInterceptor[]; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/normalize-anthropic-beta.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/normalize-anthropic-beta.ts deleted file mode 100644 index 70d6986216..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/normalize-anthropic-beta.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { AnthropicMessagesBoundaryCtx } from './types.ts'; - -const ALLOWED_ANTHROPIC_BETAS = new Set([ - 'interleaved-thinking-2025-05-14', - 'context-management-2025-06-27', - 'advanced-tool-use-2025-11-20', -]); -const INTERLEAVED_THINKING_BETA = 'interleaved-thinking-2025-05-14'; -const CONTEXT_MANAGEMENT_BETA = 'context-management-2025-06-27'; - -// Copilot rejects unknown beta values. Preserve the supported subset of the -// caller's Anthropic Messages beta intent, synthesize VS Code's thinking default only -// when the caller supplied no beta intent, and keep context_management paired -// with its required token. -// https://github.com/microsoft/vscode/blob/a234109a108ad2ca78b7d0883688b0a84e3fab42/extensions/copilot/src/platform/endpoint/node/chatEndpoint.ts#L262-L282 -// https://github.com/microsoft/vscode/blob/a234109a108ad2ca78b7d0883688b0a84e3fab42/extensions/copilot/src/extension/chatSessions/claude/node/claudeLanguageModelServer.ts#L413-L427 -export const withAnthropicBetaNormalized = async ( - ctx: AnthropicMessagesBoundaryCtx, - _env: object, - run: () => Promise, -): Promise => { - const callerSuppliedBeta = ctx.anthropicBeta.length > 0; - ctx.anthropicBeta = [...new Set(ctx.anthropicBeta.filter(beta => ALLOWED_ANTHROPIC_BETAS.has(beta)))]; - - const isAdaptiveThinking = ctx.payload.thinking?.type === 'adaptive'; - if (!callerSuppliedBeta && ctx.payload.thinking?.budget_tokens && !isAdaptiveThinking) { - ctx.anthropicBeta.push(INTERLEAVED_THINKING_BETA); - } - - const payload = ctx.payload as typeof ctx.payload & { context_management?: unknown }; - if (payload.context_management !== undefined && !ctx.anthropicBeta.includes(CONTEXT_MANAGEMENT_BETA)) { - ctx.anthropicBeta.push(CONTEXT_MANAGEMENT_BETA); - } - - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/rewrite-context-window-error.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/rewrite-context-window-error.ts deleted file mode 100644 index 62a20057c9..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/rewrite-context-window-error.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; -import { buildPromptTooLongBody } from '@floway-dev/protocols/anthropic-messages'; - -const isContextWindowError = (text: string): boolean => text.includes('Request body is too large for model context window') || text.includes('context_length_exceeded'); - -/** - * Copilot's `/v1/messages` endpoint reports context-window failures with an - * Anthropic-shape body carrying a Copilot-specific message string; Claude - * Code's detector matches on the message substring alone (case-insensitive - * `error.message.toLowerCase().includes('prompt is too long')`), so we - * rewrite this body to the canonical prompt-too-long envelope and trigger - * auto-compaction. The envelope + client-bundle evidence live in - * `@floway-dev/protocols/anthropic-messages` (`buildPromptTooLongBody` / - * `PROMPT_TOO_LONG_MESSAGE`). This interceptor exists in addition to the - * translate-layer rewriter because Copilot's Anthropic Messages endpoint never - * traverses the `anthropic-messages-via-*` translation pairs — the Copilot Anthropic Messages - * substring set here (`Request body is too large...`) is disjoint from - * the OpenAI Responses/OpenAI Chat Completions shapes those pairs match on. - */ -export const rewriteContextWindowError: CopilotAnthropicMessagesBoundaryInterceptor = async (_ctx, _env, run) => { - const result = await run(); - if (result.type !== 'api-error' || result.source !== 'upstream') return result; - - const body = new TextDecoder().decode(result.body); - if (!isContextWindowError(body)) return result; - - return { - ...result, - status: 400, - headers: new Headers({ 'content-type': 'application/json' }), - body: buildPromptTooLongBody(), - }; -}; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/set-initiator-header.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/set-initiator-header.ts deleted file mode 100644 index e1aaff3a66..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/set-initiator-header.ts +++ /dev/null @@ -1,41 +0,0 @@ -import type { AnthropicMessagesBoundaryCtx } from './types.ts'; - -/** - * Copilot's `x-initiator` header distinguishes turns that the human user just - * triggered (`user`) from turns the agent triggered to consume a tool result - * (`agent`). The header gates Copilot-side abuse controls and conversation - * accounting. We classify by the last message: a user turn whose content - * mixes plain blocks with tool_results is still user-initiated; a user turn - * whose content is *only* tool_results is the agent consuming results, and - * any assistant-final turn (which only happens on count-tokens replays) is - * always agent. - * - * The header name is lowercase `x-initiator`; HTTP header names are - * case-insensitive on the wire, so the casing is cosmetic. - * - * Generic in the run-result type so the count_tokens boundary chain - * (`Response`) and the streaming Anthropic Messages boundary chain (`ExecuteResult<...>`) - * can share one definition, matching the established behavior where - * x-initiator is set on every Copilot Anthropic Messages HTTP call. - * - * References: - * - https://github.com/caozhiyuan/copilot-api/blob/master/src/services/copilot/create-chat-completions.ts - */ -export const withInitiatorHeaderSet = async ( - ctx: AnthropicMessagesBoundaryCtx, - _env: object, - run: () => Promise, -): Promise => { - const lastMessage = ctx.payload.messages[ctx.payload.messages.length - 1]; - let initiator: 'user' | 'agent'; - if (lastMessage?.role !== 'user') { - initiator = 'agent'; - } else if (!Array.isArray(lastMessage.content)) { - initiator = 'user'; - } else { - initiator = lastMessage.content.some(block => block.type !== 'tool_result') ? 'user' : 'agent'; - } - ctx.headers.set('x-initiator', initiator); - - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/set-vision-header.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/set-vision-header.ts deleted file mode 100644 index 805a9fb0a8..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/set-vision-header.ts +++ /dev/null @@ -1,41 +0,0 @@ -import type { AnthropicMessagesBoundaryCtx } from './types.ts'; -import type { AnthropicMessagesAssistantMessage, AnthropicMessagesUserMessage } from '@floway-dev/protocols/anthropic-messages'; - -/** - * Copilot rejects Anthropic `image` blocks as plain text unless the private - * `copilot-vision-request: true` header is set. Detection must scan the final - * post-mutation payload (after other Anthropic Messages boundary interceptors have run) - * and cover both the top-level `message.content` and the nested - * `tool_result.content[]` shape; Anthropic allows images in both positions. - * - * Generic in the run-result type because the Copilot provider historically - * applied equivalent vision detection to every Anthropic Messages HTTP exchange (chat - * AND count_tokens). Keeping a single generic interceptor lets both the streaming - * Anthropic Messages boundary chain (`ExecuteResult<...>`) and the count_tokens chain - * (`Response`) share one definition. - * - * References: - * - https://github.com/caozhiyuan/copilot-api/commit/1f6b98924ae092db9b2010846c32e5cbf10817df - */ -const contentHasImage = (content: AnthropicMessagesUserMessage['content'] | AnthropicMessagesAssistantMessage['content']): boolean => { - if (!Array.isArray(content)) return false; - return content.some(block => { - if (block.type === 'image') return true; - if (block.type === 'tool_result' && Array.isArray(block.content)) { - return block.content.some(inner => inner.type === 'image'); - } - return false; - }); -}; - -export const withVisionHeaderSet = async ( - ctx: AnthropicMessagesBoundaryCtx, - _env: object, - run: () => Promise, -): Promise => { - if (ctx.payload.messages.some(message => contentHasImage(message.content))) { - ctx.headers.set('copilot-vision-request', 'true'); - } - - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/strip-cache-control-extensions.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/strip-cache-control-extensions.ts deleted file mode 100644 index 99ac929c50..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/strip-cache-control-extensions.ts +++ /dev/null @@ -1,66 +0,0 @@ -import type { CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; - -/** - * Two `cache_control` sub-fields are beta extensions to the base - * `CacheControlEphemeral` shape that Copilot's stricter Anthropic-Messages-upstream - * deployments (claude-haiku-4.5, claude-sonnet-4.5/4.6, intermittently - * claude-opus-4.5) reject: - * - * - `scope`: added by Claude Code's `prompt-caching-scope-2025-11-27` beta. - * Copilot returns `cache_control.scope: Extra inputs are not permitted`. - * - `ttl`: added by the `extended-cache-ttl-2025-04-11` beta. Floway does - * not enable that beta for Copilot, so any `ttl` value trips the same - * schema rejection on the body. - * - * Walk every position where `cache_control` may appear — system blocks, - * tools, message content blocks including `tool_use` and `tool_result` — - * and strip both sub-fields, keeping `{ type: 'ephemeral' }` so prompt - * caching still primes on slots that do honour the marker. - * - * The top-level `cache_control` field is handled by - * `withTopLevelCacheControlApplied`, which runs first and either ports it - * onto the last cacheable block (where this interceptor cleans it) or - * deletes it when no cacheable block exists. - * - * Custom providers that speak Anthropic Messages directly may accept these - * betas natively; this interceptor is Copilot-only. - * - * References: - * - https://github.com/anthropics/anthropic-sdk-typescript/blob/a53f60d59ca904f3e79296586642aac3ce68ae02/src/resources/messages/messages.ts#L2909-L2913 - * - https://github.com/caozhiyuan/copilot-api/issues/143 - * - https://github.com/caozhiyuan/copilot-api/issues/144 - * - https://github.com/caozhiyuan/copilot-api/issues/269 - * - https://github.com/caozhiyuan/copilot-api/commit/ce8224c55933f811abe5bf9ba42f9336a7852997 - */ -const stripExtensions = (block: Record): void => { - const cacheControl = block.cache_control; - if (!cacheControl || typeof cacheControl !== 'object') return; - - const { scope: _scope, ttl: _ttl, ...rest } = cacheControl as Record; - if (Object.keys(rest).length > 0) block.cache_control = rest; - else delete block.cache_control; -}; - -export const withCacheControlExtensionsStripped: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, _env, run) => { - if (Array.isArray(ctx.payload.system)) { - for (const block of ctx.payload.system as unknown as Record[]) { - stripExtensions(block); - } - } - - if (ctx.payload.tools) { - for (const tool of ctx.payload.tools as unknown as Record[]) { - stripExtensions(tool); - } - } - - for (const message of ctx.payload.messages) { - if (!Array.isArray(message.content)) continue; - - for (const block of message.content as unknown as Record[]) { - stripExtensions(block); - } - } - - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/strip-eager-input-streaming.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/strip-eager-input-streaming.ts deleted file mode 100644 index aea6423163..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/strip-eager-input-streaming.ts +++ /dev/null @@ -1,25 +0,0 @@ -import type { CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; - -/** - * `eager_input_streaming` is a per-tool property in the Anthropic Messages API - * that enables fine-grained tool input streaming. Copilot's native Anthropic Messages - * target has been observed to reject it with - * `"tools.N.custom.eager_input_streaming: Extra inputs are not permitted"`, so - * strip it only at the Copilot target boundary and leave other providers - * untouched. - * - * References: - * - https://github.com/anthropics/anthropic-sdk-typescript/blob/a53f60d59ca904f3e79296586642aac3ce68ae02/src/resources/messages/messages.ts#L1761 - */ -export const withEagerInputStreamingStripped: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, _env, run) => { - if (ctx.payload.tools) { - ctx.payload.tools = ctx.payload.tools.map(tool => { - const { eager_input_streaming: _, ...rest } = tool as typeof tool & { - eager_input_streaming?: unknown; - }; - return rest; - }); - } - - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/types.ts b/packages/provider-copilot/src/interceptors/anthropic-messages/types.ts deleted file mode 100644 index b03c1e8eef..0000000000 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/types.ts +++ /dev/null @@ -1,39 +0,0 @@ -import type { Interceptor } from '@floway-dev/interceptor'; -import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ExecuteResult, ProviderModel } from '@floway-dev/provider'; - -// Boundary ctx for Copilot Anthropic Messages interceptors. The chain runs inside -// `provider.callAnthropicMessages` after the gateway has handed control to the -// provider, so the gateway main flow no longer needs to know that Copilot -// has interceptors at all. -// -// `payload` is the source-shape body with `model` re-attached so interceptors -// that read the public model id (e.g. claude-opus-4-8 carve-outs) keep -// working unchanged; the terminal strips it before serializing to the wire. -// `headers` carries the provider's ordinary admitted headers. -// Parsed `anthropic-beta` tokens arrive through the typed Anthropic Messages call -// contract and are copied into `anthropicBeta` so variant selection and -// interceptors share one token list. The terminal serializes the normalized -// list back onto a fresh wire-header bag. -export interface AnthropicMessagesBoundaryCtx { - payload: AnthropicMessagesPayload; - headers: Headers; - anthropicBeta: string[]; - readonly model: ProviderModel; -} - -export type CopilotAnthropicMessagesBoundaryInterceptor = Interceptor< - AnthropicMessagesBoundaryCtx, - object, - ExecuteResult> ->; - -// count_tokens is a one-shot, non-streaming HTTP exchange: the terminal -// returns the raw upstream `Response` directly. Pure header/payload mutators -// only — post-`run()` event-stream inspection is not portable to this result. -export type CopilotAnthropicMessagesCountTokensBoundaryInterceptor = Interceptor< - AnthropicMessagesBoundaryCtx, - object, - Response ->; diff --git a/packages/provider-copilot/src/interceptors/openai-chat-completions/index.ts b/packages/provider-copilot/src/interceptors/openai-chat-completions/index.ts deleted file mode 100644 index 586e019a8b..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-chat-completions/index.ts +++ /dev/null @@ -1,23 +0,0 @@ -// Copilot-only OpenAI Chat Completions workarounds. The boundary chain runs inside -// `provider.callOpenAIChatCompletions`, so the gateway main flow never knows that -// Copilot has OpenAI Chat Completions interceptors at all. - -import { withToolArgumentWhitespaceAborted } from './abort-on-tool-argument-whitespace.ts'; -import { withCacheControlMarkersAttached } from './attach-cache-control-markers.ts'; -import { withInlineImagesCompressed } from './compress-images.ts'; -import { withInitiatorHeaderSet } from './set-initiator-header.ts'; -import { withVisionHeaderSet } from './set-vision-header.ts'; -import type { CopilotOpenAIChatCompletionsBoundaryInterceptor } from './types.ts'; - -// Order matters: payload-mutating interceptors run first so the header -// interceptors see the final outgoing payload, then the header interceptors -// populate `ctx.headers` for the upstream call. Cache-control marker -// attachment is a payload mutator, so it sits with the other payload mutators -// and before any header derivation. -export const COPILOT_OPENAI_CHAT_COMPLETIONS_BOUNDARY = [ - withInlineImagesCompressed, - withToolArgumentWhitespaceAborted, - withCacheControlMarkersAttached, - withInitiatorHeaderSet, - withVisionHeaderSet, -] as const satisfies readonly CopilotOpenAIChatCompletionsBoundaryInterceptor[]; diff --git a/packages/provider-copilot/src/interceptors/openai-chat-completions/set-vision-header.ts b/packages/provider-copilot/src/interceptors/openai-chat-completions/set-vision-header.ts deleted file mode 100644 index 7c56469aa8..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-chat-completions/set-vision-header.ts +++ /dev/null @@ -1,19 +0,0 @@ -import type { CopilotOpenAIChatCompletionsBoundaryInterceptor } from './types.ts'; - -/** - * Copilot's OpenAI Chat Completions endpoint requires the private - * `copilot-vision-request: true` header before it accepts OpenAI-style - * `image_url` content parts; without it images are silently dropped or - * rejected. - * - * References: - * - https://github.com/caozhiyuan/copilot-api/blob/cd0d0182eb4b9bf68a3376dc79728afa7f42ce07/src/services/copilot/create-chat-completions.ts#L28-L49 - */ -export const withVisionHeaderSet: CopilotOpenAIChatCompletionsBoundaryInterceptor = async (ctx, _env, run) => { - const hasImage = ctx.payload.messages.some( - message => Array.isArray(message.content) && message.content.some(part => part.type === 'image_url'), - ); - if (hasImage) ctx.headers.set('copilot-vision-request', 'true'); - - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/openai-chat-completions/types.ts b/packages/provider-copilot/src/interceptors/openai-chat-completions/types.ts deleted file mode 100644 index 183ae4cef8..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-chat-completions/types.ts +++ /dev/null @@ -1,18 +0,0 @@ -import type { Interceptor } from '@floway-dev/interceptor'; -import type { ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import type { ExecuteResult, ProviderModel } from '@floway-dev/provider'; - -// Boundary ctx for Copilot OpenAI Chat Completions interceptors. See anthropic-messages/types.ts -// for the boundary-isolation rationale. -export interface OpenAIChatCompletionsBoundaryCtx { - payload: OpenAIChatCompletionsPayload; - headers: Headers; - readonly model: ProviderModel; -} - -export type CopilotOpenAIChatCompletionsBoundaryInterceptor = Interceptor< - OpenAIChatCompletionsBoundaryCtx, - object, - ExecuteResult> ->; diff --git a/packages/provider-copilot/src/interceptors/openai-responses/abort-on-tool-argument-whitespace.ts b/packages/provider-copilot/src/interceptors/openai-responses/abort-on-tool-argument-whitespace.ts deleted file mode 100644 index 3530fcf9ad..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-responses/abort-on-tool-argument-whitespace.ts +++ /dev/null @@ -1,69 +0,0 @@ -import type { CopilotOpenAIResponsesBoundaryInterceptor } from './types.ts'; -import { checkWhitespaceOverflow } from '../shared/whitespace-overflow.ts'; -import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; - -/** - * Copilot has been observed to emit only whitespace (`\r`, `\n`, `\t`) inside - * `response.function_call_arguments.delta` events until `max_tokens`, never - * producing valid JSON arguments. Detect that pattern per function call output - * index and abort the upstream stream before the client times out. - * - * Behaviour: when any single output index's argument deltas accumulate more - * than `MAX_CONSECUTIVE_WHITESPACE` consecutive whitespace characters, emit a - * OpenAI Responses `error` event followed by a done frame, then end the stream. - * The OpenAI Responses source layer surfaces `error` events as a stream failure. - * - * Lives at the Copilot provider boundary so other OpenAI-Responses-capable providers - * are not slowed by per-delta whitespace inspection. - * - * References: - * - https://github.com/caozhiyuan/copilot-api/commit/4c0d775e1dc6b8648c7ad5f21fb783fc3246facf - * - https://github.com/caozhiyuan/copilot-api/commit/3cdc32c0811469da9eebec5ca3892caf068df542 - */ -const ABORT_MESSAGE = 'Tool call arguments contained excessive whitespace, indicating a degenerate response.'; - -const isArgumentsDelta = (event: OpenAIResponsesStreamEvent): event is OpenAIResponsesStreamEvent & { type: 'response.function_call_arguments.delta'; output_index: number; delta: string } => - event.type === 'response.function_call_arguments.delta'; - -const errorEvent = (): OpenAIResponsesStreamEvent => - ({ - type: 'error', - message: ABORT_MESSAGE, - code: 'api_error', - }) as OpenAIResponsesStreamEvent; - -export const withToolArgumentWhitespaceAborted: CopilotOpenAIResponsesBoundaryInterceptor = async (_invocation, _env, run) => { - const result = await run(); - // Only the streaming generate branch produces events worth inspecting. - // The compact branch is a single value envelope; pass it through unchanged. - if (result.action !== 'generate' || !result.ok) return result; - - return { - ...result, - events: (async function* (): AsyncGenerator> { - const whitespaceByIndex = new Map(); - - for await (const frame of result.events) { - if (frame.type !== 'event' || !isArgumentsDelta(frame.event)) { - yield frame; - continue; - } - - const event = frame.event; - const current = whitespaceByIndex.get(event.output_index) ?? 0; - const { count, exceeded } = checkWhitespaceOverflow(event.delta, current); - whitespaceByIndex.set(event.output_index, count); - - if (exceeded) { - console.warn('Copilot: infinite whitespace detected in OpenAI Responses function call arguments, aborting stream'); - yield eventFrame(errorEvent()); - yield doneFrame(); - return; - } - - yield frame; - } - })(), - }; -}; diff --git a/packages/provider-copilot/src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts b/packages/provider-copilot/src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts deleted file mode 100644 index 52ee48498e..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-responses/fill-empty-namespace-descriptions.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { OpenAIResponsesBoundaryCtx } from './types.ts'; -import { mapOpenAIResponsesTools } from '@floway-dev/protocols/openai-responses'; - -/** - * Some Copilot data planes reject empty namespace descriptions permitted by - * the current OpenAI schema. Fill the permitted-empty text at the Copilot - * boundary so the same request remains portable across those deployments. - * - * References: - * - https://github.com/openai/openai-openapi/commit/466c74a42f51c02f1927bc666815251dc53845dc - * - https://github.com/caozhiyuan/copilot-api/issues/345 - * - https://github.com/engineersamuel/copilot-proxy-rs/commit/1c23d7237bd1bd7c24045d0d426412851bfd57f6 - */ -export const withEmptyNamespaceDescriptionsFilled = async ( - ctx: OpenAIResponsesBoundaryCtx, - _env: object, - run: () => Promise, -): Promise => { - ctx.payload = mapOpenAIResponsesTools(ctx.payload, tool => tool.type === 'namespace' && tool.description === '' - ? { ...tool, description: `Tools in the ${tool.name} namespace.` } - : tool); - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/openai-responses/force-store-false.ts b/packages/provider-copilot/src/interceptors/openai-responses/force-store-false.ts deleted file mode 100644 index e50c64e5c0..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-responses/force-store-false.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { OpenAIResponsesBoundaryCtx } from './types.ts'; - -/** - * Copilot's `/responses` rejects `store: true` with - * `400 {"error":{"message":"store is not supported","code":"unsupported_value","param":"store"}}`. - * Force `store: false` on the outgoing payload once planning has committed to - * the Copilot OpenAI Responses target so the upstream accepts the request. The - * gateway's own stored-items persistence keys off the caller's original `store` - * value captured at parse time and is unaffected by this upstream-only flag. - * - * Generic in the run-result type so the same definition feeds both the - * streaming `/responses` chain and the non-streaming compaction chain — the - * synth-via-trigger compact call also rejects `store: true`. - */ -export const withStoreForcedFalse = async ( - ctx: OpenAIResponsesBoundaryCtx, - _env: object, - run: () => Promise, -): Promise => { - ctx.payload = { ...ctx.payload, store: false }; - - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/openai-responses/index.ts b/packages/provider-copilot/src/interceptors/openai-responses/index.ts deleted file mode 100644 index b65f9b03f0..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-responses/index.ts +++ /dev/null @@ -1,34 +0,0 @@ -// Copilot-only OpenAI Responses workarounds. The chain is a boundary the Copilot -// provider runs inside its own `callX` methods, so the gateway main flow -// never knows that Copilot has OpenAI Responses interceptors at all. - -import { withToolArgumentWhitespaceAborted } from './abort-on-tool-argument-whitespace.ts'; -import { withInlineImagesCompressed } from './compress-images.ts'; -import { withEmptyNamespaceDescriptionsFilled } from './fill-empty-namespace-descriptions.ts'; -import { withStoreForcedFalse } from './force-store-false.ts'; -import { withCopilotOpenAIResponsesItemIdMembrane } from './item-id-membrane.ts'; -import { withInitiatorHeaderSet } from './set-initiator-header.ts'; -import { withVisionHeaderSet } from './set-vision-header.ts'; -import { withImageGenerationStripped } from './strip-image-generation.ts'; -import { withServiceTierStripped } from './strip-service-tier.ts'; -import type { CopilotOpenAIResponsesBoundaryInterceptor } from './types.ts'; - -// Single chain wraps both the streaming `/responses` call and the -// non-streaming synth-via-trigger compaction call — the chain terminal -// switches on `ctx.action` to pick the wire shape. Order matters: -// payload-mutating interceptors run first so the header interceptors see -// the final outgoing payload, then the header interceptors populate -// `ctx.headers` for the upstream call. Result mutators sit between: the -// whitespace guard acts only on generate streams, while the item-id membrane -// also normalizes the generated item in compact value envelopes. -export const COPILOT_OPENAI_RESPONSES_BOUNDARY = [ - withInlineImagesCompressed, - withEmptyNamespaceDescriptionsFilled, - withServiceTierStripped, - withImageGenerationStripped, - withStoreForcedFalse, - withCopilotOpenAIResponsesItemIdMembrane, - withToolArgumentWhitespaceAborted, - withVisionHeaderSet, - withInitiatorHeaderSet, -] as const satisfies readonly CopilotOpenAIResponsesBoundaryInterceptor[]; diff --git a/packages/provider-copilot/src/interceptors/openai-responses/strip-image-generation.ts b/packages/provider-copilot/src/interceptors/openai-responses/strip-image-generation.ts deleted file mode 100644 index 64fb396fea..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-responses/strip-image-generation.ts +++ /dev/null @@ -1,73 +0,0 @@ -import type { OpenAIResponsesBoundaryCtx } from './types.ts'; -import { collectOpenAIResponsesTools, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesTool, type OpenAIResponsesToolChoice } from '@floway-dev/protocols/openai-responses'; - -/** - * A Copilot gateway filters public `image_generation` from Responses requests, - * while OpenAI supports it. Apply this provider-specific rule after target - * selection to every declaration carrier and selector, retaining unrelated - * tools in their declaration containers and relative order. - * - * https://developers.openai.com/api/docs/guides/tools-image-generation - * https://github.com/caozhiyuan/copilot-api/blob/5d37d5b1ac6566c935a5c26d046396ee5fa423cc/src/routes/responses/handler.ts#L187-L204 - */ -const isImageGenerationTool = (tool: OpenAIResponsesTool): boolean => tool.type === 'image_generation'; - -const isImageGenerationToolChoice = (choice: OpenAIResponsesToolChoice | null | undefined): boolean => - typeof choice === 'object' && choice !== null && choice.type === 'image_generation'; - -export const stripImageGenerationFromPayload = (payload: CanonicalOpenAIResponsesPayload): void => { - let removedTool = false; - - if (Array.isArray(payload.tools)) { - const tools = payload.tools.filter(tool => { - const drop = isImageGenerationTool(tool); - removedTool ||= drop; - return !drop; - }); - - if (tools.length === 0) { - delete payload.tools; - } else { - payload.tools = tools; - } - } - - payload.input = payload.input.map(item => { - if (item.type !== 'additional_tools' && item.type !== 'tool_search_output') return item; - const tools = item.tools.filter(tool => { - const drop = isImageGenerationTool(tool); - removedTool ||= drop; - return !drop; - }); - return tools.length === item.tools.length ? item : { ...item, tools }; - }); - - if (isImageGenerationToolChoice(payload.tool_choice)) { - if (collectOpenAIResponsesTools(payload).length === 0) delete payload.tool_choice; - else payload.tool_choice = 'none'; - return; - } - - if (typeof payload.tool_choice === 'object' && payload.tool_choice !== null && payload.tool_choice.type === 'allowed_tools') { - const allowed = payload.tool_choice.tools.filter(tool => tool.type !== 'image_generation'); - if (allowed.length === payload.tool_choice.tools.length) return; - if (allowed.length === 0 && payload.tool_choice.mode === 'auto') payload.tool_choice = 'none'; - else payload.tool_choice = { ...payload.tool_choice, tools: allowed }; - return; - } - - // A forced `required` choice with no surviving tools would tell Copilot to - // invoke a tool that no longer exists; drop the choice along with the tools. - if (removedTool && payload.tool_choice === 'required' && collectOpenAIResponsesTools(payload).length === 0) { - delete payload.tool_choice; - } -}; - -export const withImageGenerationStripped = async ( - ctx: OpenAIResponsesBoundaryCtx, - _env: object, - run: () => Promise, -): Promise => { - stripImageGenerationFromPayload(ctx.payload); - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/openai-responses/strip-service-tier.ts b/packages/provider-copilot/src/interceptors/openai-responses/strip-service-tier.ts deleted file mode 100644 index ac0da73ff4..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-responses/strip-service-tier.ts +++ /dev/null @@ -1,28 +0,0 @@ -import type { OpenAIResponsesBoundaryCtx } from './types.ts'; - -/** - * Copilot has no request-side `service_tier`: `/responses` answers any value - * of the field with HTTP 400 - * `{"code":"unsupported_value","param":"service_tier"}`, on a base id and a - * `-fast` id alike. The tier reaches the upstream as the raw model id - * `callOpenAIResponses` selected instead, so by the time this runs the value - * has already been consumed and only the unusable field is left to drop. - * Strip it only after planning has committed to the OpenAI Responses target - * so source-side behavior and telemetry still see the caller's original - * request. Generic in the run-result type so the same definition feeds both - * the streaming `/responses` chain and the non-streaming compaction chain. - * - * References: - * - https://github.com/caozhiyuan/copilot-api/commit/f7835a44f06976cab874700e4d94a5f5c0379369 - * - https://platform.openai.com/docs/api-reference/responses/create - */ -export const withServiceTierStripped = async ( - ctx: OpenAIResponsesBoundaryCtx, - _env: object, - run: () => Promise, -): Promise => { - const { service_tier: _, ...payload } = ctx.payload; - ctx.payload = payload; - - return await run(); -}; diff --git a/packages/provider-copilot/src/interceptors/openai-responses/types.ts b/packages/provider-copilot/src/interceptors/openai-responses/types.ts deleted file mode 100644 index 7a985bf188..0000000000 --- a/packages/provider-copilot/src/interceptors/openai-responses/types.ts +++ /dev/null @@ -1,27 +0,0 @@ -import type { Interceptor } from '@floway-dev/interceptor'; -import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; -import type { ProviderModel, ProviderOpenAIResponsesResult, OpenAIResponsesAction } from '@floway-dev/provider'; - -// Boundary ctx for Copilot OpenAI Responses interceptors. See anthropic-messages/types.ts for -// the boundary-isolation rationale. A single chain wraps both the streaming -// `/responses` call and the non-streaming synth-via-trigger compaction call; -// the chain terminal dispatches on `ctx.action` to pick the wire shape. -// `action` mirrors the gateway-side OpenAIResponsesInvocation.action — interceptors -// MAY mutate it during the chain to re-route dispatch in the terminal. -export interface OpenAIResponsesBoundaryCtx { - payload: CanonicalOpenAIResponsesPayload; - headers: Headers; - readonly model: ProviderModel; - action: OpenAIResponsesAction; -} - -// Single chain feeds both the streaming generate terminal and the compact -// terminal; the terminal switches on `ctx.action` and emits the matching -// ProviderOpenAIResponsesResult variant. Pure payload/header mutators are written -// with a `` generic so they fit; event-stream mutators (whitespace -// abort, item-id membrane) inspect the result variant directly. -export type CopilotOpenAIResponsesBoundaryInterceptor = Interceptor< - OpenAIResponsesBoundaryCtx, - object, - ProviderOpenAIResponsesResult ->; diff --git a/packages/provider-copilot/src/model-selection.ts b/packages/provider-copilot/src/model-selection.ts index 6fd9f19dc0..e170220c75 100644 --- a/packages/provider-copilot/src/model-selection.ts +++ b/packages/provider-copilot/src/model-selection.ts @@ -45,7 +45,7 @@ const firstPreferred = (models: readonly CopilotRawModel[]): CopilotRawModel | u // A narrowing filter that rolls back to the original pool when it would empty // it. On the OpenAI Responses path that rollback is the feature: OpenAI // answers an unavailable Fast mode by serving the standard lane and saying -// so, and `callOpenAIResponses` mirrors it. `callAnthropicMessages` never +// so, and OpenAI Responses selection mirrors it. Anthropic Messages selection never // reaches the rollback because Anthropic makes Fast Mode a hard contract and // the entry point pre-checks it. const narrow = (pool: readonly CopilotRawModel[], predicate: (model: CopilotRawModel) => boolean): readonly CopilotRawModel[] => { @@ -98,7 +98,7 @@ export const resolveCopilotRawModel = (models: CopilotModelsResponse, modelId: s }; // Whether the family can serve the accelerated lane at all. -// `callAnthropicMessages` pre-checks this because Anthropic rejects +// Anthropic Messages model selection pre-checks this because Anthropic rejects // `speed: 'fast'` on a model that cannot serve it, and Copilot never echoes // `usage.speed` for us to notice a downgrade afterwards. The OpenAI spelling // of the same lane needs no pre-check: that upstream reports the tier it diff --git a/packages/provider-copilot/src/interceptors/openai-responses/item-id-carrier.ts b/packages/provider-copilot/src/opaque-item-id.ts similarity index 100% rename from packages/provider-copilot/src/interceptors/openai-responses/item-id-carrier.ts rename to packages/provider-copilot/src/opaque-item-id.ts diff --git a/packages/provider-copilot/src/pipeline-facts.ts b/packages/provider-copilot/src/pipeline-facts.ts index cf269f3767..1029fe7b43 100644 --- a/packages/provider-copilot/src/pipeline-facts.ts +++ b/packages/provider-copilot/src/pipeline-facts.ts @@ -13,5 +13,6 @@ export interface CopilotRequest extends EmbeddingsRequest { } export interface CopilotHttpFacts extends HttpRequestFacts { 'request.copilot.modelKey': string; + 'request.provider.modelKey': string; } export type QuotaResponse = HttpResponseFacts & { 'response.copilot.quota': Deferred }; diff --git a/packages/provider-copilot/src/pipelines.ts b/packages/provider-copilot/src/pipelines.ts index 73636dc2d8..4aa984e6af 100644 --- a/packages/provider-copilot/src/pipelines.ts +++ b/packages/provider-copilot/src/pipelines.ts @@ -1,12 +1,48 @@ import type { CopilotAuth } from './auth.ts'; import type { EmbeddingsRequest } from './pipeline-facts.ts'; +import { copilotAnthropicMessagesApplyTopLevelCacheControl } from './stages/anthropic-messages/apply-top-level-cache-control.ts'; +import { copilotAnthropicMessagesCompressImages } from './stages/anthropic-messages/compress-images.ts'; +import { copilotAnthropicMessagesSpeedFast } from './stages/anthropic-messages/handle-speed-fast.ts'; +import { copilotAnthropicMessagesNormalizeAnthropicBeta } from './stages/anthropic-messages/normalize-anthropic-beta.ts'; +import { copilotAnthropicMessagesThinkingDisplay } from './stages/anthropic-messages/promote-thinking-display.ts'; +import { rewriteCopilotContextWindowError } from './stages/anthropic-messages/rewrite-context-window-error.ts'; +import { copilotAnthropicMessagesSetClaudeAgentHeaders } from './stages/anthropic-messages/set-claude-agent-headers.ts'; +import { copilotAnthropicMessagesSetCompactHeaders } from './stages/anthropic-messages/set-compact-headers.ts'; +import { copilotAnthropicMessagesSetInitiatorHeader } from './stages/anthropic-messages/set-initiator-header.ts'; +import { copilotAnthropicMessagesSetInteractionIdHeader } from './stages/anthropic-messages/set-interaction-id-header.ts'; +import { copilotAnthropicMessagesSetVisionHeader } from './stages/anthropic-messages/set-vision-header.ts'; +import { copilotAnthropicMessagesStripCacheControlExtensions } from './stages/anthropic-messages/strip-cache-control-extensions.ts'; +import { copilotAnthropicMessagesStripEagerInputStreaming } from './stages/anthropic-messages/strip-eager-input-streaming.ts'; +import { authenticateCopilotChat } from './stages/authenticate-chat.ts'; import { authenticateCopilot } from './stages/authenticate.ts'; import { observeCopilotQuota } from './stages/observe-quota.ts'; +import { copilotOpenAIChatCompletionsAbortToolWhitespace } from './stages/openai-chat-completions/abort-on-tool-argument-whitespace.ts'; +import { copilotOpenAIChatCompletionsAttachCacheControlMarkers } from './stages/openai-chat-completions/attach-cache-control-markers.ts'; +import { copilotOpenAIChatCompletionsCompressImages } from './stages/openai-chat-completions/compress-images.ts'; +import { copilotOpenAIChatCompletionsSetInitiatorHeader } from './stages/openai-chat-completions/set-initiator-header.ts'; +import { copilotOpenAIChatCompletionsSetVisionHeader } from './stages/openai-chat-completions/set-vision-header.ts'; +import { copilotOpenAIResponsesAbortToolWhitespace } from './stages/openai-responses/abort-on-tool-argument-whitespace.ts'; +import { copilotOpenAIResponsesCompressImages } from './stages/openai-responses/compress-images.ts'; +import { copilotOpenAIResponsesFillEmptyNamespaceDescriptions } from './stages/openai-responses/fill-empty-namespace-descriptions.ts'; +import { copilotOpenAIResponsesForceStoreFalse } from './stages/openai-responses/force-store-false.ts'; +import { copilotOpenAIResponsesItemIdMembrane } from './stages/openai-responses/item-id-membrane.ts'; +import { copilotOpenAIResponsesSetInitiatorHeader } from './stages/openai-responses/set-initiator-header.ts'; +import { copilotOpenAIResponsesSetVisionHeader } from './stages/openai-responses/set-vision-header.ts'; +import { copilotOpenAIResponsesStripImageGeneration } from './stages/openai-responses/strip-image-generation.ts'; +import { copilotOpenAIResponsesStripServiceTier } from './stages/openai-responses/strip-service-tier.ts'; +import { prepareCopilotChatRequest } from './stages/prepare-chat-request.ts'; import { prepareCopilotEmbeddings } from './stages/prepare-request.ts'; +import { reshapeCopilotCompaction } from './stages/reshape-compaction.ts'; +import { selectCopilotChatModel } from './stages/select-chat-model.ts'; import { http } from '@floway-dev/http/pipeline'; import { compose } from '@floway-dev/pipeline'; -import type { ProviderPipelines, ProviderResponse } from '@floway-dev/provider'; +import { decodeProviderResponse, observeProviderCall, observeProviderResponsesCall, type ProviderOperationRequest, type ProviderOperationResponse, type ProviderPipelines, type ProviderResponse } from '@floway-dev/provider'; export const createCopilotPipelines = (auth: CopilotAuth): ProviderPipelines => ({ - openaiEmbeddings: compose('copilot.openaiEmbeddings', [authenticateCopilot(auth), prepareCopilotEmbeddings(), observeCopilotQuota(auth.id), http]), + openaiChatCompletions: compose, ProviderOperationResponse<'openaiChatCompletions'>>('copilot.openaiChatCompletions', [selectCopilotChatModel('openaiChatCompletions'), authenticateCopilotChat(auth), copilotOpenAIChatCompletionsCompressImages, copilotOpenAIChatCompletionsAbortToolWhitespace, copilotOpenAIChatCompletionsAttachCacheControlMarkers, copilotOpenAIChatCompletionsSetInitiatorHeader, copilotOpenAIChatCompletionsSetVisionHeader, decodeProviderResponse('openaiChatCompletions'), prepareCopilotChatRequest('openaiChatCompletions'), observeCopilotQuota(auth.id), observeProviderCall, http]), + openaiResponses: compose, ProviderOperationResponse<'openaiResponses'>>('copilot.openaiResponses', [selectCopilotChatModel('openaiResponses'), authenticateCopilotChat(auth), copilotOpenAIResponsesCompressImages, copilotOpenAIResponsesFillEmptyNamespaceDescriptions, copilotOpenAIResponsesStripServiceTier, copilotOpenAIResponsesStripImageGeneration, copilotOpenAIResponsesForceStoreFalse, copilotOpenAIResponsesItemIdMembrane, copilotOpenAIResponsesAbortToolWhitespace, copilotOpenAIResponsesSetVisionHeader, copilotOpenAIResponsesSetInitiatorHeader, decodeProviderResponse('openaiResponses'), prepareCopilotChatRequest('openaiResponses'), observeCopilotQuota(auth.id), observeProviderResponsesCall, http]), + anthropicMessages: compose, ProviderOperationResponse<'anthropicMessages'>>('copilot.anthropicMessages', [selectCopilotChatModel('anthropicMessages'), authenticateCopilotChat(auth), rewriteCopilotContextWindowError, copilotAnthropicMessagesSetCompactHeaders, copilotAnthropicMessagesSetClaudeAgentHeaders, copilotAnthropicMessagesSetInteractionIdHeader, copilotAnthropicMessagesCompressImages, copilotAnthropicMessagesSpeedFast, copilotAnthropicMessagesThinkingDisplay, copilotAnthropicMessagesApplyTopLevelCacheControl, copilotAnthropicMessagesStripCacheControlExtensions, copilotAnthropicMessagesStripEagerInputStreaming, copilotAnthropicMessagesSetVisionHeader, copilotAnthropicMessagesSetInitiatorHeader, copilotAnthropicMessagesNormalizeAnthropicBeta, decodeProviderResponse('anthropicMessages'), prepareCopilotChatRequest('anthropicMessages'), observeCopilotQuota(auth.id), observeProviderCall, http]), + anthropicMessagesCountTokens: compose, ProviderOperationResponse<'anthropicMessagesCountTokens'>>('copilot.anthropicMessagesCountTokens', [selectCopilotChatModel('anthropicMessagesCountTokens'), authenticateCopilotChat(auth), copilotAnthropicMessagesCompressImages, copilotAnthropicMessagesSetVisionHeader, copilotAnthropicMessagesSetInitiatorHeader, copilotAnthropicMessagesNormalizeAnthropicBeta, decodeProviderResponse('anthropicMessagesCountTokens'), prepareCopilotChatRequest('anthropicMessagesCountTokens'), observeCopilotQuota(auth.id), observeProviderCall, http]), + openaiResponsesCompact: compose, ProviderOperationResponse<'openaiResponsesCompact'>>('copilot.openaiResponsesCompact', [selectCopilotChatModel('openaiResponsesCompact'), authenticateCopilotChat(auth), copilotOpenAIResponsesCompressImages, copilotOpenAIResponsesFillEmptyNamespaceDescriptions, copilotOpenAIResponsesStripServiceTier, copilotOpenAIResponsesStripImageGeneration, copilotOpenAIResponsesForceStoreFalse, copilotOpenAIResponsesItemIdMembrane, copilotOpenAIResponsesAbortToolWhitespace, copilotOpenAIResponsesSetVisionHeader, copilotOpenAIResponsesSetInitiatorHeader, reshapeCopilotCompaction, decodeProviderResponse('openaiResponsesCompact'), prepareCopilotChatRequest('openaiResponsesCompact'), observeCopilotQuota(auth.id), observeProviderResponsesCall, http]), + openaiEmbeddings: compose('copilot.openaiEmbeddings', [authenticateCopilot(auth), prepareCopilotEmbeddings(), observeCopilotQuota(auth.id), observeProviderCall, http]), }); diff --git a/packages/provider-copilot/src/provider.ts b/packages/provider-copilot/src/provider.ts index 1fdd720c0a..f4a9ab0f7b 100644 --- a/packages/provider-copilot/src/provider.ts +++ b/packages/provider-copilot/src/provider.ts @@ -1,30 +1,17 @@ import { chatFromCopilotRaw } from './chat-from-raw.ts'; -import { COMPACTION_TRIGGER, compactionResponse } from './compaction.ts'; import { assertCopilotUpstreamRecord } from './config.ts'; import { COPILOT_DEFAULT_FLAGS, defaultFlagsForCopilotModel } from './defaults.ts'; import { fetchCopilotModels } from './fetch-models.ts'; -import { copilotFetchOpenAIChatCompletions, copilotFetchOpenAIEmbeddings, copilotFetchAnthropicMessages, copilotFetchAnthropicMessagesCountTokens, copilotFetchOpenAIResponses, type CopilotDataPlaneFetchOptions } from './fetch.ts'; -import { COPILOT_ANTHROPIC_MESSAGES_BOUNDARY, COPILOT_ANTHROPIC_MESSAGES_COUNT_TOKENS_BOUNDARY } from './interceptors/anthropic-messages/index.ts'; -import type { AnthropicMessagesBoundaryCtx } from './interceptors/anthropic-messages/types.ts'; -import { COPILOT_OPENAI_CHAT_COMPLETIONS_BOUNDARY } from './interceptors/openai-chat-completions/index.ts'; -import type { OpenAIChatCompletionsBoundaryCtx } from './interceptors/openai-chat-completions/types.ts'; -import { COPILOT_OPENAI_RESPONSES_BOUNDARY } from './interceptors/openai-responses/index.ts'; -import type { OpenAIResponsesBoundaryCtx } from './interceptors/openai-responses/types.ts'; import { emptyKnownModels, mergeKnownModels, projectKnownModels } from './known-models.ts'; import { mergeCopilotVariants } from './merge-variants.ts'; -import { CONTEXT_1M_BETA, copilotModelSupportsFastVariant } from './model-selection.ts'; import { copilotVariantIndex } from './model-variants.ts'; -import { copilotOpenAIEmbeddingsBody, rawModelFor, rawModelSupportsEndpoint } from './operation-model.ts'; +import { rawModelSupportsEndpoint } from './operation-model.ts'; import { createCopilotPipelines } from './pipelines.ts'; import { pricingForCopilotPublicModelId } from './pricing.ts'; import { readCopilotUpstreamState, type CopilotUpstreamState } from './state.ts'; import type { CopilotRawModel } from './types.ts'; -import { runInterceptors } from '@floway-dev/interceptor'; -import { parseAnthropicMessagesStream, type AnthropicMessagesPayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import { type ModelEndpoints, type ProtocolFrame, isFastServiceTier, kindForEndpoints } from '@floway-dev/protocols/common'; -import { parseOpenAIChatCompletionsStream, type OpenAIChatCompletionsPayload, type OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import { parseOpenAIResponsesStream, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesResult } from '@floway-dev/protocols/openai-responses'; -import { eventResult, getProviderRepo, headersForAnthropicMessagesCall, jsonRequestBody, readUpstreamApiError, streamingProviderCall, apiErrorToResponse, resolveEffectiveFlags, type ExecuteResult, type FetchInit, type FlagOverrides, type HttpHeaderLines, type ProviderInstance, type Provider, type ProviderCallResult, type ProviderModel, type ProviderOpenAIResponsesResult, type ProviderStreamResult, type TelemetryModelIdentity, type UpstreamCallOptions, type UpstreamRecord } from '@floway-dev/provider'; +import { type ModelEndpoints, kindForEndpoints } from '@floway-dev/protocols/common'; +import { getProviderRepo, resolveEffectiveFlags, type FlagOverrides, type ProviderInstance, type Provider, type ProviderModel, type UpstreamRecord } from '@floway-dev/provider'; interface CopilotProviderData { rawModels: CopilotRawModel[]; @@ -71,29 +58,6 @@ const copilotModelEndpoints = (rawModels: readonly CopilotRawModel[]): ModelEndp return rawModels.some(model => rawModelSupportsEndpoint(model, 'openaiEmbeddings')) ? { openaiEmbeddings: {} } : {}; }; -const chatReasoningEffort = (body: Omit): string | undefined => (body.reasoning_effort && body.reasoning_effort !== 'none' ? body.reasoning_effort : undefined); - -const anthropicMessagesReasoningEffort = (body: Omit): string | undefined => body.output_config?.effort; - -const openaiResponsesReasoningEffort = (body: Omit): string | undefined => (body.reasoning?.effort && body.reasoning.effort !== 'none' ? body.reasoning.effort : undefined); - -const anthropicMessagesBoundaryContext = ( - body: Omit, - model: ProviderModel, - headers: Headers, - anthropicBeta: readonly string[], -): AnthropicMessagesBoundaryCtx => { - return { - payload: { ...body, model: model.id }, - headers: new Headers(headers), - anthropicBeta: [...anthropicBeta], - model, - }; -}; - -const rejectUnsupported = (capability: string) => (): Promise => - Promise.reject(new Error(`Copilot provider does not implement ${capability}`)); - const finalizeCopilotModels = ( rawModels: CopilotRawModel[], upstreamOverrides: FlagOverrides, @@ -144,103 +108,7 @@ export const createCopilotProvider = (record: UpstreamRecord): Provider => { const copilot = assertCopilotUpstreamRecord(record); const upstreamConfig = { id: copilot.id, githubHost: copilot.config.githubHost, githubToken: copilot.config.githubToken }; - const call = async ( - transport: (config: typeof upstreamConfig, init: FetchInit, options: CopilotDataPlaneFetchOptions) => Promise, - body: Record, - signal: AbortSignal | undefined, - rawModel: CopilotRawModel, - headers: HttpHeaderLines, - opts: UpstreamCallOptions, - ): Promise => { - const response = await transport( - upstreamConfig, - { - method: 'POST', - body: jsonRequestBody({ ...body, model: rawModel.id }), - signal, - }, - { extraHeaders: headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall, waitUntil: opts.waitUntil }, - ); - return { response, modelKey: rawModel.id }; - }; - - const callStreaming = ( - transport: (config: typeof upstreamConfig, init: FetchInit, options: CopilotDataPlaneFetchOptions) => Promise, - body: Record, - signal: AbortSignal | undefined, - rawModel: CopilotRawModel, - headers: HttpHeaderLines, - parser: Parameters>[1], - opts: UpstreamCallOptions, - ) => - streamingProviderCall( - transport( - upstreamConfig, - { - method: 'POST', - body: jsonRequestBody({ ...body, stream: true, model: rawModel.id }), - signal, - }, - { extraHeaders: headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall, waitUntil: opts.waitUntil }, - ), - parser, - rawModel.id, - signal, - ); - - // The boundary chain expects ExecuteResult shape so post-`run()` inspectors - // (e.g. rewriteContextWindowError) can pattern-match on `result.type`. The - // placeholder here only has to satisfy the EventResult contract while the - // chain runs inside the provider boundary; real telemetry identity is - // rebuilt downstream with pricing. - const placeholderIdentity = (modelKey: string): TelemetryModelIdentity => ({ - model: modelKey, - upstream: copilot.id, - modelKey, - pricing: null, - }); - - // Materialize an upstream error body up-front so any interceptor that - // inspects `result.body` (e.g. rewriteContextWindowError) sees the bytes. - const liftStream = async ( - streamPromise: Promise>, - ): Promise>> => { - const stream = await streamPromise; - if (stream.ok) { - return eventResult( - stream.events as AsyncIterable>, - placeholderIdentity(stream.modelKey), - { headers: stream.headers }, - ); - } - return await readUpstreamApiError(stream.response); - }; - - // Lowering rebuilds a ProviderStreamResult so callers continue to relay - // status/headers/body verbatim on errors and forward the typed event stream - // on success. `internal-error` is not a shape any Copilot boundary - // interceptor produces today; an explicit throw makes a future regression - // noisy instead of silently dropping the result. - const lowerToStream = ( - result: ExecuteResult>, - modelKey: string, - ): ProviderStreamResult => { - if (result.type === 'events') { - return { - ok: true, - events: result.events as AsyncIterable>, - modelKey, - ...(result.headers ? { headers: result.headers } : {}), - }; - } - if (result.type === 'api-error') { - return { ok: false, response: apiErrorToResponse(result), modelKey }; - } - throw new Error(`Copilot boundary chain produced unexpected ExecuteResult shape '${result.type}'`); - }; - const instance: ProviderInstance = { - callAlphaSearch: rejectUnsupported('callAlphaSearch'), getProvidedModels: async fetcher => { const fresh = await getProviderRepo().upstreams.getById(copilot.id); if (!fresh) throw new Error(`Copilot upstream ${copilot.id} disappeared mid-request`); @@ -267,166 +135,6 @@ export const createCopilotProvider = (record: UpstreamRecord): Provider => { } return finalizeCopilotModels(projectKnownModels(merged, now), copilot.flagOverrides); }, - // Copilot's catalog never declares endpoints.openaiCompletions, so this - // stub is unreachable; the rejection surfaces a routing bug. - callOpenAICompletions: rejectUnsupported('callOpenAICompletions'), - callOpenAIChatCompletions: async (model, body, signal, opts) => { - // No accelerated-lane hint here on purpose. Copilot's /chat/completions - // accepts `service_tier` only to ignore it, answering - // `service_tier: "default"` even on a request that carried `priority`. - // Selecting a `-fast` raw variant from this entry point would have - // GitHub charge us the fast rates while the response reports the base - // tier, which is the one combination our billing cannot see through. - const rawModel = rawModelFor(model, 'openaiChatCompletions', { reasoningEffort: chatReasoningEffort(body) }); - const ctx: OpenAIChatCompletionsBoundaryCtx = { - payload: { ...body, model: model.id }, - headers: new Headers(opts.headers), - model, - }; - const result = await runInterceptors>>( - ctx, {}, COPILOT_OPENAI_CHAT_COMPLETIONS_BOUNDARY, async () => { - const { model: _ignored, ...wireBody } = ctx.payload; - return await liftStream(callStreaming(copilotFetchOpenAIChatCompletions, wireBody, signal, rawModel, [...ctx.headers], parseOpenAIChatCompletionsStream, opts)); - }, - ); - return lowerToStream(result, rawModel.id); - }, - callOpenAIResponses: async (model, body, action, signal, opts) => { - // `service_tier` naming the accelerated lane — `priority`, or its - // post-rename synonym `fast` — picks the family's `-fast` raw variant. - // Copilot rejects the field itself with HTTP 400 `unsupported_value` - // (see `strip-service-tier.ts`), so the raw id is the only way to reach - // the lane. Unlike Anthropic Messages below there is no hard pre-check: - // OpenAI declines an unavailable Fast mode silently and reports the tier - // it actually served, and Copilot does the same, so a family without a - // `-fast` variant falls back to its base and answers - // `service_tier: "default"`. The caller is told which tier ran and - // `billableUsageFromOpenAIResponsesResult` prices that one. - // https://github.com/openai/codex/issues/32191 - // https://learn.microsoft.com/en-sg/answers/questions/5921564/we-send-service-tier-priority-on-a-gpt-4-1-mini-gl - const rawModel = rawModelFor(model, 'openaiResponses', { - reasoningEffort: openaiResponsesReasoningEffort(body), - fast: isFastServiceTier(body.service_tier), - }); - const ctx: OpenAIResponsesBoundaryCtx = { - payload: { ...body, model: model.id }, - headers: new Headers(opts.headers), - model, - action, - }; - // Single chain wraps both branches; the terminal dispatches on - // `ctx.action` (the post-chain value), so a mid-chain interceptor can - // flip it and steer dispatch end-to-end. Copilot has no native - // /v1/responses/compact, so the compact branch drives the same - // /responses upstream with stream:false + a compaction_trigger input - // item and reshapes the envelope via `compactionResponse`. Every - // payload/header workaround in the chain — force-store-false, - // strip-service-tier, strip-image-generation, inline-image - // compression, vision/initiator headers — applies to both branches - // identically. The item-id membrane also normalizes the compact value - // envelope, while the whitespace guard only inspects generate streams. - return await runInterceptors( - ctx, {}, COPILOT_OPENAI_RESPONSES_BOUNDARY, async () => { - const { model: _ignored, ...wireBody } = ctx.payload; - switch (ctx.action) { - case 'generate': { - const stream = await callStreaming(copilotFetchOpenAIResponses, wireBody, signal, rawModel, [...ctx.headers], parseOpenAIResponsesStream, opts); - return stream.ok - ? { action: 'generate', ok: true, events: stream.events, modelKey: stream.modelKey, ...(stream.headers ? { headers: stream.headers } : {}) } - : { action: 'generate', ok: false, response: stream.response, modelKey: stream.modelKey }; - } - case 'compact': { - const input = wireBody.input; - const triggered = { ...wireBody, input: [...input, COMPACTION_TRIGGER], stream: false, model: rawModel.id }; - const response = await copilotFetchOpenAIResponses( - upstreamConfig, - { method: 'POST', body: jsonRequestBody(triggered), signal }, - { extraHeaders: [...ctx.headers], fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall, waitUntil: opts.waitUntil }, - ); - if (!response.ok) return { action: 'compact', ok: false, response, modelKey: rawModel.id }; - const generated = (await response.json()) as OpenAIResponsesResult; - return { action: 'compact', ok: true, result: compactionResponse(input, generated), modelKey: rawModel.id }; - } - default: - ctx.action satisfies never; - throw new Error(`Unhandled OpenAIResponsesAction: ${ctx.action as string}`); - } - }, - ); - }, - callAnthropicMessages: async (model, body, signal, opts) => { - // Fast Mode is a hard contract on the request side: Anthropic returns - // HTTP 400 invalid_request_error when a model does not support it, with - // no silent fallback to standard speed. We mirror that at the gateway - // boundary before any per-Copilot workaround runs — selection alone is - // best-effort, and Copilot never echoes `usage.speed`, so an unchecked - // downgrade would be invisible to the caller and to billing alike. The - // OpenAI spelling of the same lane takes the opposite path in - // `callOpenAIResponses`, where the upstream reports the tier it served. - // https://docs.claude.com/en/build-with-claude/fast-mode - // - // The `error.message` is byte-identical to the string Anthropic emits - // on the real wire, recorded verbatim from a live response by an - // independent gateway's regression test: - // https://github.com/Yeachan-Heo/gajae-code/blob/main/packages/ai/test/anthropic-fast-mode.test.ts - if (body.speed === 'fast') { - const providerData = model.providerData as CopilotProviderData; - if (!copilotModelSupportsFastVariant(providerData.rawModels)) { - return { - ok: false, - response: Response.json( - { - type: 'error', - error: { - type: 'invalid_request_error', - message: `'${model.id}' does not support the \`speed\` parameter.`, - }, - }, - { status: 400 }, - ), - modelKey: model.id, - }; - } - } - - // Both the native Anthropic Messages call and count_tokens select the same raw - // `messages` variant; they differ only in the upstream endpoint path. - const ctx = anthropicMessagesBoundaryContext(body, model, opts.headers, opts.anthropicBeta); - const rawModel = rawModelFor(model, 'anthropicMessages', { - context1m: ctx.anthropicBeta.includes(CONTEXT_1M_BETA), - reasoningEffort: anthropicMessagesReasoningEffort(body), - fast: body.speed === 'fast', - }); - const result = await runInterceptors>>( - ctx, {}, COPILOT_ANTHROPIC_MESSAGES_BOUNDARY, async () => { - const { model: _ignored, ...wireBody } = ctx.payload; - return await liftStream(callStreaming(copilotFetchAnthropicMessages, wireBody, signal, rawModel, headersForAnthropicMessagesCall([...ctx.headers], ctx.anthropicBeta), parseAnthropicMessagesStream, opts)); - }, - ); - return lowerToStream(result, rawModel.id); - }, - callAnthropicMessagesCountTokens: async (model, body, signal, opts) => { - const ctx = anthropicMessagesBoundaryContext(body, model, opts.headers, opts.anthropicBeta); - const rawModel = rawModelFor(model, 'anthropicMessages', { - context1m: ctx.anthropicBeta.includes(CONTEXT_1M_BETA), - reasoningEffort: anthropicMessagesReasoningEffort(body), - }); - const response = await runInterceptors( - ctx, {}, COPILOT_ANTHROPIC_MESSAGES_COUNT_TOKENS_BOUNDARY, async () => { - const { model: _ignored, ...wireBody } = ctx.payload; - const { response } = await call(copilotFetchAnthropicMessagesCountTokens, wireBody, signal, rawModel, headersForAnthropicMessagesCall([...ctx.headers], ctx.anthropicBeta), opts); - return response; - }, - ); - return { response, modelKey: rawModel.id }; - }, - callOpenAIEmbeddings: (model, body, signal, opts) => call(copilotFetchOpenAIEmbeddings, copilotOpenAIEmbeddingsBody(body), signal, rawModelFor(model, 'openaiEmbeddings'), [...opts.headers], opts), - // Copilot has no /images/* upstream; catalog never emits a kind='image' - // model, so these stubs are unreachable. - callOpenAIImagesGenerations: rejectUnsupported('callOpenAIImagesGenerations'), - callOpenAIImagesEdits: rejectUnsupported('callOpenAIImagesEdits'), - callOpenAIAudioTranscriptions: rejectUnsupported('callOpenAIAudioTranscriptions'), - callRerank: rejectUnsupported('callRerank'), }; return { diff --git a/packages/provider-copilot/src/stages/anthropic-messages/apply-top-level-cache-control.ts b/packages/provider-copilot/src/stages/anthropic-messages/apply-top-level-cache-control.ts new file mode 100644 index 0000000000..0920d907bf --- /dev/null +++ b/packages/provider-copilot/src/stages/anthropic-messages/apply-top-level-cache-control.ts @@ -0,0 +1,96 @@ +import type { MessagesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { + AnthropicMessagesAssistantInputContentBlock, + AnthropicMessagesPayload, + AnthropicMessagesTextBlock, + AnthropicMessagesUserContentBlock, +} from '@floway-dev/protocols/anthropic-messages'; + +/** + * Anthropic's Messages API defines a top-level `cache_control` field that + * "automatically applies a cache_control marker to the last cacheable block + * in the request" (per `MessageCreateParamsBase` in the official SDK). + * Copilot's `/v1/messages` deployment validates against an older schema for + * several model slots (claude-haiku-4.5, claude-sonnet-4.5, claude-sonnet-4.6, + * and intermittently claude-opus-4.5) and rejects the top-level field with + * `cache_control: Extra inputs are not permitted`. Newer slots (opus 4.6/4.7) + * silently accept it. + * + * Port the marker onto the last cacheable content block (mirroring the + * documented semantics), then drop the top-level field. If the last cacheable + * block already carries its own `cache_control`, leave it alone — an explicit + * marker wins over the auto-apply. Sub-field extensions (`scope`, `ttl`) + * carried in the ported value are cleaned up by + * `withCacheControlExtensionsStripped`, which runs immediately after. + * + * References: + * - https://github.com/anthropics/anthropic-sdk-typescript/blob/a53f60d59ca904f3e79296586642aac3ce68ae02/src/resources/messages/messages.ts#L2909-L2913 + * - https://github.com/caozhiyuan/copilot-api/issues/269 + */ + +type CacheableBlock = Extract< + AnthropicMessagesUserContentBlock | AnthropicMessagesAssistantInputContentBlock, + { cache_control?: unknown } +>; + +const isCacheableBlock = (block: AnthropicMessagesUserContentBlock | AnthropicMessagesAssistantInputContentBlock): block is CacheableBlock => + block.type === 'text' || block.type === 'image' || block.type === 'tool_use' || block.type === 'tool_result'; + +/** Where the marker lands: the last message that can carry one, and within it either the whole + * string content or the last cacheable block. Reading the position out first is what lets the + * rewrite below rebuild one path and leave every other message alone. */ +const markerSite = (messages: AnthropicMessagesPayload['messages']): { readonly message: number; readonly block: number | 'whole' } | undefined => { + for (let m = messages.length - 1; m >= 0; m--) { + const content = messages[m].content; + if (typeof content === 'string') return { message: m, block: 'whole' }; + for (let b = content.length - 1; b >= 0; b--) { + if (isCacheableBlock(content[b])) return { message: m, block: b }; + } + } + return undefined; +}; + +export const copilotAnthropicMessagesApplyTopLevelCacheControl = defineStage({ + name: 'copilotAnthropicMessagesApplyTopLevelCacheControl', + through: { + request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + let payload = facts['request.provider.payload'] as typeof facts['request.provider.payload'] & { cache_control?: { type: 'ephemeral' } }; + + const topLevel = payload.cache_control; + if (topLevel === undefined) return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + + const { cache_control: _ported, ...withoutTopLevel } = payload; + const site = markerSite(payload.messages); + if (site === undefined) { + payload = withoutTopLevel; + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + } + + payload = { + ...withoutTopLevel, + // Each rewrite keeps its message's own role and only re-homes that role's own blocks, so + // the array is handed back at its element type; spreading alone widens it to the union. + messages: payload.messages.map((message, m): AnthropicMessagesPayload['messages'][number] => { + if (m !== site.message) return message; + if (site.block === 'whole') { + const block: AnthropicMessagesTextBlock = { type: 'text', text: message.content as string, cache_control: topLevel }; + return { ...message, content: [block] } as AnthropicMessagesPayload['messages'][number]; + } + const content = message.content as readonly CacheableBlock[]; + // `??=` semantics: a block that already states its own marker keeps it. + if (content[site.block].cache_control !== undefined) return message; + return { + ...message, + content: content.map((block, b) => (b === site.block ? { ...block, cache_control: topLevel } : block)), + } as AnthropicMessagesPayload['messages'][number]; + }), + }; + + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + + }, +}); diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/compress-images.ts b/packages/provider-copilot/src/stages/anthropic-messages/compress-images.ts similarity index 74% rename from packages/provider-copilot/src/interceptors/anthropic-messages/compress-images.ts rename to packages/provider-copilot/src/stages/anthropic-messages/compress-images.ts index e8874fb1ca..fd0ae5af88 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/compress-images.ts +++ b/packages/provider-copilot/src/stages/anthropic-messages/compress-images.ts @@ -1,7 +1,9 @@ -import { memoizedBase64Compressor } from '../image-compression.ts'; -import type { AnthropicMessagesBoundaryCtx } from './types.ts'; +import type { MessagesFacts } from '../../chat-facts.ts'; +import { memoizedBase64Compressor } from '../../image-compression.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; import { type ImageSizeCalculator, type SizeCaps, fitWithin } from '@floway-dev/platform'; -import type { AnthropicMessagesImageBlock, AnthropicMessagesMessage, AnthropicMessagesToolResultBlock, AnthropicMessagesToolResultContentBlock, AnthropicMessagesUserContentBlock } from '@floway-dev/protocols/anthropic-messages'; +import type { AnthropicMessagesPayload, AnthropicMessagesImageBlock, AnthropicMessagesMessage, AnthropicMessagesToolResultBlock, AnthropicMessagesToolResultContentBlock, AnthropicMessagesUserContentBlock } from '@floway-dev/protocols/anthropic-messages'; +import { type ProviderModelFacts } from '@floway-dev/provider'; // Per-model image caps for the Claude (Anthropic Messages) egress, measured from the real // /v1/messages generation path (count_tokens misreports the downscale here). @@ -48,11 +50,11 @@ const collectImageBlocks = (messages: AnthropicMessagesMessage[]): AnthropicMess return blocks; }; -const compressInlineImages = async (ctx: AnthropicMessagesBoundaryCtx): Promise => { - const blocks = collectImageBlocks(ctx.payload.messages); - if (blocks.length === 0) return; +const compressInlineImages = async (payload: AnthropicMessagesPayload, model: ProviderModelFacts): Promise => { + const blocks = collectImageBlocks(payload.messages); + if (blocks.length === 0) return payload; - const caps = claudeImageCaps(ctx.model.id); + const caps = claudeImageCaps(model.id); const targetSize: ImageSizeCalculator = source => fitWithin(source, caps); const compress = memoizedBase64Compressor(targetSize); const compressedData = new Map(); @@ -87,9 +89,9 @@ const compressInlineImages = async (ctx: AnthropicMessagesBoundaryCtx): Promise< }; }); - ctx.payload = { - ...ctx.payload, - messages: ctx.payload.messages.map(message => + return { + ...payload, + messages: payload.messages.map(message => message.role === 'user' && Array.isArray(message.content) && message.content.some(block => hasCompressedImage(block) || hasCompressedToolResultImage(block)) @@ -103,14 +105,19 @@ const compressInlineImages = async (ctx: AnthropicMessagesBoundaryCtx): Promise< // same definition serves both the streaming Anthropic Messages boundary chain and the // count_tokens boundary chain, so count_tokens sizes the same recompressed // payload the chat path sends. -export const withInlineImagesCompressed = async ( - ctx: AnthropicMessagesBoundaryCtx, - _env: object, - run: () => Promise, -): Promise => { - // Finish this nested activation before starting the upstream call. Its - // request-local memoizer keys are the full source base64 strings, which can - // be several megabytes each and must not stay live for the response stream. - await compressInlineImages(ctx); - return await run(); -}; + +export const copilotAnthropicMessagesCompressImages = defineStage({ + name: 'copilotAnthropicMessagesCompressImages', + through: { + request: { needs: ['request.provider.payload', 'request.provider.model'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + let payload = facts['request.provider.payload']; + const model = facts['request.provider.model']; + + payload = await compressInlineImages(payload, model); + + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + }, +}); diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/handle-speed-fast.ts b/packages/provider-copilot/src/stages/anthropic-messages/handle-speed-fast.ts similarity index 58% rename from packages/provider-copilot/src/interceptors/anthropic-messages/handle-speed-fast.ts rename to packages/provider-copilot/src/stages/anthropic-messages/handle-speed-fast.ts index 29f3175c30..f89892f5a1 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/handle-speed-fast.ts +++ b/packages/provider-copilot/src/stages/anthropic-messages/handle-speed-fast.ts @@ -1,17 +1,19 @@ -import type { CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; +import type { MessagesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; import type { AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; +import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; /** * Anthropic Fast Mode is a per-request opt-in carried by `speed: "fast"` on * the Anthropic Messages request and echoed back as `usage.speed: "fast"`. Copilot * does not speak Fast Mode on the wire — the upstream resolves the tier via * the `-fast` raw model id picked by `model-selection.ts` and never echoes - * `usage.speed`. This interceptor bridges the two contracts at the Copilot + * `usage.speed`. This stage bridges the two contracts at the Copilot * boundary: * * - Strip `speed: 'fast' | 'standard'` from the outbound payload. The - * value was already consumed by `callAnthropicMessages` to pick the raw variant + * value was already consumed by model selection to pick the raw variant * and pre-validate Fast Mode support, so passing it through to Copilot * would just trigger an unknown-field 400 from a strict upstream. * - Leave any other `speed` value untouched. Unknown values mean the @@ -27,23 +29,6 @@ import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; * - https://docs.claude.com/en/build-with-claude/fast-mode * - https://docs.claude.com/en/api/service-tiers */ -export const withSpeedFast: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, _env, run) => { - const speed = ctx.payload.speed; - const stampFast = speed === 'fast'; - - if (speed === 'fast' || speed === 'standard') { - const { speed: _stripped, ...payload } = ctx.payload; - ctx.payload = payload; - } - - const result = await run(); - - if (!stampFast || result.type !== 'events') return result; - return { - ...result, - events: stampFastSpeedOntoUsage(result.events), - }; -}; const stampFastSpeedOntoUsage = async function* ( frames: AsyncIterable>, @@ -74,3 +59,18 @@ const stampFastSpeedOntoUsage = async function* ( yield frame; } }; + +export const copilotAnthropicMessagesSpeedFast = defineStage, ProviderChatResponse<'anthropicMessages'>, ProviderChatServices>({ + name: 'copilotAnthropicMessagesSpeedFast', + through: { request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, response: { needs: ['response.provider.output'], consumes: ['response.provider.output'], provides: ['response.provider.output'] } }, + execute: async (facts, next, use) => { + let payload = facts['request.provider.payload']; + const speed = payload.speed; + const stamp = speed === 'fast'; + if (speed === 'fast' || speed === 'standard') { const { speed: _, ...withoutSpeed } = payload; payload = withoutSpeed; } + const back = await next(move({ ...facts, 'request.provider.payload': payload })); + const output = back['response.provider.output']; + if (!stamp || output === null || !('kind' in output) || output.kind !== 'stream') return move({ ...back }); + return move({ ...back, 'response.provider.output': { ...output, frames: use.recordProtocolFrames(stampFastSpeedOntoUsage(output.frames)) } }); + }, +}); diff --git a/packages/provider-copilot/src/stages/anthropic-messages/normalize-anthropic-beta.ts b/packages/provider-copilot/src/stages/anthropic-messages/normalize-anthropic-beta.ts new file mode 100644 index 0000000000..5224be068e --- /dev/null +++ b/packages/provider-copilot/src/stages/anthropic-messages/normalize-anthropic-beta.ts @@ -0,0 +1,44 @@ +import type { MessagesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; + +const ALLOWED_ANTHROPIC_BETAS = new Set([ + 'interleaved-thinking-2025-05-14', + 'context-management-2025-06-27', + 'advanced-tool-use-2025-11-20', +]); +const INTERLEAVED_THINKING_BETA = 'interleaved-thinking-2025-05-14'; +const CONTEXT_MANAGEMENT_BETA = 'context-management-2025-06-27'; + +// Copilot rejects unknown beta values. Preserve the supported subset of the +// caller's Anthropic Messages beta intent, synthesize VS Code's thinking default only +// when the caller supplied no beta intent, and keep context_management paired +// with its required token. +// https://github.com/microsoft/vscode/blob/a234109a108ad2ca78b7d0883688b0a84e3fab42/extensions/copilot/src/platform/endpoint/node/chatEndpoint.ts#L262-L282 +// https://github.com/microsoft/vscode/blob/a234109a108ad2ca78b7d0883688b0a84e3fab42/extensions/copilot/src/extension/chatSessions/claude/node/claudeLanguageModelServer.ts#L413-L427 + +export const copilotAnthropicMessagesNormalizeAnthropicBeta = defineStage({ + name: 'copilotAnthropicMessagesNormalizeAnthropicBeta', + through: { + request: { needs: ['request.provider.payload', 'request.provider.anthropicBeta'], consumes: ['request.provider.anthropicBeta'], provides: ['request.provider.anthropicBeta'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.provider.payload'] as typeof facts['request.provider.payload'] & { context_management?: unknown }; + let beta = facts['request.provider.anthropicBeta']; + + const callerSuppliedBeta = beta.length > 0; + beta = [...new Set(beta.filter(beta => ALLOWED_ANTHROPIC_BETAS.has(beta)))]; + + const isAdaptiveThinking = payload.thinking?.type === 'adaptive'; + if (!callerSuppliedBeta && payload.thinking?.budget_tokens && !isAdaptiveThinking) { + beta = [...beta, INTERLEAVED_THINKING_BETA]; + } + + if (payload.context_management !== undefined && !beta.includes(CONTEXT_MANAGEMENT_BETA)) { + beta = [...beta, CONTEXT_MANAGEMENT_BETA]; + } + + return move({ ...await next(move({ ...facts, 'request.provider.anthropicBeta': beta })) }); + + }, +}); diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/promote-thinking-display.ts b/packages/provider-copilot/src/stages/anthropic-messages/promote-thinking-display.ts similarity index 65% rename from packages/provider-copilot/src/interceptors/anthropic-messages/promote-thinking-display.ts rename to packages/provider-copilot/src/stages/anthropic-messages/promote-thinking-display.ts index d869a42fd8..ee56db9dba 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/promote-thinking-display.ts +++ b/packages/provider-copilot/src/stages/anthropic-messages/promote-thinking-display.ts @@ -1,7 +1,9 @@ -import type { AnthropicMessagesBoundaryCtx, CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; +import type { MessagesFacts } from '../../chat-facts.ts'; import { copilotRawModelId } from '../../model-name.ts'; -import type { AnthropicMessagesStreamEvent, AnthropicMessagesThinkingDisplay } from '@floway-dev/protocols/anthropic-messages'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent, AnthropicMessagesThinkingDisplay } from '@floway-dev/protocols/anthropic-messages'; import { eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; +import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; const CLAUDE_VERSION_PATTERN = /(?:^|-)(\d+)\.(\d+)(?=-|$)/; @@ -20,10 +22,10 @@ const isClaudeVersionAtLeast = (model: string, major: number, minor: number): bo return modelMajor > major || (modelMajor === major && modelMinor >= minor); }; -export const resolveAnthropicMessagesDownstreamThinkingDisplay = (ctx: Pick): AnthropicMessagesThinkingDisplay | undefined => { +export const resolveAnthropicMessagesDownstreamThinkingDisplay = (ctx: { payload: AnthropicMessagesPayload }): AnthropicMessagesThinkingDisplay | undefined => { const display = ctx.payload.thinking?.display; if (display !== undefined) { - // Request JSON is not runtime-validated before boundary interceptors; leave + // Request JSON is not runtime-validated before boundary stages; leave // unknown display values untouched so upstream, not this workaround, owns // rejecting or accepting future values. return isAnthropicMessagesThinkingDisplay(display) ? display : undefined; @@ -73,7 +75,7 @@ const omitThinkingTextFromProtocolFrames = async function* (frames: AsyncIterabl * the model is reasoning. Our Copilot probes found Claude 4.7 defaults to * omitted display, while 4.6/4.5 default to summarized; forcing summarized * upstream keeps data flowing during thinking and avoids the idle gap. To keep - * downstream omitted semantics, this boundary interceptor removes only thinking + * downstream omitted semantics, this boundary stage removes only thinking * text/deltas after the upstream attempt and preserves every `signature` byte; * the same probes showed blank thinking text is accepted, while any signature * tampering makes the next Anthropic Messages request fail with 400. Those probes justify @@ -91,24 +93,20 @@ const omitThinkingTextFromProtocolFrames = async function* (frames: AsyncIterabl * - https://github.com/anthropics/claude-code/issues/46987 * - https://github.com/anthropics/claude-code/issues/50477 */ -export const withThinkingDisplayPromoted: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, _env, run) => { - const downstreamDisplay = resolveAnthropicMessagesDownstreamThinkingDisplay(ctx); - const thinking = ctx.payload.thinking; - const hasActiveThinking = !!thinking && thinking.type !== 'disabled'; - const shouldExposeOmitted = hasActiveThinking && downstreamDisplay === 'omitted'; - if (hasActiveThinking && downstreamDisplay !== undefined && downstreamDisplay !== 'full') { - ctx.payload.thinking = { - ...thinking, - display: 'summarized', - }; - } - - const result = await run(); - - if (!shouldExposeOmitted || result.type !== 'events') return result; - return { - ...result, - events: omitThinkingTextFromProtocolFrames(result.events), - }; -}; +export const copilotAnthropicMessagesThinkingDisplay = defineStage, ProviderChatResponse<'anthropicMessages'>, ProviderChatServices>({ + name: 'copilotAnthropicMessagesThinkingDisplay', + through: { request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, response: { needs: ['response.provider.output'], consumes: ['response.provider.output'], provides: ['response.provider.output'] } }, + execute: async (facts, next, use) => { + let payload = facts['request.provider.payload']; + const display = resolveAnthropicMessagesDownstreamThinkingDisplay({ payload }); + const thinking = payload.thinking; + const active = !!thinking && thinking.type !== 'disabled'; + const stamp = active && display === 'omitted'; + if (active && display !== undefined && display !== 'full') payload = { ...payload, thinking: { ...thinking, display: 'summarized' } }; + const back = await next(move({ ...facts, 'request.provider.payload': payload })); + const output = back['response.provider.output']; + if (!stamp || output === null || !('kind' in output) || output.kind !== 'stream') return move({ ...back }); + return move({ ...back, 'response.provider.output': { ...output, frames: use.recordProtocolFrames(omitThinkingTextFromProtocolFrames(output.frames)) } }); + }, +}); diff --git a/packages/provider-copilot/src/stages/anthropic-messages/rewrite-context-window-error.ts b/packages/provider-copilot/src/stages/anthropic-messages/rewrite-context-window-error.ts new file mode 100644 index 0000000000..f846e1d43f --- /dev/null +++ b/packages/provider-copilot/src/stages/anthropic-messages/rewrite-context-window-error.ts @@ -0,0 +1,36 @@ +import { exchangeResponse, takeHttpResponse } from '@floway-dev/http/pipeline'; +import { defineStage, move, setRelease } from '@floway-dev/pipeline'; +import { buildPromptTooLongBody } from '@floway-dev/protocols/anthropic-messages'; +import type { ProviderChatResponse } from '@floway-dev/provider'; + +export const rewriteCopilotContextWindowError = defineStage, ProviderChatResponse<'anthropicMessages'> & { 'response.copilot.errorBody': unknown }>({ + name: 'rewriteCopilotContextWindowError', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { needs: ['response.http.exchange', 'response.http.body', 'response.provider.output'], consumes: ['response.http.exchange', 'response.http.body', 'response.provider.output'], provides: ['response.http.exchange', 'response.http.body', 'response.provider.output', 'response.copilot.errorBody'] }, + }, + execute: async (facts, next) => { + const back = await next(move({ ...facts })); + const exchange = back['response.http.exchange']; + if (exchange.type !== 'response' || exchange.status >= 200 && exchange.status < 300) return move({ ...back, 'response.copilot.errorBody': null }); + const response = exchangeResponse(exchange); + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + const bytes = new Uint8Array(await response.arrayBuffer()); + const text = new TextDecoder().decode(bytes); + let body: unknown; + try { body = JSON.parse(text); } catch (error) { if (!(error instanceof SyntaxError)) throw error; body = text; } + const tooLong = text.includes('Request body is too large for model context window') || text.includes('context_length_exceeded'); + if (!tooLong) { + const replay = takeHttpResponse(new Response(exchange.body === null ? null : bytes, { status: exchange.status, statusText: exchange.statusText, headers: exchange.headers.map(([name, value]): [string, string] => [name, value]) })); + return move({ ...back, 'response.http.exchange': replay, 'response.http.body': replay.body, 'response.copilot.errorBody': body }); + } + const rewritten: unknown = JSON.parse(new TextDecoder().decode(buildPromptTooLongBody())); + return move({ + ...back, 'response.copilot.errorBody': body, 'response.provider.output': { + status: 400, + message: 'prompt is too long', + body: rewritten, + }, + }); + }, +}); diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/set-claude-agent-headers.ts b/packages/provider-copilot/src/stages/anthropic-messages/set-claude-agent-headers.ts similarity index 63% rename from packages/provider-copilot/src/interceptors/anthropic-messages/set-claude-agent-headers.ts rename to packages/provider-copilot/src/stages/anthropic-messages/set-claude-agent-headers.ts index 036af530cb..aafe51ccd5 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/set-claude-agent-headers.ts +++ b/packages/provider-copilot/src/stages/anthropic-messages/set-claude-agent-headers.ts @@ -1,6 +1,8 @@ -import { parseUserIdMetadata } from './detect-claude-code-metadata.ts'; -import type { CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; import { CLAUDE_AGENT_USER_AGENT } from '../../auth.ts'; +import type { MessagesFacts } from '../../chat-facts.ts'; +import { parseUserIdMetadata } from '../../claude-code-metadata.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { replaceHttpHeader } from '@floway-dev/provider'; /** * When Anthropic Messages traffic comes from the Claude Code SDK proxy, VSCode @@ -42,17 +44,28 @@ import { CLAUDE_AGENT_USER_AGENT } from '../../auth.ts'; */ const UPSTREAM_REJECTS_CLAUDE_AGENT_IDENTITY = new Set(['claude-opus-4-8']); -export const withClaudeAgentHeadersSet: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, _env, run) => { - if (UPSTREAM_REJECTS_CLAUDE_AGENT_IDENTITY.has(ctx.payload.model)) { - return await run(); - } +export const copilotAnthropicMessagesSetClaudeAgentHeaders = defineStage({ + name: 'copilotAnthropicMessagesSetClaudeAgentHeaders', + through: { + request: { needs: ['request.provider.payload', 'request.http.headers', 'request.provider.model'], consumes: ['request.http.headers'], provides: ['request.http.headers'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.provider.payload']; + let headers = facts['request.http.headers']; - const { safetyIdentifier, sessionId } = parseUserIdMetadata(ctx.payload.metadata?.user_id); - if (safetyIdentifier && sessionId) { - ctx.headers.set('x-interaction-type', 'messages-proxy'); - ctx.headers.set('openai-intent', 'messages-proxy'); - ctx.headers.set('user-agent', CLAUDE_AGENT_USER_AGENT); - ctx.headers.set('copilot-integration-id', ''); - } - return await run(); -}; + if (UPSTREAM_REJECTS_CLAUDE_AGENT_IDENTITY.has(payload.model)) { + return move({ ...await next(move({ ...facts, 'request.http.headers': headers })) }); + } + + const { safetyIdentifier, sessionId } = parseUserIdMetadata(payload.metadata?.user_id); + if (safetyIdentifier && sessionId) { + headers = replaceHttpHeader(headers, 'x-interaction-type', 'messages-proxy'); + headers = replaceHttpHeader(headers, 'openai-intent', 'messages-proxy'); + headers = replaceHttpHeader(headers, 'user-agent', CLAUDE_AGENT_USER_AGENT); + headers = replaceHttpHeader(headers, 'copilot-integration-id', ''); + } + return move({ ...await next(move({ ...facts, 'request.http.headers': headers })) }); + + }, +}); diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/set-compact-headers.ts b/packages/provider-copilot/src/stages/anthropic-messages/set-compact-headers.ts similarity index 84% rename from packages/provider-copilot/src/interceptors/anthropic-messages/set-compact-headers.ts rename to packages/provider-copilot/src/stages/anthropic-messages/set-compact-headers.ts index 4ab55d9361..43592be632 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/set-compact-headers.ts +++ b/packages/provider-copilot/src/stages/anthropic-messages/set-compact-headers.ts @@ -1,5 +1,7 @@ -import type { CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; +import type { MessagesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; import type { AnthropicMessagesMessage, AnthropicMessagesTextBlock } from '@floway-dev/protocols/anthropic-messages'; +import { replaceHttpHeader } from '@floway-dev/provider'; /** * Claude Code (and OpenCode) periodically asks the model to summarize the @@ -105,22 +107,33 @@ const classifyCompact = (payload: { messages: AnthropicMessagesMessage[]; system return null; }; -export const withCompactHeadersSet: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, _env, run) => { - const kind = classifyCompact(ctx.payload); - if (kind === 'compact-request') { - ctx.headers.set('x-initiator', 'agent'); - ctx.headers.set('x-interaction-type', 'conversation-compaction'); +export const copilotAnthropicMessagesSetCompactHeaders = defineStage({ + name: 'copilotAnthropicMessagesSetCompactHeaders', + through: { + request: { needs: ['request.provider.payload', 'request.http.headers'], consumes: ['request.http.headers'], provides: ['request.http.headers'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.provider.payload']; + let headers = facts['request.http.headers']; + + const kind = classifyCompact(payload); + if (kind === 'compact-request') { + headers = replaceHttpHeader(headers, 'x-initiator', 'agent'); + headers = replaceHttpHeader(headers, 'x-interaction-type', 'conversation-compaction'); // openai-intent stays at `copilotAuthedFetch`'s `conversation-agent` // default from `packages/provider-copilot/src/auth.ts` — the same value // caozhiyuan/copilot-api re-pins inside prepareForCompact, so setting it // here would be a no-op. - } else if (kind === 'auto-continue') { + } else if (kind === 'auto-continue') { // Auto-continue gets only the agent-initiator tag; interaction-type stays // at `copilotAuthedFetch`'s `conversation-agent` default from // `packages/provider-copilot/src/auth.ts`. // This mirrors prepareForCompact when compactType === COMPACT_AUTO_CONTINUE: // it sets x-initiator: agent and leaves x-interaction-type untouched. - ctx.headers.set('x-initiator', 'agent'); - } - return await run(); -}; + headers = replaceHttpHeader(headers, 'x-initiator', 'agent'); + } + return move({ ...await next(move({ ...facts, 'request.http.headers': headers })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/anthropic-messages/set-initiator-header.ts b/packages/provider-copilot/src/stages/anthropic-messages/set-initiator-header.ts new file mode 100644 index 0000000000..61f26175de --- /dev/null +++ b/packages/provider-copilot/src/stages/anthropic-messages/set-initiator-header.ts @@ -0,0 +1,46 @@ +import type { MessagesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { replaceHttpHeader } from '@floway-dev/provider'; + +/** + * Copilot's `x-initiator` header distinguishes turns that the human user just + * triggered (`user`) from turns the agent triggered to consume a tool result + * (`agent`). The header gates Copilot-side abuse controls and conversation + * accounting. We classify by the last message: a user turn whose content + * mixes plain blocks with tool_results is still user-initiated; a user turn + * whose content is *only* tool_results is the agent consuming results, and + * any assistant-final turn (which only happens on count-tokens replays) is + * always agent. + * + * The header name is lowercase `x-initiator`; HTTP header names are + * case-insensitive on the wire, so the casing is cosmetic. + * + * References: + * - https://github.com/caozhiyuan/copilot-api/blob/master/src/services/copilot/create-chat-completions.ts + */ + +export const copilotAnthropicMessagesSetInitiatorHeader = defineStage({ + name: 'copilotAnthropicMessagesSetInitiatorHeader', + through: { + request: { needs: ['request.provider.payload', 'request.http.headers'], consumes: ['request.http.headers'], provides: ['request.http.headers'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.provider.payload']; + let headers = facts['request.http.headers']; + + const lastMessage = payload.messages[payload.messages.length - 1]; + let initiator: 'user' | 'agent'; + if (lastMessage?.role !== 'user') { + initiator = 'agent'; + } else if (!Array.isArray(lastMessage.content)) { + initiator = 'user'; + } else { + initiator = lastMessage.content.some(block => block.type !== 'tool_result') ? 'user' : 'agent'; + } + headers = replaceHttpHeader(headers, 'x-initiator', initiator); + + return move({ ...await next(move({ ...facts, 'request.http.headers': headers })) }); + + }, +}); diff --git a/packages/provider-copilot/src/interceptors/anthropic-messages/set-interaction-id-header.ts b/packages/provider-copilot/src/stages/anthropic-messages/set-interaction-id-header.ts similarity index 50% rename from packages/provider-copilot/src/interceptors/anthropic-messages/set-interaction-id-header.ts rename to packages/provider-copilot/src/stages/anthropic-messages/set-interaction-id-header.ts index 0315638e69..841a7916fe 100644 --- a/packages/provider-copilot/src/interceptors/anthropic-messages/set-interaction-id-header.ts +++ b/packages/provider-copilot/src/stages/anthropic-messages/set-interaction-id-header.ts @@ -1,7 +1,9 @@ import { v4 } from 'uuid'; -import { parseUserIdMetadata } from './detect-claude-code-metadata.ts'; -import type { CopilotAnthropicMessagesBoundaryInterceptor } from './types.ts'; +import type { MessagesFacts } from '../../chat-facts.ts'; +import { parseUserIdMetadata } from '../../claude-code-metadata.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { replaceHttpHeader } from '@floway-dev/provider'; /** * Copilot's `x-interaction-id` header threads a conversation through its @@ -24,10 +26,21 @@ const sessionUuid = async (input: string): Promise => { return v4({ random: new Uint8Array(await crypto.subtle.digest('SHA-256', data)) }); }; -export const withInteractionIdHeaderSet: CopilotAnthropicMessagesBoundaryInterceptor = async (ctx, _env, run) => { - const { sessionId } = parseUserIdMetadata(ctx.payload.metadata?.user_id); - if (sessionId) { - ctx.headers.set('x-interaction-id', await sessionUuid(sessionId)); - } - return await run(); -}; +export const copilotAnthropicMessagesSetInteractionIdHeader = defineStage({ + name: 'copilotAnthropicMessagesSetInteractionIdHeader', + through: { + request: { needs: ['request.provider.payload', 'request.http.headers'], consumes: ['request.http.headers'], provides: ['request.http.headers'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.provider.payload']; + let headers = facts['request.http.headers']; + + const { sessionId } = parseUserIdMetadata(payload.metadata?.user_id); + if (sessionId) { + headers = replaceHttpHeader(headers, 'x-interaction-id', await sessionUuid(sessionId)); + } + return move({ ...await next(move({ ...facts, 'request.http.headers': headers })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/anthropic-messages/set-vision-header.ts b/packages/provider-copilot/src/stages/anthropic-messages/set-vision-header.ts new file mode 100644 index 0000000000..a4c1b8b929 --- /dev/null +++ b/packages/provider-copilot/src/stages/anthropic-messages/set-vision-header.ts @@ -0,0 +1,44 @@ +import type { MessagesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { AnthropicMessagesAssistantMessage, AnthropicMessagesUserMessage } from '@floway-dev/protocols/anthropic-messages'; +import { replaceHttpHeader } from '@floway-dev/provider'; + +/** + * Copilot rejects Anthropic `image` blocks as plain text unless the private + * `copilot-vision-request: true` header is set. Detection must scan the final + * shaped request payload (after other Anthropic Messages boundary stages have run) + * and cover both the top-level `message.content` and the nested + * `tool_result.content[]` shape; Anthropic allows images in both positions. + * + * References: + * - https://github.com/caozhiyuan/copilot-api/commit/1f6b98924ae092db9b2010846c32e5cbf10817df + */ +const contentHasImage = (content: AnthropicMessagesUserMessage['content'] | AnthropicMessagesAssistantMessage['content']): boolean => { + if (!Array.isArray(content)) return false; + return content.some(block => { + if (block.type === 'image') return true; + if (block.type === 'tool_result' && Array.isArray(block.content)) { + return block.content.some(inner => inner.type === 'image'); + } + return false; + }); +}; + +export const copilotAnthropicMessagesSetVisionHeader = defineStage({ + name: 'copilotAnthropicMessagesSetVisionHeader', + through: { + request: { needs: ['request.provider.payload', 'request.http.headers'], consumes: ['request.http.headers'], provides: ['request.http.headers'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.provider.payload']; + let headers = facts['request.http.headers']; + + if (payload.messages.some(message => contentHasImage(message.content))) { + headers = replaceHttpHeader(headers, 'copilot-vision-request', 'true'); + } + + return move({ ...await next(move({ ...facts, 'request.http.headers': headers })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/anthropic-messages/strip-cache-control-extensions.ts b/packages/provider-copilot/src/stages/anthropic-messages/strip-cache-control-extensions.ts new file mode 100644 index 0000000000..622cb7c3dd --- /dev/null +++ b/packages/provider-copilot/src/stages/anthropic-messages/strip-cache-control-extensions.ts @@ -0,0 +1,77 @@ +import type { MessagesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { mapKeepingIdentity, withKeysChanged } from '@floway-dev/protocols/common'; + +/** + * Two `cache_control` sub-fields are beta extensions to the base + * `CacheControlEphemeral` shape that Copilot's stricter Anthropic-Messages-upstream + * deployments (claude-haiku-4.5, claude-sonnet-4.5/4.6, intermittently + * claude-opus-4.5) reject: + * + * - `scope`: added by Claude Code's `prompt-caching-scope-2025-11-27` beta. + * Copilot returns `cache_control.scope: Extra inputs are not permitted`. + * - `ttl`: added by the `extended-cache-ttl-2025-04-11` beta. Floway does + * not enable that beta for Copilot, so any `ttl` value trips the same + * schema rejection on the body. + * + * Walk every position where `cache_control` may appear — system blocks, + * tools, message content blocks including `tool_use` and `tool_result` — + * and strip both sub-fields, keeping `{ type: 'ephemeral' }` so prompt + * caching still primes on slots that do honour the marker. + * + * The top-level `cache_control` field is handled by + * `withTopLevelCacheControlApplied`, which runs first and either ports it + * onto the last cacheable block (where this stage cleans it) or + * deletes it when no cacheable block exists. + * + * Custom providers that speak Anthropic Messages directly may accept these + * betas natively; this stage is Copilot-only. + * + * References: + * - https://github.com/anthropics/anthropic-sdk-typescript/blob/a53f60d59ca904f3e79296586642aac3ce68ae02/src/resources/messages/messages.ts#L2909-L2913 + * - https://github.com/caozhiyuan/copilot-api/issues/143 + * - https://github.com/caozhiyuan/copilot-api/issues/144 + * - https://github.com/caozhiyuan/copilot-api/issues/269 + * - https://github.com/caozhiyuan/copilot-api/commit/ce8224c55933f811abe5bf9ba42f9336a7852997 + */ +/** The block with its cache-control extensions gone, or the same block when it carried none. + * Returning the original is what lets an untouched message ride through by identity. */ +const stripExtensions = (block: T): T => { + const cacheControl = (block as Record).cache_control; + if (!cacheControl || typeof cacheControl !== 'object') return block; + + const { scope: _scope, ttl: _ttl, ...rest } = cacheControl as Record; + return withKeysChanged(block, { cache_control: Object.keys(rest).length > 0 ? rest : undefined }); +}; + +export const copilotAnthropicMessagesStripCacheControlExtensions = defineStage({ + name: 'copilotAnthropicMessagesStripCacheControlExtensions', + through: { + request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + let payload = facts['request.provider.payload']; + + const system = Array.isArray(payload.system) + ? mapKeepingIdentity(payload.system, stripExtensions) + : payload.system; + const tools = payload.tools === undefined ? payload.tools : mapKeepingIdentity(payload.tools, stripExtensions); + const messages = mapKeepingIdentity(payload.messages, message => ( + Array.isArray(message.content) + // The content arrays are a union of per-role block types, so the walk is expressed over + // the shape they share — an object — and handed back at the array's own type. + ? withKeysChanged(message, { content: mapKeepingIdentity(message.content as readonly object[], stripExtensions) }) + : message + )); + + payload = withKeysChanged(payload, { + ...(system === payload.system ? {} : { system }), + ...(tools === payload.tools ? {} : { tools }), + ...(messages === payload.messages ? {} : { messages }), + }); + + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/anthropic-messages/strip-eager-input-streaming.ts b/packages/provider-copilot/src/stages/anthropic-messages/strip-eager-input-streaming.ts new file mode 100644 index 0000000000..158b58c172 --- /dev/null +++ b/packages/provider-copilot/src/stages/anthropic-messages/strip-eager-input-streaming.ts @@ -0,0 +1,41 @@ +import type { MessagesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { mapKeepingIdentity, withKeysChanged } from '@floway-dev/protocols/common'; + +/** + * `eager_input_streaming` is a per-tool property in the Anthropic Messages API + * that enables fine-grained tool input streaming. Copilot's native Anthropic Messages + * target has been observed to reject it with + * `"tools.N.custom.eager_input_streaming: Extra inputs are not permitted"`, so + * strip it only at the Copilot target boundary and leave other providers + * untouched. + * + * References: + * - https://github.com/anthropics/anthropic-sdk-typescript/blob/a53f60d59ca904f3e79296586642aac3ce68ae02/src/resources/messages/messages.ts#L1761 + */ + +export const copilotAnthropicMessagesStripEagerInputStreaming = defineStage({ + name: 'copilotAnthropicMessagesStripEagerInputStreaming', + through: { + request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + let payload = facts['request.provider.payload']; + + if (payload.tools) { + payload = withKeysChanged(payload, { + tools: mapKeepingIdentity(payload.tools, tool => { + if (!Object.hasOwn(tool, 'eager_input_streaming')) return tool; + const { eager_input_streaming: _, ...rest } = tool as typeof tool & { + eager_input_streaming?: unknown; + }; + return rest; + }), + }); + } + + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/authenticate-chat.ts b/packages/provider-copilot/src/stages/authenticate-chat.ts new file mode 100644 index 0000000000..d6670768b8 --- /dev/null +++ b/packages/provider-copilot/src/stages/authenticate-chat.ts @@ -0,0 +1,24 @@ +import { getCopilotToken, isCopilotTokenFetchError, type CopilotAuth } from '../auth.ts'; +import type { AuthenticatedChatFacts, CopilotChatFacts } from '../chat-facts.ts'; +import { takeHttpResponse } from '@floway-dev/http/pipeline'; +import { defineStage, move, secret } from '@floway-dev/pipeline'; +import type { ChatProviderOperation, ProviderChatResponse, ProviderServices } from '@floway-dev/provider'; + +export const authenticateCopilotChat = (auth: CopilotAuth) => defineStage, ProviderChatResponse, ProviderChatResponse, ProviderServices>({ + name: 'authenticateCopilotChat', + through: { + request: { needs: ['request.http.callId', 'request.provider.modelKey'], consumes: [], provides: ['request.copilot.session'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + return: { provides: ['response.http.exchange', 'response.http.body', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.provider.output'] }, + execute: async (facts, next, use) => { + const call = use.httpCall(facts['request.http.callId']); + let entry; + try { entry = await getCopilotToken(auth.id, auth.githubHost, auth.githubToken, call.fetcher, call.signal); } catch (error) { + if (!isCopilotTokenFetchError(error)) throw error; + const exchange = takeHttpResponse(new Response(error.body, { status: error.status, headers: error.headers })); + return move({ ...facts, 'response.http.exchange': exchange, 'response.http.body': exchange.body, 'response.provider.modelKey': facts['request.provider.modelKey'], 'response.provider.called': false, 'response.provider.previousCalls': [], 'response.provider.output': null }); + } + return move({ ...await next(move({ ...facts, 'request.copilot.session': { ...entry, token: secret(entry.token) } })) }); + }, +}); diff --git a/packages/provider-copilot/src/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace.ts b/packages/provider-copilot/src/stages/openai-chat-completions/abort-on-tool-argument-whitespace.ts similarity index 58% rename from packages/provider-copilot/src/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace.ts rename to packages/provider-copilot/src/stages/openai-chat-completions/abort-on-tool-argument-whitespace.ts index 9b8219d3da..96aa785364 100644 --- a/packages/provider-copilot/src/interceptors/openai-chat-completions/abort-on-tool-argument-whitespace.ts +++ b/packages/provider-copilot/src/stages/openai-chat-completions/abort-on-tool-argument-whitespace.ts @@ -1,7 +1,9 @@ -import type { CopilotOpenAIChatCompletionsBoundaryInterceptor } from './types.ts'; -import { checkWhitespaceOverflow } from '../shared/whitespace-overflow.ts'; + +import { checkWhitespaceOverflow } from '../../tool-argument-whitespace.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; +import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; /** * Copilot has been observed to emit only whitespace (`\r`, `\n`, `\t`) inside @@ -49,21 +51,26 @@ const isWhitespaceExceeded = ( return false; }; -export const withToolArgumentWhitespaceAborted: CopilotOpenAIChatCompletionsBoundaryInterceptor = async (_invocation, _env, run) => { - const result = await run(); - if (result.type !== 'events') return result; - - return { - ...result, - events: (async function* (): AsyncGenerator> { - const whitespaceByIndex = new Map(); +export const copilotOpenAIChatCompletionsAbortToolWhitespace = defineStage, ProviderChatResponse<'openaiChatCompletions'>, ProviderChatServices>({ + name: 'copilotOpenAIChatCompletionsAbortToolWhitespace', + through: { request: { needs: [], consumes: [], provides: [] }, response: { needs: ['response.provider.output'], consumes: ['response.provider.output'], provides: ['response.provider.output'] } }, + execute: async (facts, next, use) => { + const back = await next(move({ ...facts })); + const output = back['response.provider.output']; + if (output === null || !('kind' in output) || output.kind !== 'stream') return move({ ...back }); + return move({ + ...back, 'response.provider.output': { + ...output, frames: use.recordProtocolFrames((async function* (): AsyncGenerator> { + const whitespaceByIndex = new Map(); - for await (const frame of result.events) { - if (frame.type === 'event' && isWhitespaceExceeded(frame.event, whitespaceByIndex)) { - throw new Error('Copilot tool call arguments contained excessive consecutive whitespace, indicating a degenerate response.'); - } - yield frame; - } - })(), - }; -}; + for await (const frame of output.frames) { + if (frame.type === 'event' && isWhitespaceExceeded(frame.event, whitespaceByIndex)) { + throw new Error('Copilot tool call arguments contained excessive consecutive whitespace, indicating a degenerate response.'); + } + yield frame; + } + })()), + }, + }); + }, +}); diff --git a/packages/provider-copilot/src/interceptors/openai-chat-completions/attach-cache-control-markers.ts b/packages/provider-copilot/src/stages/openai-chat-completions/attach-cache-control-markers.ts similarity index 70% rename from packages/provider-copilot/src/interceptors/openai-chat-completions/attach-cache-control-markers.ts rename to packages/provider-copilot/src/stages/openai-chat-completions/attach-cache-control-markers.ts index 0c67d7c9ce..17d9759c4b 100644 --- a/packages/provider-copilot/src/interceptors/openai-chat-completions/attach-cache-control-markers.ts +++ b/packages/provider-copilot/src/stages/openai-chat-completions/attach-cache-control-markers.ts @@ -1,4 +1,5 @@ -import type { CopilotOpenAIChatCompletionsBoundaryInterceptor } from './types.ts'; +import type { ChatCompletionsFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; import type { OpenAIChatCompletionsMessage } from '@floway-dev/protocols/openai-chat-completions'; /** @@ -63,13 +64,30 @@ const selectCacheMarkerIndexes = (messages: readonly OpenAIChatCompletionsMessag return [...new Set([...systemIndexes, ...nonSystemIndexes])].sort((a, b) => a - b); }; -export const withCacheControlMarkersAttached: CopilotOpenAIChatCompletionsBoundaryInterceptor = async (ctx, _env, run) => { - const indexes = selectCacheMarkerIndexes(ctx.payload.messages); - for (const index of indexes) { - // Fresh object per message so downstream mutations (none today, but - // cheap insurance) cannot bleed across messages. - (ctx.payload.messages[index] as CopilotCacheableMessage).copilot_cache_control = { ...COPILOT_CONTEXT_CACHE_CONTROL }; - } +export const copilotOpenAIChatCompletionsAttachCacheControlMarkers = defineStage({ + name: 'copilotOpenAIChatCompletionsAttachCacheControlMarkers', + through: { + request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + let payload = facts['request.provider.payload']; - return await run(); -}; + const marked = new Set(selectCacheMarkerIndexes(payload.messages)); + if (marked.size === 0) return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + + payload = { + ...payload, + // A marker is its own object per message, so nothing downstream can carry a change from + // one to another. The messages that were not selected ride through by identity. + messages: payload.messages.map((message, index) => ( + marked.has(index) + ? { ...message, copilot_cache_control: { ...COPILOT_CONTEXT_CACHE_CONTROL } } as CopilotCacheableMessage + : message + )), + }; + + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + + }, +}); diff --git a/packages/provider-copilot/src/interceptors/openai-chat-completions/compress-images.ts b/packages/provider-copilot/src/stages/openai-chat-completions/compress-images.ts similarity index 51% rename from packages/provider-copilot/src/interceptors/openai-chat-completions/compress-images.ts rename to packages/provider-copilot/src/stages/openai-chat-completions/compress-images.ts index 63f9c15f57..43594b1665 100644 --- a/packages/provider-copilot/src/interceptors/openai-chat-completions/compress-images.ts +++ b/packages/provider-copilot/src/stages/openai-chat-completions/compress-images.ts @@ -1,23 +1,25 @@ -import { memoizedDataUrlCompressor } from '../image-compression.ts'; -import { targetSizeForOpenAIResponsesChat } from '../image-size.ts'; -import type { OpenAIChatCompletionsBoundaryCtx, CopilotOpenAIChatCompletionsBoundaryInterceptor } from './types.ts'; -import type { OpenAIChatCompletionsContentPart, OpenAIChatCompletionsMessage } from '@floway-dev/protocols/openai-chat-completions'; +import type { ChatCompletionsFacts } from '../../chat-facts.ts'; +import { memoizedDataUrlCompressor } from '../../image-compression.ts'; +import { targetSizeForOpenAIResponsesChat } from '../../image-size.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsContentPart, OpenAIChatCompletionsMessage } from '@floway-dev/protocols/openai-chat-completions'; +import { type ProviderModelFacts } from '@floway-dev/provider'; import { isBase64ImageDataUrl } from '@floway-dev/provider'; type OpenAIChatCompletionsImagePart = Extract; -const compressInlineImages = async (ctx: OpenAIChatCompletionsBoundaryCtx): Promise => { +const compressInlineImages = async (payload: OpenAIChatCompletionsPayload, model: ProviderModelFacts): Promise => { const targets: OpenAIChatCompletionsImagePart[] = []; - for (const message of ctx.payload.messages) { + for (const message of payload.messages) { if (!Array.isArray(message.content)) continue; for (const part of message.content) { if (part.type === 'image_url' && isBase64ImageDataUrl(part.image_url.url)) targets.push(part); } } - if (targets.length === 0) return; + if (targets.length === 0) return payload; - const compress = memoizedDataUrlCompressor(targetSizeForOpenAIResponsesChat(ctx.model.id)); + const compress = memoizedDataUrlCompressor(targetSizeForOpenAIResponsesChat(model.id)); const compressedUrls = new Map(); await Promise.all( targets.map(async target => { @@ -31,9 +33,9 @@ const compressInlineImages = async (ctx: OpenAIChatCompletionsBoundaryCtx): Prom if (url === undefined) throw new Error('Missing compressed OpenAI Chat Completions image URL'); return { ...part, image_url: { ...part.image_url, url } }; }; - ctx.payload = { - ...ctx.payload, - messages: ctx.payload.messages.map((message): OpenAIChatCompletionsMessage => { + return { + ...payload, + messages: payload.messages.map((message): OpenAIChatCompletionsMessage => { if (!Array.isArray(message.content) || !message.content.some(hasCompressedImage)) return message; return { ...message, @@ -46,10 +48,19 @@ const compressInlineImages = async (ctx: OpenAIChatCompletionsBoundaryCtx): Prom // Recompresses every inline base64 image (`data:image/*;base64,...` in an // `image_url` part) in the outgoing OpenAI Chat Completions payload to WebP before // the Copilot upstream call. Remote https image references are left untouched. -export const withInlineImagesCompressed: CopilotOpenAIChatCompletionsBoundaryInterceptor = async (ctx, _env, run) => { - // Finish this nested activation before starting the upstream call. Its - // request-local memoizer keys are the full source data URLs, which can be - // several megabytes each and must not stay live for the response stream. - await compressInlineImages(ctx); - return await run(); -}; + +export const copilotOpenAIChatCompletionsCompressImages = defineStage({ + name: 'copilotOpenAIChatCompletionsCompressImages', + through: { + request: { needs: ['request.provider.payload', 'request.provider.model'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + let payload = facts['request.provider.payload']; + const model = facts['request.provider.model']; + + payload = await compressInlineImages(payload, model); + + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + }, +}); diff --git a/packages/provider-copilot/src/interceptors/openai-chat-completions/set-initiator-header.ts b/packages/provider-copilot/src/stages/openai-chat-completions/set-initiator-header.ts similarity index 69% rename from packages/provider-copilot/src/interceptors/openai-chat-completions/set-initiator-header.ts rename to packages/provider-copilot/src/stages/openai-chat-completions/set-initiator-header.ts index 6877390860..591a578b37 100644 --- a/packages/provider-copilot/src/interceptors/openai-chat-completions/set-initiator-header.ts +++ b/packages/provider-copilot/src/stages/openai-chat-completions/set-initiator-header.ts @@ -1,4 +1,6 @@ -import type { CopilotOpenAIChatCompletionsBoundaryInterceptor } from './types.ts'; +import type { ChatCompletionsFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { replaceHttpHeader } from '@floway-dev/provider'; /** * Copilot's `x-initiator` header distinguishes user-triggered turns from @@ -32,11 +34,23 @@ import type { CopilotOpenAIChatCompletionsBoundaryInterceptor } from './types.ts * - https://github.com/microsoft/vscode/blob/fb5e582d1c8edb9ad0a69e50fe6f508a8c095466/extensions/copilot/src/platform/endpoint/node/responsesApi.ts#L419-L468 * - https://github.com/microsoft/vscode/blob/fb5e582d1c8edb9ad0a69e50fe6f508a8c095466/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts#L1453-L1474 */ -export const withInitiatorHeaderSet: CopilotOpenAIChatCompletionsBoundaryInterceptor = async (ctx, _env, run) => { - const lastMessage = ctx.payload.messages.at(-1); - const agentInitiated = lastMessage?.role === 'assistant' + +export const copilotOpenAIChatCompletionsSetInitiatorHeader = defineStage({ + name: 'copilotOpenAIChatCompletionsSetInitiatorHeader', + through: { + request: { needs: ['request.provider.payload', 'request.http.headers'], consumes: ['request.http.headers'], provides: ['request.http.headers'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.provider.payload']; + let headers = facts['request.http.headers']; + + const lastMessage = payload.messages.at(-1); + const agentInitiated = lastMessage?.role === 'assistant' || lastMessage?.role === 'tool'; - ctx.headers.set('x-initiator', agentInitiated ? 'agent' : 'user'); + headers = replaceHttpHeader(headers, 'x-initiator', agentInitiated ? 'agent' : 'user'); + + return move({ ...await next(move({ ...facts, 'request.http.headers': headers })) }); - return await run(); -}; + }, +}); diff --git a/packages/provider-copilot/src/stages/openai-chat-completions/set-vision-header.ts b/packages/provider-copilot/src/stages/openai-chat-completions/set-vision-header.ts new file mode 100644 index 0000000000..8d6a5a428f --- /dev/null +++ b/packages/provider-copilot/src/stages/openai-chat-completions/set-vision-header.ts @@ -0,0 +1,33 @@ +import type { ChatCompletionsFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { replaceHttpHeader } from '@floway-dev/provider'; + +/** + * Copilot's OpenAI Chat Completions endpoint requires the private + * `copilot-vision-request: true` header before it accepts OpenAI-style + * `image_url` content parts; without it images are silently dropped or + * rejected. + * + * References: + * - https://github.com/caozhiyuan/copilot-api/blob/cd0d0182eb4b9bf68a3376dc79728afa7f42ce07/src/services/copilot/create-chat-completions.ts#L28-L49 + */ + +export const copilotOpenAIChatCompletionsSetVisionHeader = defineStage({ + name: 'copilotOpenAIChatCompletionsSetVisionHeader', + through: { + request: { needs: ['request.provider.payload', 'request.http.headers'], consumes: ['request.http.headers'], provides: ['request.http.headers'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.provider.payload']; + let headers = facts['request.http.headers']; + + const hasImage = payload.messages.some( + message => Array.isArray(message.content) && message.content.some(part => part.type === 'image_url'), + ); + if (hasImage) headers = replaceHttpHeader(headers, 'copilot-vision-request', 'true'); + + return move({ ...await next(move({ ...facts, 'request.http.headers': headers })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/openai-responses/abort-on-tool-argument-whitespace.ts b/packages/provider-copilot/src/stages/openai-responses/abort-on-tool-argument-whitespace.ts new file mode 100644 index 0000000000..fc0c3b8332 --- /dev/null +++ b/packages/provider-copilot/src/stages/openai-responses/abort-on-tool-argument-whitespace.ts @@ -0,0 +1,74 @@ + +import { checkWhitespaceOverflow } from '../../tool-argument-whitespace.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; +import type { OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; + +/** + * Copilot has been observed to emit only whitespace (`\r`, `\n`, `\t`) inside + * `response.function_call_arguments.delta` events until `max_tokens`, never + * producing valid JSON arguments. Detect that pattern per function call output + * index and abort the upstream stream before the client times out. + * + * Behaviour: when any single output index's argument deltas accumulate more + * than `MAX_CONSECUTIVE_WHITESPACE` consecutive whitespace characters, emit a + * OpenAI Responses `error` event followed by a done frame, then end the stream. + * The OpenAI Responses source layer surfaces `error` events as a stream failure. + * + * Lives at the Copilot provider boundary so other OpenAI-Responses-capable providers + * are not slowed by per-delta whitespace inspection. + * + * References: + * - https://github.com/caozhiyuan/copilot-api/commit/4c0d775e1dc6b8648c7ad5f21fb783fc3246facf + * - https://github.com/caozhiyuan/copilot-api/commit/3cdc32c0811469da9eebec5ca3892caf068df542 + */ +const ABORT_MESSAGE = 'Tool call arguments contained excessive whitespace, indicating a degenerate response.'; + +const isArgumentsDelta = (event: OpenAIResponsesStreamEvent): event is OpenAIResponsesStreamEvent & { type: 'response.function_call_arguments.delta'; output_index: number; delta: string } => + event.type === 'response.function_call_arguments.delta'; + +const errorEvent = (): OpenAIResponsesStreamEvent => + ({ + type: 'error', + message: ABORT_MESSAGE, + code: 'api_error', + }) as OpenAIResponsesStreamEvent; + +export const copilotOpenAIResponsesAbortToolWhitespace = defineStage, ProviderChatResponse<'openaiResponses'>, ProviderChatServices>({ + name: 'copilotOpenAIResponsesAbortToolWhitespace', + through: { request: { needs: [], consumes: [], provides: [] }, response: { needs: ['response.provider.output'], consumes: ['response.provider.output'], provides: ['response.provider.output'] } }, + execute: async (facts, next, use) => { + const back = await next(move({ ...facts })); + const output = back['response.provider.output']; + if (output === null || !('kind' in output) || output.kind !== 'stream') return move({ ...back }); + return move({ + ...back, 'response.provider.output': { + ...output, frames: use.recordProtocolFrames((async function* (): AsyncGenerator> { + const whitespaceByIndex = new Map(); + + for await (const frame of output.frames) { + if (frame.type !== 'event' || !isArgumentsDelta(frame.event)) { + yield frame; + continue; + } + + const event = frame.event; + const current = whitespaceByIndex.get(event.output_index) ?? 0; + const { count, exceeded } = checkWhitespaceOverflow(event.delta, current); + whitespaceByIndex.set(event.output_index, count); + + if (exceeded) { + await use.log.warn('Copilot: infinite whitespace detected in OpenAI Responses function call arguments, aborting stream'); + yield eventFrame(errorEvent()); + yield doneFrame(); + return; + } + + yield frame; + } + })()), + }, + }); + }, +}); diff --git a/packages/provider-copilot/src/interceptors/openai-responses/compress-images.ts b/packages/provider-copilot/src/stages/openai-responses/compress-images.ts similarity index 57% rename from packages/provider-copilot/src/interceptors/openai-responses/compress-images.ts rename to packages/provider-copilot/src/stages/openai-responses/compress-images.ts index 55dcb79ee4..c0068cec80 100644 --- a/packages/provider-copilot/src/interceptors/openai-responses/compress-images.ts +++ b/packages/provider-copilot/src/stages/openai-responses/compress-images.ts @@ -1,22 +1,19 @@ -import { memoizedDataUrlCompressor } from '../image-compression.ts'; -import { targetSizeForOpenAIResponsesChat } from '../image-size.ts'; -import type { OpenAIResponsesBoundaryCtx } from './types.ts'; -import type { OpenAIResponsesInputContent, OpenAIResponsesInputImage, OpenAIResponsesToolOutputContent } from '@floway-dev/protocols/openai-responses'; +import type { ResponsesFacts } from '../../chat-facts.ts'; +import { memoizedDataUrlCompressor } from '../../image-compression.ts'; +import { targetSizeForOpenAIResponsesChat } from '../../image-size.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesInputContent, OpenAIResponsesInputImage, OpenAIResponsesToolOutputContent } from '@floway-dev/protocols/openai-responses'; +import { type ProviderModelFacts } from '@floway-dev/provider'; import { isBase64ImageDataUrl } from '@floway-dev/provider'; -// A single client request can reach this boundary more than once — a -// fallback candidate attempt, or another turn of the OpenAI Responses server-tool -// shim's loop — carrying the same nested image part. Remember the exact -// generated URL on that request-owned object so the repeat neither -// re-encodes a lossy WebP nor mistakes an unrelated client WebP for our -// output. The non-enumerable property stays off the wire and does not -// cross an object-spread/JSON ownership boundary. -const compressedImageUrl = Symbol('compressedImageUrl'); -type CompressibleImagePart = OpenAIResponsesInputImage & { image_url: string; [compressedImageUrl]?: string }; +// Identity provenance prevents a repeated attempt from recompressing our own WebP output. +// A weak side table leaves handed-over image facts immutable and releases their data with them. +const compressedImageUrls = new WeakMap(); +type CompressibleImagePart = OpenAIResponsesInputImage & { image_url: string }; -const compressInlineImages = async (ctx: OpenAIResponsesBoundaryCtx): Promise => { +const compressInlineImages = async (payload: CanonicalOpenAIResponsesPayload, model: ProviderModelFacts): Promise => { const targets: Array<{ part: CompressibleImagePart; imageUrl: string }> = []; - for (const item of ctx.payload.input) { + for (const item of payload.input) { const parts = item.type === 'message' ? item.content : item.type === 'function_call_output' || item.type === 'custom_tool_call_output' ? item.output : undefined; @@ -24,14 +21,14 @@ const compressInlineImages = async (ctx: OpenAIResponsesBoundaryCtx): Promise(); await Promise.all( targets.map(async target => { @@ -44,19 +41,15 @@ const compressInlineImages = async (ctx: OpenAIResponsesBoundaryCtx): Promise(parts: TPart[]): TPart[] => parts.map(part => hasCompressedImage(part) ? rewriteImage(part) as TPart : part); - ctx.payload = { - ...ctx.payload, - input: ctx.payload.input.map(item => { + return { + ...payload, + input: payload.input.map(item => { if (item.type === 'message' && Array.isArray(item.content)) { return item.content.some(hasCompressedImage) ? { ...item, content: rewriteParts(item.content) } @@ -79,14 +72,19 @@ const compressInlineImages = async (ctx: OpenAIResponsesBoundaryCtx): Promise( - ctx: OpenAIResponsesBoundaryCtx, - _env: object, - run: () => Promise, -): Promise => { - // Finish this nested activation before starting the upstream call. Its - // request-local memoizer keys are the full source data URLs, which can be - // several megabytes each and must not stay live for the response stream. - await compressInlineImages(ctx); - return await run(); -}; + +export const copilotOpenAIResponsesCompressImages = defineStage({ + name: 'copilotOpenAIResponsesCompressImages', + through: { + request: { needs: ['request.provider.payload', 'request.provider.model'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + let payload = facts['request.provider.payload']; + const model = facts['request.provider.model']; + + payload = await compressInlineImages(payload, model); + + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + }, +}); diff --git a/packages/provider-copilot/src/stages/openai-responses/fill-empty-namespace-descriptions.ts b/packages/provider-copilot/src/stages/openai-responses/fill-empty-namespace-descriptions.ts new file mode 100644 index 0000000000..a35ecbd8ae --- /dev/null +++ b/packages/provider-copilot/src/stages/openai-responses/fill-empty-namespace-descriptions.ts @@ -0,0 +1,31 @@ +import type { ResponsesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { mapOpenAIResponsesTools } from '@floway-dev/protocols/openai-responses'; + +/** + * Some Copilot data planes reject empty namespace descriptions permitted by + * the current OpenAI schema. Fill the permitted-empty text at the Copilot + * boundary so the same request remains portable across those deployments. + * + * References: + * - https://github.com/openai/openai-openapi/commit/466c74a42f51c02f1927bc666815251dc53845dc + * - https://github.com/caozhiyuan/copilot-api/issues/345 + * - https://github.com/engineersamuel/copilot-proxy-rs/commit/1c23d7237bd1bd7c24045d0d426412851bfd57f6 + */ + +export const copilotOpenAIResponsesFillEmptyNamespaceDescriptions = defineStage({ + name: 'copilotOpenAIResponsesFillEmptyNamespaceDescriptions', + through: { + request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + let payload = facts['request.provider.payload']; + + payload = mapOpenAIResponsesTools(payload, tool => tool.type === 'namespace' && tool.description === '' + ? { ...tool, description: `Tools in the ${tool.name} namespace.` } + : tool); + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/openai-responses/force-store-false.ts b/packages/provider-copilot/src/stages/openai-responses/force-store-false.ts new file mode 100644 index 0000000000..a5dd432286 --- /dev/null +++ b/packages/provider-copilot/src/stages/openai-responses/force-store-false.ts @@ -0,0 +1,31 @@ +import type { ResponsesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; + +/** + * Copilot's `/responses` rejects `store: true` with + * `400 {"error":{"message":"store is not supported","code":"unsupported_value","param":"store"}}`. + * Force `store: false` on the outgoing payload once planning has committed to + * the Copilot OpenAI Responses target so the upstream accepts the request. The + * gateway's own stored-items persistence keys off the caller's original `store` + * value captured at parse time and is unaffected by this upstream-only flag. + * + * Generic in the run-result type so the same definition feeds both the + * streaming `/responses` chain and the non-streaming compaction chain — the + * synth-via-trigger compact call also rejects `store: true`. + */ + +export const copilotOpenAIResponsesForceStoreFalse = defineStage({ + name: 'copilotOpenAIResponsesForceStoreFalse', + through: { + request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + let payload = facts['request.provider.payload']; + + payload = { ...payload, store: false }; + + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + + }, +}); diff --git a/packages/provider-copilot/src/interceptors/openai-responses/item-id-membrane.ts b/packages/provider-copilot/src/stages/openai-responses/item-id-membrane.ts similarity index 88% rename from packages/provider-copilot/src/interceptors/openai-responses/item-id-membrane.ts rename to packages/provider-copilot/src/stages/openai-responses/item-id-membrane.ts index 2db859192f..329482a856 100644 --- a/packages/provider-copilot/src/interceptors/openai-responses/item-id-membrane.ts +++ b/packages/provider-copilot/src/stages/openai-responses/item-id-membrane.ts @@ -1,7 +1,9 @@ -import { unwrapCopilotItemId, wrapCopilotItemId } from './item-id-carrier.ts'; -import type { CopilotOpenAIResponsesBoundaryInterceptor } from './types.ts'; +import type { ResponsesFacts } from '../../chat-facts.ts'; +import { unwrapCopilotItemId, wrapCopilotItemId } from '../../opaque-item-id.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; import { encodeHex, type ProtocolFrame } from '@floway-dev/protocols/common'; import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesCompactionResult, OpenAIResponsesInputItem, OpenAIResponsesOutputItem, OpenAIResponsesResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import type { ProviderChatResponse, ProviderChatServices } from '@floway-dev/provider'; // OpenAI's published examples establish these item-specific prefixes. Keeping // the Copilot output inventory closed prevents a new upstream item kind from @@ -268,12 +270,20 @@ const normalizeCompactionResult = (response: OpenAIResponsesCompactionResult): O }), }); -export const withCopilotOpenAIResponsesItemIdMembrane: CopilotOpenAIResponsesBoundaryInterceptor = async (ctx, _env, run) => { - ctx.payload = restoreInputItemIds(ctx.payload); - const result = await run(); - if (!result.ok) return result; - - return result.action === 'generate' - ? { ...result, events: normalizeFrames(result.events) } - : { ...result, result: normalizeCompactionResult(result.result) }; -}; +export const copilotOpenAIResponsesItemIdMembrane = defineStage, ProviderChatResponse<'openaiResponses'>, ProviderChatServices>({ + name: 'copilotOpenAIResponsesItemIdMembrane', + through: { + request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: ['response.provider.output'], consumes: ['response.provider.output'], provides: ['response.provider.output'] }, + }, + execute: async (facts, next, use) => { + const back = await next(move({ ...facts, 'request.provider.payload': restoreInputItemIds(facts['request.provider.payload']) })); + const output = back['response.provider.output']; + if (output === null || !('kind' in output)) return move({ ...back }); + return move({ + ...back, 'response.provider.output': output.kind === 'stream' + ? { ...output, frames: use.recordProtocolFrames(normalizeFrames(output.frames)) } + : { ...output, body: normalizeCompactionResult(output.body) }, + }); + }, +}); diff --git a/packages/provider-copilot/src/interceptors/openai-responses/set-initiator-header.ts b/packages/provider-copilot/src/stages/openai-responses/set-initiator-header.ts similarity index 51% rename from packages/provider-copilot/src/interceptors/openai-responses/set-initiator-header.ts rename to packages/provider-copilot/src/stages/openai-responses/set-initiator-header.ts index b610c17a8f..31f3608767 100644 --- a/packages/provider-copilot/src/interceptors/openai-responses/set-initiator-header.ts +++ b/packages/provider-copilot/src/stages/openai-responses/set-initiator-header.ts @@ -1,5 +1,7 @@ -import type { OpenAIResponsesBoundaryCtx } from './types.ts'; +import type { ResponsesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; import type { OpenAIResponsesInputItem } from '@floway-dev/protocols/openai-responses'; +import { replaceHttpHeader } from '@floway-dev/provider'; /** * Copilot's `x-initiator` header distinguishes user-triggered turns from @@ -22,18 +24,26 @@ import type { OpenAIResponsesInputItem } from '@floway-dev/protocols/openai-resp * - https://github.com/caozhiyuan/copilot-api/blob/cd8207cb70ede07771bf37a04accfbf2af76d980/src/routes/responses/utils.ts#L75-L87 * (`hasAgentInitiator`) */ -export const withInitiatorHeaderSet = async ( - ctx: OpenAIResponsesBoundaryCtx, - _env: object, - run: () => Promise, -): Promise => { - const lastItem: OpenAIResponsesInputItem | undefined = ctx.payload.input.at(-1); - const role = lastItem === undefined ? undefined : (lastItem as { role?: unknown }).role; - const initiator: 'user' | 'agent' = lastItem !== undefined + +export const copilotOpenAIResponsesSetInitiatorHeader = defineStage({ + name: 'copilotOpenAIResponsesSetInitiatorHeader', + through: { + request: { needs: ['request.provider.payload', 'request.http.headers'], consumes: ['request.http.headers'], provides: ['request.http.headers'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.provider.payload']; + let headers = facts['request.http.headers']; + + const lastItem: OpenAIResponsesInputItem | undefined = payload.input.at(-1); + const role = lastItem === undefined ? undefined : (lastItem as { role?: unknown }).role; + const initiator: 'user' | 'agent' = lastItem !== undefined && (role === undefined || role === null || role === '' || role === 'assistant') - ? 'agent' - : 'user'; - ctx.headers.set('x-initiator', initiator); + ? 'agent' + : 'user'; + headers = replaceHttpHeader(headers, 'x-initiator', initiator); + + return move({ ...await next(move({ ...facts, 'request.http.headers': headers })) }); - return await run(); -}; + }, +}); diff --git a/packages/provider-copilot/src/interceptors/openai-responses/set-vision-header.ts b/packages/provider-copilot/src/stages/openai-responses/set-vision-header.ts similarity index 52% rename from packages/provider-copilot/src/interceptors/openai-responses/set-vision-header.ts rename to packages/provider-copilot/src/stages/openai-responses/set-vision-header.ts index 0b208d8ad1..67457ead26 100644 --- a/packages/provider-copilot/src/interceptors/openai-responses/set-vision-header.ts +++ b/packages/provider-copilot/src/stages/openai-responses/set-vision-header.ts @@ -1,5 +1,7 @@ -import type { OpenAIResponsesBoundaryCtx } from './types.ts'; +import type { ResponsesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; import type { OpenAIResponsesInputItem } from '@floway-dev/protocols/openai-responses'; +import { replaceHttpHeader } from '@floway-dev/provider'; /** * Copilot's OpenAI Responses endpoint requires the private @@ -19,12 +21,19 @@ const itemHasImage = (item: OpenAIResponsesInputItem): boolean => { return Array.isArray(content) && content.some(part => part.type === 'input_image'); }; -export const withVisionHeaderSet = async ( - ctx: OpenAIResponsesBoundaryCtx, - _env: object, - run: () => Promise, -): Promise => { - if (ctx.payload.input.some(itemHasImage)) ctx.headers.set('copilot-vision-request', 'true'); +export const copilotOpenAIResponsesSetVisionHeader = defineStage({ + name: 'copilotOpenAIResponsesSetVisionHeader', + through: { + request: { needs: ['request.provider.payload', 'request.http.headers'], consumes: ['request.http.headers'], provides: ['request.http.headers'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const payload = facts['request.provider.payload']; + let headers = facts['request.http.headers']; - return await run(); -}; + if (payload.input.some(itemHasImage)) headers = replaceHttpHeader(headers, 'copilot-vision-request', 'true'); + + return move({ ...await next(move({ ...facts, 'request.http.headers': headers })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/openai-responses/strip-image-generation.ts b/packages/provider-copilot/src/stages/openai-responses/strip-image-generation.ts new file mode 100644 index 0000000000..8d169450bd --- /dev/null +++ b/packages/provider-copilot/src/stages/openai-responses/strip-image-generation.ts @@ -0,0 +1,76 @@ +import type { ResponsesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { collectOpenAIResponsesTools, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesTool, type OpenAIResponsesToolChoice } from '@floway-dev/protocols/openai-responses'; + +/** + * A Copilot gateway filters public `image_generation` from Responses requests, + * while OpenAI supports it. Apply this provider-specific rule after target + * selection to every declaration carrier and selector, retaining unrelated + * tools in their declaration containers and relative order. + * + * https://developers.openai.com/api/docs/guides/tools-image-generation + * https://github.com/caozhiyuan/copilot-api/blob/5d37d5b1ac6566c935a5c26d046396ee5fa423cc/src/routes/responses/handler.ts#L187-L204 + */ +const isImageGenerationTool = (tool: OpenAIResponsesTool): boolean => tool.type === 'image_generation'; + +const isImageGenerationToolChoice = (choice: OpenAIResponsesToolChoice | null | undefined): boolean => + typeof choice === 'object' && choice !== null && choice.type === 'image_generation'; + +const withoutToolChoice = (payload: CanonicalOpenAIResponsesPayload): CanonicalOpenAIResponsesPayload => { + const { tool_choice: _choice, ...rest } = payload; + return rest; +}; + +export const stripImageGenerationFromPayload = (payload: CanonicalOpenAIResponsesPayload): CanonicalOpenAIResponsesPayload => { + let removedTool = false; + const filter = (tools: OpenAIResponsesTool[]): OpenAIResponsesTool[] => { + const filtered = tools.filter(tool => { + const drop = isImageGenerationTool(tool); + removedTool ||= drop; + return !drop; + }); + return filtered.length === tools.length ? tools : filtered; + }; + let rewritten = payload; + if (Array.isArray(payload.tools)) { + const tools = filter(payload.tools); + if (tools.length === 0) { + const { tools: _tools, ...rest } = rewritten; + rewritten = rest; + } else if (tools !== payload.tools) rewritten = { ...rewritten, tools }; + } + const input = payload.input.map(item => { + if (item.type !== 'additional_tools' && item.type !== 'tool_search_output') return item; + const tools = filter(item.tools); + return tools === item.tools ? item : { ...item, tools }; + }); + if (input.some((item, index) => item !== payload.input[index])) rewritten = { ...rewritten, input }; + const choice = rewritten.tool_choice; + if (isImageGenerationToolChoice(choice)) { + return collectOpenAIResponsesTools(rewritten).length === 0 + ? withoutToolChoice(rewritten) : { ...rewritten, tool_choice: 'none' }; + } + if (typeof choice === 'object' && choice !== null && choice.type === 'allowed_tools') { + const allowed = choice.tools.filter(tool => tool.type !== 'image_generation'); + if (allowed.length === choice.tools.length) return rewritten; + return { ...rewritten, tool_choice: allowed.length === 0 && choice.mode === 'auto' ? 'none' : { ...choice, tools: allowed } }; + } + // Filtering a required choice must not expose tools outside its declared subset. + if (removedTool && choice === 'required' && collectOpenAIResponsesTools(rewritten).length === 0) return withoutToolChoice(rewritten); + return rewritten; +}; + +export const copilotOpenAIResponsesStripImageGeneration = defineStage({ + name: 'copilotOpenAIResponsesStripImageGeneration', + through: { + request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + let payload = facts['request.provider.payload']; + + payload = stripImageGenerationFromPayload(payload); + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/openai-responses/strip-service-tier.ts b/packages/provider-copilot/src/stages/openai-responses/strip-service-tier.ts new file mode 100644 index 0000000000..645198b77b --- /dev/null +++ b/packages/provider-copilot/src/stages/openai-responses/strip-service-tier.ts @@ -0,0 +1,36 @@ +import type { ResponsesFacts } from '../../chat-facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; + +/** + * Copilot has no request-side `service_tier`: `/responses` answers any value + * of the field with HTTP 400 + * `{"code":"unsupported_value","param":"service_tier"}`, on a base id and a + * `-fast` id alike. The tier reaches the upstream as the raw model id + * model selection selected instead, so by the time this runs the value + * has already been consumed and only the unusable field is left to drop. + * Strip it only after planning has committed to the OpenAI Responses target + * so source-side behavior and telemetry still see the caller's original + * request. Generic in the run-result type so the same definition feeds both + * the streaming `/responses` chain and the non-streaming compaction chain. + * + * References: + * - https://github.com/caozhiyuan/copilot-api/commit/f7835a44f06976cab874700e4d94a5f5c0379369 + * - https://platform.openai.com/docs/api-reference/responses/create + */ + +export const copilotOpenAIResponsesStripServiceTier = defineStage({ + name: 'copilotOpenAIResponsesStripServiceTier', + through: { + request: { needs: ['request.provider.payload'], consumes: ['request.provider.payload'], provides: ['request.provider.payload'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + let payload = facts['request.provider.payload']; + + const { service_tier: _, ...withoutTier } = payload; + payload = withoutTier; + + return move({ ...await next(move({ ...facts, 'request.provider.payload': payload })) }); + + }, +}); diff --git a/packages/provider-copilot/src/stages/prepare-chat-request.ts b/packages/provider-copilot/src/stages/prepare-chat-request.ts new file mode 100644 index 0000000000..77d9c5250e --- /dev/null +++ b/packages/provider-copilot/src/stages/prepare-chat-request.ts @@ -0,0 +1,28 @@ +import { copilotRequestHeaders } from '../auth.ts'; +import type { AuthenticatedChatFacts, CopilotChatHttpFacts } from '../chat-facts.ts'; +import { COMPACTION_TRIGGER } from '../compaction.ts'; +import { defineStage, isSecret, move, secret, type Secret } from '@floway-dev/pipeline'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; +import type { ChatProviderOperation, ProviderResponse, ProviderServices } from '@floway-dev/provider'; + +export const prepareCopilotChatRequest = (operation: ChatProviderOperation) => defineStage({ + name: `prepareCopilot${operation}`, + through: { + request: { needs: [...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? ['request.provider.responsesAction' as const] : []), 'request.provider.payload', 'request.provider.modelKey', 'request.http.headers', 'request.http.callId', 'request.copilot.session'], consumes: ['request.provider.model', 'request.provider.payload', 'request.copilot.session', 'request.http.headers'], provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const { 'request.provider.payload': payload, 'request.copilot.session': session, 'request.provider.model': _, ...rest } = facts; + const admitted = facts['request.http.headers'].map(([name, value]): [string, string] => [name, isSecret(value) ? value.reveal() : value]); + if (operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens') { + const beta = facts['request.provider.anthropicBeta']!; + for (let i = admitted.length - 1; i >= 0; i--) if (admitted[i]![0].toLowerCase() === 'anthropic-beta') admitted.splice(i, 1); + if (beta.length > 0) admitted.push(['anthropic-beta', beta.join(',')]); + } + const headers = [...copilotRequestHeaders({ ...session, token: session.token.reveal() }, undefined, admitted)].map(([name, value]): [string, string | Secret] => [name, name.toLowerCase() === 'authorization' ? secret(value) : value]); + const path = operation === 'openaiChatCompletions' ? '/chat/completions' : operation === 'anthropicMessages' ? '/v1/messages' : operation === 'anthropicMessagesCountTokens' ? '/v1/messages/count_tokens' : '/responses'; + const compact = 'request.provider.responsesAction' in facts && facts['request.provider.responsesAction'] === 'compact'; + const body = { ...payload, model: facts['request.provider.modelKey'], ...(operation === 'anthropicMessagesCountTokens' ? {} : { stream: !compact }), ...(compact ? { input: [...(payload as CanonicalOpenAIResponsesPayload).input, COMPACTION_TRIGGER] } : {}) }; + return move({ ...await next(move({ ...rest, 'request.http.url': `${session.baseUrl}${path}`, 'request.http.method': 'POST', 'request.http.headers': headers, 'request.http.body': body, 'request.http.encoding': 'json' })) }); + }, +}); diff --git a/packages/provider-copilot/src/stages/prepare-request.ts b/packages/provider-copilot/src/stages/prepare-request.ts index 5d33ca5bfc..c8f38aefab 100644 --- a/packages/provider-copilot/src/stages/prepare-request.ts +++ b/packages/provider-copilot/src/stages/prepare-request.ts @@ -1,20 +1,19 @@ import { copilotRequestHeaders } from '../auth.ts'; import { copilotOpenAIEmbeddingsBody, rawModelFor } from '../operation-model.ts'; import type { CopilotRequest, CopilotHttpFacts } from '../pipeline-facts.ts'; -import type { HttpResponseFacts } from '@floway-dev/http/pipeline'; import { defineStage, isSecret, move, secret, type Secret } from '@floway-dev/pipeline'; import type { ProviderResponse, ProviderServices } from '@floway-dev/provider'; export const prepareCopilotEmbeddings = () => { - const prepare = defineStage({ + const prepare = defineStage({ name: 'prepareCopilotOpenAIEmbeddings', through: { request: { needs: ['request.provider.model', 'request.provider.payload', 'request.http.headers', 'request.http.callId', 'request.copilot.session'], consumes: ['request.provider.model', 'request.provider.payload', 'request.copilot.session', 'request.http.headers'], - provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.copilot.modelKey'], + provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.copilot.modelKey', 'request.provider.modelKey'], }, - response: { needs: ['response.http.exchange'], consumes: [], provides: ['response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'] }, + response: { needs: [], consumes: [], provides: [] }, }, execute: async (facts, next) => { const { 'request.provider.model': model, 'request.provider.payload': payload, 'request.copilot.session': session, ...rest } = facts; @@ -25,13 +24,14 @@ export const prepareCopilotEmbeddings = () => { const back = await next(move({ ...rest, 'request.copilot.modelKey': modelKey, + 'request.provider.modelKey': modelKey, 'request.http.url': `${session.baseUrl}/embeddings`, 'request.http.method': 'POST', 'request.http.headers': headers, 'request.http.body': { ...copilotOpenAIEmbeddingsBody(payload), model: modelKey }, 'request.http.encoding': 'json' as const, })); - return move({ ...back, 'response.provider.modelKey': modelKey, 'response.provider.called': back['response.http.exchange'].type === 'response', 'response.provider.previousCalls': [] }); + return move({ ...back }); }, }); return prepare; diff --git a/packages/provider-copilot/src/stages/reshape-compaction.ts b/packages/provider-copilot/src/stages/reshape-compaction.ts new file mode 100644 index 0000000000..e9e0b1904f --- /dev/null +++ b/packages/provider-copilot/src/stages/reshape-compaction.ts @@ -0,0 +1,19 @@ +import type { ResponsesFacts } from '../chat-facts.ts'; +import { compactionResponse } from '../compaction.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { OpenAIResponsesResult } from '@floway-dev/protocols/openai-responses'; +import type { ProviderChatResponse } from '@floway-dev/provider'; + +export const reshapeCopilotCompaction = defineStage, ProviderChatResponse<'openaiResponsesCompact'>>({ + name: 'reshapeCopilotCompaction', + through: { + request: { needs: ['request.provider.payload'], consumes: [], provides: [] }, + response: { needs: ['response.provider.output'], consumes: ['response.provider.output'], provides: ['response.provider.output'] }, + }, + execute: async (facts, next) => { + const back = await next(move({ ...facts })); + const output = back['response.provider.output']; + if (output === null || !('kind' in output) || output.kind !== 'value') return move({ ...back }); + return move({ ...back, 'response.provider.output': { ...output, body: compactionResponse(facts['request.provider.payload'].input, output.body as unknown as OpenAIResponsesResult) } }); + }, +}); diff --git a/packages/provider-copilot/src/stages/select-chat-model.ts b/packages/provider-copilot/src/stages/select-chat-model.ts new file mode 100644 index 0000000000..bc7a5b424b --- /dev/null +++ b/packages/provider-copilot/src/stages/select-chat-model.ts @@ -0,0 +1,46 @@ +import type { CopilotChatFacts } from '../chat-facts.ts'; +import { CONTEXT_1M_BETA, copilotModelSupportsFastVariant } from '../model-selection.ts'; +import { rawModelFor } from '../operation-model.ts'; +import type { CopilotRawModel } from '../types.ts'; +import { takeHttpResponse } from '@floway-dev/http/pipeline'; +import { defineStage, move, type Handed } from '@floway-dev/pipeline'; +import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; +import { isFastServiceTier } from '@floway-dev/protocols/common'; +import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; +import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; +import type { ChatProviderOperation, ProviderChatResponse, ProviderOperationRequest, ProviderRequest, ProviderOperationPayloads } from '@floway-dev/provider'; + +export const selectCopilotChatModel = (operation: O) => defineStage & { 'request.provider.anthropicBeta'?: readonly string[] }, CopilotChatFacts, ProviderChatResponse, ProviderChatResponse, ProviderChatResponse>({ + name: `selectCopilot${operation}`, + through: { + request: { needs: [...(operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens' ? ['request.provider.anthropicBeta' as const] : []), 'request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'], consumes: ['request.provider.payload'], provides: ['request.provider.payload', 'request.provider.modelKey', ...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? ['request.provider.responsesAction' as const] : [])] }, + response: { needs: [], consumes: [], provides: [] }, + }, + return: { provides: ['response.http.exchange', 'response.http.body', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.provider.output'] }, + execute: async (facts, next) => { + const model = facts['request.provider.model']; + const payload = facts['request.provider.payload']; + let modelKey: string; + if (operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens') { + const body = payload as Omit; + if (operation === 'anthropicMessages' && body.speed === 'fast' && !copilotModelSupportsFastVariant((model.providerData as { rawModels: CopilotRawModel[] }).rawModels)) { + // Anthropic Fast Mode requires a supporting model rather than silent downgrade. + // https://docs.claude.com/en/build-with-claude/fast-mode + const exchange = takeHttpResponse(Response.json({ type: 'error', error: { type: 'invalid_request_error', message: `'${model.id}' does not support the \`speed\` parameter.` } }, { status: 400 })); + return move({ ...facts, 'response.http.exchange': exchange, 'response.http.body': exchange.body, 'response.provider.modelKey': model.id, 'response.provider.called': false, 'response.provider.previousCalls': [], 'response.provider.output': null }); + } + modelKey = rawModelFor(model, 'anthropicMessages', { + context1m: (facts as ProviderOperationRequest<'anthropicMessages'>)['request.provider.anthropicBeta'].includes(CONTEXT_1M_BETA), + reasoningEffort: body.output_config?.effort, + fast: operation === 'anthropicMessages' && body.speed === 'fast', + }).id; + } else if (operation === 'openaiChatCompletions') { + const body = payload as Omit; + modelKey = rawModelFor(model, 'openaiChatCompletions', { reasoningEffort: body.reasoning_effort && body.reasoning_effort !== 'none' ? body.reasoning_effort : undefined }).id; + } else { + const body = payload as Omit; + modelKey = rawModelFor(model, 'openaiResponses', { reasoningEffort: body.reasoning?.effort && body.reasoning.effort !== 'none' ? body.reasoning.effort : undefined, fast: isFastServiceTier(body.service_tier) }).id; + } + return move({ ...await next(move({ ...facts, 'request.provider.payload': { ...payload, model: model.id }, 'request.provider.modelKey': modelKey, ...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? { 'request.provider.responsesAction': operation === 'openaiResponsesCompact' ? 'compact' as const : 'generate' as const } : {}) }) as Handed) }); + }, +}); diff --git a/packages/provider-copilot/src/interceptors/shared/whitespace-overflow.ts b/packages/provider-copilot/src/tool-argument-whitespace.ts similarity index 92% rename from packages/provider-copilot/src/interceptors/shared/whitespace-overflow.ts rename to packages/provider-copilot/src/tool-argument-whitespace.ts index f34fe08365..c91462cfde 100644 --- a/packages/provider-copilot/src/interceptors/shared/whitespace-overflow.ts +++ b/packages/provider-copilot/src/tool-argument-whitespace.ts @@ -3,7 +3,7 @@ * line breaks / tabs in function `arguments` until `max_tokens`, which keeps * the client hanging while never producing valid JSON. * - * Two interceptors (OpenAI Chat Completions and OpenAI Responses) share this counter and + * Two stages (OpenAI Chat Completions and OpenAI Responses) share this counter and * threshold so the defense fires the same way on both upstream shapes. * * References: diff --git a/packages/provider-custom/__tests__/ingress-header-rules_test.ts b/packages/provider-custom/__tests__/ingress-header-rules_test.ts index 293c754954..56a4f07ee4 100644 --- a/packages/provider-custom/__tests__/ingress-header-rules_test.ts +++ b/packages/provider-custom/__tests__/ingress-header-rules_test.ts @@ -4,7 +4,7 @@ import type { CustomIngressHeaderRule } from '../src/config.ts'; import { createCustomProvider } from '../src/provider.ts'; import { parseRerankRequest } from '@floway-dev/protocols/rerank'; import { directFetcher, type Fetcher, type UpstreamModelConfig, type UpstreamRecord } from '@floway-dev/provider'; -import { callProviderPipeline, assertEquals, assertExists, jsonResponse, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, sseResponse, withMockedFetch } from '@floway-dev/test-utils'; +import { collectChatProviderPipeline, callProviderPipeline, assertEquals, assertExists, jsonResponse, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, sseResponse, withMockedFetch } from '@floway-dev/test-utils'; const HEADER = 'x-route'; @@ -119,12 +119,7 @@ const upstreamLines = async (rules: CustomIngressHeaderRule[], admitted: string const [model] = await provider.instance.getProvidedModels(directFetcher); assertExists(model); - await provider.instance.callOpenAIChatCompletions( - model, - { messages: [] }, - undefined, - noopUpstreamCallOptions({ fetcher, headers: admittedBag(provider.inboundHeaderAllowlist, admitted) }), - ); + await collectChatProviderPipeline(provider, 'openaiChatCompletions', model, { messages: [] }, undefined, noopUpstreamCallOptions({ fetcher, headers: admittedBag(provider.inboundHeaderAllowlist, admitted) })); assertExists(observed); return observed; @@ -196,12 +191,12 @@ test('every endpoint resolves the same rules', async () => { const messagesBody = { max_tokens: 10, messages: [{ role: 'user' as const, content: 'hi' }] }; await callProviderPipeline(provider, 'alphaSearch', model, { query: 'hi' }, undefined, opts()); - await provider.instance.callOpenAIChatCompletions(model, { messages: [] }, undefined, opts()); + await collectChatProviderPipeline(provider, 'openaiChatCompletions', model, { messages: [] }, undefined, opts()); await callProviderPipeline(provider, 'openaiCompletions', model, { prompt: 'hi' }, undefined, opts()); - await provider.instance.callOpenAIResponses(model, { input: [] }, 'generate', undefined, opts()); - await provider.instance.callOpenAIResponses(model, { input: [] }, 'compact', undefined, opts()); - await provider.instance.callAnthropicMessages(model, messagesBody, undefined, messagesOpts()); - await provider.instance.callAnthropicMessagesCountTokens(model, messagesBody, undefined, messagesOpts()); + await collectChatProviderPipeline(provider, 'openaiResponses', model, { input: [] }, undefined, opts()); + await collectChatProviderPipeline(provider, 'openaiResponsesCompact', model, { input: [] }, undefined, opts()); + await collectChatProviderPipeline(provider, 'anthropicMessages', model, messagesBody, undefined, messagesOpts()); + await collectChatProviderPipeline(provider, 'anthropicMessagesCountTokens', model, messagesBody, undefined, messagesOpts()); await callProviderPipeline(provider, 'openaiEmbeddings', model, { input: 'hi' }, undefined, opts()); await callProviderPipeline(provider, 'openaiImagesGenerations', model, { prompt: 'hi' }, undefined, opts()); await callProviderPipeline(provider, 'openaiImagesEdits', model, { diff --git a/packages/provider-custom/__tests__/provider_test.ts b/packages/provider-custom/__tests__/provider_test.ts index 31092d1c19..24ff30e84e 100644 --- a/packages/provider-custom/__tests__/provider_test.ts +++ b/packages/provider-custom/__tests__/provider_test.ts @@ -5,7 +5,7 @@ import type { ModelPricing } from '@floway-dev/protocols/common'; import { parseRerankRequest } from '@floway-dev/protocols/rerank'; import type { UpstreamModelConfig, UpstreamRecord } from '@floway-dev/provider'; import { directFetcher } from '@floway-dev/provider'; -import { callProviderPipeline, assertEquals, assertExists, assertRejects, jsonResponse, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, sseResponse, withMockedFetch } from '@floway-dev/test-utils'; +import { collectChatProviderPipeline, callProviderPipeline, assertEquals, assertExists, assertRejects, jsonResponse, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, sseResponse, withMockedFetch } from '@floway-dev/test-utils'; interface BuildOptions { ingressHeadersRules?: { key: string; value: string | null }[]; @@ -67,7 +67,7 @@ test('Custom writes every configured header value and passes admitted client val 'x-override': 'client-override', }), }); - await provider.instance.callOpenAIChatCompletions(model, { messages: [] }, undefined, opts); + await collectChatProviderPipeline(provider, 'openaiChatCompletions', model, { messages: [] }, undefined, opts); }, ); @@ -409,10 +409,10 @@ test('Custom provider forces stream=true for streaming endpoints and leaves coun assertExists(model); const opts = noopUpstreamCallOptions(); const anthropicMessagesOpts = noopAnthropicMessagesUpstreamCallOptions({ anthropicBeta: ['context-1m', 'advanced-tool-use'] }); - await provider.callOpenAIChatCompletions(model, { messages: [{ role: 'user', content: 'hi' }] }, undefined, opts); - await provider.callOpenAIResponses(model, { input: [] }, 'generate', undefined, opts); - await provider.callAnthropicMessages(model, { max_tokens: 10, messages: [{ role: 'user', content: 'hi' }] }, undefined, anthropicMessagesOpts); - await provider.callAnthropicMessagesCountTokens(model, { max_tokens: 10, messages: [{ role: 'user', content: 'hi' }] }, undefined, anthropicMessagesOpts); + await collectChatProviderPipeline(instance, 'openaiChatCompletions', model, { messages: [{ role: 'user', content: 'hi' }] }, undefined, opts); + await collectChatProviderPipeline(instance, 'openaiResponses', model, { input: [] }, undefined, opts); + await collectChatProviderPipeline(instance, 'anthropicMessages', model, { max_tokens: 10, messages: [{ role: 'user', content: 'hi' }] }, undefined, anthropicMessagesOpts); + await collectChatProviderPipeline(instance, 'anthropicMessagesCountTokens', model, { max_tokens: 10, messages: [{ role: 'user', content: 'hi' }] }, undefined, anthropicMessagesOpts); await callProviderPipeline(instance, 'openaiEmbeddings', model, { input: 'hi' }, undefined, opts); }, ); @@ -495,7 +495,7 @@ test('Custom provider callOpenAIImagesGenerations posts JSON with model re-injec const [model] = await provider.instance.getProvidedModels(directFetcher); const result = await callProviderPipeline(provider, 'openaiImagesGenerations', model, { prompt: 'hi' }, undefined, noopUpstreamCallOptions()); assertEquals(result.modelKey, 'gpt-image-2'); - assertEquals(result.response.status, 200); + assertEquals(result.response!.status, 200); }, ); assertExists(forwarded); @@ -523,7 +523,7 @@ test('Custom provider callAlphaSearch posts JSON to /v1/alpha/search with the up { id: 'search-session', commands: { search_query: [{ q: 'Floway' }] } }, undefined, noopUpstreamCallOptions()); - assertEquals(result.response.status, 200); + assertEquals(result.response!.status, 200); assertEquals(result.modelKey, 'gpt-search'); }, ); diff --git a/packages/provider-custom/__tests__/stages/audio-request_test.ts b/packages/provider-custom/__tests__/stages/audio-request_test.ts new file mode 100644 index 0000000000..1631b6f68d --- /dev/null +++ b/packages/provider-custom/__tests__/stages/audio-request_test.ts @@ -0,0 +1,23 @@ +import { expect, test } from 'vitest'; + +import { prepareCustomRequest } from '../../src/stages/prepare-request.ts'; +import { http } from '@floway-dev/http/pipeline'; +import type { HttpMultipartBody } from '@floway-dev/http/request-content'; +import { compose } from '@floway-dev/pipeline'; +import { observeProviderCall, providerModelFacts, type ProviderRequest, type ProviderResponse } from '@floway-dev/provider'; +import { collectHttpPipeline, noopUpstreamCallOptions, stubProviderModel } from '@floway-dev/test-utils'; + +test('audio prepare preserves ordered fields and file metadata while replacing the selected model', async () => { + const file = { bytes: new Uint8Array([1, 2, 3, 4]), name: 'meeting.wav', type: 'audio/wav', lastModified: 1_700_000_000_000 }; + const entries = [{ name: 'file', value: file }, { name: 'language', value: 'en' }, { name: 'model', value: 'public-model' }, { name: 'timestamp_granularities[]', value: 'word' }, { name: 'timestamp_granularities[]', value: 'segment' }]; + const prepare = prepareCustomRequest({ baseUrl: 'https://custom.example', authStyle: 'none', endpoints: {}, ingressHeadersRules: [], modelsFetch: { enabled: false }, models: [] }, 'openaiAudioTranscriptions'); + const pipeline = compose, ProviderResponse & Record>('audio.metadata', [prepare, observeProviderCall, http]); + const executed = await collectHttpPipeline(pipeline, { 'request.provider.model': providerModelFacts(stubProviderModel({ providerData: 'upstream-model' })), 'request.provider.payload': { entries }, 'request.http.callId': 0, 'request.http.headers': [] }, { httpCall: () => ({ ...noopUpstreamCallOptions(), fetcher: async () => new Response('{}'), signal: undefined }) }); + const body = executed.facts['request.http.body'] as HttpMultipartBody; + expect(body.entries.map(entry => entry.name)).toEqual(['file', 'language', 'model', 'timestamp_granularities[]', 'timestamp_granularities[]']); + expect(body.entries[0]!.value).toBe(file); + expect(body.entries[0]!.value).toMatchObject({ name: 'meeting.wav', type: 'audio/wav', lastModified: file.lastModified, bytes: new Uint8Array([1, 2, 3, 4]) }); + expect(body.entries[2]!.value).toBe('upstream-model'); + expect(body.entries.slice(3).map(entry => entry.value)).toEqual(['word', 'segment']); + expect(entries[2]!.value).toBe('public-model'); +}); diff --git a/packages/provider-custom/__tests__/stages/pipelines_test.ts b/packages/provider-custom/__tests__/stages/pipelines_test.ts new file mode 100644 index 0000000000..4b2d0ea6c5 --- /dev/null +++ b/packages/provider-custom/__tests__/stages/pipelines_test.ts @@ -0,0 +1,81 @@ +import { expect, test } from 'vitest'; + +import { createCustomPipelines } from '../../src/pipelines.ts'; +import { getFailureFacts, move, run, type Event } from '@floway-dev/pipeline'; +import { providerModelFacts, type ProviderChatServices, type ProviderOperationRequest } from '@floway-dev/provider'; +import { jsonResponse, noopUpstreamCallOptions, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; + +const pipelines = createCustomPipelines({ baseUrl: 'https://custom.example', authStyle: 'bearer', apiKey: 'secret-key', endpoints: {}, ingressHeadersRules: [{ key: 'x-many', value: 'first' }, { key: 'x-many', value: 'second' }], pathOverrides: { '/responses': '/overridden' }, modelsFetch: { enabled: false }, models: [] }); +const services: ProviderChatServices = { httpCall: () => ({ ...noopUpstreamCallOptions(), signal: undefined }), recordProtocolFrames: frames => frames }; + +test('custom compact chain derives URL and wire payload from immutable action facts', async () => { + const pipeline = pipelines.openaiResponsesCompact; + if (pipeline === undefined) throw new Error('Missing compact chain'); + const input: ProviderOperationRequest<'openaiResponsesCompact'> = move({ + 'request.provider.model': providerModelFacts(stubProviderModel({ providerData: 'wire-model' })), + 'request.provider.payload': { input: [{ type: 'message', role: 'user', content: 'hello' }], store: true, max_output_tokens: 30 }, + 'request.http.callId': 1, + 'request.http.headers': [], + }); + await withMockedFetch(async request => { + expect(request.url).toBe('https://custom.example/overridden/compact'); + expect(request.headers.get('authorization')).toBe('Bearer secret-key'); + expect(request.headers.get('x-many')).toBe('first, second'); + expect(await request.json()).toEqual({ model: 'wire-model', input: [{ type: 'message', role: 'user', content: 'hello' }] }); + return jsonResponse({ object: 'response.compaction', output: [] }); + }, async () => { + const events: Event[] = []; + const executed = await run(pipeline, input, { ...services, dump: (event: Event) => { events.push(event); } }); + expect(events).toContainEqual(expect.objectContaining({ type: 'stage.entered', facts: expect.objectContaining({ 'request.provider.responsesAction': 'compact' }) })); + expect(executed.facts['response.provider.called']).toBe(true); + expect(executed.facts['response.provider.output']).toEqual({ kind: 'value', body: { object: 'response.compaction', output: [] } }); + expect(input['request.provider.payload'].store).toBe(true); + await executed.drain(); + }); +}); + +test('count_tokens forwards typed beta intent and retains raw HTTP response facts', async () => { + const pipeline = pipelines.anthropicMessagesCountTokens; + if (pipeline === undefined) throw new Error('Missing count chain'); + const input: ProviderOperationRequest<'anthropicMessagesCountTokens'> = move({ + 'request.provider.model': providerModelFacts(stubProviderModel({ providerData: 'wire-model' })), + 'request.provider.payload': { max_tokens: 12, messages: [] }, + 'request.provider.anthropicBeta': ['future-beta'], + 'request.http.callId': 1, + 'request.http.headers': [['anthropic-beta', 'discard-ordinary-header']], + }); + await withMockedFetch(async request => { + expect(request.url).toBe('https://custom.example/v1/messages/count_tokens'); + expect(request.headers.get('anthropic-beta')).toBe('future-beta'); + expect(await request.json()).toEqual({ model: 'wire-model', max_tokens: 12, messages: [] }); + return jsonResponse({ input_tokens: 19 }); + }, async () => { + const executed = await run(pipeline, input, services); + expect(executed.facts['response.provider.output']).toEqual({ kind: 'value', body: { input_tokens: 19 } }); + expect(executed.facts['response.http.exchange']).toMatchObject({ type: 'response', status: 200 }); + await executed.drain(); + }); +}); + +for (const operation of ['openaiChatCompletions', 'anthropicMessagesCountTokens'] as const) { + test(`finite response body failure preserves its original error and call facts (${operation})`, async () => { + const error = new Error('response body socket failed'); + const body = new ReadableStream({ start(controller) { controller.error(error); } }); + const pipeline = pipelines[operation]; + if (pipeline === undefined) throw new Error(`Missing ${operation} chain`); + const input = move({ + 'request.provider.model': providerModelFacts(stubProviderModel({ providerData: 'wire-model' })), + 'request.provider.payload': { max_tokens: 12, messages: [] }, + 'request.http.callId': 1, + 'request.http.headers': [], + 'request.provider.anthropicBeta': [], + }); + await withMockedFetch(() => new Response(body, { headers: { 'content-type': 'application/json' } }), async () => { + const executed = operation === 'openaiChatCompletions' + ? run(pipelines.openaiChatCompletions!, input, services) + : run(pipelines.anthropicMessagesCountTokens!, input, services); + await expect(executed).rejects.toBe(error); + expect(getFailureFacts(error)).toMatchObject({ 'response.provider.called': true, 'response.provider.modelKey': 'wire-model' }); + }); + }); +} diff --git a/packages/provider-custom/__tests__/fetch_test.ts b/packages/provider-custom/__tests__/stages/prepare-request_test.ts similarity index 69% rename from packages/provider-custom/__tests__/fetch_test.ts rename to packages/provider-custom/__tests__/stages/prepare-request_test.ts index 62a9b52bc8..38e89967c0 100644 --- a/packages/provider-custom/__tests__/fetch_test.ts +++ b/packages/provider-custom/__tests__/stages/prepare-request_test.ts @@ -1,21 +1,12 @@ import { test } from 'vitest'; -import { assertCustomUpstreamRecord } from '../src/config.ts'; -import { - customFetchAlphaSearch, - customFetchOpenAIAudioTranscriptions, - customFetchOpenAIChatCompletions, - customFetchOpenAIEmbeddings, - customFetchAnthropicMessages, - customFetchAnthropicMessagesCountTokens, - customFetchModels, - customFetchOpenAIResponses, - customFetchOpenAIResponsesCompact, -} from '../src/fetch.ts'; -import { createCustomProvider } from '../src/provider.ts'; +import { assertCustomUpstreamRecord } from '../../src/config.ts'; +import { customFetchModels } from '../../src/control-http.ts'; +import { createCustomProvider } from '../../src/provider.ts'; +import { prepareCustomRequest } from '../../src/stages/prepare-request.ts'; import type { UpstreamRecord } from '@floway-dev/provider'; -import { directFetcher, identityWrapUpstreamCall } from '@floway-dev/provider'; -import { callProviderPipeline, assertEquals, assertExists, jsonResponse, noopUpstreamCallOptions, withMockedFetch } from '@floway-dev/test-utils'; +import { identityWrapUpstreamCall } from '@floway-dev/provider'; +import { collectPreparedProviderRequest, testFetcher, callProviderPipeline, assertEquals, assertExists, jsonResponse, noopUpstreamCallOptions, withMockedFetch } from '@floway-dev/test-utils'; const baseRecord: UpstreamRecord = { id: 'up_test', @@ -51,15 +42,15 @@ test('typed transports use default /v1/* paths', async () => { return new Response('{}', { status: 200 }); }, async () => { - await customFetchOpenAIChatCompletions(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await customFetchOpenAIResponses(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await customFetchOpenAIResponsesCompact(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await customFetchAnthropicMessages(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await customFetchAnthropicMessagesCountTokens(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await customFetchAlphaSearch(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await customFetchOpenAIEmbeddings(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await customFetchOpenAIAudioTranscriptions(config, { method: 'POST', body: new FormData() }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await customFetchModels(config, { method: 'GET' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'openaiChatCompletions'), 'openaiChatCompletions', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'openaiResponses'), 'openaiResponses', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'openaiResponsesCompact'), 'openaiResponsesCompact', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'anthropicMessages'), 'anthropicMessages', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'anthropicMessagesCountTokens'), 'anthropicMessagesCountTokens', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'alphaSearch'), 'alphaSearch', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'openaiEmbeddings'), 'openaiEmbeddings', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'openaiAudioTranscriptions'), 'openaiAudioTranscriptions', key => key, { method: 'POST', body: new FormData() }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await customFetchModels(config, { method: 'GET' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -95,13 +86,13 @@ test('admin pathOverrides replace defaults and propagate to derived sub-paths', return new Response('{}', { status: 200 }); }, async () => { - await customFetchAnthropicMessages(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'anthropicMessages'), 'anthropicMessages', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); // count_tokens / compact follow their parent override. - await customFetchAnthropicMessagesCountTokens(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await customFetchOpenAIResponsesCompact(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await customFetchAlphaSearch(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'anthropicMessagesCountTokens'), 'anthropicMessagesCountTokens', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'openaiResponsesCompact'), 'openaiResponsesCompact', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'alphaSearch'), 'alphaSearch', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); // Endpoints without an override fall back to the OpenAI default. - await customFetchOpenAIChatCompletions(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'openaiChatCompletions'), 'openaiChatCompletions', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -129,7 +120,7 @@ test('customFetchModels resolves the path from modelsFetch.endpoint', async () = return new Response('{}', { status: 200 }); }, async () => { - await customFetchModels(config, { method: 'GET' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await customFetchModels(config, { method: 'GET' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -151,7 +142,7 @@ test('customFetchModels falls back to the default /v1/models path when modelsFet return new Response('{}', { status: 200 }); }, async () => { - await customFetchModels(config, { method: 'GET' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await customFetchModels(config, { method: 'GET' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -169,7 +160,7 @@ test('bearer authStyle sends the configured token via Authorization', async () = return new Response('{}', { status: 200 }); }, async () => { - await customFetchModels(config, { method: 'GET' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await customFetchModels(config, { method: 'GET' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -196,7 +187,7 @@ test('authStyle "anthropic" sends x-api-key + anthropic-version', async () => { return new Response('{}', { status: 200 }); }, async () => { - await customFetchAnthropicMessages(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'anthropicMessages'), 'anthropicMessages', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -220,11 +211,7 @@ test('authStyle "anthropic" preserves a caller-supplied anthropic-version', asyn return new Response('{}', { status: 200 }); }, async () => { - await customFetchAnthropicMessages( - config, - { method: 'POST', body: '{}', headers: { 'anthropic-version': '2024-01-01' } }, - { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }, - ); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'anthropicMessages'), 'anthropicMessages', key => key, { method: 'POST', body: '{}', headers: { 'anthropic-version': '2024-01-01' } }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -252,7 +239,7 @@ test('authStyle "none" sends neither Authorization nor x-api-key', async () => { return new Response('{}', { status: 200 }); }, async () => { - await customFetchOpenAIChatCompletions(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareCustomRequest(config, 'openaiChatCompletions'), 'openaiChatCompletions', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -285,7 +272,7 @@ test('Custom provider callOpenAIImagesEdits forwards multipart body with model f }, async () => { const provider = createCustomProvider(record); - const [model] = await provider.instance.getProvidedModels(directFetcher); + const [model] = await provider.instance.getProvidedModels(testFetcher); const result = await callProviderPipeline(provider, 'openaiImagesEdits', model, { parameters: { prompt: 'add a kite' }, images: [{ @@ -325,7 +312,7 @@ test('Custom provider callOpenAIAudioTranscriptions preserves multipart entries }, async () => { const provider = createCustomProvider(record); - const [model] = await provider.instance.getProvidedModels(directFetcher); + const [model] = await provider.instance.getProvidedModels(testFetcher); const result = await callProviderPipeline(provider, 'openaiAudioTranscriptions', model, { entries: [ { name: 'file', value: { bytes: new Uint8Array([7, 8]), name: 'voice.ogg', type: 'audio/ogg' } }, diff --git a/packages/provider-custom/src/control-http.ts b/packages/provider-custom/src/control-http.ts new file mode 100644 index 0000000000..daaf99554c --- /dev/null +++ b/packages/provider-custom/src/control-http.ts @@ -0,0 +1,53 @@ +import type { CustomUpstreamConfig } from './config.ts'; +import { type FetchInit, type UpstreamFetchOptions, joinBaseAndPath } from '@floway-dev/provider'; + +// https://docs.anthropic.com/en/api/versioning +const ANTHROPIC_VERSION = '2023-06-01'; + +// The header block is carried as field lines rather than a `Headers`, because +// a rule set may send one name several times and `Headers.append` merges a +// repeated name into one value on undici. +// https://github.com/nodejs/undici/blob/v8.3.0/lib/web/fetch/headers.js#L236-L258 +const has = (lines: readonly (readonly [string, string])[], name: string): boolean => + lines.some(([candidate]) => candidate.toLowerCase() === name); + +const replace = (lines: [string, string][], name: string, value: string): void => { + const kept = lines.filter(([candidate]) => candidate.toLowerCase() !== name.toLowerCase()); + lines.length = 0; + lines.push(...kept, [name, value]); +}; + +const customFetchInternal = async ( + config: CustomUpstreamConfig, + path: string, + init: FetchInit, + options: UpstreamFetchOptions, +): Promise => { + const headers: [string, string][] = init.headers ? [...new Headers(init.headers)] : []; + if (config.authStyle === 'anthropic') { + replace(headers, 'x-api-key', config.apiKey); + if (!has(headers, 'anthropic-version')) headers.push(['anthropic-version', ANTHROPIC_VERSION]); + } else if (config.authStyle === 'bearer') { + replace(headers, 'Authorization', `Bearer ${config.apiKey}`); + } + // authStyle === 'none' falls through with no auth header. The same goes + // for the /models fetch — Models Fetch shares this code path, so a 'none' + // upstream is queried anonymously end-to-end. + if (init.body && !has(headers, 'content-type') && !(init.body instanceof FormData)) { + headers.push(['Content-Type', 'application/json']); + } + if (options.extraHeaders) { + // Every line the caller supplies for a name replaces this helper's own, + // and the caller's lines keep their order and their repetitions. + const supplied = new Set(options.extraHeaders.map(([name]) => name.toLowerCase())); + const kept = headers.filter(([name]) => !supplied.has(name.toLowerCase())); + headers.length = 0; + headers.push(...kept, ...options.extraHeaders.map(([name, value]): [string, string] => [name, value])); + } + return await options.wrapUpstreamCall(() => options.fetcher(joinBaseAndPath(config.baseUrl, path), { ...init, headers })); +}; + +// /models lives on its own fetch toggle (see config.modelsFetch.endpoint), +// not in pathOverrides. +export const customFetchModels = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => + customFetchInternal(config, config.modelsFetch.endpoint ?? '/v1/models', init, options); diff --git a/packages/provider-custom/src/fetch-models.ts b/packages/provider-custom/src/fetch-models.ts index 2e490354d6..f383c7c96f 100644 --- a/packages/provider-custom/src/fetch-models.ts +++ b/packages/provider-custom/src/fetch-models.ts @@ -10,7 +10,7 @@ // effort metadata. The container is admitted if `data` is an array. import type { CustomUpstreamConfig } from './config.ts'; -import { customFetchModels } from './fetch.ts'; +import { customFetchModels } from './control-http.ts'; import { BILLING_METRICS, canonicalizePricingSelector, type BillingMetric, type ModelKind, type ModelPricing, type OpaqueBlobCompatibilityScope, parseNonNegativeDecimalString, type PriceVector, type PricingSelector, validateModelPricing } from '@floway-dev/protocols/common'; import { chatField, fetchUpstreamModels, type Fetcher, type UpstreamChatModelConfig, identityWrapUpstreamCall, opaqueBlobCompatibilityScopeField } from '@floway-dev/provider'; diff --git a/packages/provider-custom/src/fetch.ts b/packages/provider-custom/src/fetch.ts deleted file mode 100644 index 5df603f6f9..0000000000 --- a/packages/provider-custom/src/fetch.ts +++ /dev/null @@ -1,87 +0,0 @@ -import type { CustomPathOverrideKey, CustomUpstreamConfig } from './config.ts'; -import { type FetchInit, type UpstreamFetchOptions, joinBaseAndPath } from '@floway-dev/provider'; - -// https://docs.anthropic.com/en/api/versioning -const ANTHROPIC_VERSION = '2023-06-01'; - -// Endpoint key is the OpenAI-canonical path fragment (`/chat/completions`, -// `/images/generations`, ...). The default upstream URL is the key prefixed -// with `/v1`; pathOverrides (see config.ts) replace it one-for-one. The -// messages count-tokens and responses compact endpoints append a suffix -// to their parent's resolved path so an override of the parent ripples -// down to both. -const pathOverrideFor = (config: CustomUpstreamConfig, key: CustomPathOverrideKey): string => - config.pathOverrides?.[key] ?? `/v1${key}`; - -// The header block is carried as field lines rather than a `Headers`, because -// a rule set may send one name several times and `Headers.append` merges a -// repeated name into one value on undici. -// https://github.com/nodejs/undici/blob/v8.3.0/lib/web/fetch/headers.js#L236-L258 -const has = (lines: readonly (readonly [string, string])[], name: string): boolean => - lines.some(([candidate]) => candidate.toLowerCase() === name); - -const replace = (lines: [string, string][], name: string, value: string): void => { - const kept = lines.filter(([candidate]) => candidate.toLowerCase() !== name.toLowerCase()); - lines.length = 0; - lines.push(...kept, [name, value]); -}; - -const customFetchInternal = async ( - config: CustomUpstreamConfig, - path: string, - init: FetchInit, - options: UpstreamFetchOptions, -): Promise => { - const headers: [string, string][] = init.headers ? [...new Headers(init.headers)] : []; - if (config.authStyle === 'anthropic') { - replace(headers, 'x-api-key', config.apiKey); - if (!has(headers, 'anthropic-version')) headers.push(['anthropic-version', ANTHROPIC_VERSION]); - } else if (config.authStyle === 'bearer') { - replace(headers, 'Authorization', `Bearer ${config.apiKey}`); - } - // authStyle === 'none' falls through with no auth header. The same goes - // for the /models fetch — Models Fetch shares this code path, so a 'none' - // upstream is queried anonymously end-to-end. - if (init.body && !has(headers, 'content-type') && !(init.body instanceof FormData)) { - headers.push(['Content-Type', 'application/json']); - } - if (options.extraHeaders) { - // Every line the caller supplies for a name replaces this helper's own, - // and the caller's lines keep their order and their repetitions. - const supplied = new Set(options.extraHeaders.map(([name]) => name.toLowerCase())); - const kept = headers.filter(([name]) => !supplied.has(name.toLowerCase())); - headers.length = 0; - headers.push(...kept, ...options.extraHeaders.map(([name, value]): [string, string] => [name, value])); - } - return await options.wrapUpstreamCall(() => options.fetcher(joinBaseAndPath(config.baseUrl, path), { ...init, headers })); -}; - -export const customFetchRerank = (config: CustomUpstreamConfig, path: string, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, path, init, options); - -export const customFetchOpenAIChatCompletions = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, pathOverrideFor(config, '/chat/completions'), init, options); -export const customFetchOpenAIResponses = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, pathOverrideFor(config, '/responses'), init, options); -export const customFetchOpenAIResponsesCompact = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, `${pathOverrideFor(config, '/responses')}/compact`, init, options); -export const customFetchAnthropicMessages = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, pathOverrideFor(config, '/messages'), init, options); -export const customFetchAnthropicMessagesCountTokens = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, `${pathOverrideFor(config, '/messages')}/count_tokens`, init, options); -export const customFetchOpenAIEmbeddings = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, pathOverrideFor(config, '/embeddings'), init, options); -export const customFetchOpenAICompletions = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, pathOverrideFor(config, '/completions'), init, options); -export const customFetchOpenAIImagesGenerations = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, pathOverrideFor(config, '/images/generations'), init, options); -export const customFetchOpenAIImagesEdits = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, pathOverrideFor(config, '/images/edits'), init, options); -export const customFetchOpenAIAudioTranscriptions = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, pathOverrideFor(config, '/audio/transcriptions'), init, options); -export const customFetchAlphaSearch = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, pathOverrideFor(config, '/alpha/search'), init, options); -// /models lives on its own fetch toggle (see config.modelsFetch.endpoint), -// not in pathOverrides. -export const customFetchModels = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, config.modelsFetch.endpoint ?? '/v1/models', init, options); diff --git a/packages/provider-custom/src/pipelines.ts b/packages/provider-custom/src/pipelines.ts index ce3438c8d0..7278a3e650 100644 --- a/packages/provider-custom/src/pipelines.ts +++ b/packages/provider-custom/src/pipelines.ts @@ -2,14 +2,20 @@ import type { CustomUpstreamConfig } from './config.ts'; import { prepareCustomRequest } from './stages/prepare-request.ts'; import { http } from '@floway-dev/http/pipeline'; import { compose } from '@floway-dev/pipeline'; -import type { ProviderOperationPayloads, ProviderOperationResponse, ProviderPipelines, ProviderRequest } from '@floway-dev/provider'; +import { decodeProviderResponse, observeProviderCall, observeProviderResponsesCall, selectProviderResponsesAction } from '@floway-dev/provider'; +import type { ProviderOperationPayloads, ProviderOperationRequest, ProviderOperationResponse, ProviderPipelines, ProviderRequest } from '@floway-dev/provider'; export const createCustomPipelines = (config: CustomUpstreamConfig): ProviderPipelines => ({ - alphaSearch: compose, ProviderOperationResponse<'alphaSearch'>>('custom.alphaSearch', [prepareCustomRequest(config, 'alphaSearch'), http]), - openaiCompletions: compose, ProviderOperationResponse<'openaiCompletions'>>('custom.openaiCompletions', [prepareCustomRequest(config, 'openaiCompletions'), http]), - openaiEmbeddings: compose, ProviderOperationResponse<'openaiEmbeddings'>>('custom.openaiEmbeddings', [prepareCustomRequest(config, 'openaiEmbeddings'), http]), - openaiImagesGenerations: compose, ProviderOperationResponse<'openaiImagesGenerations'>>('custom.openaiImagesGenerations', [prepareCustomRequest(config, 'openaiImagesGenerations'), http]), - openaiImagesEdits: compose, ProviderOperationResponse<'openaiImagesEdits'>>('custom.openaiImagesEdits', [prepareCustomRequest(config, 'openaiImagesEdits'), http]), - openaiAudioTranscriptions: compose, ProviderOperationResponse<'openaiAudioTranscriptions'>>('custom.openaiAudioTranscriptions', [prepareCustomRequest(config, 'openaiAudioTranscriptions'), http]), - rerank: compose, ProviderOperationResponse<'rerank'>>('custom.rerank', [prepareCustomRequest(config, 'rerank'), http]), + alphaSearch: compose, ProviderOperationResponse<'alphaSearch'>>('custom.alphaSearch', [prepareCustomRequest(config, 'alphaSearch'), observeProviderCall, http]), + openaiChatCompletions: compose, ProviderOperationResponse<'openaiChatCompletions'>>('custom.openaiChatCompletions', [decodeProviderResponse('openaiChatCompletions'), prepareCustomRequest(config, 'openaiChatCompletions'), observeProviderCall, http]), + openaiResponses: compose, ProviderOperationResponse<'openaiResponses'>>('custom.openaiResponses', [selectProviderResponsesAction('generate'), decodeProviderResponse('openaiResponses'), prepareCustomRequest(config, 'openaiResponses'), observeProviderResponsesCall, http]), + openaiResponsesCompact: compose, ProviderOperationResponse<'openaiResponsesCompact'>>('custom.openaiResponsesCompact', [selectProviderResponsesAction('compact'), decodeProviderResponse('openaiResponsesCompact'), prepareCustomRequest(config, 'openaiResponsesCompact'), observeProviderResponsesCall, http]), + anthropicMessages: compose, ProviderOperationResponse<'anthropicMessages'>>('custom.anthropicMessages', [decodeProviderResponse('anthropicMessages'), prepareCustomRequest(config, 'anthropicMessages'), observeProviderCall, http]), + anthropicMessagesCountTokens: compose, ProviderOperationResponse<'anthropicMessagesCountTokens'>>('custom.anthropicMessagesCountTokens', [decodeProviderResponse('anthropicMessagesCountTokens'), prepareCustomRequest(config, 'anthropicMessagesCountTokens'), observeProviderCall, http]), + openaiCompletions: compose, ProviderOperationResponse<'openaiCompletions'>>('custom.openaiCompletions', [prepareCustomRequest(config, 'openaiCompletions'), observeProviderCall, http]), + openaiEmbeddings: compose, ProviderOperationResponse<'openaiEmbeddings'>>('custom.openaiEmbeddings', [prepareCustomRequest(config, 'openaiEmbeddings'), observeProviderCall, http]), + openaiImagesGenerations: compose, ProviderOperationResponse<'openaiImagesGenerations'>>('custom.openaiImagesGenerations', [prepareCustomRequest(config, 'openaiImagesGenerations'), observeProviderCall, http]), + openaiImagesEdits: compose, ProviderOperationResponse<'openaiImagesEdits'>>('custom.openaiImagesEdits', [prepareCustomRequest(config, 'openaiImagesEdits'), observeProviderCall, http]), + openaiAudioTranscriptions: compose, ProviderOperationResponse<'openaiAudioTranscriptions'>>('custom.openaiAudioTranscriptions', [prepareCustomRequest(config, 'openaiAudioTranscriptions'), observeProviderCall, http]), + rerank: compose, ProviderOperationResponse<'rerank'>>('custom.rerank', [prepareCustomRequest(config, 'rerank'), observeProviderCall, http]), }); diff --git a/packages/provider-custom/src/provider.ts b/packages/provider-custom/src/provider.ts index 7742d6a6dd..de3b06ca0d 100644 --- a/packages/provider-custom/src/provider.ts +++ b/packages/provider-custom/src/provider.ts @@ -1,15 +1,10 @@ -import { assertCustomUpstreamRecord, type CustomUpstreamConfig } from './config.ts'; +import { assertCustomUpstreamRecord } from './config.ts'; import { CUSTOM_DEFAULT_FLAGS } from './defaults.ts'; import { fetchCustomModels, type CustomModelsResponse, type CustomRawModel } from './fetch-models.ts'; -import { customFetchAlphaSearch, customFetchOpenAIAudioTranscriptions, customFetchOpenAIChatCompletions, customFetchOpenAICompletions, customFetchOpenAIEmbeddings, customFetchOpenAIImagesEdits, customFetchOpenAIImagesGenerations, customFetchAnthropicMessages, customFetchAnthropicMessagesCountTokens, customFetchRerank, customFetchOpenAIResponses, customFetchOpenAIResponsesCompact } from './fetch.ts'; import { inferEndpointsFromModelId } from './infer-endpoints.ts'; import { createCustomPipelines } from './pipelines.ts'; -import { parseAnthropicMessagesStream } from '@floway-dev/protocols/anthropic-messages'; import { type ModelEndpoints, kindForEndpoints } from '@floway-dev/protocols/common'; -import { parseOpenAIChatCompletionsStream } from '@floway-dev/protocols/openai-chat-completions'; -import { parseOpenAIResponsesStream, type OpenAIResponsesCompactionResult, toCompactPayloadShape } from '@floway-dev/protocols/openai-responses'; -import { DEFAULT_RERANK_PATHS, serializeRerankRequest } from '@floway-dev/protocols/rerank'; -import { headersForAnthropicMessagesCall, jsonRequestBody, serializeModelFieldOpenAIAudioTranscriptionRequest, serializeOpenAIImagesEditsRequest, publicModelId, resolveEffectiveFlags, streamingProviderCall, type FetchInit, type FlagId, type HttpHeaderLines, type ProviderInstance, type Provider, type ProviderCallResult, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamModelConfig, type UpstreamRecord } from '@floway-dev/provider'; +import { publicModelId, resolveEffectiveFlags, type FlagId, type ProviderInstance, type Provider, type ProviderModel, type UpstreamModelConfig, type UpstreamRecord } from '@floway-dev/provider'; const rawModelIdOf = (model: ProviderModel): string => model.providerData as string; @@ -151,131 +146,12 @@ export const projectCustomModels = ( export const createCustomProvider = (record: UpstreamRecord): Provider => { const { config } = assertCustomUpstreamRecord(record); - // Each name is resolved as a whole: the admitted client values are dropped - // and its rules rebuild the name's value list in rule order, so a - // passthrough rule reinstates what the client sent and every configured - // rule contributes its own value beside it. - const valuesByKey = config.ingressHeadersRules.reduce>((byKey, rule) => { - const values = byKey.get(rule.key); - if (values) values.push(rule.value); - else byKey.set(rule.key, [rule.value]); - return byKey; - }, new Map()); - - const headersForCall = (headers: Headers): HttpHeaderLines => { - const resolved: [string, string][] = []; - for (const [name, value] of headers) { - if (!valuesByKey.has(name.toLowerCase())) resolved.push([name, value]); - } - for (const [key, values] of valuesByKey) { - const admitted = headers.get(key); - for (const value of values) { - if (value !== null) resolved.push([key, value]); - else if (admitted !== null) resolved.push([key, admitted]); - } - } - return resolved; - }; - const call = ( - transport: (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions) => Promise, - model: ProviderModel, - body: Record, - signal: AbortSignal | undefined, - headers: HttpHeaderLines, - opts: UpstreamCallOptions, - ): Promise => { - const rawModelId = rawModelIdOf(model); - return transport(config, { method: 'POST', body: jsonRequestBody({ ...body, model: rawModelId }), signal }, { extraHeaders: headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }) - .then(response => ({ - response, - modelKey: rawModelId, - })); - }; - - const callStreaming = ( - transport: (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions) => Promise, - model: ProviderModel, - body: Record, - signal: AbortSignal | undefined, - headers: HttpHeaderLines, - parser: ProviderStreamParser, - opts: UpstreamCallOptions, - ) => { - const rawModelId = rawModelIdOf(model); - return streamingProviderCall( - transport( - config, - { method: 'POST', body: jsonRequestBody({ ...body, stream: true, model: rawModelId }), signal }, - { extraHeaders: headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, - ), - parser, - rawModelId, - signal, - ); - }; - const instance: ProviderInstance = { getProvidedModels: async fetcher => { if (!config.modelsFetch.enabled) return projectCustomModels(record); const response = await fetchCustomModels(config, fetcher); return projectCustomModels(record, response); }, - callAlphaSearch: (model, body, signal, opts) => call(customFetchAlphaSearch, model, body, signal, headersForCall(opts.headers), opts), - callOpenAICompletions: (model, body, signal, opts) => call(customFetchOpenAICompletions, model, body, signal, headersForCall(opts.headers), opts), - callOpenAIChatCompletions: (model, body, signal, opts) => callStreaming(customFetchOpenAIChatCompletions, model, body, signal, headersForCall(opts.headers), parseOpenAIChatCompletionsStream, opts), - callOpenAIResponses: async (model, body, action, signal, opts) => { - switch (action) { - case 'generate': { - const stream = await callStreaming(customFetchOpenAIResponses, model, body, signal, headersForCall(opts.headers), parseOpenAIResponsesStream, opts); - return stream.ok - ? { action: 'generate', ok: true, events: stream.events, modelKey: stream.modelKey, ...(stream.headers ? { headers: stream.headers } : {}) } - : { action: 'generate', ok: false, response: stream.response, modelKey: stream.modelKey }; - } - case 'compact': { - const rawModelId = rawModelIdOf(model); - const response = await customFetchOpenAIResponsesCompact( - config, - { method: 'POST', body: jsonRequestBody({ ...toCompactPayloadShape(body), model: rawModelId }), signal }, - { extraHeaders: headersForCall(opts.headers), fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, - ); - return response.ok - ? { action: 'compact', ok: true, result: (await response.json()) as OpenAIResponsesCompactionResult, modelKey: rawModelId } - : { action: 'compact', ok: false, response, modelKey: rawModelId }; - } - default: - action satisfies never; - throw new Error(`Unhandled OpenAIResponsesAction: ${action as string}`); - } - }, - callAnthropicMessages: (model, body, signal, opts) => callStreaming(customFetchAnthropicMessages, model, body, signal, headersForAnthropicMessagesCall(headersForCall(opts.headers), opts.anthropicBeta), parseAnthropicMessagesStream, opts), - callAnthropicMessagesCountTokens: (model, body, signal, opts) => call(customFetchAnthropicMessagesCountTokens, model, body, signal, headersForAnthropicMessagesCall(headersForCall(opts.headers), opts.anthropicBeta), opts), - callOpenAIEmbeddings: (model, body, signal, opts) => call(customFetchOpenAIEmbeddings, model, body, signal, headersForCall(opts.headers), opts), - callOpenAIImagesGenerations: (model, body, signal, opts) => call(customFetchOpenAIImagesGenerations, model, body, signal, headersForCall(opts.headers), opts), - callOpenAIImagesEdits: async (model, request, signal, opts) => { - const rawModelId = rawModelIdOf(model); - const body = await serializeOpenAIImagesEditsRequest(request, rawModelId); - const response = await customFetchOpenAIImagesEdits(config, { method: 'POST', body, signal }, { extraHeaders: headersForCall(opts.headers), fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }); - return { response, modelKey: rawModelId }; - }, - callOpenAIAudioTranscriptions: async (model, request, signal, opts) => { - const rawModelId = rawModelIdOf(model); - const body = serializeModelFieldOpenAIAudioTranscriptionRequest(request, rawModelId); - const response = await customFetchOpenAIAudioTranscriptions(config, { method: 'POST', body, signal }, { extraHeaders: headersForCall(opts.headers), fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }); - return { response, modelKey: rawModelId }; - }, - callRerank: async (model, request, signal, opts) => { - const target = model.rerankTarget; - if (target === undefined) throw new Error(`Rerank model ${model.id} has no outbound target`); - const rawModelId = rawModelIdOf(model); - const body = serializeRerankRequest(target.protocol, rawModelId, request); - const response = await customFetchRerank( - config, - target.path ?? DEFAULT_RERANK_PATHS[target.protocol], - { method: 'POST', body: jsonRequestBody(body), signal }, - { extraHeaders: headersForCall(opts.headers), fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, - ); - return { response, modelKey: rawModelId, target }; - }, }; return { diff --git a/packages/provider-custom/src/stages/prepare-request.ts b/packages/provider-custom/src/stages/prepare-request.ts index b8c7f5dfc9..777e660dc6 100644 --- a/packages/provider-custom/src/stages/prepare-request.ts +++ b/packages/provider-custom/src/stages/prepare-request.ts @@ -1,12 +1,14 @@ import type { CustomPathOverrideKey, CustomUpstreamConfig } from '../config.ts'; -import { type HttpHeaders, type HttpRequestFacts, type HttpResponseFacts } from '@floway-dev/http/pipeline'; +import { type HttpHeaders, type HttpRequestFacts } from '@floway-dev/http/pipeline'; import { multipartBody, type HttpBody, type HttpBodyEncoding } from '@floway-dev/http/request-content'; import { defineStage, isSecret, move, secret, type Secret } from '@floway-dev/pipeline'; import type { RerankTarget } from '@floway-dev/protocols/common'; +import { toCompactPayloadShape } from '@floway-dev/protocols/openai-responses'; import { DEFAULT_RERANK_PATHS, serializeRerankRequest } from '@floway-dev/protocols/rerank'; import { joinBaseAndPath, mergeHttpHeaders, prepareOpenAIImagesEditsBody, replaceHttpHeader, withHttpContentType, type ProviderOperation, type ProviderOperationPayloads, type ProviderRequest, type ProviderResponse, type ProviderServices } from '@floway-dev/provider'; interface CustomHttpFacts extends HttpRequestFacts { + 'request.provider.modelKey': string; 'request.custom.modelKey': string; 'request.custom.path': string; } @@ -14,6 +16,11 @@ interface CustomHttpFacts extends HttpRequestFacts { type CustomResponse = ProviderResponse & { 'response.provider.rerankTarget'?: RerankTarget }; const PATHS: Partial> = { + openaiChatCompletions: '/chat/completions', + openaiResponses: '/responses', + openaiResponsesCompact: '/responses', + anthropicMessages: '/messages', + anthropicMessagesCountTokens: '/messages', alphaSearch: '/alpha/search', openaiCompletions: '/completions', openaiEmbeddings: '/embeddings', @@ -40,18 +47,18 @@ const resolvedHeaders = (config: CustomUpstreamConfig, incoming: HttpHeaders): H }; export const prepareCustomRequest = (config: CustomUpstreamConfig, operation: O) => { - const prepare = defineStage, CustomHttpFacts, HttpResponseFacts, CustomResponse, ProviderServices>({ + const prepare = defineStage & { 'request.provider.anthropicBeta'?: readonly string[]; 'request.provider.responsesAction': 'generate' | 'compact' }, CustomHttpFacts, CustomResponse, CustomResponse, ProviderServices>({ name: `prepareCustom${operation.replace(/^openai/, 'OpenAI').replace(/^alpha/, 'Alpha').replace(/^rerank/, 'Rerank')}`, through: { request: { - needs: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'], + needs: [...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? ['request.provider.responsesAction' as const] : []), 'request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers', ...(operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens' ? ['request.provider.anthropicBeta' as const] : [])], consumes: ['request.provider.model', 'request.provider.payload', 'request.http.headers'], - provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.custom.modelKey', 'request.custom.path'], + provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.provider.modelKey', 'request.custom.modelKey', 'request.custom.path'], }, response: { - needs: ['response.http.exchange'], + needs: [], consumes: [], - provides: ['response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', ...(operation === 'rerank' ? ['response.provider.rerankTarget' as const] : [])], + provides: [...(operation === 'rerank' ? ['response.provider.rerankTarget' as const] : [])], }, }, execute: async (facts, next) => { @@ -85,7 +92,11 @@ export const prepareCustomRequest = (config: Custom default: { const key = PATHS[operation]!; path = config.pathOverrides?.[key] ?? `/v1${key}`; - body = { ...payload, model: modelKey }; + if ((operation === 'openaiResponses' || operation === 'openaiResponsesCompact') && facts['request.provider.responsesAction'] === 'compact') path += '/compact'; + if (operation === 'anthropicMessagesCountTokens') path += '/count_tokens'; + body = (operation === 'openaiResponses' || operation === 'openaiResponsesCompact') && facts['request.provider.responsesAction'] === 'compact' + ? { ...toCompactPayloadShape(payload as ProviderOperationPayloads['openaiResponsesCompact']), model: modelKey } + : { ...payload, ...(operation === 'openaiChatCompletions' || operation === 'openaiResponses' || operation === 'openaiResponsesCompact' || operation === 'anthropicMessages' ? { stream: true } : {}), model: modelKey }; } } let base: HttpHeaders = []; @@ -96,9 +107,15 @@ export const prepareCustomRequest = (config: Custom base = replaceHttpHeader(base, 'Authorization', secret(`Bearer ${config.apiKey}`)); } base = withHttpContentType(base, body, encoding); - const headers = mergeHttpHeaders(base, resolvedHeaders(config, facts['request.http.headers'])); + let headers = mergeHttpHeaders(base, resolvedHeaders(config, facts['request.http.headers'])); + if (operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens') { + const beta = facts['request.provider.anthropicBeta'] as readonly string[]; + headers = headers.filter(([name]) => name.toLowerCase() !== 'anthropic-beta'); + if (beta.length > 0) headers = [...headers, ['anthropic-beta', beta.join(',')]]; + } const back = await next(move({ ...rest, + 'request.provider.modelKey': modelKey, 'request.custom.modelKey': modelKey, 'request.custom.path': path, 'request.http.url': joinBaseAndPath(config.baseUrl, path), @@ -107,7 +124,7 @@ export const prepareCustomRequest = (config: Custom 'request.http.body': body, 'request.http.encoding': encoding, })); - return move({ ...back, 'response.provider.modelKey': modelKey, 'response.provider.called': back['response.http.exchange'].type === 'response', 'response.provider.previousCalls': [], ...(target === undefined ? {} : { 'response.provider.rerankTarget': target }) }); + return move({ ...back, ...(target === undefined ? {} : { 'response.provider.rerankTarget': target }) }); }, }); return prepare; diff --git a/packages/provider-ollama/__tests__/account-probe_test.ts b/packages/provider-ollama/__tests__/account-probe_test.ts index 51582efb7d..9402a865e0 100644 --- a/packages/provider-ollama/__tests__/account-probe_test.ts +++ b/packages/provider-ollama/__tests__/account-probe_test.ts @@ -5,7 +5,7 @@ import { assertOllamaUpstreamRecord } from '../src/config.ts'; import { createOllamaProvider } from '../src/provider.ts'; import { readOllamaUpstreamState } from '../src/state.ts'; import { directFetcher, initProviderRepo, type UpstreamRecord } from '@floway-dev/provider'; -import { assertEquals, assertRejects, noopUpstreamCallOptions, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; +import { collectChatProviderPipeline, assertEquals, assertRejects, noopUpstreamCallOptions, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; const UPSTREAM_ID = 'up_ollama_account'; @@ -130,12 +130,7 @@ const callChat = async (record: UpstreamRecord, onCall: (pathname: string) => vo return new Response('data: [DONE]\n\n', { status: 200, headers: { 'content-type': 'text/event-stream' } }); }, async () => { - await provider.instance.callOpenAIChatCompletions( - stubProviderModel({ providerData: 'gpt-oss:120b' }), - { messages: [] }, - undefined, - noopUpstreamCallOptions({ waitUntil: promise => { pending.push(promise); } }), - ); + await collectChatProviderPipeline(provider, 'openaiChatCompletions', stubProviderModel({ providerData: 'gpt-oss:120b' }), { messages: [] }, undefined, noopUpstreamCallOptions({ waitUntil: promise => { pending.push(promise); } })); await Promise.all(pending); }, ); diff --git a/packages/provider-ollama/__tests__/provider_test.ts b/packages/provider-ollama/__tests__/provider_test.ts index 9fd4aead69..d660bed4a1 100644 --- a/packages/provider-ollama/__tests__/provider_test.ts +++ b/packages/provider-ollama/__tests__/provider_test.ts @@ -2,7 +2,7 @@ import { test } from 'vitest'; import { createOllamaProvider } from '../src/provider.ts'; import { initProviderRepo, type UpstreamRecord } from '@floway-dev/provider'; -import { callProviderPipeline, assertEquals, assertExists, jsonResponse, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, testFetcher, withMockedFetch } from '@floway-dev/test-utils'; +import { collectChatProviderPipeline, callProviderPipeline, assertEquals, assertExists, jsonResponse, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, testFetcher, withMockedFetch } from '@floway-dev/test-utils'; // A cloud upstream writes its usage snapshot after the calls it serves, so the // provider needs a repo to write into wherever those calls are exercised. @@ -196,21 +196,14 @@ test('call* methods POST to /v1/ with the upstream model id and Bearer if (url.pathname === '/v1/chat/completions') { chatRequest = request; chatBody = await request.json(); - // SSE response so streamingProviderCall does not throw on the empty - // body — we only assert the request shape. return new Response('', { status: 200, headers: { 'content-type': 'text/event-stream' } }); } return new Response('unexpected', { status: 500 }); }, async () => { const [providerModel] = await instance.instance.getProvidedModels(testFetcher); - const result = await instance.instance.callOpenAIChatCompletions( - providerModel, - { messages: [{ role: 'user', content: 'hi' }] }, - undefined, - noopUpstreamCallOptions(), - ); - assertEquals(result.modelKey, 'gpt-oss:120b'); + const result = await collectChatProviderPipeline(instance, 'openaiChatCompletions', providerModel, { messages: [{ role: 'user', content: 'hi' }] }, undefined, noopUpstreamCallOptions()); + assertEquals(result.facts['response.provider.modelKey'], 'gpt-oss:120b'); }, ); @@ -250,8 +243,8 @@ test('Anthropic Messages methods serialize typed anthropic-beta metadata only on async () => { const [model] = await instance.instance.getProvidedModels(testFetcher); const opts = noopAnthropicMessagesUpstreamCallOptions({ anthropicBeta: ['context-1m', 'advanced-tool-use'] }); - await instance.instance.callAnthropicMessages(model, { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }] }, undefined, opts); - await instance.instance.callAnthropicMessagesCountTokens(model, { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }] }, undefined, opts); + await collectChatProviderPipeline(instance, 'anthropicMessages', model, { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }] }, undefined, opts); + await collectChatProviderPipeline(instance, 'anthropicMessagesCountTokens', model, { max_tokens: 16, messages: [{ role: 'user', content: 'hi' }] }, undefined, opts); }, ); diff --git a/packages/provider-ollama/__tests__/fetch_test.ts b/packages/provider-ollama/__tests__/stages/prepare-request_test.ts similarity index 50% rename from packages/provider-ollama/__tests__/fetch_test.ts rename to packages/provider-ollama/__tests__/stages/prepare-request_test.ts index a7ccea17dd..7dda74f1a5 100644 --- a/packages/provider-ollama/__tests__/fetch_test.ts +++ b/packages/provider-ollama/__tests__/stages/prepare-request_test.ts @@ -1,19 +1,11 @@ import { test } from 'vitest'; -import { assertOllamaUpstreamRecord } from '../src/config.ts'; -import { - ollamaFetchOpenAIChatCompletions, - ollamaFetchOpenAIAudioTranscriptions, - ollamaFetchOpenAIEmbeddings, - ollamaFetchAnthropicMessages, - ollamaFetchOpenAIResponses, - ollamaFetchOpenAIResponsesCompact, - ollamaFetchShow, - ollamaFetchTags, -} from '../src/fetch.ts'; +import { assertOllamaUpstreamRecord } from '../../src/config.ts'; +import { ollamaFetchShow, ollamaFetchTags } from '../../src/control-http.ts'; +import { prepareOllamaRequest } from '../../src/stages/prepare-request.ts'; import type { UpstreamRecord } from '@floway-dev/provider'; -import { directFetcher, identityWrapUpstreamCall } from '@floway-dev/provider'; -import { assertEquals, withMockedFetch } from '@floway-dev/test-utils'; +import { identityWrapUpstreamCall } from '@floway-dev/provider'; +import { collectPreparedProviderRequest, testFetcher, assertEquals, withMockedFetch } from '@floway-dev/test-utils'; const baseRecord: UpstreamRecord = { id: 'up_ollama_test', @@ -45,14 +37,14 @@ test('typed transports hit the fixed Ollama endpoint paths', async () => { return new Response('{}', { status: 200 }); }, async () => { - await ollamaFetchOpenAIChatCompletions(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await ollamaFetchOpenAIResponses(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await ollamaFetchOpenAIResponsesCompact(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await ollamaFetchAnthropicMessages(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await ollamaFetchOpenAIEmbeddings(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await ollamaFetchOpenAIAudioTranscriptions(config, { method: 'POST', body: new FormData() }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await ollamaFetchTags(config, { method: 'GET' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); - await ollamaFetchShow(config, { method: 'POST', body: '{"name":"gpt-oss:120b"}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareOllamaRequest(config, 'openaiChatCompletions'), 'openaiChatCompletions', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareOllamaRequest(config, 'openaiResponses'), 'openaiResponses', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareOllamaRequest(config, 'openaiResponsesCompact'), 'openaiResponsesCompact', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareOllamaRequest(config, 'anthropicMessages'), 'anthropicMessages', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareOllamaRequest(config, 'openaiEmbeddings'), 'openaiEmbeddings', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareOllamaRequest(config, 'openaiAudioTranscriptions'), 'openaiAudioTranscriptions', key => key, { method: 'POST', body: new FormData() }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await ollamaFetchTags(config, { method: 'GET' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await ollamaFetchShow(config, { method: 'POST', body: '{"name":"gpt-oss:120b"}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); @@ -77,7 +69,7 @@ test('Authorization: Bearer is set when apiKey is configured', async () => { return new Response('{}', { status: 200 }); }, async () => { - await ollamaFetchOpenAIChatCompletions(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareOllamaRequest(config, 'openaiChatCompletions'), 'openaiChatCompletions', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); assertEquals(auth, 'Bearer ollama_test'); @@ -95,7 +87,7 @@ test('Authorization header is omitted entirely when apiKey is absent (local daem return new Response('{}', { status: 200 }); }, async () => { - await ollamaFetchOpenAIChatCompletions(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareOllamaRequest(config, 'openaiChatCompletions'), 'openaiChatCompletions', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); assertEquals(auth, null); @@ -110,7 +102,7 @@ test('Content-Type defaults to application/json for JSON bodies', async () => { return new Response('{}', { status: 200 }); }, async () => { - await ollamaFetchOpenAIChatCompletions(config, { method: 'POST', body: '{}' }, { fetcher: directFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); + await collectPreparedProviderRequest(prepareOllamaRequest(config, 'openaiChatCompletions'), 'openaiChatCompletions', key => key, { method: 'POST', body: '{}' }, { fetcher: testFetcher, wrapUpstreamCall: identityWrapUpstreamCall }); }, ); assertEquals(contentType, 'application/json'); diff --git a/packages/provider-ollama/__tests__/usage-probe_test.ts b/packages/provider-ollama/__tests__/usage-probe_test.ts index 0e7b821128..abc661b3a5 100644 --- a/packages/provider-ollama/__tests__/usage-probe_test.ts +++ b/packages/provider-ollama/__tests__/usage-probe_test.ts @@ -10,7 +10,7 @@ import { refreshOllamaUsageProbe, } from '../src/usage-probe.ts'; import { directFetcher, initProviderRepo, type UpstreamRecord } from '@floway-dev/provider'; -import { assertEquals, assertRejects, noopUpstreamCallOptions, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; +import { collectChatProviderPipeline, assertEquals, assertRejects, noopUpstreamCallOptions, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; const UPSTREAM_ID = 'up_ollama_usage'; @@ -128,12 +128,7 @@ const callChat = async (record: UpstreamRecord, onUsageProbe: () => void): Promi return new Response('data: [DONE]\n\n', { status: 200, headers: { 'content-type': 'text/event-stream' } }); }, async () => { - await provider.instance.callOpenAIChatCompletions( - stubProviderModel({ providerData: 'gpt-oss:120b' }), - { messages: [] }, - undefined, - noopUpstreamCallOptions({ waitUntil: promise => { pending.push(promise); } }), - ); + await collectChatProviderPipeline(provider, 'openaiChatCompletions', stubProviderModel({ providerData: 'gpt-oss:120b' }), { messages: [] }, undefined, noopUpstreamCallOptions({ waitUntil: promise => { pending.push(promise); } })); await Promise.all(pending); }, ); @@ -165,12 +160,7 @@ test('token counting leaves the account windows untouched and arms no probe', as return new Response('{"input_tokens":1}', { status: 200 }); }, async () => { - await provider.instance.callAnthropicMessagesCountTokens( - stubProviderModel({ providerData: 'gpt-oss:120b' }), - { messages: [], max_tokens: 16 }, - undefined, - { ...noopUpstreamCallOptions({ waitUntil: promise => { pending.push(promise); } }), anthropicBeta: [] }, - ); + await collectChatProviderPipeline(provider, 'anthropicMessagesCountTokens', stubProviderModel({ providerData: 'gpt-oss:120b' }), { messages: [], max_tokens: 16 }, undefined, { ...noopUpstreamCallOptions({ waitUntil: promise => { pending.push(promise); } }), anthropicBeta: [] }); await Promise.all(pending); }, ); diff --git a/packages/provider-ollama/src/account-probe.ts b/packages/provider-ollama/src/account-probe.ts index f1ddc20306..c8543cbafa 100644 --- a/packages/provider-ollama/src/account-probe.ts +++ b/packages/provider-ollama/src/account-probe.ts @@ -22,7 +22,7 @@ // and the Copilot card next to this one names its account without one. import { type OllamaUpstreamConfig } from './config.ts'; -import { ollamaFetchMe } from './fetch.ts'; +import { ollamaFetchMe } from './control-http.ts'; import { type OllamaAccountEntry, type OllamaUpstreamState, readOllamaUpstreamState } from './state.ts'; import { isOllamaUsageEnabled } from './usage-probe.ts'; import { type Fetcher, getProviderRepo, identityWrapUpstreamCall } from '@floway-dev/provider'; diff --git a/packages/provider-ollama/src/fetch.ts b/packages/provider-ollama/src/control-http.ts similarity index 61% rename from packages/provider-ollama/src/fetch.ts rename to packages/provider-ollama/src/control-http.ts index a40029abd4..d4b4cf0f36 100644 --- a/packages/provider-ollama/src/fetch.ts +++ b/packages/provider-ollama/src/control-http.ts @@ -25,28 +25,12 @@ const ollamaFetchInternal = async ( return await options.wrapUpstreamCall(() => options.fetcher(joinBaseAndPath(config.baseUrl, path), { ...init, headers })); }; -export const ollamaFetchOpenAIChatCompletions = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - ollamaFetchInternal(config, '/v1/chat/completions', init, options); -export const ollamaFetchOpenAIResponses = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - ollamaFetchInternal(config, '/v1/responses', init, options); -export const ollamaFetchOpenAIResponsesCompact = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - ollamaFetchInternal(config, '/v1/responses/compact', init, options); -export const ollamaFetchAnthropicMessages = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - ollamaFetchInternal(config, '/v1/messages', init, options); -export const ollamaFetchAnthropicMessagesCountTokens = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - ollamaFetchInternal(config, '/v1/messages/count_tokens', init, options); -export const ollamaFetchOpenAIEmbeddings = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - ollamaFetchInternal(config, '/v1/embeddings', init, options); -export const ollamaFetchOpenAICompletions = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - ollamaFetchInternal(config, '/v1/completions', init, options); // Stock Ollama mounts this OpenAI-compatible route, parses its multipart body, // and adapts the audio into an internal chat request. Capability discovery does // not advertise a dedicated transcription bit, so the provider exposes it only // through manual `openaiAudioTranscriptions` model entries. // https://github.com/ollama/ollama/blob/573386c35eac76124ffce571f4b0fefa0a7fe13c/server/routes.go#L1916-L1922 // https://github.com/ollama/ollama/blob/573386c35eac76124ffce571f4b0fefa0a7fe13c/middleware/openai.go#L682-L789 -export const ollamaFetchOpenAIAudioTranscriptions = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => - ollamaFetchInternal(config, '/v1/audio/transcriptions', init, options); export const ollamaFetchTags = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => ollamaFetchInternal(config, '/api/tags', init, options); export const ollamaFetchShow = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => diff --git a/packages/provider-ollama/src/fetch-models.ts b/packages/provider-ollama/src/fetch-models.ts index f887eeef4c..63b2223c17 100644 --- a/packages/provider-ollama/src/fetch-models.ts +++ b/packages/provider-ollama/src/fetch-models.ts @@ -19,8 +19,9 @@ // /api/embed for native callers and /v1/embeddings for the OpenAI shim. import type { OllamaUpstreamConfig } from './config.ts'; -import { ollamaFetchShow, ollamaFetchTags } from './fetch.ts'; -import { discardUpstreamResponse, fetchUpstreamModels, type Fetcher, identityWrapUpstreamCall, jsonRequestBody } from '@floway-dev/provider'; +import { ollamaFetchShow, ollamaFetchTags } from './control-http.ts'; +import { discardUpstreamResponse } from '@floway-dev/http'; +import { fetchUpstreamModels, type Fetcher, identityWrapUpstreamCall, jsonRequestBody } from '@floway-dev/provider'; export interface OllamaRawModel { // The slug Ollama uses everywhere (e.g. `gpt-oss:120b`, `deepseek-v4-flash`, diff --git a/packages/provider-ollama/src/pipelines.ts b/packages/provider-ollama/src/pipelines.ts index 495674a2ab..d4583a2897 100644 --- a/packages/provider-ollama/src/pipelines.ts +++ b/packages/provider-ollama/src/pipelines.ts @@ -4,13 +4,19 @@ import { prepareOllamaRequest } from './stages/prepare-request.ts'; import type { OllamaUpstreamState } from './state.ts'; import { http } from '@floway-dev/http/pipeline'; import { compose } from '@floway-dev/pipeline'; -import type { ProviderOperationPayloads, ProviderOperationResponse, ProviderPipelines, ProviderRequest } from '@floway-dev/provider'; +import { decodeProviderResponse, observeProviderCall, observeProviderResponsesCall, selectProviderResponsesAction } from '@floway-dev/provider'; +import type { ProviderOperationPayloads, ProviderOperationRequest, ProviderOperationResponse, ProviderPipelines, ProviderRequest } from '@floway-dev/provider'; export const createOllamaPipelines = (upstreamId: string, config: OllamaUpstreamConfig, state: OllamaUpstreamState): ProviderPipelines => { const observe = observeOllamaAccount(upstreamId, config, state); return { - openaiCompletions: compose, ProviderOperationResponse<'openaiCompletions'>>('ollama.openaiCompletions', [observe, prepareOllamaRequest(config, 'openaiCompletions'), http]), - openaiEmbeddings: compose, ProviderOperationResponse<'openaiEmbeddings'>>('ollama.openaiEmbeddings', [observe, prepareOllamaRequest(config, 'openaiEmbeddings'), http]), - openaiAudioTranscriptions: compose, ProviderOperationResponse<'openaiAudioTranscriptions'>>('ollama.openaiAudioTranscriptions', [observe, prepareOllamaRequest(config, 'openaiAudioTranscriptions'), http]), + openaiChatCompletions: compose, ProviderOperationResponse<'openaiChatCompletions'>>('ollama.openaiChatCompletions', [observe, decodeProviderResponse('openaiChatCompletions'), prepareOllamaRequest(config, 'openaiChatCompletions'), observeProviderCall, http]), + openaiResponses: compose, ProviderOperationResponse<'openaiResponses'>>('ollama.openaiResponses', [observe, selectProviderResponsesAction('generate'), decodeProviderResponse('openaiResponses'), prepareOllamaRequest(config, 'openaiResponses'), observeProviderResponsesCall, http]), + openaiResponsesCompact: compose, ProviderOperationResponse<'openaiResponsesCompact'>>('ollama.openaiResponsesCompact', [observe, selectProviderResponsesAction('compact'), decodeProviderResponse('openaiResponsesCompact'), prepareOllamaRequest(config, 'openaiResponsesCompact'), observeProviderResponsesCall, http]), + anthropicMessages: compose, ProviderOperationResponse<'anthropicMessages'>>('ollama.anthropicMessages', [observe, decodeProviderResponse('anthropicMessages'), prepareOllamaRequest(config, 'anthropicMessages'), observeProviderCall, http]), + anthropicMessagesCountTokens: compose, ProviderOperationResponse<'anthropicMessagesCountTokens'>>('ollama.anthropicMessagesCountTokens', [decodeProviderResponse('anthropicMessagesCountTokens'), prepareOllamaRequest(config, 'anthropicMessagesCountTokens'), observeProviderCall, http]), + openaiCompletions: compose, ProviderOperationResponse<'openaiCompletions'>>('ollama.openaiCompletions', [observe, prepareOllamaRequest(config, 'openaiCompletions'), observeProviderCall, http]), + openaiEmbeddings: compose, ProviderOperationResponse<'openaiEmbeddings'>>('ollama.openaiEmbeddings', [observe, prepareOllamaRequest(config, 'openaiEmbeddings'), observeProviderCall, http]), + openaiAudioTranscriptions: compose, ProviderOperationResponse<'openaiAudioTranscriptions'>>('ollama.openaiAudioTranscriptions', [observe, prepareOllamaRequest(config, 'openaiAudioTranscriptions'), observeProviderCall, http]), }; }; diff --git a/packages/provider-ollama/src/provider.ts b/packages/provider-ollama/src/provider.ts index 836596b3bf..f76c51794f 100644 --- a/packages/provider-ollama/src/provider.ts +++ b/packages/provider-ollama/src/provider.ts @@ -24,25 +24,15 @@ // catalog, and an auto row carrying the same upstreamModelId is dropped so the // manual copy is the only one for that id. -import { scheduleOllamaAccountProbe } from './account-probe.ts'; import { chatFromOllamaRaw } from './chat-from-raw.ts'; -import { assertOllamaUpstreamRecord, type OllamaUpstreamConfig } from './config.ts'; +import { assertOllamaUpstreamRecord } from './config.ts'; import { OLLAMA_DEFAULT_FLAGS } from './defaults.ts'; import { fetchOllamaCatalog, type OllamaCatalog } from './fetch-models.ts'; -import { ollamaFetchOpenAIAudioTranscriptions, ollamaFetchOpenAIChatCompletions, ollamaFetchOpenAICompletions, ollamaFetchOpenAIEmbeddings, ollamaFetchAnthropicMessages, ollamaFetchAnthropicMessagesCountTokens, ollamaFetchOpenAIResponses, ollamaFetchOpenAIResponsesCompact } from './fetch.ts'; import { createOllamaPipelines } from './pipelines.ts'; import { pricingForOllamaModelKey } from './pricing.ts'; import { readOllamaUpstreamState } from './state.ts'; -import { scheduleOllamaUsageProbe } from './usage-probe.ts'; -import { parseAnthropicMessagesStream } from '@floway-dev/protocols/anthropic-messages'; import { type ModelEndpoints, kindForEndpoints } from '@floway-dev/protocols/common'; -import { parseOpenAIChatCompletionsStream } from '@floway-dev/protocols/openai-chat-completions'; -import { parseOpenAIResponsesStream, type OpenAIResponsesCompactionResult, toCompactPayloadShape } from '@floway-dev/protocols/openai-responses'; -import { headersForAnthropicMessagesCall, jsonRequestBody, publicModelId, resolveEffectiveFlags, serializeModelFieldOpenAIAudioTranscriptionRequest, streamingProviderCall, type FetchInit, type FlagId, type HttpHeaderLines, type ProviderInstance, type Provider, type ProviderCallResult, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord } from '@floway-dev/provider'; - -// providerData carries the raw upstream id verbatim — the same value /api/tags -// returns and the same value the gateway must send back on every inference call. -const rawModelIdOf = (model: ProviderModel): string => model.providerData as string; +import { publicModelId, resolveEffectiveFlags, type FlagId, type ProviderInstance, type Provider, type ProviderModel, type UpstreamRecord } from '@floway-dev/provider'; // Vision / tool / thinking capabilities live alongside `embedding` in the // /api/show response. Embedding is the only one that drives a different @@ -85,27 +75,6 @@ export const createOllamaProvider = (record: UpstreamRecord): Provider => { const upstreamFlags = resolveEffectiveFlags([OLLAMA_DEFAULT_FLAGS, record.flagOverrides]); const state = readOllamaUpstreamState(record.state); - // Ollama Cloud moves the account's session and weekly windows on inference - // calls, and exposes them nowhere but its usage endpoint, so each such call - // arms a debounced background refresh. The account behind the key is armed by - // the same calls behind its own, much longer interval. Token counting never - // reaches a model and leaves the windows untouched, so it arms neither. - const armProbes = (opts: UpstreamCallOptions): void => { - scheduleOllamaUsageProbe(record.id, config, state, opts.fetcher, opts.waitUntil); - scheduleOllamaAccountProbe(record.id, config, state, opts.fetcher, opts.waitUntil); - }; - - // Arms once the upstream round-trip has produced a response, so the usage - // probe reads windows that already account for this call. A rate-limited - // response arms it too — that is exactly when an operator wants the windows - // on screen. A transport that threw never reached the account and arms - // nothing. - const withProbes = (opts: UpstreamCallOptions, dispatched: Promise): Promise => - dispatched.then(result => { - armProbes(opts); - return result; - }); - // Manual models always emit. const overriddenIds = new Set(config.models.map(m => m.upstreamModelId)); const manualModels: ProviderModel[] = config.models.map(model => { @@ -128,49 +97,7 @@ export const createOllamaProvider = (record: UpstreamRecord): Provider => { if (kind === 'chat' && model.chat) internal.chat = model.chat; return internal; }); - const call = ( - transport: (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions) => Promise, - model: ProviderModel, - body: Record, - signal: AbortSignal | undefined, - headers: HttpHeaderLines, - opts: UpstreamCallOptions, - ): Promise => { - const rawModelId = rawModelIdOf(model); - return transport( - config, - { method: 'POST', body: jsonRequestBody({ ...body, model: rawModelId }), signal }, - { extraHeaders: headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, - ).then(response => ({ response, modelKey: rawModelId })); - }; - - const callStreaming = ( - transport: (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions) => Promise, - model: ProviderModel, - body: Record, - signal: AbortSignal | undefined, - headers: HttpHeaderLines, - parser: ProviderStreamParser, - opts: UpstreamCallOptions, - ) => { - const rawModelId = rawModelIdOf(model); - return streamingProviderCall( - transport( - config, - { method: 'POST', body: jsonRequestBody({ ...body, stream: true, model: rawModelId }), signal }, - { extraHeaders: headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, - ), - parser, - rawModelId, - signal, - ); - }; - - const rejectUnsupported = (capability: string) => () => - Promise.reject(new Error(`Ollama provider does not implement ${capability}`)); - const instance: ProviderInstance = { - callAlphaSearch: rejectUnsupported('callAlphaSearch'), getProvidedModels: async fetcher => { const catalog = await fetchOllamaCatalog(config, fetcher); const auto = finalizeOllamaModels( @@ -179,46 +106,6 @@ export const createOllamaProvider = (record: UpstreamRecord): Provider => { ); return [...manualModels, ...auto]; }, - callOpenAICompletions: (model, body, signal, opts) => withProbes(opts, call(ollamaFetchOpenAICompletions, model, body, signal, [...opts.headers], opts)), - callOpenAIChatCompletions: (model, body, signal, opts) => withProbes(opts, callStreaming(ollamaFetchOpenAIChatCompletions, model, body, signal, [...opts.headers], parseOpenAIChatCompletionsStream, opts)), - callOpenAIResponses: async (model, body, action, signal, opts) => { - switch (action) { - case 'generate': { - const stream = await withProbes(opts, callStreaming(ollamaFetchOpenAIResponses, model, body, signal, [...opts.headers], parseOpenAIResponsesStream, opts)); - return stream.ok - ? { action: 'generate', ok: true, events: stream.events, modelKey: stream.modelKey, ...(stream.headers ? { headers: stream.headers } : {}) } - : { action: 'generate', ok: false, response: stream.response, modelKey: stream.modelKey }; - } - case 'compact': { - const rawModelId = rawModelIdOf(model); - const response = await withProbes(opts, ollamaFetchOpenAIResponsesCompact( - config, - { method: 'POST', body: jsonRequestBody({ ...toCompactPayloadShape(body), model: rawModelId }), signal }, - { extraHeaders: [...opts.headers], fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, - )); - return response.ok - ? { action: 'compact', ok: true, result: (await response.json()) as OpenAIResponsesCompactionResult, modelKey: rawModelId } - : { action: 'compact', ok: false, response, modelKey: rawModelId }; - } - default: - action satisfies never; - throw new Error(`Unhandled OpenAIResponsesAction: ${action as string}`); - } - }, - callAnthropicMessages: (model, body, signal, opts) => withProbes(opts, callStreaming(ollamaFetchAnthropicMessages, model, body, signal, headersForAnthropicMessagesCall([...opts.headers], opts.anthropicBeta), parseAnthropicMessagesStream, opts)), - callAnthropicMessagesCountTokens: (model, body, signal, opts) => call(ollamaFetchAnthropicMessagesCountTokens, model, body, signal, headersForAnthropicMessagesCall([...opts.headers], opts.anthropicBeta), opts), - callOpenAIEmbeddings: (model, body, signal, opts) => withProbes(opts, call(ollamaFetchOpenAIEmbeddings, model, body, signal, [...opts.headers], opts)), - // Ollama serves no image-generation endpoint; reject if the gateway ever - // routes one here. /v1/images/* is not exposed by the upstream binary. - callOpenAIImagesGenerations: rejectUnsupported('callOpenAIImagesGenerations'), - callOpenAIImagesEdits: rejectUnsupported('callOpenAIImagesEdits'), - callOpenAIAudioTranscriptions: async (model, request, signal, opts) => { - const rawModelId = rawModelIdOf(model); - const body = serializeModelFieldOpenAIAudioTranscriptionRequest(request, rawModelId); - const response = await withProbes(opts, ollamaFetchOpenAIAudioTranscriptions(config, { method: 'POST', body, signal }, { extraHeaders: [...opts.headers], fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall })); - return { response, modelKey: rawModelId }; - }, - callRerank: rejectUnsupported('callRerank'), }; return { diff --git a/packages/provider-ollama/src/stages/prepare-request.ts b/packages/provider-ollama/src/stages/prepare-request.ts index 075563c12d..d92178508e 100644 --- a/packages/provider-ollama/src/stages/prepare-request.ts +++ b/packages/provider-ollama/src/stages/prepare-request.ts @@ -1,16 +1,23 @@ import type { OllamaUpstreamConfig } from '../config.ts'; -import type { HttpRequestFacts, HttpResponseFacts } from '@floway-dev/http/pipeline'; +import type { HttpRequestFacts } from '@floway-dev/http/pipeline'; import { multipartBody } from '@floway-dev/http/request-content'; import { defineStage, move, secret } from '@floway-dev/pipeline'; +import { toCompactPayloadShape } from '@floway-dev/protocols/openai-responses'; import { joinBaseAndPath, mergeHttpHeaders, withHttpContentType, type ProviderOperationPayloads, type ProviderRequest, type ProviderResponse, type ProviderServices } from '@floway-dev/provider'; -type OllamaOperation = 'openaiCompletions' | 'openaiEmbeddings' | 'openaiAudioTranscriptions'; +type OllamaOperation = 'openaiChatCompletions' | 'openaiResponses' | 'openaiResponsesCompact' | 'anthropicMessages' | 'anthropicMessagesCountTokens' | 'openaiCompletions' | 'openaiEmbeddings' | 'openaiAudioTranscriptions'; interface OllamaHttpFacts extends HttpRequestFacts { + 'request.provider.modelKey': string; 'request.ollama.modelKey': string; } const PATHS: Record = { + openaiChatCompletions: '/v1/chat/completions', + openaiResponses: '/v1/responses', + openaiResponsesCompact: '/v1/responses/compact', + anthropicMessages: '/v1/messages', + anthropicMessagesCountTokens: '/v1/messages/count_tokens', openaiCompletions: '/v1/completions', openaiEmbeddings: '/v1/embeddings', // https://github.com/ollama/ollama/blob/573386c35eac76124ffce571f4b0fefa0a7fe13c/server/routes.go#L1916-L1922 @@ -18,15 +25,15 @@ const PATHS: Record = { }; export const prepareOllamaRequest = (config: OllamaUpstreamConfig, operation: O) => { - const prepare = defineStage, OllamaHttpFacts, HttpResponseFacts, ProviderResponse, ProviderServices>({ + const prepare = defineStage & { 'request.provider.anthropicBeta'?: readonly string[]; 'request.provider.responsesAction': 'generate' | 'compact' }, OllamaHttpFacts, ProviderResponse, ProviderResponse, ProviderServices>({ name: `prepareOllama${operation.replace(/^openai/, 'OpenAI').replace(/^alpha/, 'Alpha').replace(/^rerank/, 'Rerank')}`, through: { request: { - needs: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'], + needs: [...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? ['request.provider.responsesAction' as const] : []), 'request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers', ...(operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens' ? ['request.provider.anthropicBeta' as const] : [])], consumes: ['request.provider.model', 'request.provider.payload', 'request.http.headers'], - provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.ollama.modelKey'], + provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.provider.modelKey', 'request.ollama.modelKey'], }, - response: { needs: ['response.http.exchange'], consumes: [], provides: ['response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'] }, + response: { needs: [], consumes: [], provides: [] }, }, execute: async (facts, next) => { const { 'request.provider.model': model, 'request.provider.payload': payload, ...rest } = facts; @@ -34,19 +41,27 @@ export const prepareOllamaRequest = (config: OllamaUp const encoding = operation === 'openaiAudioTranscriptions' ? 'multipart' as const : 'json' as const; const body = operation === 'openaiAudioTranscriptions' ? multipartBody((payload as ProviderOperationPayloads['openaiAudioTranscriptions']).entries.map(entry => entry.name === 'model' ? { ...entry, value: modelKey } : entry)) - : { ...payload, model: modelKey }; + : (operation === 'openaiResponses' || operation === 'openaiResponsesCompact') && facts['request.provider.responsesAction'] === 'compact' + ? { ...toCompactPayloadShape(payload as ProviderOperationPayloads['openaiResponsesCompact']), model: modelKey } + : { ...payload, ...(operation === 'openaiChatCompletions' || operation === 'openaiResponses' || operation === 'openaiResponsesCompact' || operation === 'anthropicMessages' ? { stream: true } : {}), model: modelKey }; const base = config.apiKey ? [['Authorization', secret(`Bearer ${config.apiKey}`)] as const] : []; - const headers = mergeHttpHeaders(withHttpContentType(base, body, encoding), facts['request.http.headers']); + let headers = mergeHttpHeaders(withHttpContentType(base, body, encoding), facts['request.http.headers']); + if (operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens') { + const beta = facts['request.provider.anthropicBeta'] as readonly string[]; + headers = headers.filter(([name]) => name.toLowerCase() !== 'anthropic-beta'); + if (beta.length > 0) headers = [...headers, ['anthropic-beta', beta.join(',')]]; + } const back = await next(move({ ...rest, + 'request.provider.modelKey': modelKey, 'request.ollama.modelKey': modelKey, - 'request.http.url': joinBaseAndPath(config.baseUrl, PATHS[operation]), + 'request.http.url': joinBaseAndPath(config.baseUrl, ((operation === 'openaiResponses' || operation === 'openaiResponsesCompact') ? PATHS[facts['request.provider.responsesAction'] === 'compact' ? 'openaiResponsesCompact' : 'openaiResponses'] : PATHS[operation])), 'request.http.method': 'POST', 'request.http.headers': headers, 'request.http.body': body, 'request.http.encoding': encoding, })); - return move({ ...back, 'response.provider.modelKey': modelKey, 'response.provider.called': back['response.http.exchange'].type === 'response', 'response.provider.previousCalls': [] }); + return move({ ...back }); }, }); return prepare; diff --git a/packages/provider-ollama/src/usage-probe.ts b/packages/provider-ollama/src/usage-probe.ts index c555b3bc58..b6f07e7afa 100644 --- a/packages/provider-ollama/src/usage-probe.ts +++ b/packages/provider-ollama/src/usage-probe.ts @@ -31,7 +31,7 @@ // https://github.com/ollama/ollama/blob/f0078ae4766d0d570e196158f20dde309bd96124/docs/api/errors.mdx import { type OllamaUpstreamConfig } from './config.ts'; -import { ollamaFetchUsage } from './fetch.ts'; +import { ollamaFetchUsage } from './control-http.ts'; import { type OllamaUsageObservation, type OllamaUsageProbeEntry, type OllamaUpstreamState, readOllamaUpstreamState } from './state.ts'; import { type Fetcher, getProviderRepo, identityWrapUpstreamCall } from '@floway-dev/provider'; diff --git a/packages/provider/__tests__/audio_test.ts b/packages/provider/__tests__/audio_test.ts deleted file mode 100644 index 25aba03ac8..0000000000 --- a/packages/provider/__tests__/audio_test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { test } from 'vitest'; - -import { serializeModelPathOpenAIAudioTranscriptionRequest, serializeModelFieldOpenAIAudioTranscriptionRequest } from '../src/audio.ts'; -import { assertEquals, assertExists } from '@floway-dev/test-utils'; - -test('serializeModelFieldOpenAIAudioTranscriptionRequest preserves ordered fields and file metadata while replacing model', async () => { - const file = { bytes: new Uint8Array([1, 2, 3, 4]), name: 'meeting.wav', type: 'audio/wav', lastModified: 1_700_000_000_000 }; - const form = serializeModelFieldOpenAIAudioTranscriptionRequest({ - entries: [ - { name: 'file', value: file }, - { name: 'language', value: 'en' }, - { name: 'model', value: 'public-model' }, - { name: 'timestamp_granularities[]', value: 'word' }, - { name: 'timestamp_granularities[]', value: 'segment' }, - ], - }, 'upstream-model'); - - assertEquals([...form.keys()], ['file', 'language', 'model', 'timestamp_granularities[]', 'timestamp_granularities[]']); - assertEquals(form.get('model'), 'upstream-model'); - assertEquals(form.getAll('timestamp_granularities[]'), ['word', 'segment']); - const serializedFile = form.get('file'); - assertEquals(serializedFile instanceof File, true); - assertExists(serializedFile as File); - assertEquals((serializedFile as File).name, 'meeting.wav'); - assertEquals((serializedFile as File).type, 'audio/wav'); - assertEquals((serializedFile as File).lastModified, file.lastModified); - assertEquals(new Uint8Array(await (serializedFile as File).arrayBuffer()), new Uint8Array([1, 2, 3, 4])); -}); - -test('serializeModelPathOpenAIAudioTranscriptionRequest omits model fields selected by the URL', () => { - const form = serializeModelPathOpenAIAudioTranscriptionRequest({ - entries: [ - { name: 'model', value: 'public-model' }, - { name: 'file', value: { bytes: new TextEncoder().encode('audio'), name: 'meeting.wav', type: 'audio/wav' } }, - { name: 'response_format', value: 'json' }, - ], - }); - - assertEquals([...form.keys()], ['file', 'response_format']); - assertEquals(form.get('model'), null); -}); diff --git a/packages/provider/__tests__/images_test.ts b/packages/provider/__tests__/images_test.ts index e8e38848f9..adc49938c2 100644 --- a/packages/provider/__tests__/images_test.ts +++ b/packages/provider/__tests__/images_test.ts @@ -1,15 +1,26 @@ import { test } from 'vitest'; -import { serializeOpenAIImagesEditsJsonPayload, serializeOpenAIImagesEditsRequest } from '../src/images.ts'; +import { serializeOpenAIImagesEditsJsonPayload, prepareOpenAIImagesEditsBody } from '../src/images.ts'; +import { isReplayableBody } from '../src/options.ts'; +import { serializeHttpBody, type HttpMultipartBody } from '@floway-dev/http/request-content'; import { assertEquals } from '@floway-dev/test-utils'; -const parseJsonBody = async (body: Awaited>): Promise => { - if (body instanceof FormData) throw new Error('expected a JSON image-edit body'); - return await new Response(body.open()).json(); +const encodedBody = (prepared: Awaited>): Response => { + const body = serializeHttpBody(prepared.body, prepared.encoding); + if (!isReplayableBody(body)) throw new Error('Expected replayable request content'); + return new Response(body.open(), { + headers: prepared.encoding === 'multipart' + ? { 'content-type': `multipart/form-data; boundary=${(prepared.body as HttpMultipartBody).boundary}` } + : { 'content-type': 'application/json' }, + }); +}; +const parseJsonBody = async (prepared: Awaited>): Promise => { + if (prepared.encoding !== 'json') throw new Error('expected a JSON image-edit body'); + return await encodedBody(prepared).json(); }; -test('serializeOpenAIImagesEditsRequest preserves reference fields and encodes mixed uploads from their bytes', async () => { - const serialized = await serializeOpenAIImagesEditsRequest({ +test('prepareOpenAIImagesEditsBody preserves reference fields and encodes mixed uploads from their bytes', async () => { + const serialized = await prepareOpenAIImagesEditsBody({ images: [ { type: 'reference', reference: { image_url: 'https://example.test/image.png', detail: 'future-field' } }, { type: 'upload', file: { bytes: new TextEncoder().encode('inline'), name: 'inline.png', type: 'image/png' } }, @@ -42,33 +53,33 @@ test('serializeOpenAIImagesEditsJsonPayload forces upload sources into data URLs }); }); -test('serializeOpenAIImagesEditsRequest uses the singular field for one upload and the array field for many', async () => { +test('prepareOpenAIImagesEditsBody uses the singular field for one upload and the array field for many', async () => { const first = { bytes: new TextEncoder().encode('first'), name: 'first.png', type: 'image/png' }; const second = { bytes: new TextEncoder().encode('second'), name: 'second.png', type: 'image/png' }; - const single = await serializeOpenAIImagesEditsRequest({ + const single = await prepareOpenAIImagesEditsBody({ images: [{ type: 'upload', file: first }], parameters: { prompt: 'single' }, }, 'gpt-image'); - assertEquals(single instanceof FormData, true); - const singleForm = single as FormData; + assertEquals(single.encoding, 'multipart'); + const singleForm = await encodedBody(single).formData(); assertEquals((singleForm.get('image') as File).name, first.name); assertEquals(new Uint8Array(await (singleForm.get('image') as File).arrayBuffer()), first.bytes); assertEquals(singleForm.getAll('image[]'), []); - const multiple = await serializeOpenAIImagesEditsRequest({ + const multiple = await prepareOpenAIImagesEditsBody({ images: [{ type: 'upload', file: first }, { type: 'upload', file: second }], parameters: { prompt: 'multiple' }, }, 'gpt-image'); - assertEquals(multiple instanceof FormData, true); - const multipleForm = multiple as FormData; + assertEquals(multiple.encoding, 'multipart'); + const multipleForm = await encodedBody(multiple).formData(); assertEquals(multipleForm.getAll('image[]').map(value => (value as File).name), [first.name, second.name]); assertEquals(await Promise.all(multipleForm.getAll('image[]').map(async value => new Uint8Array(await (value as File).arrayBuffer()))), [first.bytes, second.bytes]); assertEquals(multipleForm.get('image'), null); assertEquals(multipleForm.get('model'), 'gpt-image'); }); -test('serializeOpenAIImagesEditsRequest leaves malformed inline data for upstream JSON validation', async () => { - const serialized = await serializeOpenAIImagesEditsRequest({ +test('prepareOpenAIImagesEditsBody leaves malformed inline data for upstream JSON validation', async () => { + const serialized = await prepareOpenAIImagesEditsBody({ images: [{ type: 'inline', reference: { image_url: 'data:image/png;base64,%%%' }, @@ -82,8 +93,8 @@ test('serializeOpenAIImagesEditsRequest leaves malformed inline data for upstrea }); }); -test('serializeOpenAIImagesEditsRequest preserves extra inline reference fields through JSON', async () => { - const serialized = await serializeOpenAIImagesEditsRequest({ +test('prepareOpenAIImagesEditsBody preserves extra inline reference fields through JSON', async () => { + const serialized = await prepareOpenAIImagesEditsBody({ images: [{ type: 'inline', reference: { image_url: 'data:image/png;base64,aW1hZ2U=', future_field: 'keep' }, diff --git a/packages/provider/__tests__/stages/decode-response_test.ts b/packages/provider/__tests__/stages/decode-response_test.ts new file mode 100644 index 0000000000..6881c1677a --- /dev/null +++ b/packages/provider/__tests__/stages/decode-response_test.ts @@ -0,0 +1,93 @@ +import { expect, test } from 'vitest'; + +import type { ProviderChatResponse, ProviderChatServices } from '../../src/pipeline.ts'; +import { decodeProviderResponse } from '../../src/stages/decode-response.ts'; +import { observeProviderCall } from '../../src/stages/observe-call.ts'; +import { http, type HttpRequestFacts } from '@floway-dev/http/pipeline'; +import { compose, defineStage, getFailureFacts, move, run, setRelease } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; + +type Entry = HttpRequestFacts & { 'request.provider.modelKey': string }; +const entry = (): Entry => move({ + 'request.provider.modelKey': 'actual-wire-model', + 'request.http.callId': 1, + 'request.http.url': 'https://upstream.example/chat/completions', + 'request.http.method': 'POST', + 'request.http.headers': [], + 'request.http.body': { model: 'actual-wire-model', stream: true, messages: [] }, + 'request.http.encoding': 'json', +}); +const pipeline = compose>('provider.chat', [decodeProviderResponse('openaiChatCompletions'), observeProviderCall, http]); +const services = (response: () => Promise, record: ProviderChatServices['recordProtocolFrames'] = frames => frames): ProviderChatServices => ({ + httpCall: () => ({ fetcher: response, signal: undefined, waitUntil: () => {}, wrapUpstreamCall: call => call() }), + recordProtocolFrames: record, +}); + +test('stream decoding records its protocol stream and shares the single owned HTTP body', async () => { + let streams = 0; + const executed = await run(pipeline, entry(), services(async () => new Response('data: {"id":"reply","choices":[]}\n\ndata: [DONE]\n\n', { headers: { 'content-type': 'text/event-stream' } }), frames => { streams++; return frames; })); + const output = executed.facts['response.provider.output']; + expect(output).not.toBeNull(); + if (output === null || !('kind' in output) || output.kind !== 'stream') throw new Error('Expected decoded stream'); + const frames: ProtocolFrame[] = []; + for await (const frame of output.frames) frames.push(frame); + expect(frames.map(frame => frame.type)).toEqual(['event', 'done']); + expect(streams).toBe(1); + expect(executed.facts['response.provider.called']).toBe(true); + expect(executed.facts['response.provider.modelKey']).toBe('actual-wire-model'); + const exchange = executed.facts['response.http.exchange']; + if (exchange.type !== 'response' || exchange.body === null) throw new Error('Expected HTTP body'); + expect(executed.facts['response.http.body']).toBe(exchange.body); + setRelease(exchange.body, async () => {}); + await executed.drain(); +}); + +test('a successful non-SSE reply is a protocol failure value with its complete parsed payload', async () => { + const body = { error: { message: 'x'.repeat(4096) } }; + const executed = await run(pipeline, entry(), services(async () => Response.json(body))); + const output = executed.facts['response.provider.output']; + expect(output).toMatchObject({ status: 502, body }); + expect(executed.facts['response.provider.called']).toBe(true); + expect(executed.facts['response.http.exchange']).toMatchObject({ type: 'response', status: 200 }); + await executed.drain(); +}); + +test('actual HTTP replies remain failure facts when a higher decoder stage throws', async () => { + const error = new TypeError('programming error', { cause: new Error('original cause') }); + const fault = defineStage, ProviderChatResponse<'openaiChatCompletions'>>({ + name: 'faultAfterReply', + through: { request: { needs: [], consumes: [], provides: [] }, response: { needs: [], consumes: [], provides: [] } }, + execute: async (facts, next) => { await next(move({ ...facts })); throw error; }, + }); + try { + await run(compose>('provider.fault', [fault, decodeProviderResponse('openaiChatCompletions'), observeProviderCall, http]), entry(), services(async () => new Response('data: [DONE]\n\n', { headers: { 'content-type': 'text/event-stream' } }))); + throw new Error('Expected stage failure'); + } catch (caught) { + expect(caught).toBe(error); + expect(getFailureFacts(caught)).toMatchObject({ 'response.provider.called': true, 'response.provider.modelKey': 'actual-wire-model', 'response.provider.previousCalls': [] }); + } +}); + +test('a non2xx reply remains an unchanged HTTP exchange with no decoded stream', async () => { + const executed = await run(pipeline, entry(), services(async () => new Response('rate limited', { status: 429, headers: { 'x-upstream': 'original' } }))); + expect(executed.facts['response.provider.output']).toBeNull(); + expect(executed.facts['response.http.exchange']).toMatchObject({ type: 'response', status: 429, headers: expect.arrayContaining([['x-upstream', 'original']]) }); + expect(executed.facts['response.provider.modelKey']).toBe('actual-wire-model'); + await executed.drain(); +}); + +test.each([ + { status: 204, body: null, contentType: null, phrases: ['204', 'stream is required', 'Body: '] }, + { status: 200, body: '{"error":{"message":"azure stub"}}', contentType: 'application/json', phrases: ['200', 'application/json', 'stream is required', 'azure stub'] }, + { status: 200, body: 'not really an event stream', contentType: 'text/event-stream-fake', phrases: ['stream is required'] }, + { status: 200, body: '{"choices":[]}', contentType: null, phrases: ['"unknown"', '{"choices":[]}'] }, + { status: 200, body: 'x'.repeat(2048), contentType: 'application/json', phrases: ['...[truncated]'] }, +])('successful non-stream reply $contentType keeps diagnostic context as a failure value', async ({ status, body, contentType, phrases }) => { + const executed = await run(pipeline, entry(), services(async () => new Response(body === null ? null : new TextEncoder().encode(body), { status, headers: contentType === null ? {} : { 'content-type': contentType } }))); + const output = executed.facts['response.provider.output']; + if (output === null || 'kind' in output) throw new Error('Expected a protocol failure value'); + expect(output.status).toBe(502); + for (const phrase of phrases) expect(output.message).toContain(phrase); + expect(executed.facts['response.provider.called']).toBe(true); + await executed.drain(); +}); diff --git a/packages/provider/__tests__/streaming_test.ts b/packages/provider/__tests__/streaming_test.ts deleted file mode 100644 index 7843b9aae2..0000000000 --- a/packages/provider/__tests__/streaming_test.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { test } from 'vitest'; - -import { streamingProviderCall } from '../src/streaming.ts'; -import { doneFrame, eventFrame, type ProtocolFrame } from '@floway-dev/protocols/common'; -import { assertEquals, assertRejects, assertStringIncludes } from '@floway-dev/test-utils'; - -interface StubEvent { type: string } - -// Stub parser: feed the body bytes through TextDecoder and yield one -// eventFrame per non-empty line, plus a terminal doneFrame. Mirrors the -// shape (but not the protocol-specific logic) of parseXxxStream so we can -// assert streamingProviderCall plumbing without dragging in protocol parsers. -const stubParser = (body: ReadableStream): AsyncIterable> => (async function* () { - const reader = body.getReader(); - const decoder = new TextDecoder(); - let buffer = ''; - for (;;) { - const { done, value } = await reader.read(); - if (done) { - buffer += decoder.decode(); - break; - } - buffer += decoder.decode(value, { stream: true }); - } - for (const line of buffer.split('\n').filter(Boolean)) { - yield eventFrame({ type: line }); - } - yield doneFrame(); -})(); - -test('streamingProviderCall returns ok:false when upstream is non-2xx', async () => { - const response = new Response('rate limited', { status: 429 }); - const result = await streamingProviderCall(Promise.resolve(response), stubParser, 'm-1', undefined); - assertEquals(result.ok, false); - if (result.ok) throw new Error('expected ok:false'); - assertEquals(result.response.status, 429); - assertEquals(result.modelKey, 'm-1'); -}); - -test('streamingProviderCall throws on 2xx without a body, surfacing the status and an body marker', async () => { - // 204 is the canonical "no body" success; this is a provider-contract violation - // because the streaming endpoints always force stream:true. - const response = new Response(null, { status: 204 }); - await assertRejects(async () => { - try { - await streamingProviderCall(Promise.resolve(response), stubParser, 'm-1', undefined); - } catch (error) { - assertStringIncludes((error as Error).message, '204'); - assertStringIncludes((error as Error).message, 'stream is required'); - assertStringIncludes((error as Error).message, 'Body: '); - throw error; - } - }, Error); -}); - -test('streamingProviderCall throws when 2xx content-type is not text/event-stream, including the upstream body for diagnosis', async () => { - const response = new Response(JSON.stringify({ error: { message: 'azure stub' } }), { - status: 200, - headers: { 'content-type': 'application/json' }, - }); - await assertRejects(async () => { - try { - await streamingProviderCall(Promise.resolve(response), stubParser, 'm-1', undefined); - } catch (error) { - assertStringIncludes((error as Error).message, '200'); - assertStringIncludes((error as Error).message, 'application/json'); - assertStringIncludes((error as Error).message, 'stream is required'); - assertStringIncludes((error as Error).message, 'azure stub'); - throw error; - } - }, Error); -}); - -test('streamingProviderCall rejects a content-type that only starts with the SSE essence', async () => { - const response = new Response('not really an event stream', { - status: 200, - headers: { 'content-type': 'text/event-stream-fake' }, - }); - await assertRejects( - () => streamingProviderCall(Promise.resolve(response), stubParser, 'm-1', undefined), - Error, - ); -}); - -test('streamingProviderCall surfaces "unknown" content-type when the header is missing', async () => { - // Cloudflare Workers sometimes hands us a 200 with no content-type header - // when the upstream response is malformed; the diagnostic must label that - // as "unknown" rather than the empty string so the operator can grep for it. - const response = new Response('{"choices":[]}', { status: 200 }); - await assertRejects(async () => { - try { - await streamingProviderCall(Promise.resolve(response), stubParser, 'm-1', undefined); - } catch (error) { - assertStringIncludes((error as Error).message, '"unknown"'); - assertStringIncludes((error as Error).message, '{"choices":[]}'); - throw error; - } - }, Error); -}); - -test('streamingProviderCall truncates oversized bodies in the error message', async () => { - const big = 'x'.repeat(2048); - const response = new Response(big, { status: 200, headers: { 'content-type': 'application/json' } }); - await assertRejects(async () => { - try { - await streamingProviderCall(Promise.resolve(response), stubParser, 'm-1', undefined); - } catch (error) { - assertStringIncludes((error as Error).message, '...[truncated]'); - throw error; - } - }, Error); -}); - -test('streamingProviderCall returns ok:true with parsed frames on 2xx SSE', async () => { - const response = new Response('alpha\nbeta\n', { - status: 200, - headers: { 'content-type': 'Text/Event-Stream; charset=utf-8' }, - }); - const result = await streamingProviderCall(Promise.resolve(response), stubParser, 'm-1', undefined); - assertEquals(result.ok, true); - if (!result.ok) throw new Error('expected ok:true'); - const frames: ProtocolFrame[] = []; - for await (const frame of result.events) frames.push(frame); - assertEquals(frames, [eventFrame({ type: 'alpha' }), eventFrame({ type: 'beta' }), doneFrame()]); -}); diff --git a/packages/provider/src/audio.ts b/packages/provider/src/audio.ts index 0fa30e54d9..64a51c402f 100644 --- a/packages/provider/src/audio.ts +++ b/packages/provider/src/audio.ts @@ -5,34 +5,3 @@ export type OpenAIAudioTranscriptionFormEntry = HttpFormEntry; export interface OpenAIAudioTranscriptionRequest { readonly entries: readonly OpenAIAudioTranscriptionFormEntry[]; } - -type OpenAIAudioTranscriptionModelField = - | { readonly type: 'replace'; readonly value: string } - | { readonly type: 'omit' }; - -const serializeOpenAIAudioTranscriptionRequest = ( - request: OpenAIAudioTranscriptionRequest, - modelField: OpenAIAudioTranscriptionModelField, -): FormData => { - const form = new FormData(); - for (const entry of request.entries) { - if (entry.name === 'model') { - if (modelField.type === 'replace') form.append(entry.name, modelField.value); - } else if (typeof entry.value === 'string') { - form.append(entry.name, entry.value); - } else { - const file = entry.value; - form.append(entry.name, new File([file.bytes as Uint8Array], file.name, { type: file.type, lastModified: file.lastModified })); - } - } - return form; -}; - -export const serializeModelFieldOpenAIAudioTranscriptionRequest = ( - request: OpenAIAudioTranscriptionRequest, - model: string, -): FormData => serializeOpenAIAudioTranscriptionRequest(request, { type: 'replace', value: model }); - -export const serializeModelPathOpenAIAudioTranscriptionRequest = ( - request: OpenAIAudioTranscriptionRequest, -): FormData => serializeOpenAIAudioTranscriptionRequest(request, { type: 'omit' }); diff --git a/packages/provider/src/error.ts b/packages/provider/src/error.ts index 3fe5498872..1ccfd6ee9a 100644 --- a/packages/provider/src/error.ts +++ b/packages/provider/src/error.ts @@ -1,4 +1,4 @@ -// Errors that bubble out of source/target emit or interceptors and need a +// Errors that bubble out of source/target emit or stages and need a // structured envelope for the api debug response. The target_api lane is // typed as a free string here so the package stays decoupled from the // api-internal serve-api unions — the api always passes the narrowed value diff --git a/packages/provider/src/flags.ts b/packages/provider/src/flags.ts index 9839c52bd6..c34b4b2fe8 100644 --- a/packages/provider/src/flags.ts +++ b/packages/provider/src/flags.ts @@ -1,11 +1,11 @@ // Stable identities for every admin-toggleable per-upstream behavior flag. -// Interceptor code references a flag by id; the dependency goes interceptor +// Stage code references a flag by id; the dependency goes stage // → flag, never the other way. The dashboard owns presentation and translated // copy, while this package owns the ids shared by providers and persisted // overrides. // // Vendor-style flags (`vendor-deepseek`, `vendor-qwen`, `vendor-kimi`) are -// mutually exclusive per model: a vendor interceptor translates the +// mutually exclusive per model: a vendor stage translates the // gateway's OpenAI-canonical request and response shape into the vendor's // wire dialect; with no vendor flag set, behavior defaults to the OpenAI // standard and no vendor rewrite runs. diff --git a/packages/provider/src/images.ts b/packages/provider/src/images.ts index c5d100a074..53a1169fa7 100644 --- a/packages/provider/src/images.ts +++ b/packages/provider/src/images.ts @@ -1,6 +1,4 @@ import { base64ToBytes, bytesToBase64, parseBase64ImageDataUrl } from './image-helpers.ts'; -import { jsonRequestBody } from './json-request.ts'; -import type { ReplayableBody } from './options.ts'; import { multipartBody as httpMultipartBody, type HttpFile, type HttpFormEntry, type HttpBody, type HttpBodyEncoding } from '@floway-dev/http/request-content'; import type { OpenAIImageEditReference } from '@floway-dev/protocols/openai-images'; @@ -96,16 +94,3 @@ export const prepareOpenAIImagesEditsBody = async (request: OpenAIImagesEditsReq ? { body: await serializeOpenAIImagesEditsJsonPayload(request, model), encoding: 'json' } : { body: httpMultipartBody(entries), encoding: 'multipart' }; }; - -export const serializeOpenAIImagesEditsRequest = async (request: OpenAIImagesEditsRequest, model: string): Promise => { - const entries = multipartEntries(request, model); - if (entries !== null) { - const form = new FormData(); - for (const { name, value } of entries) { - if (typeof value === 'string') form.append(name, value); - else form.append(name, new File([value.bytes as Uint8Array], value.name, { type: value.type, lastModified: value.lastModified })); - } - return form; - } - return jsonRequestBody(await serializeOpenAIImagesEditsJsonPayload(request, model)); -}; diff --git a/packages/provider/src/index.ts b/packages/provider/src/index.ts index 2ffdd520d8..e9782ad74d 100644 --- a/packages/provider/src/index.ts +++ b/packages/provider/src/index.ts @@ -7,31 +7,16 @@ export type { OpenAIResponsesInvocation, } from './invocation.ts'; export { providerModelOf } from './invocation.ts'; -export type { ProviderCallResponse, ProviderModelFacts, ProviderOperation, ProviderOperationPayloads, ProviderOperationResponse, ProviderPipeline, ProviderPipelines, ProviderRequest, ProviderResponse, ProviderRerankResponse, ProviderServices } from './pipeline.ts'; +export type { AnthropicMessagesCountTokensBody, ChatProviderOperation, NonChatProviderOperation, ProviderCallResponse, ProviderChatResponse, ProviderChatServices, ProviderModelFacts, ProviderOperation, ProviderOperationOutputs, ProviderOperationPayloads, ProviderOperationRequest, ProviderOperationResponse, ProviderPipeline, ProviderPipelines, ProviderProtocolFailure, ProviderRequest, ProviderResponse, ProviderRerankResponse, ProviderServices, ProviderStreamOutput, ProviderValueOutput } from './pipeline.ts'; export { providerModelFacts } from './pipeline.ts'; +export { decodeProviderResponse } from './stages/decode-response.ts'; +export { selectProviderResponsesAction } from './stages/select-responses-action.ts'; +export { observeProviderCall, observeProviderResponsesCall } from './stages/observe-call.ts'; export { mergeHttpHeaders, replaceHttpHeader, withHttpContentType } from './http-headers.ts'; export type { InternalDebugError } from './error.ts'; export { toInternalDebugError } from './error.ts'; -export type { - ApiErrorResult, - EventResult, - EventResultMetadata, - ExecuteResult, - InternalErrorResult, - PlainResult, -} from './result.ts'; -export { - apiErrorToResponse, - decodeApiErrorBody, - discardUpstreamResponse, - eventResult, - internalErrorResult, - plainResult, - readUpstreamApiError, -} from './result.ts'; - export type { InternalAliasedFrom, InternalModel, @@ -52,22 +37,15 @@ export type { Provider, InboundHeaderMatcher, ProviderInstance, - ProviderCallResult, - ProviderRerankCallResult, ProviderModule, AnthropicMessagesUpstreamCallOptions, - ProviderOpenAIResponsesResult, - ProviderStreamResult, OpenAIResponsesAction, UpstreamCallOptions, } from './provider.ts'; export { headersForAnthropicMessagesCall } from './anthropic-messages.ts'; export type { OpenAIImagesEditsRequest, OpenAIImagesEditsSource } from './images.ts'; -export { prepareOpenAIImagesEditsBody, serializeOpenAIImagesEditsJsonPayload, serializeOpenAIImagesEditsRequest } from './images.ts'; +export { prepareOpenAIImagesEditsBody, serializeOpenAIImagesEditsJsonPayload } from './images.ts'; export type { OpenAIAudioTranscriptionFormEntry, OpenAIAudioTranscriptionRequest } from './audio.ts'; -export { serializeModelFieldOpenAIAudioTranscriptionRequest, serializeModelPathOpenAIAudioTranscriptionRequest } from './audio.ts'; -export type { ProviderStreamParser } from './streaming.ts'; -export { streamingProviderCall } from './streaming.ts'; export type { ProviderRepo, UpstreamsRepoSlim } from './repo.ts'; export { getProviderRepo, initProviderRepo, UpstreamGoneError } from './repo.ts'; diff --git a/packages/provider/src/invocation.ts b/packages/provider/src/invocation.ts index 081c9e8a48..cb7d1b9224 100644 --- a/packages/provider/src/invocation.ts +++ b/packages/provider/src/invocation.ts @@ -9,23 +9,8 @@ import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/open export type ChatTargetApi = 'anthropicMessages' | 'openaiResponses' | 'openaiChatCompletions'; -// One (provider, model) pair the resolver produced for an inbound id, -// plus the per-request `Fetcher` minted for the provider's upstream. The -// pair is the smallest unit the dispatch layer needs to make a wire call: -// `provider.instance.callXxx(providerModelOf(candidate), body, ...)` — -// upstream id / upstream name / provider kind / capability flags come off -// `provider.*`, the merged public metadata (id, endpoints, limits, ...) off -// `model.*`, and the per-upstream `ProviderModel` (providerData, -// enabledFlags) off `providerModelOf(candidate)`. -// -// Resolution narrows by `model.kind` only — choosing the inbound target -// protocol from `model.endpoints` is the attempt layer's job, not part of -// the candidate. -// -// `rules` is set only for candidates minted by the alias walk — it carries -// the picked target's rule overlay so the attempt's terminal wire call can -// apply it against the target IR. Absent (undefined) for direct-resolution -// candidates and present (possibly `{}`) for alias-origin candidates. +// A resolver's live candidate belongs to services. Pipeline facts retain its identifier and +// portable model/rule content; the selected operation hands off into provider.pipelines. export interface ModelCandidate { readonly provider: Provider; readonly model: InternalModel; @@ -33,19 +18,7 @@ export interface ModelCandidate { readonly rules?: AliasRules; } -// Pull the emitting upstream's `ProviderModel` off the candidate. Dispatch -// hands this to the provider's `callXxx`; interceptor gates read -// `.enabledFlags`, boundary shims read `.providerData`, etc. The candidate -// always names exactly one upstream via `provider.upstreamId`; for real-row -// candidates the resolver populates `model.providerModels` with an entry -// under that key at candidate-creation time. -// -// Two error paths, distinguished so a caller reading the message can tell -// which invariant broke: an alias row was mistakenly used as a dispatch -// target (the resolver should have expanded it to its target's real row -// first), or a real row is missing the entry for the candidate's upstream -// (the candidate was assembled outside the resolver, or the row was merged -// after the upstream stopped contributing). +// Alias rows must be expanded before selecting one upstream's catalog metadata. export const providerModelOf = (candidate: ModelCandidate): ProviderModel => { const { model, provider } = candidate; if (model.providerModels === undefined) { @@ -58,13 +31,8 @@ export const providerModelOf = (candidate: ModelCandidate): ProviderModel => { return providerModel; }; -// Per-protocol invocation shape passed to interceptors. Carries the -// source-shape request body (mutable, so the body can be cleaned), the -// candidate the attempt is dispatching against, the chat target protocol -// the attempt picked for this candidate, and a mutable `Headers` instance -// carried into the boundary chain — so workarounds that only need to set -// or drop a header stay at the owning interceptor boundary instead of -// widening the provider call signature. +// Local planning views carry immutable payload content and live candidate/transport handles. +// Stages materialize their resulting payload, action and header lines into facts. export interface AnthropicMessagesInvocation { payload: AnthropicMessagesPayload; readonly candidate: ModelCandidate; @@ -74,10 +42,7 @@ export interface AnthropicMessagesInvocation { export interface OpenAIResponsesInvocation { payload: CanonicalOpenAIResponsesPayload; - // Mutable action tag — interceptors can flip 'compact' to 'generate' so the - // inner provider call runs a normal summarization turn (see the - // openai-responses-compact-shim) and the gateway derives snapshot mode from the - // post-chain action carried on the provider's tagged result. + // Planning may choose a different dispatched action while preserving source intent. action: OpenAIResponsesAction; readonly candidate: ModelCandidate; readonly targetApi: ChatTargetApi; diff --git a/packages/provider/src/options.ts b/packages/provider/src/options.ts index b98d35ad18..20251c0e79 100644 --- a/packages/provider/src/options.ts +++ b/packages/provider/src/options.ts @@ -21,8 +21,7 @@ export interface UpstreamFetchOptions { } // Identity wrapper for callers that don't participate in per-request TTFT -// timing — model-listing helpers and interceptor sub-calls that dispatch -// outside the primary data-plane fetch. +// timing — catalog and control calls outside the primary model-endpoint timing window. export const identityWrapUpstreamCall = (dispatch: () => Promise): Promise => dispatch(); // Transfer a request into the fetcher exactly once. The wrapper used for diff --git a/packages/provider/src/pipeline.ts b/packages/provider/src/pipeline.ts index 2dfaaedee9..14c3971fd1 100644 --- a/packages/provider/src/pipeline.ts +++ b/packages/provider/src/pipeline.ts @@ -4,13 +4,21 @@ import type { OpenAIImagesEditsRequest } from './images.ts'; import type { ProviderModel } from './model.ts'; import type { HttpResponseFacts, HttpHeaders, HttpServices } from '@floway-dev/http/pipeline'; import type { Pipeline } from '@floway-dev/pipeline'; -import type { RerankTarget } from '@floway-dev/protocols/common'; +import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; +import type { ProtocolFrame, RerankTarget } from '@floway-dev/protocols/common'; +import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import type { OpenAICompletionsPayload } from '@floway-dev/protocols/openai-completions'; import type { OpenAIEmbeddingsPayload } from '@floway-dev/protocols/openai-embeddings'; import type { OpenAIImagesGenerationsPayload } from '@floway-dev/protocols/openai-images'; +import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesCompactionResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; import type { CanonicalRerankRequest } from '@floway-dev/protocols/rerank'; export interface ProviderOperationPayloads { + openaiChatCompletions: Omit; + openaiResponses: Omit; + openaiResponsesCompact: Omit; + anthropicMessages: Omit; + anthropicMessagesCountTokens: Omit; alphaSearch: Record; openaiCompletions: Omit; openaiEmbeddings: Omit; @@ -21,6 +29,8 @@ export interface ProviderOperationPayloads { } export type ProviderOperation = keyof ProviderOperationPayloads; +export type ChatProviderOperation = 'openaiChatCompletions' | 'openaiResponses' | 'openaiResponsesCompact' | 'anthropicMessages' | 'anthropicMessagesCountTokens'; +export type NonChatProviderOperation = Exclude; export type ProviderModelFacts = Omit & { readonly enabledFlags: readonly FlagId[] }; @@ -33,6 +43,9 @@ export interface ProviderRequest

{ 'request.http.headers': HttpHeaders; } +export type ProviderOperationRequest = ProviderRequest & + (O extends 'anthropicMessages' | 'anthropicMessagesCountTokens' ? { 'request.provider.anthropicBeta': readonly string[] } : object); + export interface ProviderCallResponse extends HttpResponseFacts { /** A preflight reply does not represent a call to the model endpoint. */ 'response.provider.called': boolean; @@ -48,13 +61,42 @@ export interface ProviderRerankResponse extends ProviderResponse { 'response.provider.rerankTarget': RerankTarget; } +export interface ProviderProtocolFailure { + readonly status: number; + readonly message: string; + readonly body?: unknown; +} + +export type ProviderStreamOutput = { readonly kind: 'stream'; readonly frames: AsyncIterable> }; +export type ProviderValueOutput = { readonly kind: 'value'; readonly body: T }; +export interface AnthropicMessagesCountTokensBody { + readonly input_tokens: number; + readonly [key: string]: unknown; +} + +export interface ProviderOperationOutputs { + openaiChatCompletions: ProviderStreamOutput; + openaiResponses: ProviderStreamOutput | ProviderValueOutput; + openaiResponsesCompact: ProviderStreamOutput | ProviderValueOutput; + anthropicMessages: ProviderStreamOutput; + anthropicMessagesCountTokens: ProviderValueOutput; +} + +export interface ProviderChatResponse extends ProviderResponse { + 'response.provider.output': ProviderOperationOutputs[O] | ProviderProtocolFailure | null; +} + export type ProviderOperationResponse = - O extends 'rerank' ? ProviderRerankResponse : ProviderResponse; + O extends ChatProviderOperation ? ProviderChatResponse : O extends 'rerank' ? ProviderRerankResponse : ProviderResponse; export type ProviderPipeline = Pipeline< - ProviderRequest, + ProviderOperationRequest, ProviderOperationResponse >; export type ProviderPipelines = { readonly [O in ProviderOperation]?: ProviderPipeline }; export type ProviderServices = HttpServices; + +export type ProviderChatServices = ProviderServices & { + readonly recordProtocolFrames: >(frames: AsyncIterable) => AsyncIterable; +}; diff --git a/packages/provider/src/provider.ts b/packages/provider/src/provider.ts index fbce3b1561..62068509a8 100644 --- a/packages/provider/src/provider.ts +++ b/packages/provider/src/provider.ts @@ -1,27 +1,10 @@ -import type { OpenAIAudioTranscriptionRequest } from './audio.ts'; import type { FlagDefaults } from './flags.ts'; -import type { OpenAIImagesEditsRequest } from './images.ts'; import type { ModelPrefixConfig } from './model-prefix.ts'; import type { ProviderModel, UpstreamModelsCache, UpstreamProviderKind, UpstreamRecord } from './model.ts'; import type { Fetcher } from './options.ts'; import type { ProviderPipelines } from './pipeline.ts'; -import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import type { ProtocolFrame, RerankTarget } from '@floway-dev/protocols/common'; -import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; -import type { OpenAICompletionsPayload } from '@floway-dev/protocols/openai-completions'; -import type { OpenAIEmbeddingsPayload } from '@floway-dev/protocols/openai-embeddings'; -import type { OpenAIImagesGenerationsPayload } from '@floway-dev/protocols/openai-images'; -import type { CanonicalOpenAIResponsesPayload, OpenAIResponsesCompactionResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; -import type { CanonicalRerankRequest } from '@floway-dev/protocols/rerank'; -// Action tag threaded through the OpenAI Responses pipeline. `generate` is a normal -// streaming /responses turn; `compact` is the summarize-and-replace-history -// turn that some upstreams expose natively (`/v1/responses/compact`, -// chatgpt.com's RemoteCompactionV2 over /codex/responses) and others have to -// simulate. The same `callOpenAIResponses` method dispatches on this tag, and -// interceptors are free to flip it (the openai-responses-compact-shim turns 'compact' -// into 'generate' so the inner upstream call runs an ordinary summarization -// turn against the SUMMARIZATION_PROMPT). +// The dispatched action can change independently of the source endpoint's original intent. export type OpenAIResponsesAction = 'generate' | 'compact'; export type InboundHeaderMatcher = string | RegExp; @@ -46,48 +29,6 @@ export interface Provider { instance: ProviderInstance; } -export interface ProviderCallResult { - response: Response; - modelKey: string; -} - -export interface ProviderRerankCallResult extends ProviderCallResult { - target: RerankTarget; -} - -// Streaming endpoints (Anthropic Messages / OpenAI Responses / OpenAI Chat Completions) return decoded -// protocol frames directly — the provider drives the upstream fetch, parses -// the SSE wire via @floway-dev/protocols, and emits the typed event stream. -// `ok: true` optionally carries the raw upstream `Headers` so the source-side -// `respond` layer can forward them to the downstream client (blocklist in -// gateway `shared/respond.ts` — hop-by-hop, body framing, cookies). Absent -// on lifted/synthesized streams that have no upstream Response behind them, -// matching the same shape on `EventResult`. -// `ok: false` carries the raw upstream Response verbatim so the gateway -// boundary can relay status + body + headers unchanged. Non-2xx-but-not-SSE -// responses throw from the provider as a contract violation (provider always -// forces stream=true on streaming endpoints). -export type ProviderStreamResult = - | { ok: true; events: AsyncIterable>; modelKey: string; headers?: Headers } - | { ok: false; response: Response; modelKey: string }; - -// `action: 'generate'` is a normal streaming /responses turn — its frames -// flow through the per-frame event stream like every other streaming endpoint. -// `action: 'compact'` is non-streaming — the upstream returns a single -// `response.compaction` envelope. Some upstreams expose a native compaction -// endpoint and produce the envelope directly; others synthesize the envelope -// from a regular /responses turn — both return the typed value rather than a -// re-parsed synthesized SSE body. The discriminated result tags which branch -// actually ran so the gateway's shape-lowering can pick between the streaming -// and value-envelope arms; snapshot mode itself reads `invocation.action` (the -// post-chain caller intent), not the result tag. -// The `ok: false` contract is identical to ProviderStreamResult above. -export type ProviderOpenAIResponsesResult = - | { action: 'generate'; ok: true; events: AsyncIterable>; modelKey: string; headers?: Headers } - | { action: 'generate'; ok: false; response: Response; modelKey: string } - | { action: 'compact'; ok: true; result: OpenAIResponsesCompactionResult; modelKey: string } - | { action: 'compact'; ok: false; response: Response; modelKey: string }; - // Per-call options the gateway threads through to the provider. // // `fetcher` is the per-upstream proxy-aware indirection for outbound HTTP. @@ -105,9 +46,8 @@ export type ProviderOpenAIResponsesResult = // The gateway filters the source request through the selected provider // instance's `inboundHeaderAllowlist` before constructing this bag. Protocol-owned // metadata is carried by its owning invocation boundary and does not widen -// this provider-level policy. A provider may clone and mutate the bag for -// request-specific wire shaping, but must not retain the gateway-owned -// reference past the call. +// this provider-level policy. Pipeline request shaping uses immutable header-line facts; this bag is the +// transport fixture and control-call boundary. export interface UpstreamCallOptions { fetcher: Fetcher; waitUntil: (promise: Promise) => void; @@ -117,7 +57,7 @@ export interface UpstreamCallOptions { // TLS, and CONNECT. Further dispatches after output preserve the measured // first-token interval, including internal server-tool continuations. // This interval includes data-plane egress and excludes model routing, translation, - // and interceptor preparation before dispatch. Candidate iteration clears + // and stage preparation before dispatch. Candidate iteration clears // both timing anchors on failover, so the recorded interval can be shorter // than the latency the client observed. wrapUpstreamCall: (dispatch: () => Promise) => Promise; @@ -131,28 +71,7 @@ export interface AnthropicMessagesUpstreamCallOptions extends UpstreamCallOption } export interface ProviderInstance { - // Catalog refresh fetches a single resource and never enters the per-request - // latency budget, so it takes the per-upstream fetcher directly instead of - // the broader `UpstreamCallOptions` bag the data-plane `call*` methods use. getProvidedModels(fetcher: Fetcher): Promise; - callAlphaSearch(model: ProviderModel, body: Record, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; - // /v1/completions text completions. Passthrough. Providers whose - // upstream doesn't expose /v1/completions set `endpoints.openaiCompletions` - // to absent in getProvidedModels, so this method is unreachable for - // those upstreams; the rejecting stubs in those providers are pure - // defense-in-depth. - callOpenAICompletions(model: ProviderModel, body: Omit, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; - callOpenAIChatCompletions(model: ProviderModel, body: Omit, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise>; - callOpenAIResponses(model: ProviderModel, body: Omit, action: OpenAIResponsesAction, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; - callAnthropicMessages(model: ProviderModel, body: Omit, signal: AbortSignal | undefined, opts: AnthropicMessagesUpstreamCallOptions): Promise>; - // count_tokens is non-streaming JSON; the gateway relays the upstream - // Response verbatim. - callAnthropicMessagesCountTokens(model: ProviderModel, body: Omit, signal: AbortSignal | undefined, opts: AnthropicMessagesUpstreamCallOptions): Promise; - callOpenAIEmbeddings(model: ProviderModel, body: Omit, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; - callOpenAIImagesGenerations(model: ProviderModel, body: Omit, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; - callOpenAIImagesEdits(model: ProviderModel, request: OpenAIImagesEditsRequest, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; - callOpenAIAudioTranscriptions(model: ProviderModel, request: OpenAIAudioTranscriptionRequest, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; - callRerank(model: ProviderModel, request: CanonicalRerankRequest, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; } // Static, module-shaped surface each provider package exports. The gateway @@ -163,8 +82,7 @@ export interface ProviderInstance { export interface ProviderModule { // Instance factory: capture the record and return closures. Sync — any // I/O the provider needs (token refresh, state persistence, catalog - // fetch) happens on demand inside the per-request methods on the - // returned ProviderInstance. + // fetch) happens on demand inside the operation pipelines and the catalog control callback. create: (record: UpstreamRecord) => Provider; // Exhaustive default map over every catalog flag id for a fresh // upstream of this kind; see each provider package's `defaults.ts`. diff --git a/packages/provider/src/result.ts b/packages/provider/src/result.ts deleted file mode 100644 index ba8c445446..0000000000 --- a/packages/provider/src/result.ts +++ /dev/null @@ -1,129 +0,0 @@ -import type { InternalDebugError } from './error.ts'; -import type { PerformanceTelemetryContext, TelemetryModelIdentity } from './telemetry.ts'; -import type { BillableUsage } from '@floway-dev/protocols/common'; - -export interface EventResult { - type: 'events'; - events: AsyncIterable; - modelIdentity: TelemetryModelIdentity; - performance?: PerformanceTelemetryContext; - finalMetadata?: Promise; - // Raw upstream response headers for the source-side `respond` layer to - // forward (blocklist in gateway `shared/respond.ts`). Absent on - // lifted/synthesized streams that have no upstream Response behind them. - headers?: Headers; -} - -export interface EventResultMetadata { - modelIdentity: TelemetryModelIdentity; - performance?: PerformanceTelemetryContext; - // What the upstream turn cost, read from the upstream's own usage. Absent - // until the stream that reports it has been consumed. - billableUsage?: BillableUsage; -} - -// HTTP-shaped error envelope the respond layer forwards to the client -// verbatim (status + headers + body). `source` distinguishes a real upstream -// non-2xx from a gateway-synthesized envelope (model not routable, missing -// stored item, server-tool input rejected, etc.) so observers like the -// request dump can record the failure category truthfully rather than -// labelling every 4xx as `upstream error N`. `upstreamId` is set on real -// upstream 4xx/5xx (`source === 'upstream'`) so the dump row can attribute the -// failure to its source; it is absent on gateway-synthesized envelopes that -// never reached an upstream. -export interface ApiErrorResult { - type: 'api-error'; - source: 'upstream' | 'gateway'; - status: number; - headers: Headers; - body: Uint8Array; - performance?: PerformanceTelemetryContext; - upstreamId?: string; -} - -// Gateway-side bug surface (parser crash, interceptor throw, etc.). The -// protocol's respond layer renders a debug envelope around `error` -// (stack, cause, target_api) rather than passing through a wire body — -// the shape differs from `ApiErrorResult` for that reason. -export interface InternalErrorResult { - type: 'internal-error'; - status: number; - error: InternalDebugError; - performance?: PerformanceTelemetryContext; -} - -// A fully-shaped non-streaming success body — the output of a source endpoint -// that measures rather than generates (count_tokens). It is NOT an -// `ExecuteResult`: the target emit/interceptor layer never produces one. The -// orchestrator passes it straight to `respond` without persistence, and -// `respond` emits it verbatim. `upstreamId` is present when the body came from -// a real upstream call and absent for gateway-synthesized envelopes (rewrite -// failures, internal-debug bodies). -export interface PlainResult { - type: 'plain'; - status: number; - headers: Headers; - body: Uint8Array; - upstreamId?: string; -} - -export type ExecuteResult = EventResult | ApiErrorResult | InternalErrorResult; - -export interface EventResultOptions { - performance?: PerformanceTelemetryContext; - finalMetadata?: Promise; - headers?: Headers; -} - -export const eventResult = ( - events: AsyncIterable, - modelIdentity: TelemetryModelIdentity, - options: EventResultOptions = {}, -): EventResult => { - const result: EventResult = { type: 'events', events, modelIdentity }; - if (options.performance !== undefined) result.performance = options.performance; - if (options.finalMetadata !== undefined) result.finalMetadata = options.finalMetadata; - if (options.headers !== undefined) result.headers = options.headers; - return result; -}; - -export const internalErrorResult = (status: number, error: InternalDebugError, performance?: PerformanceTelemetryContext): InternalErrorResult => ({ - type: 'internal-error', - status, - error, - ...(performance ? { performance } : {}), -}); - -export const plainResult = (status: number, headers: Headers, body: Uint8Array, upstreamId?: string): PlainResult => ({ - type: 'plain', - status, - headers, - body, - ...(upstreamId !== undefined ? { upstreamId } : {}), -}); - -export const readUpstreamApiError = async (response: Response, upstreamId?: string): Promise => ({ - type: 'api-error', - source: 'upstream', - status: response.status, - headers: new Headers(response.headers), - body: new Uint8Array(await response.arrayBuffer()), - ...(upstreamId !== undefined ? { upstreamId } : {}), -}); - -// A response whose body is neither read nor cancelled keeps its transport -// alive: on the direct-connect egress the dialed socket is closed only from the -// body's own read-to-end or cancel path, so an abandoned response strands one -// socket for the life of the process. Every path that walks away from an -// upstream response goes through here. -export const discardUpstreamResponse = async (response: Response): Promise => { - await response.body?.cancel(); -}; - -export const apiErrorToResponse = (error: ApiErrorResult): Response => - new Response(error.body.slice().buffer, { - status: error.status, - headers: new Headers(error.headers), - }); - -export const decodeApiErrorBody = (error: ApiErrorResult): string => new TextDecoder().decode(error.body); diff --git a/packages/provider/src/stages/decode-response.ts b/packages/provider/src/stages/decode-response.ts new file mode 100644 index 0000000000..1ffd9f4994 --- /dev/null +++ b/packages/provider/src/stages/decode-response.ts @@ -0,0 +1,61 @@ +import type { ChatProviderOperation, ProviderChatResponse, ProviderChatServices, ProviderOperationOutputs, ProviderProtocolFailure } from '../pipeline.ts'; +import { exchangeResponse, type HttpRequestFacts, type HttpResponseFacts, type HttpResponseExchange } from '@floway-dev/http/pipeline'; +import { defineStage, move, setRelease } from '@floway-dev/pipeline'; +import { parseAnthropicMessagesStream } from '@floway-dev/protocols/anthropic-messages'; +import { isEventStreamMediaType } from '@floway-dev/protocols/common'; +import { parseOpenAIChatCompletionsStream } from '@floway-dev/protocols/openai-chat-completions'; +import { parseOpenAIResponsesStream } from '@floway-dev/protocols/openai-responses'; + +type Request = Pick; +type Up = HttpResponseFacts & { 'response.provider.responsesAction': 'generate' | 'compact' }; +type Answer = HttpResponseFacts & Pick, 'response.provider.output'>; + +const parsedBody = (text: string): unknown => { + try { return JSON.parse(text); } catch (error) { if (!(error instanceof SyntaxError)) throw error; return text; } +}; + +const protocolFailure = (exchange: HttpResponseExchange, text: string): ProviderProtocolFailure => { + const type = exchange.headers.find(([name]) => name.toLowerCase() === 'content-type')?.[1] ?? ''; + const snippet = text.length === 0 ? '' : text.length > 1024 ? `${text.slice(0, 1024)}...[truncated]` : text; + return { + status: 502, + message: `Upstream returned ${exchange.status} with content-type "${type || 'unknown'}" but stream is required (provider must force stream=true and return text/event-stream when response.ok). Body: ${snippet}`, + body: parsedBody(text), + }; +}; + +export const decodeProviderResponse = (operation: O) => defineStage, ProviderChatServices>({ + name: `decodeProvider${operation.replace(/^openai/, 'OpenAI').replace(/^anthropic/, 'Anthropic')}`, + through: { + request: { needs: ['request.http.callId'], consumes: [], provides: [] }, + response: { needs: ['response.http.exchange', 'response.http.body', ...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? ['response.provider.responsesAction' as const] : [])], consumes: ['response.http.body'], provides: ['response.http.body', 'response.provider.output'] }, + }, + execute: async (facts, next, use) => { + const back = await next(move({ ...facts })); + const exchange = back['response.http.exchange']; + const output = (answer: ProviderOperationOutputs[O] | ProviderProtocolFailure | null) => move({ ...back, 'response.provider.output': answer }); + if (exchange.type === 'transportFailure' || exchange.status < 200 || exchange.status >= 300) return output(null); + if ((operation === 'openaiResponses' || operation === 'openaiResponsesCompact') && back['response.provider.responsesAction'] === 'compact' || operation === 'anthropicMessagesCountTokens') { + const response = exchangeResponse(exchange); + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + const text = await response.text(); + let body: unknown; + try { body = JSON.parse(text); } catch (error) { + if (!(error instanceof SyntaxError)) throw error; + return output({ status: 502, message: `Upstream returned invalid ${operation} JSON`, body: text }); + } + return output({ kind: 'value', body } as ProviderOperationOutputs[O]); + } + const contentType = exchange.headers.find(([name]) => name.toLowerCase() === 'content-type')?.[1]; + if (exchange.body === null || !isEventStreamMediaType(contentType)) { + const response = exchangeResponse(exchange); + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + const text = await response.text(); + return output(protocolFailure(exchange, text)); + } + const signal = use.httpCall(facts['request.http.callId']).signal; + if (operation === 'anthropicMessages') return output({ kind: 'stream', frames: use.recordProtocolFrames(parseAnthropicMessagesStream(exchange.body, { signal })) } as ProviderOperationOutputs[O]); + if (operation === 'openaiChatCompletions') return output({ kind: 'stream', frames: use.recordProtocolFrames(parseOpenAIChatCompletionsStream(exchange.body, { signal })) } as ProviderOperationOutputs[O]); + return output({ kind: 'stream', frames: use.recordProtocolFrames(parseOpenAIResponsesStream(exchange.body, { signal })) } as ProviderOperationOutputs[O]); + }, +}); diff --git a/packages/provider/src/stages/observe-call.ts b/packages/provider/src/stages/observe-call.ts new file mode 100644 index 0000000000..100cc6ef69 --- /dev/null +++ b/packages/provider/src/stages/observe-call.ts @@ -0,0 +1,27 @@ +import type { ProviderResponse, ProviderServices } from '../pipeline.ts'; +import type { HttpResponseFacts } from '@floway-dev/http/pipeline'; +import { defineStage, move } from '@floway-dev/pipeline'; + +type Request = { 'request.provider.modelKey': string; 'request.provider.responsesAction': 'generate' | 'compact' }; +type Response = ProviderResponse & { 'response.provider.responsesAction'?: 'generate' | 'compact' }; + +const observeCall = (responses: boolean) => defineStage({ + name: responses ? 'observeProviderResponsesCall' : 'observeProviderCall', + through: { + request: { needs: ['request.provider.modelKey', ...(responses ? ['request.provider.responsesAction' as const] : [])], consumes: [], provides: [] }, + response: { needs: ['response.http.exchange'], consumes: [], provides: ['response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', ...(responses ? ['response.provider.responsesAction' as const] : [])] }, + }, + execute: async (facts, next) => { + const back = await next(move({ ...facts })); + return move({ + ...back, + 'response.provider.modelKey': facts['request.provider.modelKey'], + 'response.provider.called': back['response.http.exchange'].type === 'response', + 'response.provider.previousCalls': [], + ...(responses ? { 'response.provider.responsesAction': facts['request.provider.responsesAction'] } : {}), + }); + }, +}); + +export const observeProviderCall = observeCall(false); +export const observeProviderResponsesCall = observeCall(true); diff --git a/packages/provider/src/stages/select-responses-action.ts b/packages/provider/src/stages/select-responses-action.ts new file mode 100644 index 0000000000..2fcf72bd2b --- /dev/null +++ b/packages/provider/src/stages/select-responses-action.ts @@ -0,0 +1,12 @@ +import { defineStage, move } from '@floway-dev/pipeline'; + +type Facts = { 'request.provider.responsesAction': 'generate' | 'compact' }; + +export const selectProviderResponsesAction = (action: Facts['request.provider.responsesAction']) => defineStage({ + name: 'selectProviderResponsesAction', + through: { + request: { needs: [], consumes: [], provides: ['request.provider.responsesAction'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => move({ ...await next(move({ ...facts, 'request.provider.responsesAction': action })) }), +}); diff --git a/packages/provider/src/streaming.ts b/packages/provider/src/streaming.ts deleted file mode 100644 index 0e6fcec48d..0000000000 --- a/packages/provider/src/streaming.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { ProviderStreamResult } from './provider.ts'; -import { isEventStreamMediaType, type ProtocolFrame } from '@floway-dev/protocols/common'; - -export type ProviderStreamParser = ( - body: ReadableStream, - options?: { signal?: AbortSignal }, -) => AsyncIterable>; - -const BODY_SNIPPET_CHARS = 1024; - -const readBodySnippet = async (response: Response): Promise => { - try { - const text = await response.text(); - if (text.length === 0) return ''; - return text.length > BODY_SNIPPET_CHARS ? `${text.slice(0, BODY_SNIPPET_CHARS)}...[truncated]` : text; - } catch { - return ''; - } -}; - -// A 2xx non-SSE upstream is a provider-contract violation: every streaming -// endpoint is called with stream=true. The throw bubbles to the target -// boundary, which turns it into a 502. The upstream's body (or a snippet -// of it) is folded into the error message so what the upstream actually -// returned reaches the operator instead of being discarded — content-type -// "unknown" with no body context is otherwise impossible to debug. -export const streamingProviderCall = async ( - upstreamFetch: Promise, - parser: ProviderStreamParser, - modelKey: string, - signal: AbortSignal | undefined, -): Promise> => { - const response = await upstreamFetch; - if (!response.ok) { - return { ok: false, response, modelKey }; - } - const contentType = response.headers.get('content-type') ?? ''; - if (!response.body || !isEventStreamMediaType(contentType)) { - const snippet = await readBodySnippet(response); - throw new Error(`Upstream returned ${response.status} with content-type "${contentType || 'unknown'}" but stream is required (provider must force stream=true and return text/event-stream when response.ok). Body: ${snippet}`); - } - return { ok: true, events: parser(response.body, { signal }), modelKey, headers: response.headers }; -}; diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index f2c0b82448..1a0364c764 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -12,6 +12,7 @@ "dependencies": { "@floway-dev/http": "workspace:*", "@floway-dev/pipeline": "workspace:*", + "@floway-dev/protocols": "workspace:*", "@floway-dev/provider": "workspace:*" } } diff --git a/packages/test-utils/src/chat-provider-pipeline.ts b/packages/test-utils/src/chat-provider-pipeline.ts new file mode 100644 index 0000000000..8b43512d05 --- /dev/null +++ b/packages/test-utils/src/chat-provider-pipeline.ts @@ -0,0 +1,52 @@ +import { noopUpstreamCallOptions } from './stubs.ts'; +import { exchangeResponse } from '@floway-dev/http/pipeline'; +import { move, run, setRelease, type Handed, type RunServices } from '@floway-dev/pipeline'; +import { providerModelFacts, type AnthropicMessagesUpstreamCallOptions, type ChatProviderOperation, type Provider, type ProviderChatResponse, type ProviderChatServices, type ProviderModel, type ProviderOperationOutputs, type ProviderOperationPayloads, type ProviderOperationRequest, type ProviderOperationResponse, type UpstreamCallOptions } from '@floway-dev/provider'; + +type Frame = Extract extends { frames: AsyncIterable } ? F : never; + +export const collectChatProviderPipeline = async ( + provider: Provider, + operation: O, + model: ProviderModel, + payload: ProviderOperationPayloads[O], + signal: AbortSignal | undefined = undefined, + options: UpstreamCallOptions | AnthropicMessagesUpstreamCallOptions = noopUpstreamCallOptions(), + observers: Pick = {}, +): Promise<{ facts: ProviderOperationResponse & Record; output: ProviderChatResponse['response.provider.output']; frames: Frame[]; response: Response | null }> => { + const pipeline = provider.pipelines[operation]; + if (pipeline === undefined) throw new Error(`${provider.kind} has no ${operation} pipeline`); + const entry = move({ + 'request.provider.model': providerModelFacts(model), + 'request.provider.payload': payload, + 'request.http.callId': 0, + 'request.http.headers': [...options.headers], + ...(operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens' ? { 'request.provider.anthropicBeta': (options as AnthropicMessagesUpstreamCallOptions).anthropicBeta } : {}), + }) as Handed>; + const services: ProviderChatServices & Pick = { ...observers, httpCall: () => ({ ...options, signal }), recordProtocolFrames: frames => frames }; + const executed = await run, ProviderOperationResponse, ProviderChatServices>(pipeline, entry, services); + const facts = executed.facts as ProviderOperationResponse & Record; + const exchange = facts['response.http.exchange']; + const output = facts['response.provider.output'] as ProviderChatResponse['response.provider.output']; + const frames: Frame[] = []; + let response: Response | null = null; + try { + if (exchange.type === 'transportFailure') throw exchange.error; + if (output !== null && 'kind' in output && output.kind === 'stream') { + for await (const frame of output.frames) frames.push(frame as Frame); + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + } else if (output === null) { + const raw = exchangeResponse(exchange); + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + const bytes = await raw.arrayBuffer(); + response = new Response(exchange.body === null ? null : bytes, { status: exchange.status, statusText: exchange.statusText, headers: exchange.headers.map(([name, value]): [string, string] => [name, value]) }); + } + } catch (error) { + try { await executed.drain(); } catch (cleanupError) { + if (cleanupError !== error) throw new AggregateError([error, cleanupError], 'Provider test call and cleanup failed', { cause: error }); + } + throw error; + } + await executed.drain(); + return { facts, output, frames, response }; +}; diff --git a/packages/test-utils/src/http-pipeline.ts b/packages/test-utils/src/http-pipeline.ts new file mode 100644 index 0000000000..3e1320aed1 --- /dev/null +++ b/packages/test-utils/src/http-pipeline.ts @@ -0,0 +1,20 @@ +import { exchangeResponse, type HttpResponseFacts, type HttpServices } from '@floway-dev/http/pipeline'; +import { move, run, setRelease, type Pipeline } from '@floway-dev/pipeline'; + +export const collectHttpPipeline = async (pipeline: Pipeline, entry: Entry, services: HttpServices): Promise<{ facts: Exit; response: Response }> => { + const executed = await run(pipeline, move(entry), services); + const exchange = executed.facts['response.http.exchange']; + try { + if (exchange.type === 'transportFailure') throw exchange.error; + const response = exchangeResponse(exchange); + const bytes = await response.arrayBuffer(); + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + await executed.drain(); + return { facts: executed.facts, response: new Response(exchange.body === null ? null : bytes, { status: exchange.status, statusText: exchange.statusText, headers: response.headers }) }; + } catch (error) { + try { await executed.drain(); } catch (cleanupError) { + if (cleanupError !== error) throw new AggregateError([error, cleanupError], 'HTTP test call and cleanup failed', { cause: error }); + } + throw error; + } +}; diff --git a/packages/test-utils/src/index.ts b/packages/test-utils/src/index.ts index a88fcf7e28..76abeb9966 100644 --- a/packages/test-utils/src/index.ts +++ b/packages/test-utils/src/index.ts @@ -1,3 +1,9 @@ +export { eventResult, type EventResult, type EventResultMetadata, type ExecuteResult, type ProviderCallResult, type ProviderRerankCallResult, type ProviderStreamResult, type ProviderOpenAIResponsesResult } from './provider-results.ts'; +export { collectPreparedProviderRequest } from './prepared-provider-request.ts'; +export { collectHttpPipeline } from './http-pipeline.ts'; +export { applyProviderStage, type ProviderStageProbe, type OpenAIChatCompletionsProbe, type OpenAIResponsesProbe, type AnthropicMessagesProbe } from './provider-stage.ts'; +export { stubChatProviderPipeline, type StubChatProviderCall } from './stub-chat-provider-pipeline.ts'; +export { collectChatProviderPipeline } from './chat-provider-pipeline.ts'; export { assert, assertEquals, diff --git a/packages/test-utils/src/prepared-provider-request.ts b/packages/test-utils/src/prepared-provider-request.ts new file mode 100644 index 0000000000..8c1c1779dd --- /dev/null +++ b/packages/test-utils/src/prepared-provider-request.ts @@ -0,0 +1,28 @@ +import { collectHttpPipeline } from './http-pipeline.ts'; +import { stubProviderModel } from './stubs.ts'; +import { http } from '@floway-dev/http/pipeline'; +import { compose, type Stage } from '@floway-dev/pipeline'; +import { observeProviderCall, providerModelFacts, type FetchInit, type ProviderOperation, type ProviderRequest, type ProviderResponse, type UpstreamFetchOptions } from '@floway-dev/provider'; + +export const collectPreparedProviderRequest = async (prepare: Stage, operation: ProviderOperation, modelData: (modelKey: string) => unknown, init: FetchInit, options: UpstreamFetchOptions, selectedModel?: string): Promise => { + const fields: Record = typeof init.body === 'string' ? JSON.parse(init.body) as Record : {}; + const modelKey = selectedModel ?? (typeof fields.model === 'string' ? fields.model : 'test-model'); + const { model: _, ...json } = fields; + let payload: object = json; + if (init.body instanceof FormData) { + const entries = await Promise.all([...init.body].map(async ([name, value]) => ({ name, value: typeof value === 'string' ? value : { name: value.name, type: value.type, lastModified: value.lastModified, bytes: new Uint8Array(await value.arrayBuffer()) } }))); + payload = { entries }; + } + const headers = [...new Headers(init.headers), ...options.extraHeaders ?? []].map(([name, value]): [string, string] => [name, value]); + const beta = new Headers(headers).get('anthropic-beta'); + const pipeline = compose, ProviderResponse>('testPreparedProviderRequest', [prepare, observeProviderCall, http]); + const result = await collectHttpPipeline(pipeline, { + 'request.provider.model': providerModelFacts(stubProviderModel({ id: modelKey, providerData: modelData(modelKey) })), + 'request.provider.payload': payload, + 'request.http.callId': 0, + 'request.http.headers': headers, + ...(operation === 'openaiResponses' || operation === 'openaiResponsesCompact' ? { 'request.provider.responsesAction': operation === 'openaiResponsesCompact' ? 'compact' : 'generate' } : {}), + ...(operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens' ? { 'request.provider.anthropicBeta': beta === null ? [] : beta.split(',') } : {}), + }, { httpCall: () => ({ ...options, signal: init.signal ?? undefined, waitUntil: () => {} }) }); + return result.response; +}; diff --git a/packages/test-utils/src/provider-pipeline.ts b/packages/test-utils/src/provider-pipeline.ts index b6dae79a9f..9b1b75f77b 100644 --- a/packages/test-utils/src/provider-pipeline.ts +++ b/packages/test-utils/src/provider-pipeline.ts @@ -1,13 +1,14 @@ +import type { ProviderCallResult, ProviderRerankCallResult } from './provider-results.ts'; import { noopUpstreamCallOptions } from './stubs.ts'; import { exchangeResponse, type HttpServices } from '@floway-dev/http/pipeline'; import { move, run, setRelease } from '@floway-dev/pipeline'; -import type { Provider, ProviderCallResult, ProviderModel, ProviderOperation, ProviderOperationPayloads, ProviderOperationResponse, ProviderPipeline, ProviderRerankCallResult, ProviderRequest, UpstreamCallOptions } from '@floway-dev/provider'; +import type { NonChatProviderOperation, Provider, ProviderModel, ProviderOperationPayloads, ProviderOperationResponse, ProviderPipeline, ProviderOperationRequest, UpstreamCallOptions } from '@floway-dev/provider'; import { providerModelFacts } from '@floway-dev/provider'; -type CallResult = +type CallResult = (O extends 'rerank' ? ProviderRerankCallResult : ProviderCallResult) & { readonly called: boolean; readonly previousCalls: readonly { readonly modelKey: string }[] }; -export const callProviderPipeline = async ( +export const callProviderPipeline = async ( provider: Provider, operation: O, model: ProviderModel, @@ -17,14 +18,14 @@ export const callProviderPipeline = async ( ): Promise> => { const pipeline = provider.pipelines[operation]; if (pipeline === undefined) throw new Error(`${provider.kind} has no ${operation} pipeline`); - const entry: ProviderRequest = move({ + const entry = move({ 'request.provider.model': providerModelFacts(model), 'request.provider.payload': payload, 'request.http.callId': 0, 'request.http.headers': [...options.headers], - }); + }) as ProviderOperationRequest; const services: HttpServices = { httpCall: () => ({ ...options, signal }) }; - const executed = await run, ProviderOperationResponse, HttpServices>(pipeline as ProviderPipeline, entry, services); + const executed = await run, ProviderOperationResponse, HttpServices>(pipeline as ProviderPipeline, entry, services); const facts = executed.facts; const exchange = facts['response.http.exchange']; if (exchange.type === 'transportFailure') { @@ -32,13 +33,17 @@ export const callProviderPipeline = async ( throw exchange.error; } const response = exchangeResponse(exchange); + if (exchange.body !== null) setRelease(exchange.body, async () => {}); let bytes: ArrayBuffer; try { bytes = await response.arrayBuffer(); - if (exchange.body !== null) setRelease(exchange.body, async () => {}); - } finally { - await executed.drain(); + } catch (error) { + try { await executed.drain(); } catch (cleanupError) { + if (cleanupError !== error) throw new AggregateError([error, cleanupError], 'Provider test call and cleanup failed', { cause: error }); + } + throw error; } + await executed.drain(); const result = { response: new Response(exchange.body === null ? null : bytes, { status: exchange.status, statusText: exchange.statusText, headers: response.headers }), modelKey: facts['response.provider.modelKey'], diff --git a/packages/test-utils/src/provider-results.ts b/packages/test-utils/src/provider-results.ts new file mode 100644 index 0000000000..5847caf322 --- /dev/null +++ b/packages/test-utils/src/provider-results.ts @@ -0,0 +1,41 @@ +import type { BillableUsage, ProtocolFrame, RerankTarget } from '@floway-dev/protocols/common'; +import type { OpenAIResponsesCompactionResult, OpenAIResponsesStreamEvent } from '@floway-dev/protocols/openai-responses'; +import type { InternalDebugError, PerformanceTelemetryContext, TelemetryModelIdentity } from '@floway-dev/provider'; + +export interface ProviderCallResult { + response: Response; + modelKey: string; +} +export interface ProviderRerankCallResult extends ProviderCallResult { target: RerankTarget } +export type ProviderStreamResult = + | { ok: true; events: AsyncIterable>; modelKey: string; headers?: Headers } + | { ok: false; response: Response; modelKey: string }; +export type ProviderOpenAIResponsesResult = + | (ProviderStreamResult & { action: 'generate' }) + | { action: 'compact'; ok: true; result: OpenAIResponsesCompactionResult; modelKey: string } + | { action: 'compact'; ok: false; response: Response; modelKey: string }; + +export interface EventResultMetadata { + modelIdentity: TelemetryModelIdentity; + performance?: PerformanceTelemetryContext; + billableUsage?: BillableUsage; +} +export interface EventResult { + type: 'events'; + events: AsyncIterable; + modelIdentity: TelemetryModelIdentity; + performance?: PerformanceTelemetryContext; + finalMetadata?: Promise; + headers?: Headers; +} +export type ExecuteResult = EventResult + | { type: 'api-error'; source: 'upstream' | 'gateway'; status: number; headers: Headers; body: Uint8Array; performance?: PerformanceTelemetryContext; upstreamId?: string } + | { type: 'internal-error'; status: number; error: InternalDebugError; performance?: PerformanceTelemetryContext }; + +export const eventResult = (events: AsyncIterable, modelIdentity: TelemetryModelIdentity, options: Omit, 'type' | 'events' | 'modelIdentity'> = {}): EventResult => { + const result: EventResult = { type: 'events', events, modelIdentity }; + if (options.performance !== undefined) result.performance = options.performance; + if (options.finalMetadata !== undefined) result.finalMetadata = options.finalMetadata; + if (options.headers !== undefined) result.headers = options.headers; + return result; +}; diff --git a/packages/test-utils/src/provider-stage.ts b/packages/test-utils/src/provider-stage.ts new file mode 100644 index 0000000000..5dd42f5ff6 --- /dev/null +++ b/packages/test-utils/src/provider-stage.ts @@ -0,0 +1,67 @@ +import { takeHttpResponse, type HttpExchange, type HttpHeaders } from '@floway-dev/http/pipeline'; +import { compose, defineStage, isSecret, move, run, type Stage } from '@floway-dev/pipeline'; +import { providerModelFacts, type ProviderModel, type ProviderOperationOutputs, type ProviderOperationPayloads, type ProviderProtocolFailure } from '@floway-dev/provider'; + +type ProtocolFrame = Extract['frames'] extends AsyncIterable + ? F extends { type: 'event'; event: unknown } ? Omit & { event: T } : F + : never; +type OpenAIChatCompletionsPayload = ProviderOperationPayloads['openaiChatCompletions'] & { model: string }; +type CanonicalOpenAIResponsesPayload = ProviderOperationPayloads['openaiResponses'] & { model: string }; +type AnthropicMessagesPayload = ProviderOperationPayloads['anthropicMessages'] & { model: string }; + +export interface ProviderStageProbe

{ + payload: P; + headers: Headers; + model: ProviderModel; + action?: 'generate' | 'compact'; + anthropicBeta?: readonly string[]; +} +export type OpenAIChatCompletionsProbe = ProviderStageProbe; +export type OpenAIResponsesProbe = ProviderStageProbe & { action: 'generate' | 'compact' }; +export type AnthropicMessagesProbe = ProviderStageProbe & { anthropicBeta: readonly string[] }; + +type Output = { kind: 'stream'; frames: AsyncIterable> } | { kind: 'value'; body: unknown } | ProviderProtocolFailure | null; +type Facts = Record & { 'response.http.exchange': HttpExchange; 'response.provider.output': Output }; + +export const applyProviderStage = async

(stage: Stage, probe: ProviderStageProbe

, reply: () => Promise, extraFacts: Readonly> = {}): Promise => { + let template: unknown; + const terminal = defineStage, Facts>({ + name: 'testProtocolReply', + return: { provides: ['response.http.exchange', 'response.http.body', 'response.provider.called', 'response.provider.previousCalls', 'response.provider.modelKey', 'response.provider.output'] }, + execute: async facts => { + probe.payload = facts['request.provider.payload'] as P; + probe.headers = new Headers((facts['request.http.headers'] as HttpHeaders).map(([name, value]): [string, string] => [name, isSecret(value) ? value.reveal() : value])); + if ('request.provider.anthropicBeta' in facts) probe.anthropicBeta = facts['request.provider.anthropicBeta'] as readonly string[]; + template = await reply(); + const value = typeof template === 'object' && template !== null ? template as Record : null; + let exchange: HttpExchange = { type: 'response', status: 200, statusText: 'OK', headers: [], body: null }; + let output: Output = null; + if (value !== null) { + if (value.type === 'events' || value.ok === true && value.action !== 'compact') output = { kind: 'stream', frames: value.events as AsyncIterable> }; + else if (value.ok === true && value.action === 'compact') output = { kind: 'value', body: value.result }; + else if (value.response instanceof Response) exchange = takeHttpResponse(value.response); + else if (value.type === 'api-error') exchange = takeHttpResponse(new Response(value.body as Uint8Array, { status: value.status as number, headers: value.headers as Headers })); + } + return move({ ...facts, 'response.http.exchange': exchange, 'response.http.body': exchange.body, 'response.provider.called': true, 'response.provider.previousCalls': [], 'response.provider.modelKey': probe.model.id, 'response.provider.output': output }); + }, + }); + const executed = await run(compose, Facts>('testProviderStage', [stage, terminal]), move({ + 'request.provider.payload': probe.payload, + 'request.provider.model': providerModelFacts(probe.model), + 'request.provider.modelKey': probe.model.id, + 'request.http.headers': [...probe.headers], + 'request.http.callId': 0, + ...(probe.action === undefined ? {} : { 'request.provider.responsesAction': probe.action }), + ...(probe.anthropicBeta === undefined ? {} : { 'request.provider.anthropicBeta': probe.anthropicBeta }), + ...extraFacts, + }), { recordProtocolFrames: >(frames: AsyncIterable): AsyncIterable => frames }); + const output = executed.facts['response.provider.output']; + const value = typeof template === 'object' && template !== null ? template as Record : null; + let result: unknown = template; + if (value !== null && output !== null) { + if ('kind' in output) result = output.kind === 'stream' ? { ...value, events: output.frames } : { ...value, result: output.body }; + else if (value.type === 'api-error') result = { ...value, status: output.status, headers: new Headers({ 'content-type': 'application/json' }), body: new TextEncoder().encode(JSON.stringify(output.body)) }; + } + await executed.drain(); + return result as R; +}; diff --git a/packages/test-utils/src/stub-chat-provider-pipeline.ts b/packages/test-utils/src/stub-chat-provider-pipeline.ts new file mode 100644 index 0000000000..ccb469186c --- /dev/null +++ b/packages/test-utils/src/stub-chat-provider-pipeline.ts @@ -0,0 +1,84 @@ +import { takeHttpResponse } from '@floway-dev/http/pipeline'; +import { compose, defineStage, isSecret, move, type Handed } from '@floway-dev/pipeline'; +import type { AnthropicMessagesUpstreamCallOptions, ChatProviderOperation, ProviderChatServices, ProviderModel, ProviderOperationOutputs, ProviderOperationPayloads, ProviderOperationRequest, ProviderOperationResponse, ProviderPipeline, ProviderResponse, UpstreamCallOptions } from '@floway-dev/provider'; +import { decodeProviderResponse } from '@floway-dev/provider'; + +type Frame = Extract extends { frames: AsyncIterable } ? F : never; +type TransportFailure = { type: 'transportFailure'; error: unknown }; +type Reply = ResponsesReply | StreamReply<'openaiChatCompletions'> | StreamReply<'anthropicMessages'> | CountReply | TransportFailure; +type AnyFrame = Frame<'openaiChatCompletions'> | Frame<'openaiResponses'> | Frame<'anthropicMessages'>; +type StreamReply = { ok: true; events: AsyncIterable>; modelKey: string; headers?: Headers } | { ok: false; response: Response; modelKey: string }; +type ResponsesReply = (StreamReply<'openaiResponses'> & { action: 'generate' }) | { action: 'compact'; ok: true; result: Extract['body']; modelKey: string } | { action: 'compact'; ok: false; response: Response; modelKey: string }; +type CountReply = { response: Response; modelKey: string }; +export type StubChatProviderCall = + O extends 'openaiResponses' | 'openaiResponsesCompact' + ? (model: ProviderModel, payload: ProviderOperationPayloads[O], action: 'generate' | 'compact', signal: AbortSignal | undefined, options: UpstreamCallOptions) => Promise + : O extends 'anthropicMessagesCountTokens' + ? (model: ProviderModel, payload: ProviderOperationPayloads[O], signal: AbortSignal | undefined, options: AnthropicMessagesUpstreamCallOptions) => Promise + : O extends 'anthropicMessages' + ? (model: ProviderModel, payload: ProviderOperationPayloads[O], signal: AbortSignal | undefined, options: AnthropicMessagesUpstreamCallOptions) => Promise | TransportFailure> + : (model: ProviderModel, payload: ProviderOperationPayloads[O], signal: AbortSignal | undefined, options: UpstreamCallOptions) => Promise | TransportFailure>; + +const responseFromFrames = (operation: ChatProviderOperation, frames: AsyncIterable, sourceHeaders: Headers | undefined): Response => { + const iterator = frames[Symbol.asyncIterator](); + const encoder = new TextEncoder(); + const body = new ReadableStream({ + pull: async controller => { + try { + for (;;) { + const item = await iterator.next(); + if (item.done) { controller.close(); return; } + const frame = item.value; + if (frame.type === 'done') { + if (operation === 'anthropicMessages') continue; + controller.enqueue(encoder.encode('data: [DONE]\n\n')); + return; + } + const eventName = operation === 'openaiChatCompletions' ? '' : `event: ${(frame.event as { type: string }).type}\n`; + controller.enqueue(encoder.encode(`${eventName}data: ${JSON.stringify(frame.event)}\n\n`)); + return; + } + } catch (error) { controller.error(error); } + }, + cancel: async () => { await iterator.return?.(); }, + }); + const headers = new Headers(sourceHeaders); + headers.set('content-type', 'text/event-stream'); + return new Response(body, { headers }); +}; + +export const stubChatProviderPipeline = (operation: O, call: StubChatProviderCall): ProviderPipeline => { + type Entry = ProviderOperationRequest & { 'request.provider.anthropicBeta': readonly string[] }; + type Ready = Entry & ProviderResponse & { 'request.provider.responsesAction': 'generate' | 'compact'; 'response.provider.responsesAction': 'generate' | 'compact' }; + const invoke = defineStage, ProviderOperationResponse, ProviderChatServices>({ + name: `stubProviderInvoke${operation}`, + through: { + request: { needs: [...(operation === 'anthropicMessages' || operation === 'anthropicMessagesCountTokens' ? ['request.provider.anthropicBeta' as const] : []), 'request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'], consumes: [], provides: ['request.provider.responsesAction', 'response.provider.responsesAction', 'response.http.exchange', 'response.http.body', 'response.provider.called', 'response.provider.previousCalls', 'response.provider.modelKey'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next, use) => { + const modelFacts = facts['request.provider.model']; + const model: ProviderModel = { ...modelFacts, enabledFlags: new Set(modelFacts.enabledFlags) }; + const httpCall = use.httpCall(facts['request.http.callId']); + const options: AnthropicMessagesUpstreamCallOptions = { ...httpCall, headers: new Headers(facts['request.http.headers'].map(([name, value]): [string, string] => [name, isSecret(value) ? value.reveal() : value])), anthropicBeta: facts['request.provider.anthropicBeta'] }; + let reply: Reply; + if (operation === 'openaiResponses' || operation === 'openaiResponsesCompact') { + reply = await httpCall.wrapUpstreamCall(() => (call as StubChatProviderCall<'openaiResponses'>)(model, facts['request.provider.payload'] as ProviderOperationPayloads['openaiResponses'], operation === 'openaiResponsesCompact' ? 'compact' : 'generate', httpCall.signal, options)); + } else { + reply = await httpCall.wrapUpstreamCall(() => (call as unknown as (model: ProviderModel, payload: unknown, signal: AbortSignal | undefined, options: AnthropicMessagesUpstreamCallOptions) => Promise)(model, facts['request.provider.payload'] as ProviderOperationPayloads['anthropicMessages'], httpCall.signal, options)); + } + if ('type' in reply) return move({ ...await next(move({ ...facts, 'request.provider.responsesAction': operation === 'openaiResponsesCompact' ? 'compact' : 'generate', 'response.provider.responsesAction': operation === 'openaiResponsesCompact' ? 'compact' : 'generate', 'response.http.exchange': reply, 'response.http.body': null, 'response.provider.called': false, 'response.provider.previousCalls': [], 'response.provider.modelKey': model.id }) as Ready & Record) }) as Handed>; + const response = 'response' in reply ? reply.response : 'events' in reply + ? responseFromFrames(operation, reply.events, reply.headers) + : Response.json(reply.result); + const exchange = takeHttpResponse(response); + return move({ ...await next(move({ ...facts, 'request.provider.responsesAction': 'action' in reply ? reply.action : operation === 'openaiResponsesCompact' ? 'compact' : 'generate', 'response.provider.responsesAction': 'action' in reply ? reply.action : operation === 'openaiResponsesCompact' ? 'compact' : 'generate', 'response.http.exchange': exchange, 'response.http.body': exchange.body, 'response.provider.called': true, 'response.provider.previousCalls': [], 'response.provider.modelKey': reply.modelKey }) as Ready & Record) }) as Handed>; + }, + }); + const reply = defineStage({ + name: 'stubProviderReply', + return: { provides: ['response.provider.responsesAction', 'response.http.exchange', 'response.http.body', 'response.provider.called', 'response.provider.previousCalls', 'response.provider.modelKey'] }, + execute: async facts => move({ ...facts }) as Ready & Record, + }); + return compose, ProviderOperationResponse>(`stub.${operation}`, [invoke, decodeProviderResponse(operation), reply]); +}; diff --git a/packages/test-utils/src/stubs.ts b/packages/test-utils/src/stubs.ts index 7b2fd727a3..8321e1e4ba 100644 --- a/packages/test-utils/src/stubs.ts +++ b/packages/test-utils/src/stubs.ts @@ -1,11 +1,7 @@ import { testFetcher } from './mock-fetch.ts'; import { type FlagId, type InternalModel, type PerformanceTelemetryContext, type ProviderInstance, type Provider, type ProviderModel, type ModelCandidate, type TelemetryModelIdentity, type AnthropicMessagesUpstreamCallOptions, type UpstreamCallOptions, identityWrapUpstreamCall } from '@floway-dev/provider'; -// No-op UpstreamCallOptions factory for tests calling provider methods -// directly: the fetcher uses runtime fetch so `globalThis.fetch` spies still -// intercept; waitUntil drops the promise (the runtime would have absorbed it -// in production). The Headers bag is per-call so tests that mutate it do not -// bleed state across cases. +// The runtime fetch spy remains visible behind replayable bodies, and each fixture owns its headers. export const noopUpstreamCallOptions = (overrides: Partial = {}): UpstreamCallOptions => ({ fetcher: testFetcher, waitUntil: () => {}, @@ -20,10 +16,7 @@ export const noopAnthropicMessagesUpstreamCallOptions = (overrides: Partial = {}): ProviderModel => ({ id: 'test-model', upstreamModelId: 'test-model', @@ -78,24 +71,13 @@ export const mockPerfTelemetryContext = (overrides: Partial = {}): ProviderInstance => ({ getProvidedModels: overrides.getProvidedModels ?? (() => Promise.resolve([])), - callAlphaSearch: overrides.callAlphaSearch ?? (() => Promise.reject(new Error('stubProvider.callAlphaSearch was called'))), - callOpenAICompletions: overrides.callOpenAICompletions ?? (() => Promise.reject(new Error('stubProvider.callOpenAICompletions was called'))), - callOpenAIChatCompletions: overrides.callOpenAIChatCompletions ?? (() => Promise.reject(new Error('stubProvider.callOpenAIChatCompletions was called'))), - callOpenAIResponses: overrides.callOpenAIResponses ?? (() => Promise.reject(new Error('stubProvider.callOpenAIResponses was called'))), - callAnthropicMessages: overrides.callAnthropicMessages ?? (() => Promise.reject(new Error('stubProvider.callAnthropicMessages was called'))), - callAnthropicMessagesCountTokens: overrides.callAnthropicMessagesCountTokens ?? (() => Promise.reject(new Error('stubProvider.callAnthropicMessagesCountTokens was called'))), - callOpenAIEmbeddings: overrides.callOpenAIEmbeddings ?? (() => Promise.reject(new Error('stubProvider.callOpenAIEmbeddings was called'))), - callOpenAIImagesGenerations: overrides.callOpenAIImagesGenerations ?? (() => Promise.reject(new Error('stubProvider.callOpenAIImagesGenerations was called'))), - callOpenAIImagesEdits: overrides.callOpenAIImagesEdits ?? (() => Promise.reject(new Error('stubProvider.callOpenAIImagesEdits was called'))), - callOpenAIAudioTranscriptions: overrides.callOpenAIAudioTranscriptions ?? (() => Promise.reject(new Error('stubProvider.callOpenAIAudioTranscriptions was called'))), - callRerank: overrides.callRerank ?? (() => Promise.reject(new Error('stubProvider.callRerank was called'))), }); // Stitches together a candidate whose `model.providerModels` map carries an // entry under the wired provider's upstream id — that's what // `providerModelOf(candidate)` resolves to at dispatch time. The // `enabledFlags` / `providerData` shortcuts populate that ProviderModel -// directly — the common case for interceptor tests that just need a flag set +// directly — the common case for stage tests that just need a flag set // for the resolver's own upstream key. Any `model.providerModels` supplied // through `overrides.model` replaces both the default entry and those // shortcuts wholesale. diff --git a/packages/translate/__tests__/index_test.ts b/packages/translate/__tests__/index_test.ts index 679ab97be7..3572aebbfe 100644 --- a/packages/translate/__tests__/index_test.ts +++ b/packages/translate/__tests__/index_test.ts @@ -1,4 +1,4 @@ -import { test } from 'vitest'; +import { expect, test } from 'vitest'; import { translateAnthropicMessagesViaOpenAIChatCompletions, @@ -15,7 +15,6 @@ import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-m import type { GeminiGenerateContentPayload } from '@floway-dev/protocols/gemini-generate-content'; import type { OpenAIChatCompletionsPayload } from '@floway-dev/protocols/openai-chat-completions'; import type { CanonicalOpenAIResponsesPayload } from '@floway-dev/protocols/openai-responses'; -import { assertEquals } from '@floway-dev/test-utils'; const schema = { type: 'object', properties: { x: { type: 'string' } } }; const formatSchema = { type: 'object', properties: { y: { type: 'string' } } }; @@ -82,18 +81,34 @@ const translations: Array<{ name: string; source: unknown; translate: () => Prom { name: 'Gemini to Anthropic Messages', source: gemini, translate: async () => (await translateGeminiGenerateContentViaAnthropicMessages(gemini, { model: 'm', fallbackMaxOutputTokens: 16 })).target }, ]; -test.each(translations)('$name owns its target payload without retaining source or prior target objects', async ({ source, translate }) => { - const original = structuredClone(source); - const first = await translate(); - const second = await translate(); +const freezeJson = (value: T): T => { + for (const object of objectsIn(value)) Object.freeze(object); + return value; +}; + +test.each(translations)('$name preserves frozen sources and shares unchanged schemas across trips', async ({ source, translate }) => { + freezeJson(source); + const original = JSON.stringify(source); + const first = freezeJson(await translate()); + const second = freezeJson(await translate()); const sourceObjects = objectsIn(source); - const firstObjects = objectsIn(first); - const secondObjects = objectsIn(second); + const shared = [...objectsIn(first)].filter(object => sourceObjects.has(object)); + expect(shared).toContain(schema); + expect(objectsIn(second).has(schema)).toBe(true); + expect(JSON.stringify(source)).toBe(original); - assertEquals(source, original); - for (const object of firstObjects) assertEquals(sourceObjects.has(object), false); - for (const object of secondObjects) { - assertEquals(sourceObjects.has(object), false); - assertEquals(firstObjects.has(object), false); - } + const firstBody = first as { tools: Array> }; + const tool = firstBody.tools[0]; + const nested = tool.function as Record | undefined; + const field = nested ? 'parameters' : 'input_schema' in tool ? 'input_schema' : 'parameters'; + const originalSchema = (nested ?? tool)[field] as typeof schema; + expect(() => { originalSchema.properties.x.type = 'number'; }).toThrow(TypeError); + const rewrittenSchema = { ...originalSchema, properties: { ...originalSchema.properties, providerOnly: { type: 'boolean' } } }; + const rewrittenTool = nested ? { ...tool, function: { ...nested, [field]: rewrittenSchema } } : { ...tool, [field]: rewrittenSchema }; + const rewritten = { ...firstBody, tools: [rewrittenTool, ...firstBody.tools.slice(1)] }; + expect(rewritten.tools[0]).not.toBe(tool); + expect(originalSchema).toBe(schema); + expect(originalSchema.properties).not.toHaveProperty('providerOnly'); + expect(JSON.stringify(source)).toBe(original); + expect(objectsIn(second).has(schema)).toBe(true); }); diff --git a/packages/translate/__tests__/openai-chat-completions-via-anthropic-messages/request_test.ts b/packages/translate/__tests__/openai-chat-completions-via-anthropic-messages/request_test.ts index 410934128f..a4df5576f3 100644 --- a/packages/translate/__tests__/openai-chat-completions-via-anthropic-messages/request_test.ts +++ b/packages/translate/__tests__/openai-chat-completions-via-anthropic-messages/request_test.ts @@ -273,7 +273,7 @@ test('simple user message → string content', async () => { assertEquals(result.messages.length, 1); assertEquals(result.messages[0].role, 'user'); // Last message of the request gets promoted to a single text block with a - // cache breakpoint by applyLastMessageCacheBreakpoint. + // cache breakpoint by withLastMessageCacheBreakpoint. assertEquals(result.messages[0].content, [{ type: 'text', text: 'Hello', cache_control: { type: 'ephemeral' } }]); }); diff --git a/packages/translate/__tests__/openai-responses-via-anthropic-messages/request_test.ts b/packages/translate/__tests__/openai-responses-via-anthropic-messages/request_test.ts index b451bf5cd2..575b0e0ddc 100644 --- a/packages/translate/__tests__/openai-responses-via-anthropic-messages/request_test.ts +++ b/packages/translate/__tests__/openai-responses-via-anthropic-messages/request_test.ts @@ -533,7 +533,7 @@ test('buildTargetRequest wraps custom tools as single-string function tools and }, }, // Single (only) function-tool entry receives the breakpoint as the last - // function tool; applyLastToolCacheBreakpoint walks right-to-left. + // function tool; withLastToolCacheBreakpoint walks right-to-left. cache_control: { type: 'ephemeral' }, }, ]); @@ -592,7 +592,7 @@ test.each([ tool_use_id: 'call_1', content: expected, // Last block of the last message — cache_control attached by - // applyLastMessageCacheBreakpoint. + // withLastMessageCacheBreakpoint. cache_control: { type: 'ephemeral' }, }, ], diff --git a/packages/translate/__tests__/shared/via-anthropic-messages/cache-breakpoints_test.ts b/packages/translate/__tests__/shared/via-anthropic-messages/cache-breakpoints_test.ts index 628b6621b6..eb065b5742 100644 --- a/packages/translate/__tests__/shared/via-anthropic-messages/cache-breakpoints_test.ts +++ b/packages/translate/__tests__/shared/via-anthropic-messages/cache-breakpoints_test.ts @@ -1,30 +1,33 @@ -import { test } from 'vitest'; +import { expect, test } from 'vitest'; -import { applyLastMessageCacheBreakpoint, applyLastSystemCacheBreakpoint, applyLastToolCacheBreakpoint } from '../../../src/shared/via-anthropic-messages/cache-breakpoints.ts'; +import { withLastMessageCacheBreakpoint, withLastSystemCacheBreakpoint, withLastToolCacheBreakpoint } from '../../../src/shared/via-anthropic-messages/cache-breakpoints.ts'; import type { AnthropicMessagesAssistantMessage, AnthropicMessagesMessage, AnthropicMessagesTextBlock, AnthropicMessagesTool, AnthropicMessagesUserMessage } from '@floway-dev/protocols/anthropic-messages'; import { assert, assertEquals } from '@floway-dev/test-utils'; const cacheControlOf = (value: unknown): unknown => (value as { cache_control?: unknown }).cache_control; -test('applyLastToolCacheBreakpoint marks the last custom tool, skipping native web search', () => { +test('withLastToolCacheBreakpoint marks the last custom tool, skipping native web search', () => { const tools: AnthropicMessagesTool[] = [ { type: 'custom', name: 'a', input_schema: {} }, { type: 'custom', name: 'b', input_schema: {} }, { type: 'web_search_20250305', name: 'web_search' }, ]; - applyLastToolCacheBreakpoint(tools); + const marked = withLastToolCacheBreakpoint(tools)!; + expect(marked[0]).toBe(tools[0]); + expect(marked[2]).toBe(tools[2]); + expect(cacheControlOf(tools[1])).toBeUndefined(); assertEquals(cacheControlOf(tools[0]), undefined); - assertEquals(cacheControlOf(tools[1]), { type: 'ephemeral' }); + assertEquals(cacheControlOf(marked[1]), { type: 'ephemeral' }); assertEquals(cacheControlOf(tools[2]), undefined); }); -test('applyLastMessageCacheBreakpoint promotes a string last message to a text block', () => { +test('withLastMessageCacheBreakpoint promotes a string last message to a text block', () => { const messages: AnthropicMessagesMessage[] = [{ role: 'user', content: 'hello' }]; - applyLastMessageCacheBreakpoint(messages); - assertEquals(messages[0].content, [{ type: 'text', text: 'hello', cache_control: { type: 'ephemeral' } }]); + const marked = withLastMessageCacheBreakpoint(messages); + assertEquals(marked[0].content, [{ type: 'text', text: 'hello', cache_control: { type: 'ephemeral' } }]); }); -test('applyLastMessageCacheBreakpoint marks an image as the trailing block', () => { +test('withLastMessageCacheBreakpoint marks an image as the trailing block', () => { const message: AnthropicMessagesUserMessage = { role: 'user', content: [ @@ -32,50 +35,54 @@ test('applyLastMessageCacheBreakpoint marks an image as the trailing block', () { type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'x' } }, ], }; - applyLastMessageCacheBreakpoint([message]); - assert(Array.isArray(message.content)); - assertEquals(cacheControlOf(message.content[0]), undefined); - assertEquals(cacheControlOf(message.content[1]), { type: 'ephemeral' }); + const marked = withLastMessageCacheBreakpoint([message])[0]; + assert(Array.isArray(marked.content)); + assertEquals(cacheControlOf(marked.content[0]), undefined); + assertEquals(cacheControlOf(marked.content[1]), { type: 'ephemeral' }); }); -test('applyLastMessageCacheBreakpoint marks a trailing assistant tool_use block', () => { +test('withLastMessageCacheBreakpoint marks a trailing assistant tool_use block', () => { const message: AnthropicMessagesAssistantMessage = { role: 'assistant', content: [{ type: 'tool_use', id: 't1', name: 'run', input: {} }], }; - applyLastMessageCacheBreakpoint([message]); - assert(Array.isArray(message.content)); - assertEquals(cacheControlOf(message.content[0]), { type: 'ephemeral' }); + const marked = withLastMessageCacheBreakpoint([message])[0]; + assert(Array.isArray(marked.content)); + assertEquals(cacheControlOf(marked.content[0]), { type: 'ephemeral' }); }); -test('applyLastMessageCacheBreakpoint falls back to an earlier message when the last has no cacheable block', () => { +test('withLastMessageCacheBreakpoint falls back to an earlier message when the last has no cacheable block', () => { const messages: AnthropicMessagesMessage[] = [ { role: 'user', content: [{ type: 'text', text: 'q' }] }, { role: 'assistant', content: [{ type: 'thinking', thinking: 'reasoning…' }] }, ]; - applyLastMessageCacheBreakpoint(messages); - const userContent = messages[0].content; - const assistantContent = messages[1].content; + const marked = withLastMessageCacheBreakpoint(messages); + const userContent = marked[0].content; + const assistantContent = marked[1].content; + expect(marked[1]).toBe(messages[1]); assert(Array.isArray(userContent) && Array.isArray(assistantContent)); assertEquals(cacheControlOf(userContent[0]), { type: 'ephemeral' }); assertEquals(cacheControlOf(assistantContent[0]), undefined); }); -test('applyLastSystemCacheBreakpoint is a no-op on undefined or empty input', () => { - applyLastSystemCacheBreakpoint(undefined); +test('withLastSystemCacheBreakpoint is a no-op on undefined or empty input', () => { + expect(withLastSystemCacheBreakpoint(undefined)).toBeUndefined(); const empty: AnthropicMessagesTextBlock[] = []; - applyLastSystemCacheBreakpoint(empty); + expect(withLastSystemCacheBreakpoint(empty)).toBe(empty); assertEquals(empty, []); }); -test('applyLastSystemCacheBreakpoint marks only the last block when multiple are present', () => { +test('withLastSystemCacheBreakpoint marks only the last block when multiple are present', () => { const system: AnthropicMessagesTextBlock[] = [ { type: 'text', text: 'instructions' }, { type: 'text', text: 'leading note' }, { type: 'text', text: 'final block' }, ]; - applyLastSystemCacheBreakpoint(system); + const marked = withLastSystemCacheBreakpoint(system)!; + expect(marked[0]).toBe(system[0]); + expect(marked[1]).toBe(system[1]); + expect(cacheControlOf(system[2])).toBeUndefined(); assertEquals(cacheControlOf(system[0]), undefined); assertEquals(cacheControlOf(system[1]), undefined); - assertEquals(cacheControlOf(system[2]), { type: 'ephemeral' }); + assertEquals(cacheControlOf(marked[2]), { type: 'ephemeral' }); }); diff --git a/packages/translate/package.json b/packages/translate/package.json index ef19b94491..694806f1fe 100644 --- a/packages/translate/package.json +++ b/packages/translate/package.json @@ -10,8 +10,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@floway-dev/protocols": "workspace:*", - "klona": "2.0.6" + "@floway-dev/protocols": "workspace:*" }, "devDependencies": { "@floway-dev/test-utils": "workspace:*" diff --git a/packages/translate/src/anthropic-messages-via-openai-chat-completions/request.ts b/packages/translate/src/anthropic-messages-via-openai-chat-completions/request.ts index c005449c61..2761b585a6 100644 --- a/packages/translate/src/anthropic-messages-via-openai-chat-completions/request.ts +++ b/packages/translate/src/anthropic-messages-via-openai-chat-completions/request.ts @@ -1,4 +1,3 @@ -import { klona } from 'klona/json'; import { filterAnthropicMessagesClientTools } from '../shared/anthropic-messages-via/client-tools.ts'; import { resolveAnthropicMessagesReasoningEffort } from '../shared/anthropic-messages-via/reasoning-effort.ts'; @@ -197,9 +196,9 @@ const translateAnthropicMessagesAssistant = (message: AnthropicMessagesAssistant }; // Anthropic Messages system blocks are prompt boundaries; preserve each one -// as a separate OpenAI Chat Completions text part so a CC→Anthropic Messages→CC round trip -// does not silently merge them. Falls back to the simple string form when -// the source is already a single-string field. +// as a separate OpenAI Chat Completions text part so an OpenAI Chat Completions → +// Anthropic Messages → OpenAI Chat Completions round trip does not merge them. +// Preserve the simple string form when the source is already a string. const systemContentFromBlocks = (system: string | AnthropicMessagesTextBlock[]): string | OpenAIChatCompletionsContentPart[] => typeof system === 'string' ? system @@ -267,7 +266,7 @@ const translateAnthropicMessagesToolChoice = (toolChoice?: AnthropicMessagesPayl export const buildTargetRequest = (payload: AnthropicMessagesPayload): OpenAIChatCompletionsPayload => { const clientTools = filterAnthropicMessagesClientTools(payload.tools); // Pass effort through verbatim; per-upstream enum acceptance (e.g. some - // backends rejecting `xhigh`/`max`) is the target interceptor's concern. + // backends rejecting `xhigh`/`max`) is the target provider's concern. const reasoningEffort = resolveAnthropicMessagesReasoningEffort(payload); const jsonSchema = openAiJsonSchemaCoreFromAnthropicMessagesFormat(payload.output_config?.format); const responseFormat = jsonSchema ? { type: 'json_schema' as const, json_schema: jsonSchema } : undefined; @@ -279,7 +278,7 @@ export const buildTargetRequest = (payload: AnthropicMessagesPayload): OpenAICha messages: translateAnthropicMessagesInput(payload.messages, payload.system), ...(reasoningEffort !== undefined ? { reasoning_effort: reasoningEffort } : {}), max_tokens: payload.max_tokens, - stop: klona(payload.stop_sequences), + stop: payload.stop_sequences, stream: true, // Ask the upstream for usage on every streaming chunk, not just the final // one. `include_usage` is the OpenAI-standard flag; `continuous_usage_stats` diff --git a/packages/translate/src/anthropic-messages-via-openai-responses/events.ts b/packages/translate/src/anthropic-messages-via-openai-responses/events.ts index e8a8e2dc28..e1d67b65eb 100644 --- a/packages/translate/src/anthropic-messages-via-openai-responses/events.ts +++ b/packages/translate/src/anthropic-messages-via-openai-responses/events.ts @@ -396,7 +396,7 @@ const handleCompleted = (response: OpenAIResponsesResult, state: OpenAIResponses return events; }; -// A OpenAI Responses upstream can report a context-exceeded failure inside the SSE +// An OpenAI Responses upstream can report a context-exceeded failure inside the SSE // stream (Codex emits `response.failed` with `error.code = // context_length_exceeded`; some Copilot fronts surface a stream `error` // event with the same code). We rewrite those into the same Anthropic diff --git a/packages/translate/src/anthropic-messages-via-openai-responses/request.ts b/packages/translate/src/anthropic-messages-via-openai-responses/request.ts index 5b421aa208..3ad36da510 100644 --- a/packages/translate/src/anthropic-messages-via-openai-responses/request.ts +++ b/packages/translate/src/anthropic-messages-via-openai-responses/request.ts @@ -1,4 +1,3 @@ -import { klona } from 'klona/json'; import { anthropicMessagesReasoningBlockToOpenAIResponsesReasoning } from '../shared/anthropic-messages-and-openai-responses/reasoning.ts'; import { filterAnthropicMessagesClientTools } from '../shared/anthropic-messages-via/client-tools.ts'; @@ -246,7 +245,7 @@ export const buildTargetRequest = (payload: AnthropicMessagesPayload): Canonical max_output_tokens: payload.max_tokens, ...(payload.tools !== undefined ? { tools: translateTools(clientTools) } : {}), ...(toolChoice !== undefined ? { tool_choice: toolChoice } : {}), - ...(payload.metadata ? { metadata: klona(payload.metadata) } : {}), + ...(payload.metadata ? { metadata: payload.metadata } : {}), stream: true, ...(reasoning ? { reasoning } : {}), ...(text ? { text } : {}), diff --git a/packages/translate/src/gemini-generate-content-via-anthropic-messages/request.ts b/packages/translate/src/gemini-generate-content-via-anthropic-messages/request.ts index 9e098804e2..8ac0b0c414 100644 --- a/packages/translate/src/gemini-generate-content-via-anthropic-messages/request.ts +++ b/packages/translate/src/gemini-generate-content-via-anthropic-messages/request.ts @@ -1,4 +1,3 @@ -import { klona } from 'klona/json'; import { geminiGenerateContentFunctionCallingIntent, @@ -14,7 +13,7 @@ import { type GeminiGenerateContentToolCallIds, geminiGenerateContentVisibleText, } from '../shared/gemini-generate-content-via/gemini-generate-content.ts'; -import { applyLastMessageCacheBreakpoint, applyLastSystemCacheBreakpoint, applyLastToolCacheBreakpoint } from '../shared/via-anthropic-messages/cache-breakpoints.ts'; +import { withLastMessageCacheBreakpoint, withLastSystemCacheBreakpoint, withLastToolCacheBreakpoint } from '../shared/via-anthropic-messages/cache-breakpoints.ts'; import { TranslatorInputError } from '../translator-input-error.ts'; import { ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS, @@ -120,7 +119,7 @@ const buildAssistantMessage = (content: GeminiGenerateContentContent, turnIndex: type: 'tool_use', id, name: call.name, - input: klona(call.args), + input: call.args, }); return; } @@ -190,21 +189,21 @@ const applyGenerationConfig = (request: AnthropicMessagesPayload, generationConf request.top_k = generationConfig.topK; } if (generationConfig.stopSequences !== undefined) { - request.stop_sequences = klona(generationConfig.stopSequences); + request.stop_sequences = generationConfig.stopSequences; } // Gemini generateContent's `responseSchema` is the bare JSON Schema; Anthropic carries it // as `output_config.format = { type: 'json_schema', schema }`. `responseMimeType: // application/json` without a schema has no Anthropic equivalent and is // dropped — the routing fallback degrades gracefully rather than fails. return generationConfig.responseSchema !== undefined - ? { format: { type: 'json_schema', schema: klona(generationConfig.responseSchema as Record) } } + ? { format: { type: 'json_schema', schema: generationConfig.responseSchema as Record } } : {}; }; const inputSchemaForDeclaration = (parameters: Record | undefined): Record => { // AnthropicMessagesClientTool requires input_schema, so parameterless Gemini generateContent function // declarations use the smallest object schema rather than dropping the tool. - return klona(parameters) ?? { type: 'object', properties: {} }; + return parameters ?? { type: 'object', properties: {} }; }; const buildTools = (payload: GeminiGenerateContentPayload): AnthropicMessagesTool[] | undefined => { @@ -239,8 +238,7 @@ export const buildTargetRequest = ( const system = geminiGenerateContentText(payload.systemInstruction); if (system !== null) { const systemBlocks: AnthropicMessagesTextBlock[] = [{ type: 'text', text: system }]; - applyLastSystemCacheBreakpoint(systemBlocks); - request.system = systemBlocks; + request.system = withLastSystemCacheBreakpoint(systemBlocks); } payload.contents?.forEach((content, turnIndex) => { @@ -273,10 +271,9 @@ export const buildTargetRequest = ( if (thinking !== undefined) request.thinking = thinking; if (!hasGenerationOutputConfig && Object.keys(outputConfig).length > 0) attachOutputConfig(); - const tools = buildTools(payload); + const tools = withLastToolCacheBreakpoint(buildTools(payload)); if (tools) request.tools = tools; - applyLastToolCacheBreakpoint(request.tools); - applyLastMessageCacheBreakpoint(request.messages); + request.messages = withLastMessageCacheBreakpoint(request.messages); const intent = geminiGenerateContentFunctionCallingIntent(payload.toolConfig?.functionCallingConfig); switch (intent?.type) { diff --git a/packages/translate/src/gemini-generate-content-via-openai-chat-completions/request.ts b/packages/translate/src/gemini-generate-content-via-openai-chat-completions/request.ts index 1ef773f04e..f960bf55bd 100644 --- a/packages/translate/src/gemini-generate-content-via-openai-chat-completions/request.ts +++ b/packages/translate/src/gemini-generate-content-via-openai-chat-completions/request.ts @@ -1,4 +1,3 @@ -import { klona } from 'klona/json'; import { geminiGenerateContentFunctionCallingIntent, @@ -162,7 +161,7 @@ const applyGenerationConfig = (request: OpenAIChatCompletionsPayload, generation request.top_p = generationConfig.topP; } if (generationConfig.stopSequences !== undefined) { - request.stop = klona(generationConfig.stopSequences); + request.stop = generationConfig.stopSequences; } if (generationConfig.candidateCount !== undefined) { request.n = generationConfig.candidateCount; @@ -182,7 +181,7 @@ const applyGenerationConfig = (request: OpenAIChatCompletionsPayload, generation type: 'json_schema', json_schema: { name: 'gemini_response', - schema: klona(generationConfig.responseSchema), + schema: generationConfig.responseSchema, }, }; } else if (generationConfig.responseMimeType === 'application/json') { @@ -199,7 +198,7 @@ const buildTools = (payload: GeminiGenerateContentPayload): OpenAIChatCompletion function: { name: declaration.name, ...(declaration.description !== undefined ? { description: declaration.description } : {}), - ...(declaration.parameters !== undefined ? { parameters: klona(declaration.parameters) } : {}), + ...(declaration.parameters !== undefined ? { parameters: declaration.parameters } : {}), }, })); diff --git a/packages/translate/src/gemini-generate-content-via-openai-responses/request.ts b/packages/translate/src/gemini-generate-content-via-openai-responses/request.ts index d0d1519b62..29fb03d19a 100644 --- a/packages/translate/src/gemini-generate-content-via-openai-responses/request.ts +++ b/packages/translate/src/gemini-generate-content-via-openai-responses/request.ts @@ -1,4 +1,3 @@ -import { klona } from 'klona/json'; import { geminiGenerateContentFunctionCallingIntent, @@ -140,7 +139,7 @@ const applyGenerationConfig = (request: CanonicalOpenAIResponsesPayload, generat type: 'json_schema', json_schema: { name: 'gemini_response', - schema: klona(generationConfig.responseSchema), + schema: generationConfig.responseSchema, }, }, }; @@ -162,7 +161,7 @@ const buildTools = (payload: GeminiGenerateContentPayload): OpenAIResponsesTool[ type: 'function' as const, name: declaration.name, ...(declaration.description !== undefined ? { description: declaration.description } : {}), - parameters: klona(declaration.parameters) ?? { type: 'object', properties: {} }, + parameters: declaration.parameters ?? { type: 'object', properties: {} }, strict: false, })); diff --git a/packages/translate/src/openai-chat-completions-via-anthropic-messages/request.ts b/packages/translate/src/openai-chat-completions-via-anthropic-messages/request.ts index c80be7026c..3089332dc8 100644 --- a/packages/translate/src/openai-chat-completions-via-anthropic-messages/request.ts +++ b/packages/translate/src/openai-chat-completions-via-anthropic-messages/request.ts @@ -1,8 +1,7 @@ -import { klona } from 'klona/json'; import { anthropicMessagesThinkingBlockFromOpenAIChatCompletionsScalarReasoning } from '../shared/openai-chat-completions-and-anthropic-messages/reasoning.ts'; import { openAIChatCompletionsScalarReasoningText } from '../shared/openai-chat-completions-and-openai-responses/reasoning.ts'; -import { applyLastMessageCacheBreakpoint, applyLastSystemCacheBreakpoint, applyLastToolCacheBreakpoint } from '../shared/via-anthropic-messages/cache-breakpoints.ts'; +import { withLastMessageCacheBreakpoint, withLastSystemCacheBreakpoint, withLastToolCacheBreakpoint } from '../shared/via-anthropic-messages/cache-breakpoints.ts'; import { anthropicMessagesReasoningFieldsFromEffort } from '../shared/via-anthropic-messages/reasoning-effort.ts'; import { resolveImageUrlToAnthropicMessagesImage, unavailableRemoteImageLoader } from '../shared/via-anthropic-messages/remote-images.ts'; import { anthropicMessagesServiceTierFieldsFromOpenAI } from '../shared/via-anthropic-messages/service-tier.ts'; @@ -154,7 +153,7 @@ const buildAnthropicMessagesInput = async (messages: OpenAIChatCompletionsMessag // was hoisted earlier). Anthropic upstreams diverge on inline // role:'system' here (Bedrock accepts it under placement rules; // Vertex rejects it outright), so the gateway's - // `rewrite-mid-conv-system-to-user` interceptor flag is the safety + // `rewrite-mid-conv-system-to-user` role-compatibility flag is the safety // net for any inline system that would otherwise reach an upstream // that does not accept it. const blocks = convertSystemContent(message.content); @@ -176,12 +175,12 @@ const translateOpenAIChatCompletionsTools = (tools: OpenAIChatCompletionsTool[]) tools.map(tool => ({ name: tool.function.name, description: tool.function.description, - input_schema: klona(tool.function.parameters) ?? { type: 'object', properties: {} }, + input_schema: tool.function.parameters ?? { type: 'object', properties: {} }, ...(tool.function.strict !== undefined ? { strict: tool.function.strict } : {}), })); const translateOpenAIChatCompletionsToolChoice = (toolChoice: NonNullable): AnthropicMessagesPayload['tool_choice'] => { - if (typeof toolChoice === 'string') return klona(CHAT_TOOL_CHOICES[toolChoice]); + if (typeof toolChoice === 'string') return { ...CHAT_TOOL_CHOICES[toolChoice] }; return { type: 'tool', name: toolChoice.function.name }; }; @@ -210,9 +209,9 @@ export const buildTargetRequest = async (payload: OpenAIChatCompletionsPayload, const maxTokens = payload.max_tokens ?? options.fallbackMaxOutputTokens ?? ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS; const tools = payload.tools?.length ? translateOpenAIChatCompletionsTools(payload.tools) : undefined; - applyLastSystemCacheBreakpoint(systemBlocks); - applyLastToolCacheBreakpoint(tools); - applyLastMessageCacheBreakpoint(messages); + const cachedSystem = withLastSystemCacheBreakpoint(systemBlocks); + const cachedTools = withLastToolCacheBreakpoint(tools); + const cachedMessages = withLastMessageCacheBreakpoint(messages); // Merge OpenAI Chat Completions `reasoning_effort` + `response_format` into a single Anthropic Messages // `output_config` so a chat-source structured-output request survives @@ -229,7 +228,7 @@ export const buildTargetRequest = async (payload: OpenAIChatCompletionsPayload, jsonSchema?.schema && typeof jsonSchema.schema === 'object' && !Array.isArray(jsonSchema.schema) ? (jsonSchema.schema as Record) : undefined; const outputConfig: NonNullable = {}; if (reasoningEffort !== undefined) outputConfig.effort = reasoningEffort; - if (formatSchema) outputConfig.format = { type: 'json_schema', schema: klona(formatSchema) }; + if (formatSchema) outputConfig.format = { type: 'json_schema', schema: formatSchema }; const hasOutputConfig = Object.keys(outputConfig).length > 0; const serviceTierFields = anthropicMessagesServiceTierFieldsFromOpenAI(payload.service_tier); @@ -238,18 +237,18 @@ export const buildTargetRequest = async (payload: OpenAIChatCompletionsPayload, // of treating them as a backchannel for Anthropic `metadata.user_id`. return { model: payload.model, - messages, + messages: cachedMessages, max_tokens: maxTokens, - ...(systemBlocks.length > 0 ? { system: systemBlocks } : {}), + ...(cachedSystem?.length ? { system: cachedSystem } : {}), ...(payload.temperature != null ? { temperature: payload.temperature } : {}), ...(payload.top_p != null ? { top_p: payload.top_p } : {}), ...(payload.stop != null ? { - stop_sequences: Array.isArray(payload.stop) ? klona(payload.stop) : [payload.stop], + stop_sequences: Array.isArray(payload.stop) ? payload.stop : [payload.stop], } : {}), stream: true, - ...(tools ? { tools } : {}), + ...(cachedTools ? { tools: cachedTools } : {}), ...(payload.tool_choice != null ? { tool_choice: translateOpenAIChatCompletionsToolChoice(payload.tool_choice) } : {}), ...(thinking ? { thinking } : {}), ...(hasOutputConfig ? { output_config: outputConfig } : {}), diff --git a/packages/translate/src/openai-chat-completions-via-openai-responses/request.ts b/packages/translate/src/openai-chat-completions-via-openai-responses/request.ts index 501eae6d91..3e9d8c2535 100644 --- a/packages/translate/src/openai-chat-completions-via-openai-responses/request.ts +++ b/packages/translate/src/openai-chat-completions-via-openai-responses/request.ts @@ -1,4 +1,3 @@ -import { klona } from 'klona/json'; import { openaiChatCompletionsContentToOpenAIResponsesInputContent, openaiChatCompletionsContentToText } from '../shared/openai-chat-completions-and-openai-responses/content.ts'; import { openAIChatCompletionsScalarReasoningText, scalarToOpenAIResponsesReasoningItem, translateOpenAIChatCompletionsReasoningItems } from '../shared/openai-chat-completions-and-openai-responses/reasoning.ts'; @@ -11,7 +10,7 @@ const translateChatTools = (tools?: OpenAIChatCompletionsTool[] | null): OpenAIR ? tools.map(tool => ({ type: 'function', name: tool.function.name, - parameters: klona(tool.function.parameters) ?? { type: 'object', properties: {} }, + parameters: tool.function.parameters ?? { type: 'object', properties: {} }, // OpenAI Chat Completions function tools are non-strict by default while OpenAI Responses function // tools default strict; make omission explicit to preserve OpenAI Chat Completions semantics. strict: tool.function.strict ?? false, @@ -135,7 +134,7 @@ export const buildTargetRequest = (payload: OpenAIChatCompletionsPayload): Canon }); } - const responseTextConfig = payload.response_format === undefined ? undefined : payload.response_format === null ? null : { format: klona(payload.response_format) }; + const responseTextConfig = payload.response_format === undefined ? undefined : payload.response_format === null ? null : { format: payload.response_format }; // OpenAI Chat Completions' `reasoning_effort: 'none'` disables reasoning without an OpenAI Responses // equivalent (OpenAI Responses `reasoning.effort` has no 'none' member); drop the @@ -162,7 +161,7 @@ export const buildTargetRequest = (payload: OpenAIChatCompletionsPayload): Canon // Same-purpose OpenAI fields are normal OpenAI Chat Completions/OpenAI Responses adapter surface; // provider-specific policy filtering belongs at the target boundary, not in // pairwise translation. - ...(payload.metadata !== undefined ? { metadata: klona(payload.metadata) } : {}), + ...(payload.metadata !== undefined ? { metadata: payload.metadata } : {}), stream: true, // Preserve OpenAI Chat Completions' omitted `store` as omitted instead of synthesizing // `store: false`. OpenAI's migration guide treats storage as the default diff --git a/packages/translate/src/openai-responses-via-anthropic-messages/request.ts b/packages/translate/src/openai-responses-via-anthropic-messages/request.ts index fce130519d..d707b4f4c4 100644 --- a/packages/translate/src/openai-responses-via-anthropic-messages/request.ts +++ b/packages/translate/src/openai-responses-via-anthropic-messages/request.ts @@ -1,4 +1,3 @@ -import { klona } from 'klona/json'; import { canonicalizeOpenAIResponsesPayload } from '../canonicalize-openai-responses-payload.ts'; import { openaiResponsesReasoningToAnthropicMessagesUpstreamBlock } from '../shared/anthropic-messages-and-openai-responses/reasoning.ts'; @@ -7,7 +6,7 @@ import { restrictAllowedTools } from '../shared/openai-responses-via/allowed-too import { buildCustomToolInputSchema } from '../shared/openai-responses-via/custom-tool-wrap.ts'; import { flattenNamespaceTools, type NamespaceToolNames } from '../shared/openai-responses-via/namespace-tools.ts'; import { rejectProgramCaller, rejectProgrammaticOpenAIResponsesPayload } from '../shared/openai-responses-via/programmatic-tooling.ts'; -import { applyLastMessageCacheBreakpoint, applyLastSystemCacheBreakpoint, applyLastToolCacheBreakpoint } from '../shared/via-anthropic-messages/cache-breakpoints.ts'; +import { withLastMessageCacheBreakpoint, withLastSystemCacheBreakpoint, withLastToolCacheBreakpoint } from '../shared/via-anthropic-messages/cache-breakpoints.ts'; import { anthropicMessagesReasoningFieldsFromEffort } from '../shared/via-anthropic-messages/reasoning-effort.ts'; import { resolveImageUrlToAnthropicMessagesImage, unavailableRemoteImageLoader } from '../shared/via-anthropic-messages/remote-images.ts'; import { anthropicMessagesServiceTierFieldsFromOpenAI } from '../shared/via-anthropic-messages/service-tier.ts'; @@ -321,7 +320,7 @@ const translateTools = ( // spelling for a tool that takes no arguments. // https://github.com/anthropics/anthropic-sdk-typescript/blob/3b45cd3b69c956ac63384fdb09ce1d8109f3fa80/src/resources/messages/messages.ts#L1845-L1852 // https://github.com/anthropics/anthropic-sdk-typescript/blob/3b45cd3b69c956ac63384fdb09ce1d8109f3fa80/examples/managed-agents-self-hosted-sandbox-worker.ts#L34-L41 - input_schema: klona(tool.parameters) ?? { type: 'object', properties: {} }, + input_schema: tool.parameters ?? { type: 'object', properties: {} }, ...(tool.strict == null ? {} : { strict: tool.strict }), }); continue; @@ -387,9 +386,9 @@ export const buildTargetRequest = async (source: OpenAIResponsesRequestPayload, ]; const effort = payload.reasoning?.effort; const maxTokens = payload.max_output_tokens ?? options.fallbackMaxOutputTokens ?? ANTHROPIC_MESSAGES_FALLBACK_MAX_TOKENS; - applyLastSystemCacheBreakpoint(systemBlocks); - applyLastToolCacheBreakpoint(tools); - applyLastMessageCacheBreakpoint(messages); + const cachedSystem = withLastSystemCacheBreakpoint(systemBlocks); + const cachedTools = withLastToolCacheBreakpoint(tools); + const cachedMessages = withLastMessageCacheBreakpoint(messages); // Merge reasoning effort + structured-output format into a single // `output_config`. `effort === 'none'` still maps to `thinking: {type: @@ -406,7 +405,7 @@ export const buildTargetRequest = async (source: OpenAIResponsesRequestPayload, const { thinking, effort: outputConfigEffort } = anthropicMessagesReasoningFieldsFromEffort(effort); const outputConfig: NonNullable = {}; if (outputConfigEffort !== undefined) outputConfig.effort = outputConfigEffort; - if (formatSchema) outputConfig.format = { type: 'json_schema', schema: klona(formatSchema) }; + if (formatSchema) outputConfig.format = { type: 'json_schema', schema: formatSchema }; const hasOutputConfig = Object.keys(outputConfig).length > 0; const serviceTierFields = anthropicMessagesServiceTierFieldsFromOpenAI(payload.service_tier); @@ -416,13 +415,13 @@ export const buildTargetRequest = async (source: OpenAIResponsesRequestPayload, // semantics. const target: AnthropicMessagesPayload = { model: payload.model, - messages, + messages: cachedMessages, max_tokens: maxTokens, - ...(systemBlocks.length > 0 ? { system: systemBlocks } : {}), + ...(cachedSystem?.length ? { system: cachedSystem } : {}), ...(payload.temperature != null ? { temperature: payload.temperature } : {}), ...(payload.top_p != null ? { top_p: payload.top_p } : {}), stream: true, - tools, + tools: cachedTools, tool_choice: translateToolChoice(allowed.choice), ...(thinking ? { thinking } : {}), ...(hasOutputConfig ? { output_config: outputConfig } : {}), diff --git a/packages/translate/src/openai-responses-via-openai-chat-completions/request.ts b/packages/translate/src/openai-responses-via-openai-chat-completions/request.ts index a4e3ef0134..0b05aede74 100644 --- a/packages/translate/src/openai-responses-via-openai-chat-completions/request.ts +++ b/packages/translate/src/openai-responses-via-openai-chat-completions/request.ts @@ -1,4 +1,3 @@ -import { klona } from 'klona/json'; import { canonicalizeOpenAIResponsesPayload } from '../canonicalize-openai-responses-payload.ts'; import { openaiResponsesContentToOpenAIChatCompletionsContent, openaiResponsesContentToText } from '../shared/openai-chat-completions-and-openai-responses/content.ts'; @@ -89,8 +88,8 @@ const translateOpenAIResponsesTools = (tools: OpenAIResponsesTool[] | null | und // After allowed_tools selection, Chat Completions can represent only flat // function and custom declarations. Custom tools are wrapped as functions // and recorded so response events can restore their freeform shape. The - // server-tool shim rewrites hosted web_search declarations to ordinary - // function tools before this translation, so retained shim tools need no + // hosted-tools stages rewrite web_search declarations to ordinary + // function tools before this translation, so retained hosted tools need no // special handling here. const out: OpenAIChatCompletionsTool[] = []; @@ -102,7 +101,7 @@ const translateOpenAIResponsesTools = (tools: OpenAIResponsesTool[] | null | und name: tool.name, // OpenAI Responses spells "unspecified" as an omitted key or an explicit // `null`; OpenAI Chat Completions has only the omitted-key spelling. - ...(tool.parameters == null ? {} : { parameters: klona(tool.parameters) }), + ...(tool.parameters == null ? {} : { parameters: tool.parameters }), ...(tool.strict == null ? {} : { strict: tool.strict }), ...(tool.description ? { description: tool.description } : {}), }, @@ -140,7 +139,7 @@ const buildOpenAIChatCompletionsResponseFormat = (text: OpenAIResponsesPayload[' if (text === null) return null; // `text: {}` means no explicit format. Keep it omitted instead of converting // absence into an explicit OpenAI Chat Completions `response_format: null`. - const format = klona(text.format); + const format = text.format; if (!Object.hasOwn(text, 'format') || format === undefined) return undefined; if (format === null) return null; // OpenAI Responses API uses a flat json_schema shape @@ -290,7 +289,7 @@ export const buildTargetRequest = (source: OpenAIResponsesRequestPayload): Targe stream: true, ...(payload.temperature !== undefined ? { temperature: payload.temperature } : {}), ...(payload.top_p !== undefined ? { top_p: payload.top_p } : {}), - ...(payload.metadata !== undefined ? { metadata: klona(payload.metadata) } : {}), + ...(payload.metadata !== undefined ? { metadata: payload.metadata } : {}), ...(payload.store !== undefined ? { store: payload.store } : {}), ...(payload.parallel_tool_calls !== undefined ? { parallel_tool_calls: payload.parallel_tool_calls } : {}), ...(responseFormat !== undefined ? { response_format: responseFormat } : {}), diff --git a/packages/translate/src/shared/anthropic-messages-via/context-window-error.ts b/packages/translate/src/shared/anthropic-messages-via/context-window-error.ts index a873b62e51..01f30158d3 100644 --- a/packages/translate/src/shared/anthropic-messages-via/context-window-error.ts +++ b/packages/translate/src/shared/anthropic-messages-via/context-window-error.ts @@ -19,7 +19,7 @@ import { buildPromptTooLongBody } from '@floway-dev/protocols/anthropic-messages // - Copilot `/v1/messages` (Anthropic-shaped body carrying a Copilot-specific // message string): // {"type":"error","error":{"type":"invalid_request_error","message":"Request body is too large for model context window ..."}} -// The provider-copilot Anthropic Messages interceptor handles this at the provider +// The provider-copilot Anthropic Messages error-rewrite stage handles this at the provider // layer; the substring is listed here so an Anthropic-Messages-via-OpenAI-Responses trip that // ever encounters the same phrasing behaves consistently. diff --git a/packages/translate/src/shared/anthropic-messages-via/reasoning-effort.ts b/packages/translate/src/shared/anthropic-messages-via/reasoning-effort.ts index 17ef179645..e101588815 100644 --- a/packages/translate/src/shared/anthropic-messages-via/reasoning-effort.ts +++ b/packages/translate/src/shared/anthropic-messages-via/reasoning-effort.ts @@ -6,7 +6,7 @@ import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-m // // 1. `output_config.effort` — gateway-canonical OpenAI-style override; passes // through verbatim. Per-upstream enum acceptance is the target -// interceptor's concern. +// provider's concern. // 2. `thinking.type === 'disabled'` — emit the `'none'` sentinel. // 3. `thinking.type === 'enabled'` / `'adaptive'` — caller asked for thinking // but did not pin an effort. Emit `'medium'` (the OpenAI/Azure default diff --git a/packages/translate/src/shared/anthropic-messages-via/structured-output.ts b/packages/translate/src/shared/anthropic-messages-via/structured-output.ts index afc6c718e3..b26d9f7301 100644 --- a/packages/translate/src/shared/anthropic-messages-via/structured-output.ts +++ b/packages/translate/src/shared/anthropic-messages-via/structured-output.ts @@ -15,8 +15,6 @@ // // Anthropic spec: https://platform.claude.com/docs/en/build-with-claude/structured-outputs -import { klona } from 'klona/json'; - import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; export const ANTHROPIC_MESSAGES_OPENAI_JSON_SCHEMA_NAME = 'messages_response'; @@ -31,5 +29,5 @@ export interface OpenAiJsonSchemaCore { export const openAiJsonSchemaCoreFromAnthropicMessagesFormat = (format: AnthropicMessagesOutputFormat | undefined): OpenAiJsonSchemaCore | undefined => { if (format?.type !== 'json_schema') return undefined; - return { name: ANTHROPIC_MESSAGES_OPENAI_JSON_SCHEMA_NAME, strict: true, schema: klona(format.schema) }; + return { name: ANTHROPIC_MESSAGES_OPENAI_JSON_SCHEMA_NAME, strict: true, schema: format.schema }; }; diff --git a/packages/translate/src/shared/anthropic-messages-via/tool-schema.ts b/packages/translate/src/shared/anthropic-messages-via/tool-schema.ts index cf1319e74e..ac698a0558 100644 --- a/packages/translate/src/shared/anthropic-messages-via/tool-schema.ts +++ b/packages/translate/src/shared/anthropic-messages-via/tool-schema.ts @@ -7,10 +7,6 @@ // defaults `parameters` to `{type: 'object', properties: {}}`, so this closes a // real asymmetry. Ref: // https://github.com/caozhiyuan/copilot-api/commit/ad57069826843c5d17d7b0e5ef2f75050128893c -import { klona } from 'klona/json'; -export const normalizeAnthropicMessagesToolInputSchema = (schema: Record): Record => { - const target = klona(schema); - if (target.type !== 'object' || target.properties !== undefined) return target; - return { ...target, properties: {} }; -}; +export const normalizeAnthropicMessagesToolInputSchema = (schema: Record): Record => + schema.type !== 'object' || schema.properties !== undefined ? schema : { ...schema, properties: {} }; diff --git a/packages/translate/src/shared/openai-chat-completions-and-openai-responses/reasoning.ts b/packages/translate/src/shared/openai-chat-completions-and-openai-responses/reasoning.ts index ae342cf769..c3ddd62348 100644 --- a/packages/translate/src/shared/openai-chat-completions-and-openai-responses/reasoning.ts +++ b/packages/translate/src/shared/openai-chat-completions-and-openai-responses/reasoning.ts @@ -1,4 +1,3 @@ -import { klona } from 'klona/json'; import type { OpenAIChatCompletionsReasoningItem } from '@floway-dev/protocols/openai-chat-completions'; import { createRandomOpenAIResponsesItemId, type OpenAIResponsesInputItem, type OpenAIResponsesOutputReasoning, type OpenAIResponsesReasoningItem } from '@floway-dev/protocols/openai-responses'; @@ -42,7 +41,7 @@ export const toOpenAIChatCompletionsReasoningItem = (item: OpenAIChatCompletions }); export const addOpenAIResponsesReasoningToOpenAIChatCompletionsProjection = (projection: OpenAIChatCompletionsReasoningProjection, item: OpenAIChatCompletionsReasoningSourceItem): void => { - projection.items.push({ ...toOpenAIChatCompletionsReasoningItem(item), summary: klona(item.summary) }); + projection.items.push({ ...toOpenAIChatCompletionsReasoningItem(item), summary: item.summary }); const text = item.summary.map(part => part.text).join(''); if (projection.text === undefined && text) projection.text = text; @@ -82,7 +81,7 @@ export const translateOpenAIChatCompletionsReasoningItems = (hasReadableSummary(item) - ? [{ ...toOpenAIResponsesReasoningItem(item), summary: klona(item.summary ?? []) } as T] + ? [{ ...toOpenAIResponsesReasoningItem(item), summary: item.summary ?? [] } as T] : [])); return translated.length > 0 ? translated : null; }; diff --git a/packages/translate/src/shared/via-anthropic-messages/cache-breakpoints.ts b/packages/translate/src/shared/via-anthropic-messages/cache-breakpoints.ts index c0e9615c3c..36fb483d7f 100644 --- a/packages/translate/src/shared/via-anthropic-messages/cache-breakpoints.ts +++ b/packages/translate/src/shared/via-anthropic-messages/cache-breakpoints.ts @@ -21,36 +21,32 @@ // - https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching import type { - AnthropicMessagesAssistantContentBlock, AnthropicMessagesCacheControl, AnthropicMessagesMessage, AnthropicMessagesTextBlock, AnthropicMessagesTool, - AnthropicMessagesUserContentBlock, } from '@floway-dev/protocols/anthropic-messages'; const ephemeralCacheControl = (): AnthropicMessagesCacheControl => ({ type: 'ephemeral' }); -export const applyLastToolCacheBreakpoint = (tools: AnthropicMessagesTool[] | undefined): void => { - if (!tools || tools.length === 0) return; +export const withLastToolCacheBreakpoint = (tools: AnthropicMessagesTool[] | undefined): AnthropicMessagesTool[] | undefined => { + if (!tools || tools.length === 0) return tools; for (let i = tools.length - 1; i >= 0; i--) { const tool = tools[i]; // Native web-search tools carry a `web_search_*` discriminant and are not // part of the stable prefix, so the breakpoint lands on the last custom // tool instead. if (!tool.type || tool.type === 'custom') { - tool.cache_control = ephemeralCacheControl(); - return; + return tools.with(i, { ...tool, cache_control: ephemeralCacheControl() }); } } + return tools; }; -export const applyLastSystemCacheBreakpoint = (system: AnthropicMessagesTextBlock[] | undefined): void => { - if (!system || system.length === 0) return; - system[system.length - 1].cache_control = ephemeralCacheControl(); -}; +export const withLastSystemCacheBreakpoint = (system: AnthropicMessagesTextBlock[] | undefined): AnthropicMessagesTextBlock[] | undefined => + system?.length ? system.with(system.length - 1, { ...system[system.length - 1], cache_control: ephemeralCacheControl() }) : system; -export const applyLastMessageCacheBreakpoint = (messages: AnthropicMessagesMessage[]): void => { +export const withLastMessageCacheBreakpoint = (messages: AnthropicMessagesMessage[]): AnthropicMessagesMessage[] => { for (let m = messages.length - 1; m >= 0; m--) { const message = messages[m]; @@ -58,16 +54,16 @@ export const applyLastMessageCacheBreakpoint = (messages: AnthropicMessagesMessa // AnthropicMessagesTextBlock is valid in the user, assistant, and system content // unions, so the union cast lets one literal serve any of the three roles. const block: AnthropicMessagesTextBlock = { type: 'text', text: message.content, cache_control: ephemeralCacheControl() }; - message.content = [block] as AnthropicMessagesUserContentBlock[] | AnthropicMessagesAssistantContentBlock[] | AnthropicMessagesTextBlock[]; - return; + return messages.with(m, { ...message, content: [block] } as AnthropicMessagesMessage); } for (let b = message.content.length - 1; b >= 0; b--) { const block = message.content[b]; if (block.type === 'text' || block.type === 'image' || block.type === 'tool_use' || block.type === 'tool_result') { - block.cache_control = ephemeralCacheControl(); - return; + const content = message.content.map((part, index) => index === b ? { ...part, cache_control: ephemeralCacheControl() } : part); + return messages.with(m, { ...message, content } as AnthropicMessagesMessage); } } } + return messages; }; diff --git a/packages/translate/src/types.ts b/packages/translate/src/types.ts index 545654523a..6f404b693b 100644 --- a/packages/translate/src/types.ts +++ b/packages/translate/src/types.ts @@ -16,7 +16,7 @@ export type RemoteImageLoader = (url: string) => Promise * * The client's stream preference is intentionally not in this context. * Translation always emits `stream: true` on the target payload; the LLM - * upstream layer enforces SSE streaming and source `respond.ts` boundaries + * upstream layer enforces SSE streaming and source protocol edges * collect a non-streamed downstream response when the client did not ask * for SSE. */ @@ -24,35 +24,20 @@ export type TranslationContext = { readonly model: string; } & TExtras; -/** - * A wire-shaped upstream error body handed to `TranslateTrip.apiError`. The - * pair returns a same-shaped object to rewrite the outbound envelope, or - * `undefined` to pass it through unchanged. The provider layer's - * `ApiErrorResult` shape is intentionally not imported here — translate is a - * leaf below provider, so we express the contract in bare HTTP-response - * primitives and let the gateway compose it back into an `ApiErrorResult`. - */ +/** HTTP primitives supplied to a translation pair's optional error rewriter. Provider + * runtime types stay outside this package; the gateway owns the default source envelope. */ export interface TranslatedApiError { readonly status: number; readonly headers: Headers; readonly body: Uint8Array; } -/** - * What one translation trip hands back: the target payload, an events - * translator closure mapping target-protocol events into source-protocol - * events, and an optional upstream-error rewriter. +/** A translation's immutable target and trip-scoped response translators. * - * The target owns its mutable JSON objects and arrays. Downstream rules and - * providers can mutate it while the source request and other trips remain - * live, so pair builders clone values borrowed from their source. - * - * `apiError` is optional: when the target upstream returns a non-2xx HTTP - * body (rather than an SSE stream), the pair may rewrite it into the source - * protocol's envelope. Returning `undefined` — or omitting the field - * entirely — passes the upstream body through verbatim, which is what most - * pairs want. - */ + * Source and target values are immutable. Translation rebuilds changed protocol nodes and + * shares unchanged subtrees; downstream rules and providers rebuild the paths they change. + * An omitted error rewriter, or an undefined result, leaves source-envelope rendering to + * the gateway. An explicit result supplies the pair's protocol-specific error mapping. */ export interface TranslateTripResult { target: TgtPayload; events: (frames: AsyncIterable>) => AsyncIterable>; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f8863dbe4f..ef03e96fcd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -363,9 +363,6 @@ importers: '@floway-dev/http': specifier: workspace:* version: link:../http - '@floway-dev/interceptor': - specifier: workspace:* - version: link:../interceptor '@floway-dev/pipeline': specifier: workspace:* version: link:../pipeline @@ -458,12 +455,6 @@ importers: specifier: ^2.2.0 version: 2.2.0 - packages/interceptor: - devDependencies: - '@floway-dev/test-utils': - specifier: workspace:* - version: link:../test-utils - packages/pipeline: dependencies: '@noble/hashes': @@ -541,9 +532,6 @@ importers: '@floway-dev/http': specifier: workspace:* version: link:../http - '@floway-dev/interceptor': - specifier: workspace:* - version: link:../interceptor '@floway-dev/pipeline': specifier: workspace:* version: link:../pipeline @@ -560,9 +548,12 @@ importers: packages/provider-claude-code: dependencies: - '@floway-dev/interceptor': + '@floway-dev/http': + specifier: workspace:* + version: link:../http + '@floway-dev/pipeline': specifier: workspace:* - version: link:../interceptor + version: link:../pipeline '@floway-dev/protocols': specifier: workspace:* version: link:../protocols @@ -585,9 +576,6 @@ importers: '@floway-dev/http': specifier: workspace:* version: link:../http - '@floway-dev/interceptor': - specifier: workspace:* - version: link:../interceptor '@floway-dev/pipeline': specifier: workspace:* version: link:../pipeline @@ -610,9 +598,6 @@ importers: '@floway-dev/http': specifier: workspace:* version: link:../http - '@floway-dev/interceptor': - specifier: workspace:* - version: link:../interceptor '@floway-dev/pipeline': specifier: workspace:* version: link:../pipeline @@ -706,6 +691,9 @@ importers: '@floway-dev/pipeline': specifier: workspace:* version: link:../pipeline + '@floway-dev/protocols': + specifier: workspace:* + version: link:../protocols '@floway-dev/provider': specifier: workspace:* version: link:../provider @@ -715,9 +703,6 @@ importers: '@floway-dev/protocols': specifier: workspace:* version: link:../protocols - klona: - specifier: 2.0.6 - version: 2.0.6 devDependencies: '@floway-dev/test-utils': specifier: workspace:*