From 3214cdb918a6552042f2b3052a782f48f06960a2 Mon Sep 17 00:00:00 2001 From: Menci Date: Sun, 16 Aug 2026 20:14:11 +0800 Subject: [PATCH 01/79] feat(gateway): add the fact space and shared stages the families run on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first half of migrating the non-chat families: the space every family's pipeline extends, the services a run is given, and the two stages every family's serve pipeline needs. Neither shared stage names a family's own key. Branching is a capability of the framework and what something branches on is a concept in the domain, so a family hands in the two domain-shaped things — how to narrow a candidate, and how to read an attempt's outcome — and the stages stay written against the shared space. That is what lets them compose into a pipeline over any family's larger space with no variance question to lose, since assembly reasons over declarations and declarations are strings. `resolveCandidates` carries both traits, and it is what found the modelling error corrected in the commit below it: refusing a request no upstream can serve means answering with a key the descend path never carries. The families themselves follow. Nothing is routed through this yet, so the gateway's behaviour is unchanged. --- packages/gateway/package.json | 1 + .../gateway/src/data-plane/pipeline/facts.ts | 65 ++++++++ .../src/data-plane/pipeline/services.ts | 25 ++++ .../gateway/src/data-plane/pipeline/stages.ts | 141 ++++++++++++++++++ pnpm-lock.yaml | 3 + 5 files changed, 235 insertions(+) create mode 100644 packages/gateway/src/data-plane/pipeline/facts.ts create mode 100644 packages/gateway/src/data-plane/pipeline/services.ts create mode 100644 packages/gateway/src/data-plane/pipeline/stages.ts diff --git a/packages/gateway/package.json b/packages/gateway/package.json index 69fd159774..3baff3e40a 100644 --- a/packages/gateway/package.json +++ b/packages/gateway/package.json @@ -42,6 +42,7 @@ "@floway-dev/agent-setup": "workspace:*", "@floway-dev/http": "workspace:*", "@floway-dev/interceptor": "workspace:*", + "@floway-dev/pipeline": "workspace:*", "@floway-dev/platform": "workspace:*", "@floway-dev/protocols": "workspace:*", "@floway-dev/provider": "workspace:*", diff --git a/packages/gateway/src/data-plane/pipeline/facts.ts b/packages/gateway/src/data-plane/pipeline/facts.ts new file mode 100644 index 0000000000..df942de0fa --- /dev/null +++ b/packages/gateway/src/data-plane/pipeline/facts.ts @@ -0,0 +1,65 @@ +// The keys every family's pipeline shares. A family extends this space with its own +// protocol keys by intersection and never merges into it, so a stage written here drops +// into any family's pipeline and a family's own keys are unreachable from a stage that +// was not written against them. +// +// Keys are namespaced, camelCase and family-first. `ingress.*` is what the client sent and +// stays put across a protocol switch; `serve.*` belongs to the served request as a whole +// and outlives an attempt; `request.*` and `response.*` are the two directions, mirrored +// key for key and semantically disjoint. + +import type { UsageQuantities } from '../../repo/types.ts'; +import type { Secret } from '@floway-dev/pipeline'; +import type { ModelCandidate, TelemetryModelIdentity } from '@floway-dev/provider'; + +/** What an upstream call is answerable for. Keyed by billed entity, because one call can + * bill in units that are not commensurable, and an entity with no quantities at all is + * how "the upstream was called and reported nothing" is said. */ +export interface BillableEntity { + readonly identity: TelemetryModelIdentity; + readonly quantities: UsageQuantities; +} + +/** A failure is a value, never a throw, so an earlier stage can fail over a later stage's + * fault — even a 400, because the path and the flags may differ on the next candidate. */ +export interface Failure { + readonly status: number; + readonly message: string; + /** The upstream's own body, when there was one. A client is not owed the upstream's + * exact bytes, but a dump reader is owed what actually came back. */ + readonly body?: unknown; +} + +export const isFailure = (value: unknown): value is Failure => + typeof value === 'object' && value !== null && 'status' in value && 'message' in value; + +export interface GatewayFacts { + /** What the client sent, before anything read it. Recorded on every request, so the + * dump can always show it. */ + 'ingress.http.method': string; + 'ingress.http.path': string; + 'ingress.http.headers': readonly (readonly [string, string])[]; + 'ingress.http.body': Uint8Array; + + /** The public model id the client asked for, and the candidates it resolves to. Nothing + * consumes these: they outlive an attempt. */ + 'serve.model': string; + 'serve.candidates': readonly ModelCandidate[]; + + /** Which upstream this attempt targets. Provided per attempt by the stage that forks. */ + 'route.candidate': ModelCandidate; + + /** There is exactly one url and one headers. Headers are rewritten the whole way down, + * so the dump shows a header's entire history in one place, and a value may be secret. */ + 'request.http.url': string; + 'request.http.headers': readonly (readonly [string, string | Secret])[]; + + 'response.http.status': number; + 'response.http.headers': readonly (readonly [string, string])[]; + /** The upstream's body, still open. Whoever declares it consumed owns draining it. */ + 'response.http.body': ReadableStream & AsyncDisposable; + + /** The authoritative reading, provided closest to the upstream on the dialect it + * actually spoke. Every step that changes usage re-provides it. */ + 'response.usage.billable': readonly BillableEntity[]; +} diff --git a/packages/gateway/src/data-plane/pipeline/services.ts b/packages/gateway/src/data-plane/pipeline/services.ts new file mode 100644 index 0000000000..2aaf285960 --- /dev/null +++ b/packages/gateway/src/data-plane/pipeline/services.ts @@ -0,0 +1,25 @@ +// What a run is given beside its facts. Services are wiring: they are fixed for the run at +// the prologue and never change on a handoff, because what a stage hands to `next` is the +// next segment's facts — if it also supplied capabilities, the same pipeline value would +// run with different capabilities depending on who called it. +// +// Everything in facts is dumpable, and that is the test: a live handle dumps as nothing, +// so a live handle is never a fact. + +import type { GatewayCtx } from '../shared/gateway-ctx.ts'; +import type { Event, Logger } from '@floway-dev/pipeline'; + +export interface GatewayServices { + /** The global sink. Every stage's lines reach it, tagged with the stage's name. */ + readonly log?: Logger; + /** Present only when this request is being dumped, which is what keeps recording + * conditional: with no sink resolved here, the runner does none of it. */ + readonly dump?: (event: Event) => void; + + /** The request-scoped context the settlement and telemetry stages read. It is a service + * and not a fact because it holds live handles — the scheduler, the abort signal. */ + readonly gateway: GatewayCtx; + /** Binds a promise to the request's lifetime: `waitUntil` on Workers, the event loop on + * Node. What the drain is handed to, so an answer is not held up by it. */ + readonly background: (work: Promise) => void; +} diff --git a/packages/gateway/src/data-plane/pipeline/stages.ts b/packages/gateway/src/data-plane/pipeline/stages.ts new file mode 100644 index 0000000000..1f64161103 --- /dev/null +++ b/packages/gateway/src/data-plane/pipeline/stages.ts @@ -0,0 +1,141 @@ +// Resolving a model to the upstreams that can serve it, and running the suffix once per +// candidate until one answers. Both are ordinary stages: nothing in the framework knows +// what a retry is, and array position is the whole of the statement that what follows +// `failover` is the per-attempt segment. +// +// Neither names a family's own key. Branching is a capability of the framework and what +// something branches on is a concept in the domain, so the family hands in the two +// domain-shaped things — how to narrow a candidate, and how to read an attempt's outcome — +// and these stages stay written against the shared space alone. That is also what lets +// them compose into a pipeline over any family's larger space with no variance question to +// lose: assembly reasons over declarations, which are strings. + +import type { GatewayFacts } from './facts.ts'; +import type { GatewayServices } from './services.ts'; +import { enumerateModelCandidates } from '../providers/resolution.ts'; +import { appendFailedUpstreams } from '../shared/failed-upstreams.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { Facts } from '@floway-dev/pipeline'; +import type { ModelKind } from '@floway-dev/protocols/common'; +import type { ModelCandidate } from '@floway-dev/provider'; + +type Slice = { [P in K]: GatewayFacts[P] }; + +/** What a family narrows its candidates by, and what it says when nothing is left. A + * candidate that resolves but cannot serve this request — no endpoint for the kind, or a + * payload the target protocol cannot express — is not a candidate, and saying why is what + * turns an empty list into a usable 400 rather than a bare 404. */ +export interface Narrowing { + readonly kind: ModelKind; + /** Keeps a candidate, or says in one phrase why it cannot serve this request. */ + readonly reject: (candidate: ModelCandidate) => string | null; + /** What this family answers with when there is no candidate to try. The family decides + * which of its own keys carries the failure, which is why the refusal slice is a type + * parameter rather than a shared key: a reusable stage whose short-circuit provides + * family-specific keys cannot be written against the shared space alone. */ + readonly refuse: (status: number, message: string) => Refusal; + /** The same keys as strings, so assembly can check that a short-circuit here covers what + * the stages above it need. */ + readonly refuses: readonly (keyof Refusal)[]; +} + +/** + * Provides `serve.candidates`, or answers with the failure that says why there are none — + * both traits, because "no upstream serves this model" is an answer this stage already + * holds and nothing below it could produce one. + */ +export const resolveCandidates = (narrowing: Narrowing) => defineStage< + Slice<'serve.model'>, // what arrives + Slice<'serve.model' | 'serve.candidates'>, // what it hands down + Slice<'response.usage.billable'>, // what comes back + Slice<'response.usage.billable'>, // what it hands up, having descended + Slice<'response.usage.billable'> & Refusal, // and what it answers with instead + GatewayServices +>({ + name: 'resolveCandidates', + through: { + request: { needs: ['serve.model'], consumes: [], provides: ['serve.candidates'] }, + response: { needs: ['response.usage.billable'], consumes: [], provides: [] }, + }, + return: { provides: ['response.usage.billable', ...narrowing.refuses] }, + execute: async (facts, next, use) => { + const model = facts['serve.model']; + const { candidates, sawModel, failedUpstreams } = await enumerateModelCandidates({ + upstreamIds: use.gateway.upstreamIds, + model, + kind: narrowing.kind, + scheduler: use.gateway.backgroundScheduler, + runtimeLocation: use.gateway.runtimeLocation, + }); + + // An empty billed set is what "we did not call an upstream" looks like. The settlement + // stages still run and still write; the row simply names no billed entity. + const refuse = (status: number, message: string) => + move({ ...facts, 'response.usage.billable': [], ...narrowing.refuse(status, message) }); + + if (candidates.length === 0) { + const missing = sawModel + ? `Model ${model} does not support ${narrowing.kind}.` + : `Model ${model} is not available on any configured upstream.`; + return refuse(sawModel ? 400 : 404, appendFailedUpstreams(missing, failedUpstreams)); + } + + const refused = new Set(); + const viable = candidates.filter(candidate => { + const why = narrowing.reject(candidate); + if (why !== null) refused.add(why); + return why === null; + }); + if (viable.length === 0) { + return refuse(400, appendFailedUpstreams( + `Model ${model} does not support this request: ${[...refused].join('; ')}.`, + failedUpstreams, + )); + } + + use.log.debug('resolved candidates', { model, viable: viable.length, resolved: candidates.length }); + return await next({ ...facts, 'serve.candidates': move(viable) }); + }, +}); + +/** + * Runs what follows it once per candidate and returns the first that did not fail. + * + * The losing attempts' bodies are its own — that is what `consumes` declares on the way up + * — and the winner's rides onward, which is what `provides` declares. Failover never + * breaks a stream: once one has opened there is nothing to fail over to, and the family's + * edge is what settles that by sitting above this stage rather than below it. + */ +export const failover = (failed: (handedUp: Facts) => boolean) => defineStage< + Slice<'serve.candidates'>, + Slice<'serve.candidates' | 'route.candidate'>, + Slice<'response.http.body' | 'response.usage.billable'>, + Slice<'response.http.body' | 'response.usage.billable'>, + GatewayServices +>({ + name: 'failover', + through: { + request: { needs: ['serve.candidates'], consumes: [], provides: ['route.candidate'] }, + response: { + needs: ['response.usage.billable'], + consumes: ['response.http.body'], + provides: ['response.http.body'], + }, + }, + execute: async (facts, next, use) => { + let last: Slice<'response.http.body' | 'response.usage.billable'> | undefined; + for (const candidate of facts['serve.candidates']) { + // Per-attempt telemetry state, cleared before control leaves, so a mid-attempt throw + // still attributes its performance row to the candidate that was being tried. + use.gateway.attempt.upstreamCallStartedAt = null; + use.gateway.attempt.firstOutputTokenAt = null; + last = await next({ ...facts, 'route.candidate': move(candidate) }); + if (!failed(last as Facts)) return last; + use.log.info('candidate failed, trying the next', { upstream: candidate.provider.upstreamId }); + } + if (last === undefined) throw new Error('failover: assembly handed it an empty candidate list'); + // Every candidate failed, and the last failure is the base — so the client sees real + // upstream telemetry rather than a synthesized gateway envelope. + return last; + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b0b6fd61e6..51094c6e2a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -330,6 +330,9 @@ importers: '@floway-dev/interceptor': specifier: workspace:* version: link:../interceptor + '@floway-dev/pipeline': + specifier: workspace:* + version: link:../pipeline '@floway-dev/platform': specifier: workspace:* version: link:../platform From 72baf7eb8a277e041e5702d7c6f859a687227278 Mon Sep 17 00:00:00 2001 From: Menci Date: Sun, 16 Aug 2026 20:33:13 +0800 Subject: [PATCH 02/79] feat(rerank): serve rerank as a pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first family, and the one worth doing first: rerank already has a canonical contract — parse, render, serialize, and a usage reading — so it proves the shared stages on real business without needing a protocol written first. Four stages, and the shape every family will repeat. `emitRerank` is the edge: it renders the canonical answer back into whichever of the four protocols the client spoke, which is why the source protocol is an ingress fact and not a request one — it has to survive the switch to whatever the upstream turned out to speak. `callRerankUpstream` is the ending: it dials, parses the body, and provides both the answer and what the call is billable for. Two behaviours change, both because the architecture has no passthrough. The same-protocol path used to forward the upstream's bytes unread; it is now parsed and re-serialized like the translating path, so one code path serves both and the dump shows a parsed answer either way. And an upstream that refused is a value rather than a forwarded Response, so an earlier stage can fail over it. The client's headers reach the upstream from the record rather than from a live request object, which is what lets the dump show what was there to be filtered. The test that matters most is the one that records a hole. `callRerankUpstream` reads `ingress.http.headers`, and the derived entry contract does not mention it: a stage whose only trait is `return` declares no request side, by ruling, so assembly cannot see what an ending stage reads — and every family's ending stage reads something. The consequence is concrete: a caller omitting that key gets a runtime failure at the deepest stage instead of an assembly error, which is the thing the entry contract exists to prevent. The type layer still catches it at the definition site, so it is a gap and not a break. Nothing is routed here yet; the route table still runs the passthrough serve. --- .../data-plane/rerank-pipeline_test.ts | 46 +++++ .../gateway/src/data-plane/rerank/pipeline.ts | 183 ++++++++++++++++++ 2 files changed, 229 insertions(+) create mode 100644 packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts create mode 100644 packages/gateway/src/data-plane/rerank/pipeline.ts diff --git a/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts b/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts new file mode 100644 index 0000000000..4fbf352534 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts @@ -0,0 +1,46 @@ +// Rerank's pipeline, assembled. `compose` derives the entry contract and rejects an array +// that cannot work, so most of what this file establishes is established by the assembly +// succeeding at all — and what is worth writing down is the entry contract it derives, +// including the one key it cannot see. + +import { describe, expect, it } from 'vitest'; + +import { rerankServePipeline } from '../../src/data-plane/rerank/pipeline.ts'; +import { move, run } from '@floway-dev/pipeline'; +import type { CanonicalRerankRequest } from '@floway-dev/protocols/rerank'; + +const request: CanonicalRerankRequest = { + sourceProtocol: 'cohere-v2', + raw: {}, + query: 'what is a pipeline', + documents: ['one', 'two'], +}; + +describe('the rerank pipeline', () => { + it('assembles, and asks its caller for what the descending stages need', () => { + expect([...rerankServePipeline(request).entryNeeds].sort()).toEqual([ + 'ingress.rerank.sourceProtocol', + 'request.rerank.canonical', + 'serve.model', + ]); + }); + + // `callRerankUpstream` reads `ingress.http.headers`, and the entry contract does not + // mention it. That is not this family's defect: a stage whose only trait is `return` + // declares no request side at all, by ruling — "when it short-circuits, only `provides`" + // — so assembly cannot see what an ending stage reads, and every family's ending stage + // reads something. + // + // Written as a test rather than a comment because the hole has a consequence: a caller + // who omits that key gets a runtime failure at the deepest stage instead of an assembly + // error, and the entry contract exists to stop exactly that. The type layer still + // catches it at the definition site, which is why this is a gap and not a break. + it('cannot see what an ending stage reads, because a return-only stage declares no needs', () => { + expect(rerankServePipeline(request).entryNeeds).not.toContain('ingress.http.headers'); + }); + + it('names the entry key a caller did not bring, before any stage runs', async () => { + await expect(run(rerankServePipeline(request), move({ 'serve.model': 'rerank-v3' }) as never, {})) + .rejects.toThrow('run(rerankServe): rerankServe needs'); + }); +}); diff --git a/packages/gateway/src/data-plane/rerank/pipeline.ts b/packages/gateway/src/data-plane/rerank/pipeline.ts new file mode 100644 index 0000000000..869dc12d26 --- /dev/null +++ b/packages/gateway/src/data-plane/rerank/pipeline.ts @@ -0,0 +1,183 @@ +// Rerank as a pipeline. The family with a canonical contract already — parse, render, +// serialize and a usage reading — so it is the one that proves the shared stages on real +// business before the families that need their contracts written first. +// +// The shape every family repeats: +// +// emitRerank the edge: serializes the answer into the client's protocol +// writeSettlement above the fork, so a run bills once however many attempts it made +// resolveCandidates narrows to the upstreams that can serve this request +// failover runs what follows once per candidate +// callRerankUpstream the ending: dials, and provides what came back + +import type { UsageQuantities } from '../../repo/types.ts'; +import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; +import { isFailure } from '../pipeline/facts.ts'; +import type { GatewayServices } from '../pipeline/services.ts'; +import { failover, resolveCandidates } from '../pipeline/stages.ts'; +import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; +import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; +import type { Pipeline } from '@floway-dev/pipeline'; +import { defineStage, move, compose } from '@floway-dev/pipeline'; +import { parseDecimalString, type RerankProtocol, type RerankSourceProtocol } from '@floway-dev/protocols/common'; +import { + parseRerankResponse, + parseRerankUsage, + renderRerankResponse, + rerankRequestIncompatibility, + type CanonicalRerankRequest, + type CanonicalRerankResponse, +} from '@floway-dev/protocols/rerank'; +import { providerModelOf } from '@floway-dev/provider'; + +/** Rerank's own keys. They extend the shared space and never merge into it, so a stage + * written against the gateway alone cannot name one. */ +export interface RerankFacts extends GatewayFacts { + /** Which of the four rerank protocols the client spoke. It belongs to the ingress and + * stays put: the answer is rendered back into it whatever the upstream spoke. */ + 'ingress.rerank.sourceProtocol': RerankSourceProtocol; + 'request.rerank.canonical': CanonicalRerankRequest; + 'response.rerank.canonical': CanonicalRerankResponse | Failure; + /** Which protocol the upstream turned out to speak. A response fact, because it is not + * known until one answered, and the edge needs it to render back into the client's. */ + 'response.rerank.targetProtocol': RerankProtocol; + /** What the client is actually sent, in its own protocol. The edge provides it, so a + * dump shows the bytes the client received rather than the gateway's canonical form. */ + 'response.rerank.rendered': Record; +} + +type R = { [P in K]: RerankFacts[P] }; + +/** + * The outermost edge. Renders the canonical answer into the protocol the client spoke — + * which is why `ingress.rerank.sourceProtocol` is an ingress fact and not a request one: + * it survives the switch to whatever protocol the upstream turned out to speak. + */ +const emitRerank = defineStage< + R<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical'>, + R<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical'>, + R<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical' | 'response.rerank.canonical' | 'response.rerank.targetProtocol'>, + R<'response.rerank.rendered'> +>({ + name: 'emitRerank', + through: { + request: { + needs: ['ingress.rerank.sourceProtocol', 'request.rerank.canonical'], + consumes: [], + provides: [], + }, + response: { + needs: ['response.rerank.canonical'], + consumes: ['response.rerank.canonical', 'response.rerank.targetProtocol'], + provides: ['response.rerank.rendered'], + }, + }, + execute: async (facts, next) => { + const back = await next(facts); + const { 'response.rerank.canonical': answer, 'response.rerank.targetProtocol': target, ...rest } = back; + if (isFailure(answer)) { + return { ...rest, 'response.rerank.rendered': move({ error: { message: answer.message, type: 'api_error' } }) }; + } + return { + ...rest, + 'response.rerank.rendered': move(renderRerankResponse( + back['ingress.rerank.sourceProtocol'], + target, + answer, + back['request.rerank.canonical'], + )), + }; + }, +}); + +/** + * The ending. It dials, reads the upstream's body, and provides the canonical answer and + * what the call is billable for. A failure is a value: a 429 here is what an earlier stage + * fails over, and even a 400 can be, because the next candidate's path and flags may differ. + */ +const callRerankUpstream = defineStage< + R<'request.rerank.canonical' | 'route.candidate' | 'ingress.http.headers'>, + R<'response.rerank.canonical' | 'response.rerank.targetProtocol' | 'response.usage.billable'>, + GatewayServices +>({ + name: 'callRerankUpstream', + return: { + provides: ['response.rerank.canonical', 'response.rerank.targetProtocol', 'response.usage.billable'], + }, + execute: async (facts, use) => { + const candidate = facts['route.candidate']; + const request = facts['request.rerank.canonical']; + const result = await candidate.provider.instance.callRerank( + providerModelOf(candidate), + request, + use.gateway.abortSignal, + // The client's own headers reach the upstream from the record, not from a live + // request object: what a provider is allowed to forward is filtered per provider, + // and the dump shows what was there to filter. + buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]) => [name, value]))), + ); + use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'rerank'); + const identity = telemetryModelIdentity(candidate, result.modelKey); + + if (!result.response.ok) { + use.log.warn('upstream refused', { status: result.response.status }); + return move({ + ...facts, + 'response.rerank.canonical': { + status: result.response.status, + message: await result.response.text(), + }, + 'response.rerank.targetProtocol': result.target.protocol, + // The upstream was called and reported nothing, which is a different situation + // from reporting zero — so the entity is present with no quantities. + 'response.usage.billable': [{ identity, quantities: {} }], + }); + } + + // Every protocol the gateway carries is one it fully understands: the body is parsed + // here and re-serialized at the edge, whether or not the two protocols match. + const body = await result.response.json() as unknown; + const canonical = parseRerankResponse(result.target.protocol, body); + const usage = parseRerankUsage(result.target.protocol, body); + return move({ + ...facts, + 'response.rerank.canonical': canonical, + 'response.rerank.targetProtocol': result.target.protocol, + 'response.usage.billable': [{ identity, quantities: billed(usage) }], + }); + }, +}); + +const billed = (usage: Pick | undefined): UsageQuantities => { + const quantities: UsageQuantities = {}; + if (usage?.searchUnits !== undefined) quantities.rerank_searches = parseDecimalString(String(usage.searchUnits)); + if (usage?.totalTokens !== undefined) quantities.input_tokens = parseDecimalString(String(usage.totalTokens)); + return quantities; +}; + +/** A candidate that cannot serve *this* request is not a candidate. Saying why is what + * turns an empty list into a 400 a client can act on. */ +const narrowing = (request: CanonicalRerankRequest) => ({ + kind: 'rerank' as const, + reject: (candidate: Parameters[0]) => { + const model = providerModelOf(candidate); + if (candidate.model.endpoints.rerank === undefined || model.rerankTarget === undefined) { + return 'the upstream does not expose a rerank endpoint'; + } + return rerankRequestIncompatibility(model.rerankTarget.protocol, request); + }, + refuse: (status: number, message: string) => ({ 'response.rerank.canonical': { status, message } }), + refuses: ['response.rerank.canonical'] as const, +}); + +export const rerankServePipeline = (request: CanonicalRerankRequest): Pipeline< + R<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical' | 'serve.model'>, + R<'response.rerank.rendered' | 'response.usage.billable'> +> => compose('rerankServe', [ + emitRerank, + resolveCandidates(narrowing(request)), + failover(handedUp => isFailure((handedUp as { 'response.rerank.canonical'?: unknown })['response.rerank.canonical'])), + callRerankUpstream, +]); + +export type { BillableEntity }; From 682c89e206c4fb0deffe23daf14a4545ba08c81b Mon Sep 17 00:00:00 2001 From: Menci Date: Sun, 16 Aug 2026 21:04:59 +0800 Subject: [PATCH 03/79] feat(embeddings): serve embeddings as a pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The simplest family, and the one that shows the shape with nothing else in the way: one protocol, no translation and no stream, so the array is exactly the four stages every family has. The narrowing is a constant rather than a function of the request, because an embeddings request carries nothing a candidate could be incompatible with. `packages/protocols/src/embeddings` was five lines of index signature — a passthrough shape, not a contract — so the real one is written here: the request's four input arms, `encoding_format`, `dimensions` and `user`, and the response's embeddings, model and usage, each cited against the OpenAI specification. `encoding_format` is why the canonical response holds numbers rather than whatever the upstream wrote. Both official OpenAI SDKs send `encoding_format: base64` when their caller did not choose one, so base64 is the common case on the wire; an upstream that ignores the field and answers with float arrays hands such a client a body it decodes as base64 and turns into noise. Parsing whatever arrived and writing what the client asked for is what closes that, which is why the encoding is an ingress fact and stays put. A base64 embedding is the vector's float32 elements packed little-endian, and reading it into numbers is exact: every float32 is a float64, so a vector survives any number of trips. Two behaviours change. A field outside the protocol is now named in a 400 rather than forwarded, because the request schema is `additionalProperties: false` and both gateways compared against — LiteLLM and copilot-api — draw the same line; this is the one decision worth a second look, since vLLM and Jina document supersets of the endpoint. And an upstream that reports no usage now bills an entity with no quantities, which is how "called and reported nothing" is said, rather than no row at all. The test records what assembly cannot see, and this family shows it twice over: a return-only stage declares no request side, so neither `ingress.http.headers` nor `request.embeddings.canonical` reaches the derived entry contract — and unlike rerank, whose edge needed the payload for rendering, nothing here puts the payload in the contract by accident. The route still runs the passthrough serve. The old handler parses and serializes through the new contract so the two agree until it is deleted. --- .../data-plane/embeddings-pipeline_test.ts | 42 ++++ .../gateway/src/data-plane/embeddings/http.ts | 36 ++-- .../src/data-plane/embeddings/pipeline.ts | 166 ++++++++++++++++ .../__tests__/embeddings/translate_test.ts | 151 ++++++++++++++ packages/protocols/src/embeddings/index.ts | 99 +++++++++- .../protocols/src/embeddings/translate.ts | 184 ++++++++++++++++++ 6 files changed, 665 insertions(+), 13 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/embeddings-pipeline_test.ts create mode 100644 packages/gateway/src/data-plane/embeddings/pipeline.ts create mode 100644 packages/protocols/__tests__/embeddings/translate_test.ts create mode 100644 packages/protocols/src/embeddings/translate.ts diff --git a/packages/gateway/__tests__/data-plane/embeddings-pipeline_test.ts b/packages/gateway/__tests__/data-plane/embeddings-pipeline_test.ts new file mode 100644 index 0000000000..c42d762300 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/embeddings-pipeline_test.ts @@ -0,0 +1,42 @@ +// Embeddings' pipeline, assembled. `compose` derives the entry contract and rejects an +// array that cannot work, so most of what this file establishes is established by the +// assembly succeeding at all — and what is worth writing down is the entry contract it +// derives, including the two keys it cannot see. + +import { describe, expect, it } from 'vitest'; + +import { embeddingsServePipeline } from '../../src/data-plane/embeddings/pipeline.ts'; +import { move, run } from '@floway-dev/pipeline'; + +describe('the embeddings pipeline', () => { + it('assembles, and asks its caller for what the descending stages need', () => { + expect([...embeddingsServePipeline.entryNeeds].sort()).toEqual([ + 'ingress.embeddings.encodingFormat', + 'serve.model', + ]); + }); + + // Rerank records the same hole against `ingress.http.headers`; embeddings shows it twice + // over, and the second one is the sharper case. A stage whose only trait is `return` + // declares no request side at all, by ruling — "when it short-circuits, only `provides`" + // — so assembly cannot see what an ending stage reads. `callEmbeddingsUpstream` reads + // both the headers and the request payload, and neither reaches the entry contract. + // + // Rerank's edge happened to need `request.rerank.canonical` for rendering, which put it + // in the contract for an unrelated reason. This family's edge needs only the encoding, + // so nothing above the ending stage names the payload — and the pipeline that cannot run + // without it does not ask for it. Written as a test rather than a comment because the + // hole has a consequence: a caller who omits either key gets a runtime failure at the + // deepest stage instead of an assembly error, and the entry contract exists to stop + // exactly that. The type layer still catches it at the definition site, which is why + // this is a gap and not a break. + it('cannot see what an ending stage reads, because a return-only stage declares no needs', () => { + expect(embeddingsServePipeline.entryNeeds).not.toContain('request.embeddings.canonical'); + expect(embeddingsServePipeline.entryNeeds).not.toContain('ingress.http.headers'); + }); + + it('names the entry key a caller did not bring, before any stage runs', async () => { + await expect(run(embeddingsServePipeline, move({ 'serve.model': 'text-embedding-3-small' }) as never, {})) + .rejects.toThrow('run(embeddingsServe): embeddingsServe needs'); + }); +}); diff --git a/packages/gateway/src/data-plane/embeddings/http.ts b/packages/gateway/src/data-plane/embeddings/http.ts index de5c06801a..253209808c 100644 --- a/packages/gateway/src/data-plane/embeddings/http.ts +++ b/packages/gateway/src/data-plane/embeddings/http.ts @@ -1,37 +1,51 @@ // POST /v1/embeddings — route embedding requests to the provider that // declares the requested model and embeddings capability. +// +// Transitional: the request is already parsed against the embeddings contract and written +// again for the upstream, which is what the pipeline does, while the response is still +// forwarded by the passthrough scaffold. The whole file goes when the route is wired to +// `embeddingsServePipeline`. import type { Context } from 'hono'; import { tokenUsageFromEmbeddingsBody } from './usage.ts'; import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; import { createGatewayCtxFromHono, finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; -import { prepareJsonModelRequest } from '../shared/passthrough-request.ts'; import { passthroughApiError, passthroughServe } from '../shared/passthrough-serve.ts'; import { readRequestBody, takeRequestBody } from '../shared/request-body.ts'; +import { parseEmbeddingsRequest, serializeEmbeddingsRequest, type ParsedEmbeddingsRequest } from '@floway-dev/protocols/embeddings'; + +// The contract reports a malformed request by throwing; what the client is owed is a 400 +// carrying the reason. Reading has to finish before the context takes the bytes. +const readRequest = (bytes: Uint8Array): { type: 'ok'; parsed: ParsedEmbeddingsRequest } | { type: 'invalid'; message: string } => { + try { + return { type: 'ok', parsed: parseEmbeddingsRequest(JSON.parse(new TextDecoder().decode(bytes)) as unknown) }; + } catch (e) { + return { type: 'invalid', message: e instanceof Error ? e.message : String(e) }; + } +}; export const embeddings = async (c: Context): Promise => { const requestBody = await readRequestBody(c); - const request = prepareJsonModelRequest(requestBody.bytes, 'Embeddings'); + const result = readRequest(requestBody.bytes); const ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - if (request.type === 'invalid') { + if (result.type === 'invalid') { ctx.dump?.error('gateway'); - return finalizeGatewayResponse(ctx, passthroughApiError(c, request.message, 400)); + return finalizeGatewayResponse(ctx, passthroughApiError(c, result.message, 400)); } + const { model, request } = result.parsed; - ctx.dump?.requestedModel(request.model); + ctx.dump?.requestedModel(model); const response = await passthroughServe({ c, ctx, sourceApi: '/embeddings', operation: 'embeddings', - model: request.model, + model, kind: 'embedding', - modelServesEndpoint: model => model.endpoints.embeddings !== undefined, - call: async (provider, model, opts) => { - const { model: _model, ...body } = request.body; - return await provider.instance.callEmbeddings(model, body, undefined, opts); - }, + modelServesEndpoint: candidate => candidate.endpoints.embeddings !== undefined, + call: async (provider, providerModel, opts) => + await provider.instance.callEmbeddings(providerModel, serializeEmbeddingsRequest(request), undefined, opts), response: { format: 'json', extractBilling: tokenUsageFromEmbeddingsBody }, }); return finalizeGatewayResponse(ctx, response); diff --git a/packages/gateway/src/data-plane/embeddings/pipeline.ts b/packages/gateway/src/data-plane/embeddings/pipeline.ts new file mode 100644 index 0000000000..d390c8454d --- /dev/null +++ b/packages/gateway/src/data-plane/embeddings/pipeline.ts @@ -0,0 +1,166 @@ +// Embeddings as a pipeline. The smallest family there is: one protocol, no translation and +// no stream — so what is left is the four stages every family has, and nothing else. +// +// emitEmbeddings the edge: writes the answer in the encoding the client asked for +// resolveCandidates narrows to the upstreams that can serve this request +// failover runs what follows once per candidate +// callEmbeddingsUpstream the ending: dials, and provides what came back +// +// The narrowing is a constant rather than a function of the request, and that is what makes +// this the simple family: an embeddings request carries nothing a candidate could be +// incompatible with, so the only question is whether the upstream exposes the endpoint. + +import type { UsageQuantities } from '../../repo/types.ts'; +import type { Failure, GatewayFacts } from '../pipeline/facts.ts'; +import { isFailure } from '../pipeline/facts.ts'; +import type { GatewayServices } from '../pipeline/services.ts'; +import { failover, resolveCandidates, type Narrowing } from '../pipeline/stages.ts'; +import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; +import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; +import type { Pipeline } from '@floway-dev/pipeline'; +import { compose, defineStage, move } from '@floway-dev/pipeline'; +import { parseDecimalString } from '@floway-dev/protocols/common'; +import { + parseEmbeddingsResponse, + renderEmbeddingsResponse, + serializeEmbeddingsRequest, + type CanonicalEmbeddingsRequest, + type CanonicalEmbeddingsResponse, + type CanonicalEmbeddingsUsage, + type EmbeddingsEncodingFormat, +} from '@floway-dev/protocols/embeddings'; +import { providerModelOf } from '@floway-dev/provider'; + +/** Embeddings' own keys. They extend the shared space and never merge into it, so a stage + * written against the gateway alone cannot name one. */ +export interface EmbeddingsFacts extends GatewayFacts { + /** Which encoding the client is able to read. It belongs to the ingress and stays put: + * the answer is written in it whichever encoding the upstream chose to answer in, and + * the two differ whenever an upstream ignores the field. */ + 'ingress.embeddings.encodingFormat': EmbeddingsEncodingFormat; + 'request.embeddings.canonical': CanonicalEmbeddingsRequest; + 'response.embeddings.canonical': CanonicalEmbeddingsResponse | Failure; + /** What the client is actually sent. The edge provides it, so a dump shows the bytes the + * client received rather than the gateway's canonical form. */ + 'response.embeddings.rendered': Record; +} + +type E = { [P in K]: EmbeddingsFacts[P] }; + +/** + * The outermost edge. Writes the vectors in the encoding the client asked for — which is + * why the encoding is an ingress fact and not a request one: it has to survive an upstream + * that answered in the other one. A client on an official OpenAI SDK asked for `base64` + * without its caller choosing to, and will decode as base64 whatever it is handed. + */ +const emitEmbeddings = defineStage< + E<'ingress.embeddings.encodingFormat'>, + E<'ingress.embeddings.encodingFormat'>, + E<'ingress.embeddings.encodingFormat' | 'response.embeddings.canonical'>, + E<'response.embeddings.rendered'> +>({ + name: 'emitEmbeddings', + through: { + request: { + needs: ['ingress.embeddings.encodingFormat'], + consumes: [], + provides: [], + }, + response: { + needs: ['response.embeddings.canonical'], + consumes: ['response.embeddings.canonical'], + provides: ['response.embeddings.rendered'], + }, + }, + execute: async (facts, next) => { + const back = await next(facts); + const { 'response.embeddings.canonical': answer, ...rest } = back; + if (isFailure(answer)) { + return { ...rest, 'response.embeddings.rendered': move({ error: { message: answer.message, type: 'api_error' } }) }; + } + return { + ...rest, + 'response.embeddings.rendered': move(renderEmbeddingsResponse(back['ingress.embeddings.encodingFormat'], answer)), + }; + }, +}); + +/** + * The ending. It dials, reads the upstream's body, and provides the canonical answer and + * what the call is billable for. A failure is a value: a 429 here is what an earlier stage + * fails over, and even a 400 can be, because the next candidate's path and flags may differ. + */ +const callEmbeddingsUpstream = defineStage< + E<'request.embeddings.canonical' | 'route.candidate' | 'ingress.http.headers'>, + E<'response.embeddings.canonical' | 'response.usage.billable'>, + GatewayServices +>({ + name: 'callEmbeddingsUpstream', + return: { + provides: ['response.embeddings.canonical', 'response.usage.billable'], + }, + execute: async (facts, use) => { + const candidate = facts['route.candidate']; + const result = await candidate.provider.instance.callEmbeddings( + providerModelOf(candidate), + serializeEmbeddingsRequest(facts['request.embeddings.canonical']), + use.gateway.abortSignal, + // The client's own headers reach the upstream from the record, not from a live + // request object: what a provider is allowed to forward is filtered per provider, + // and the dump shows what was there to filter. + buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]) => [name, value]))), + ); + use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'embeddings'); + const identity = telemetryModelIdentity(candidate, result.modelKey); + + if (!result.response.ok) { + use.log.warn('upstream refused', { status: result.response.status }); + return move({ + ...facts, + 'response.embeddings.canonical': { + status: result.response.status, + message: await result.response.text(), + }, + // The upstream was called and reported nothing, which is a different situation + // from reporting zero — so the entity is present with no quantities. + 'response.usage.billable': [{ identity, quantities: {} }], + }); + } + + // Every protocol the gateway carries is one it fully understands: the body is parsed + // here and written again at the edge, in whichever encoding the client can read. + const canonical = parseEmbeddingsResponse(await result.response.json() as unknown); + return move({ + ...facts, + 'response.embeddings.canonical': canonical, + 'response.usage.billable': [{ identity, quantities: billed(canonical.usage) }], + }); + }, +}); + +// An embeddings call has no output side, so `prompt_tokens` is the whole of what the +// upstream metered and `total_tokens` restates it. An upstream that reported nothing bills +// nothing, and says so by leaving the entity's quantities empty. +const billed = (usage: CanonicalEmbeddingsUsage | undefined): UsageQuantities => + usage === undefined ? {} : { input_tokens: parseDecimalString(String(usage.promptTokens)) }; + +/** A candidate that cannot serve *this* request is not a candidate. Saying why is what + * turns an empty list into a 400 a client can act on. */ +const narrowing: Narrowing> = { + kind: 'embedding', + reject: candidate => candidate.model.endpoints.embeddings === undefined + ? 'the upstream does not expose an embeddings endpoint' + : null, + refuse: (status, message) => ({ 'response.embeddings.canonical': { status, message } }), + refuses: ['response.embeddings.canonical'], +}; + +export const embeddingsServePipeline: Pipeline< + E<'ingress.embeddings.encodingFormat' | 'request.embeddings.canonical' | 'serve.model'>, + E<'response.embeddings.rendered' | 'response.usage.billable'> +> = compose('embeddingsServe', [ + emitEmbeddings, + resolveCandidates(narrowing), + failover(handedUp => isFailure((handedUp as { 'response.embeddings.canonical'?: unknown })['response.embeddings.canonical'])), + callEmbeddingsUpstream, +]); diff --git a/packages/protocols/__tests__/embeddings/translate_test.ts b/packages/protocols/__tests__/embeddings/translate_test.ts new file mode 100644 index 0000000000..1026689d8c --- /dev/null +++ b/packages/protocols/__tests__/embeddings/translate_test.ts @@ -0,0 +1,151 @@ +import { describe, expect, test } from 'vitest'; + +import { + parseEmbeddingsRequest, + parseEmbeddingsResponse, + renderEmbeddingsResponse, + serializeEmbeddingsRequest, +} from '../../src/embeddings/translate.ts'; + +// The float32 little-endian packing of [0.0023064255, -0.009327292, 1, -0.0028842222], +// and the float64 values those bits denote. Produced by Float32Array + Buffer on Node 24, +// which is the encoding both official OpenAI SDKs read a base64 embedding with. +const PACKED = 'ZicXO4DRGLwAAIA/OAU9uw=='; +const VECTOR = [0.002306425478309393, -0.009327292442321777, 1, -0.0028842221945524216]; + +describe('embeddings request ingress', () => { + test('a single text stays a single text, so the wire shape the upstream sees is unchanged', () => { + const parsed = parseEmbeddingsRequest({ model: 'text-embedding-3-small', input: 'a sentence' }); + + expect(parsed.model).toBe('text-embedding-3-small'); + expect(parsed.request).toEqual({ input: 'a sentence' }); + expect(serializeEmbeddingsRequest(parsed.request)).toEqual({ input: 'a sentence' }); + }); + + test('the four input arms each survive', () => { + const arms = [ + 'one', + ['one', 'two'], + [1212, 318, 257], + [[1212, 318], [257, 1332]], + ]; + for (const input of arms) { + expect(parseEmbeddingsRequest({ model: 'm', input }).request.input).toEqual(input); + } + }); + + test('an omitted encoding_format is resolved for the client and left off the wire', () => { + const parsed = parseEmbeddingsRequest({ model: 'm', input: 'a' }); + + expect(parsed.encodingFormat).toBe('float'); + expect(parsed.request.encodingFormat).toBeUndefined(); + expect(serializeEmbeddingsRequest(parsed.request)).not.toHaveProperty('encoding_format'); + }); + + test('a chosen encoding_format is both what the client reads and what the upstream is asked for', () => { + const parsed = parseEmbeddingsRequest({ model: 'm', input: 'a', encoding_format: 'base64', dimensions: 256, user: 'u' }); + + expect(parsed.encodingFormat).toBe('base64'); + expect(serializeEmbeddingsRequest(parsed.request)).toEqual({ + input: 'a', + encoding_format: 'base64', + dimensions: 256, + user: 'u', + }); + }); + + test('a field the protocol has no place for is named rather than dropped', () => { + expect(() => parseEmbeddingsRequest({ model: 'm', input: 'a', truncate: true, task: 'retrieval' })) + .toThrow('Embeddings does not support truncate, task'); + }); + + // An empty array satisfies three of the specification's four input arms at once, so the + // canonical value would be ambiguous even though the JSON is well-formed. + test('an empty input is refused, which is what keeps the parsed arms distinguishable', () => { + expect(() => parseEmbeddingsRequest({ model: 'm', input: [] })).toThrow('input must not be empty'); + }); + + test('a mixed array is refused rather than read as whichever arm came first', () => { + expect(() => parseEmbeddingsRequest({ model: 'm', input: ['a', 1] })).toThrow('input[1] must be a non-empty string'); + expect(() => parseEmbeddingsRequest({ model: 'm', input: [1, 'a'] })).toThrow('input[1] must be an integer'); + }); + + test('a non-integer dimensions is refused', () => { + expect(() => parseEmbeddingsRequest({ model: 'm', input: 'a', dimensions: 0 })).toThrow('dimensions must be a positive integer'); + expect(() => parseEmbeddingsRequest({ model: 'm', input: 'a', dimensions: 1.5 })).toThrow('dimensions must be an integer'); + }); +}); + +describe('embeddings response egress', () => { + test('a base64 vector is read as the numbers it denotes', () => { + const parsed = parseEmbeddingsResponse({ + object: 'list', + model: 'text-embedding-3-small', + data: [{ object: 'embedding', index: 0, embedding: PACKED }], + usage: { prompt_tokens: 8, total_tokens: 8 }, + }); + + expect(parsed.embeddings).toEqual([{ index: 0, values: VECTOR }]); + expect(parsed.usage).toEqual({ promptTokens: 8, totalTokens: 8 }); + }); + + test('a truncated base64 vector is refused rather than read as a shorter one', () => { + expect(() => parseEmbeddingsResponse({ + model: 'm', + data: [{ index: 0, embedding: PACKED.slice(0, 8) }], + })).toThrow('data[0].embedding is not a whole number of float32 values'); + }); + + // An upstream that reports no usage is a fact the gateway has to be able to carry: the + // billed entity is present with no quantities, which is not the same as reporting zero. + test('a missing usage block leaves the answer intact and reports nothing', () => { + const parsed = parseEmbeddingsResponse({ model: 'm', data: [{ index: 0, embedding: [0.5] }] }); + + expect(parsed.usage).toBeUndefined(); + expect(renderEmbeddingsResponse('float', parsed)).not.toHaveProperty('usage'); + }); + + // The case the encoding split exists for. An OpenAI SDK client asked for base64 without + // saying so and will decode whatever it is handed; an upstream that ignores the field + // and answers with float arrays would otherwise hand it noise. + test('the answer is written in the encoding the client asked for, not the one it arrived in', () => { + const arrived = parseEmbeddingsResponse({ model: 'm', data: [{ index: 0, embedding: VECTOR }] }); + + expect(renderEmbeddingsResponse('base64', arrived)).toMatchObject({ + object: 'list', + data: [{ object: 'embedding', index: 0, embedding: PACKED }], + }); + expect(renderEmbeddingsResponse('float', parseEmbeddingsResponse({ + model: 'm', + data: [{ index: 0, embedding: PACKED }], + }))).toMatchObject({ data: [{ embedding: VECTOR }] }); + }); + + // Every float32 is a float64, so widening loses nothing and narrowing a value that came + // from a float32 gives the same bits back. Holding vectors as numbers is therefore exact + // and not an approximation, however many times a vector crosses the gateway. + test('a vector survives any number of trips through the two encodings', () => { + const once = parseEmbeddingsResponse({ model: 'm', data: [{ index: 0, embedding: PACKED }] }); + const twice = parseEmbeddingsResponse(renderEmbeddingsResponse('base64', once)); + const thrice = parseEmbeddingsResponse(renderEmbeddingsResponse('base64', twice)); + + expect(renderEmbeddingsResponse('base64', thrice)).toMatchObject({ data: [{ embedding: PACKED }] }); + expect(thrice.embeddings).toEqual(once.embeddings); + }); + + test('object is written by the gateway at both levels rather than repeated back', () => { + const rendered = renderEmbeddingsResponse('float', parseEmbeddingsResponse({ + object: 'not-a-list', + model: 'm', + data: [{ object: 'not-an-embedding', index: 0, embedding: [0.5] }], + usage: { prompt_tokens: 1, total_tokens: 1 }, + })); + + expect(rendered).toEqual({ + object: 'list', + data: [{ object: 'embedding', index: 0, embedding: [0.5] }], + model: 'm', + usage: { prompt_tokens: 1, total_tokens: 1 }, + }); + }); +}); diff --git a/packages/protocols/src/embeddings/index.ts b/packages/protocols/src/embeddings/index.ts index e336f07948..41524c486a 100644 --- a/packages/protocols/src/embeddings/index.ts +++ b/packages/protocols/src/embeddings/index.ts @@ -1,5 +1,100 @@ +// OpenAI's `/v1/embeddings` contract, and the canonical form the gateway holds it in. +// Specification: https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34214-L34331 +// +// There is one embeddings protocol, so the canonical form and the wire differ in one +// substantive place: `encoding_format` decides how a vector is *written*, and a vector is +// the same vector either way — so the canonical response holds numbers, and the encoding +// is a rendering decision the edge makes from what the client asked for. +// +// That divergence is load-bearing rather than tidy. Both official OpenAI SDKs send +// `encoding_format: base64` when the caller did not choose one, so base64 is the common +// case on the wire and not an exotic one: +// https://github.com/openai/openai-python/blob/10ee3f0da2ac6f93345c1204bd7bb1a2faa79ff2/src/openai/resources/embeddings.py#L111-L112 +// https://github.com/openai/openai-node/blob/cc7dbfa9b9dd6fe0ff72141e9c0c3d82b18ba9aa/src/resources/embeddings.ts#L36-L42 +// An upstream that ignores the field and answers with float arrays hands such a client a +// body it will decode as base64 and turn into noise. Parsing whatever arrived and +// rendering what the client asked for is what closes that. + +/** How a vector is written on the wire. Under `base64` an embedding is one string, not an + * array of numbers. + * https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34272-L34280 */ +export type EmbeddingsEncodingFormat = 'float' | 'base64'; + +/** One text, a batch of texts, one pre-tokenized text, or a batch of those — the + * specification's own four-armed `oneOf`: + * https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34218-L34254 + * + * The arms are told apart by reading an element, which works only because every array arm + * is `minItems: 1` there — an empty array would belong to three of them at once. Parsing + * rejects it, so a value of this type is always unambiguous. */ +export type EmbeddingsInput = string | readonly string[] | readonly number[] | readonly (readonly number[])[]; + +export interface CanonicalEmbeddingsRequest { + input: EmbeddingsInput; + /** What the upstream is asked for, absent included — a client that did not ask must not + * grow a field on the wire it never sent. */ + encodingFormat?: EmbeddingsEncodingFormat; + /** Only `text-embedding-3` and later models honour it + * (https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34281-L34286), + * and which models those are is the upstream's knowledge and not the catalog's — so it + * travels to whichever upstream is chosen and that upstream answers for it. */ + dimensions?: number; + /** https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34287-L34293 */ + user?: string; +} + +export interface ParsedEmbeddingsRequest { + model: string; + /** What the client will be able to read: `encoding_format` resolved against the + * protocol's own default of `float` + * (https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34276). + * Concrete, because the answer has to be written in one encoding or the other — and + * distinct from the request's own field, which stays absent when the client omitted it. */ + encodingFormat: EmbeddingsEncodingFormat; + request: CanonicalEmbeddingsRequest; +} + +export interface CanonicalEmbedding { + /** Which input this vector answers, carried by the specification as a field rather than + * left to array position: + * https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L38168-L38170 */ + index: number; + values: readonly number[]; +} + +export interface CanonicalEmbeddingsUsage { + promptTokens: number; + totalTokens: number; +} + +export interface CanonicalEmbeddingsResponse { + /** What the upstream says it generated with, which need not be the id the client + * addressed — an upstream is free to answer under a dated or internal name. */ + model: string; + embeddings: readonly CanonicalEmbedding[]; + /** The specification requires it + * (https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34327-L34331) + * and it is optional here regardless, because an upstream that reports nothing is a + * situation the gateway has to be able to state: the billed entity is then present with + * no quantities, which is a different fact from reporting zero. Rejecting the body + * instead would turn an answered call into a 502 and throw away the vectors it + * returned. */ + usage?: CanonicalEmbeddingsUsage; +} + +/** The request as it goes out. `model` is the upstream's own id and the provider + * substitutes it, so what the gateway hands a provider is everything but that. */ export interface EmbeddingsPayload { model: string; - input?: unknown; - [key: string]: unknown; + input: EmbeddingsInput; + encoding_format?: EmbeddingsEncodingFormat; + dimensions?: number; + user?: string; } + +export { + parseEmbeddingsRequest, + parseEmbeddingsResponse, + renderEmbeddingsResponse, + serializeEmbeddingsRequest, +} from './translate.ts'; diff --git a/packages/protocols/src/embeddings/translate.ts b/packages/protocols/src/embeddings/translate.ts new file mode 100644 index 0000000000..651113b514 --- /dev/null +++ b/packages/protocols/src/embeddings/translate.ts @@ -0,0 +1,184 @@ +import type { + CanonicalEmbedding, + CanonicalEmbeddingsRequest, + CanonicalEmbeddingsResponse, + CanonicalEmbeddingsUsage, + EmbeddingsEncodingFormat, + EmbeddingsInput, + EmbeddingsPayload, + ParsedEmbeddingsRequest, +} from './index.ts'; +import { decodeForgivingBase64, encodeBase64 } from '../common/base-encoding.ts'; + +const isRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +const requiredString = (value: unknown, field: string): string => { + if (typeof value !== 'string' || value.length === 0) throw new Error(`${field} must be a non-empty string`); + return value; +}; + +const optionalString = (value: unknown, field: string): string | undefined => { + if (value === undefined) return undefined; + if (typeof value !== 'string') throw new Error(`${field} must be a string`); + return value; +}; + +const requiredInteger = (value: unknown, field: string): number => { + if (typeof value !== 'number' || !Number.isInteger(value)) throw new Error(`${field} must be an integer`); + return value; +}; + +const optionalPositiveInteger = (value: unknown, field: string): number | undefined => { + if (value === undefined) return undefined; + const integer = requiredInteger(value, field); + if (integer < 1) throw new Error(`${field} must be a positive integer`); + return integer; +}; + +const parseEncodingFormat = (value: unknown): EmbeddingsEncodingFormat | undefined => { + if (value === undefined) return undefined; + if (value !== 'float' && value !== 'base64') throw new Error('encoding_format must be float or base64'); + return value; +}; + +const parseTokenIds = (value: readonly unknown[], field: string): readonly number[] => { + if (value.length === 0) throw new Error(`${field} must not be empty`); + return value.map((token, index) => requiredInteger(token, `${field}[${index}]`)); +}; + +const parseInput = (value: unknown): EmbeddingsInput => { + if (typeof value === 'string') return value; + if (!Array.isArray(value)) throw new Error('input must be a string or an array'); + if (value.length === 0) throw new Error('input must not be empty'); + const head: unknown = value[0]; + if (typeof head === 'string') return value.map((text, index) => requiredString(text, `input[${index}]`)); + if (!Array.isArray(head)) return parseTokenIds(value as readonly unknown[], 'input'); + return value.map((tokens, index) => { + if (!Array.isArray(tokens)) throw new Error(`input[${index}] must be an array of integers`); + return parseTokenIds(tokens as readonly unknown[], `input[${index}]`); + }); +}; + +// The request schema is `additionalProperties: false` +// (https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34216), +// so the protocol has no extension point and a field outside this set is one the gateway +// cannot carry. Naming it beats dropping it silently and beats guessing what it meant. The +// two gateways compared against draw the same line: LiteLLM's embeddings surface is +// `user` / `encoding_format` / `dimensions`, and anything else raises unless the operator +// opts into dropping — +// https://github.com/BerriAI/litellm/blob/bc6e7df05b018eefe6c7293790ca3f4de38709ac/litellm/utils.py#L3259-L3273 +// — while copilot-api's request type is narrower still, `input` and `model` alone: +// https://github.com/ericc-ch/copilot-api/blob/0ea08febdd7e3e055b03dd298bf57e669500b5c1/src/services/copilot/create-embeddings.ts#L19-L22 +const EMBEDDINGS_REQUEST_FIELDS = ['model', 'input', 'encoding_format', 'dimensions', 'user']; + +export const parseEmbeddingsRequest = (value: unknown): ParsedEmbeddingsRequest => { + if (!isRecord(value)) throw new Error('Embeddings request body must be an object'); + const unsupported = Object.keys(value).filter(field => !EMBEDDINGS_REQUEST_FIELDS.includes(field)); + if (unsupported.length > 0) throw new Error(`Embeddings does not support ${unsupported.join(', ')}`); + + const encodingFormat = parseEncodingFormat(value.encoding_format); + const dimensions = optionalPositiveInteger(value.dimensions, 'dimensions'); + const user = optionalString(value.user, 'user'); + return { + model: requiredString(value.model, 'model'), + encodingFormat: encodingFormat ?? 'float', + request: { + input: parseInput(value.input), + ...(encodingFormat === undefined ? {} : { encodingFormat }), + ...(dimensions === undefined ? {} : { dimensions }), + ...(user === undefined ? {} : { user }), + }, + }; +}; + +export const serializeEmbeddingsRequest = (request: CanonicalEmbeddingsRequest): Omit => ({ + input: request.input, + ...(request.encodingFormat === undefined ? {} : { encoding_format: request.encodingFormat }), + ...(request.dimensions === undefined ? {} : { dimensions: request.dimensions }), + ...(request.user === undefined ? {} : { user: request.user }), +}); + +// A base64 embedding is the vector's float32 elements packed little-endian. Both official +// SDKs establish that by reading the decoded bytes straight into a native float32 view — +// `array.array('f', ...)` and `Float32Array`, each host-endian and each correct only +// because every platform they run on is: +// https://github.com/openai/openai-python/blob/10ee3f0da2ac6f93345c1204bd7bb1a2faa79ff2/src/openai/resources/embeddings.py#L122-L131 +// https://github.com/openai/openai-node/blob/cc7dbfa9b9dd6fe0ff72141e9c0c3d82b18ba9aa/src/internal/utils/base64.ts#L45-L52 +// `DataView` states the byte order rather than inheriting it. +// +// Holding a vector as numbers is exact and not an approximation: every float32 is a +// float64, so widening loses nothing and narrowing a value that came from a float32 gives +// the same bits back. A vector survives any number of trips through this pair. +const FLOAT32_BYTES = 4; + +const parseEmbedding = (value: unknown, field: string): readonly number[] => { + if (Array.isArray(value)) { + return value.map((element, index) => { + if (typeof element !== 'number') throw new Error(`${field}[${index}] must be a number`); + return element; + }); + } + if (typeof value !== 'string') throw new Error(`${field} must be an array of numbers or a base64 string`); + const bytes = decodeForgivingBase64(value); + if (bytes.length % FLOAT32_BYTES !== 0) throw new Error(`${field} is not a whole number of float32 values`); + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + return Array.from({ length: bytes.length / FLOAT32_BYTES }, (_, index) => view.getFloat32(index * FLOAT32_BYTES, true)); +}; + +const renderEmbedding = (values: readonly number[]): string => { + const bytes = new Uint8Array(values.length * FLOAT32_BYTES); + const view = new DataView(bytes.buffer); + values.forEach((value, index) => view.setFloat32(index * FLOAT32_BYTES, value, true)); + return encodeBase64(bytes); +}; + +const parseUsage = (value: unknown): CanonicalEmbeddingsUsage | undefined => { + if (value === undefined || value === null) return undefined; + if (!isRecord(value)) throw new Error('usage must be an object'); + return { + promptTokens: requiredInteger(value.prompt_tokens, 'usage.prompt_tokens'), + totalTokens: requiredInteger(value.total_tokens, 'usage.total_tokens'), + }; +}; + +export const parseEmbeddingsResponse = (value: unknown): CanonicalEmbeddingsResponse => { + if (!isRecord(value)) throw new Error('Embeddings response body must be an object'); + const { data } = value; + if (!Array.isArray(data)) throw new Error('data must be an array'); + const usage = parseUsage(value.usage); + const embeddings: CanonicalEmbedding[] = data.map((entry, position) => { + if (!isRecord(entry)) throw new Error(`data[${position}] must be an object`); + return { + index: requiredInteger(entry.index, `data[${position}].index`), + values: parseEmbedding(entry.embedding, `data[${position}].embedding`), + }; + }); + return { + model: requiredString(value.model, 'model'), + embeddings, + ...(usage === undefined ? {} : { usage }), + }; +}; + +/** + * Writes the answer in the encoding the client asked for, whichever one the upstream + * answered in. `object` is `x-stainless-const` at both levels, so it is a constant of the + * protocol rather than something an upstream varies, and it is written here rather than + * repeated back from the body. + */ +export const renderEmbeddingsResponse = ( + format: EmbeddingsEncodingFormat, + response: CanonicalEmbeddingsResponse, +): Record => ({ + object: 'list', + data: response.embeddings.map(embedding => ({ + object: 'embedding', + index: embedding.index, + embedding: format === 'base64' ? renderEmbedding(embedding.values) : embedding.values, + })), + model: response.model, + ...(response.usage === undefined ? {} : { + usage: { prompt_tokens: response.usage.promptTokens, total_tokens: response.usage.totalTokens }, + }), +}); From 53b9736113883ec2dda45200d5e535a15aae60c6 Mon Sep 17 00:00:00 2001 From: Menci Date: Sun, 16 Aug 2026 21:39:04 +0800 Subject: [PATCH 04/79] feat(gateway): serve the remaining non-chat families as pipelines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five families at once, each with a real protocol contract where it had none. `completions`, `embeddings`, `images` and `audio` ran on `passthrough-serve.ts`, which forwards a body it has not parsed — and the architecture has no such concept, so three of them needed a contract written before they could have a pipeline at all. `packages/protocols/src/{audio,images,completions,embeddings}/` carries those now, cited to the OpenAI specification. `alpha-search` is shaped differently and is not forced into the others' shape: it has no upstream model and no candidate list, executing locally through the configured search provider or relaying a selected provider's response. That is an ending, not a missing family — the same distinction ruling 5 settled for HTTP, where a provider that does not ultimately speak plain HTTP supplies its own. Nothing is routed here yet: the route table still mounts the passthrough handlers, so the gateway's behaviour is unchanged. The wiring, and deleting the passthrough helpers, is one commit once every family is sound. Two defects the families found in what they were built on are recorded in the Pull Request and fixed in the commits that follow. Both were found by building on the core rather than by reading it, which is what this layer is for. --- .../data-plane/audio-pipeline_test.ts | 51 +++ .../data-plane/completions-pipeline_test.ts | 211 ++++++++++ .../data-plane/images-pipeline_test.ts | 65 +++ .../data-plane/search-pipeline_test.ts | 58 +++ .../src/data-plane/alpha-search/pipeline.ts | 356 +++++++++++++++++ .../src/data-plane/alpha-search/protocol.ts | 115 ++++++ .../src/data-plane/alpha-search/routes.ts | 74 +--- .../gateway/src/data-plane/audio/pipeline.ts | 375 +++++++++++++++++ .../src/data-plane/completions/pipeline.ts | 377 ++++++++++++++++++ .../src/data-plane/completions/usage.ts | 9 +- .../gateway/src/data-plane/images/pipeline.ts | 266 ++++++++++++ .../__tests__/audio/transcription_test.ts | 102 +++++ .../__tests__/images/request_test.ts | 129 ++++++ .../__tests__/images/response_test.ts | 75 ++++ packages/protocols/src/audio/index.ts | 43 +- packages/protocols/src/audio/stream.ts | 39 ++ packages/protocols/src/audio/subtitles.ts | 107 +++++ packages/protocols/src/audio/transcription.ts | 179 +++++++++ packages/protocols/src/completions/index.ts | 80 +++- packages/protocols/src/completions/parse.ts | 22 + packages/protocols/src/completions/stream.ts | 26 ++ packages/protocols/src/images/index.ts | 77 ++++ packages/protocols/src/images/request.ts | 112 ++++++ packages/protocols/src/images/response.ts | 109 +++++ 24 files changed, 2952 insertions(+), 105 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/audio-pipeline_test.ts create mode 100644 packages/gateway/__tests__/data-plane/completions-pipeline_test.ts create mode 100644 packages/gateway/__tests__/data-plane/images-pipeline_test.ts create mode 100644 packages/gateway/__tests__/data-plane/search-pipeline_test.ts create mode 100644 packages/gateway/src/data-plane/alpha-search/pipeline.ts create mode 100644 packages/gateway/src/data-plane/alpha-search/protocol.ts create mode 100644 packages/gateway/src/data-plane/audio/pipeline.ts create mode 100644 packages/gateway/src/data-plane/completions/pipeline.ts create mode 100644 packages/gateway/src/data-plane/images/pipeline.ts create mode 100644 packages/protocols/__tests__/audio/transcription_test.ts create mode 100644 packages/protocols/__tests__/images/request_test.ts create mode 100644 packages/protocols/__tests__/images/response_test.ts create mode 100644 packages/protocols/src/audio/stream.ts create mode 100644 packages/protocols/src/audio/subtitles.ts create mode 100644 packages/protocols/src/audio/transcription.ts create mode 100644 packages/protocols/src/completions/parse.ts create mode 100644 packages/protocols/src/completions/stream.ts create mode 100644 packages/protocols/src/images/request.ts create mode 100644 packages/protocols/src/images/response.ts diff --git a/packages/gateway/__tests__/data-plane/audio-pipeline_test.ts b/packages/gateway/__tests__/data-plane/audio-pipeline_test.ts new file mode 100644 index 0000000000..09950d5d37 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/audio-pipeline_test.ts @@ -0,0 +1,51 @@ +// Audio transcription's pipeline, assembled. `compose` derives the entry contract and +// rejects an array that cannot work, so most of what this file establishes is established +// by the module importing at all — the assembly runs at load. What is worth writing down is +// the entry contract it derives, the keys it cannot see, and the one runtime property this +// family's code is shaped around that no declaration expresses. + +import { describe, expect, it } from 'vitest'; + +import { audioTranscriptionServePipeline } from '../../src/data-plane/audio/pipeline.ts'; +import { isReleasable, move, run } from '@floway-dev/pipeline'; + +describe('the audio transcription pipeline', () => { + it('assembles, and asks its caller for what the descending stages need', () => { + expect([...audioTranscriptionServePipeline.entryNeeds].sort()).toEqual([ + 'ingress.audioTranscription.responseFormat', + 'serve.model', + ]); + }); + + // The ending stage reads `request.audioTranscription.form` and `ingress.http.headers`, and + // the entry contract mentions neither. That is not this family's defect: a stage whose only + // trait is `return` declares no request side at all, by ruling — "when it short-circuits, + // only `provides`" — so assembly cannot see what an ending stage reads. + // + // It bites harder here than it does for a family whose edge needs the request payload to + // render: this family's edge does not, so the payload the whole endpoint exists to send is + // among the keys assembly cannot ask for. A caller who omits it reaches the deepest stage + // before failing. The type layer still catches it at the definition site, which is why this + // is a gap and not a break. + it('cannot see the request payload, because only a return-only stage reads it', () => { + expect(audioTranscriptionServePipeline.entryNeeds).not.toContain('request.audioTranscription.form'); + expect(audioTranscriptionServePipeline.entryNeeds).not.toContain('ingress.http.headers'); + }); + + it('names the entry key a caller did not bring, before any stage runs', async () => { + await expect(run(audioTranscriptionServePipeline, move({ 'serve.model': 'whisper-1' }) as never, {})) + .rejects.toThrow('run(audioTranscriptionServe): audioTranscriptionServe needs'); + }); + + // Why the streamed answer is handed over as a wrapper around its generator rather than as + // the generator: an async generator carries `Symbol.asyncDispose`, so the runner would adopt + // one in the record as a resource and the top-level sweep would call it — and for a + // generator that call is `return()`, which cancels the iteration rather than draining it. + // The resource is the upstream body, at `response.http.body`, and it is the only thing here + // that should answer to release. + it('makes an events view that the runner cannot mistake for the resource', () => { + const generator = (async function* () { yield 1; })(); + expect(isReleasable(generator)).toBe(true); + expect(isReleasable({ [Symbol.asyncIterator]: () => generator })).toBe(false); + }); +}); diff --git a/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts new file mode 100644 index 0000000000..5836225431 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts @@ -0,0 +1,211 @@ +// Completions' pipeline, assembled and run. `compose` derives the entry contract and rejects +// an array that cannot work, so the assembly succeeding is itself most of what a test of the +// wiring would say — what is written down here is the entry contract, the two things the +// assembly cannot see, and one run of each shape the answer can take, because this is the +// first family whose answer can be a stream and none of that is visible in a declaration. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { completionsServePipeline } from '../../src/data-plane/completions/pipeline.ts'; +import { enumerateModelCandidates } from '../../src/data-plane/providers/resolution.ts'; +import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts'; +import { move, run } from '@floway-dev/pipeline'; +import type { SseFrame } from '@floway-dev/protocols/common'; +import { directFetcher, type ModelCandidate, type ProviderCallResult, type UpstreamCallOptions } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; + +vi.mock('../../src/data-plane/providers/resolution.ts', async importOriginal => ({ + ...(await importOriginal()), + enumerateModelCandidates: vi.fn(), +})); + +const resolves = (candidates: readonly ModelCandidate[]): void => { + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] }); +}; + +const candidate = ( + upstream: string, + callCompletions: (model: unknown, body: unknown, signal: AbortSignal | undefined, opts: UpstreamCallOptions) => Promise, +): ModelCandidate => { + const endpoints = { chatCompletions: {}, completions: {} }; + return { + provider: { + upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + instance: stubProvider({ callCompletions }), + }, + model: stubInternalModel({ id: 'text-model', endpoints, providerModels: { [upstream]: stubProviderModel({ id: 'text-model', endpoints }) } }, upstream), + fetcher: directFetcher, + }; +}; + +const sse = (...events: readonly string[]): Response => + new Response(events.map(event => `data: ${event}\n\n`).join(''), { status: 200, headers: { 'content-type': 'text/event-stream' } }); + +const chunk = (text: string): string => + JSON.stringify({ id: 'cmpl_1', object: 'text_completion', created: 1, model: 'text-model', choices: [{ index: 0, text }] }); + +const usageChunk = JSON.stringify({ + id: 'cmpl_1', object: 'text_completion', created: 1, model: 'text-model', choices: [], + usage: { prompt_tokens: 5, completion_tokens: 7, total_tokens: 12 }, +}); + +const serve = async (facts: Record) => await run( + completionsServePipeline, + move(facts) as never, + { gateway: mockGatewayCtx({ wantsStream: facts['ingress.completions.wantsStream'] === true }), background: () => {} } as never, +); + +const entryFacts = (overrides: Record = {}) => ({ + 'ingress.completions.wantsStream': true, + 'ingress.completions.wantsUsageChunk': false, + 'ingress.http.headers': [], + 'request.completions.payload': { model: 'text-model', prompt: 'hello', stream: true }, + 'serve.model': 'text-model', + ...overrides, +}); + +const collect = async (rendered: unknown): Promise => { + const frames: SseFrame[] = []; + for await (const frame of rendered as AsyncIterable) frames.push(frame); + return frames; +}; + +beforeEach(() => { vi.mocked(enumerateModelCandidates).mockReset(); }); + +describe('the completions pipeline', () => { + it('assembles, and asks its caller for what the descending stages need', () => { + expect([...completionsServePipeline.entryNeeds].sort()).toEqual([ + 'ingress.completions.wantsStream', + 'ingress.completions.wantsUsageChunk', + 'request.completions.payload', + 'serve.model', + ]); + }); + + // `callCompletionsUpstream` reads `ingress.http.headers`, and the entry contract does not + // mention it. That is not this family's defect: a stage whose only trait is `return` + // declares no request side at all, by ruling — "when it short-circuits, only `provides`" — + // so assembly cannot see what an ending stage reads, and every family's ending stage reads + // something. A caller who omits that key gets a runtime failure at the deepest stage + // instead of the assembly error the entry contract exists to give it. + it('cannot see what an ending stage reads, because a return-only stage declares no needs', () => { + expect(completionsServePipeline.entryNeeds).not.toContain('ingress.http.headers'); + }); + + it('renders the upstream frames as SSE, hiding the usage chunk the client did not ask for', async () => { + let sent: Record | undefined; + resolves([candidate('up_a', async (_model, body) => { + sent = body as Record; + return { response: sse(chunk('he'), chunk('llo'), usageChunk, '[DONE]'), modelKey: 'text-model-key' }; + })]); + + const { facts, drain } = await serve(entryFacts()); + + // The answer comes back before the drain runs, which is what lets a streaming family + // hand its stream on: the frames are still there to read. + expect(await collect(facts['response.completions.rendered'])).toEqual([ + { type: 'sse', event: undefined, data: chunk('he') }, + { type: 'sse', event: undefined, data: chunk('llo') }, + { type: 'sse', event: undefined, data: '[DONE]' }, + ]); + // Metering is the gateway's, not the client's: the upstream is always asked for the + // chunk the client is not shown. + expect(sent).toMatchObject({ stream_options: { include_usage: true } }); + expect(sent).not.toHaveProperty('model'); + await drain(); + }); + + it('says the upstream was called and reported nothing until the frames run out', async () => { + resolves([candidate('up_a', async () => ({ response: sse(chunk('hi'), usageChunk, '[DONE]'), modelKey: 'text-model-key' }))]); + + const { facts, drain } = await serve(entryFacts()); + + // What is known when the ending stage hands up: an entity, and no quantities. The + // numbers arrive with the last chunk, which is after this run has answered. + expect(facts['response.usage.billable']).toEqual([ + { identity: { model: 'text-model', upstream: 'up_a', modelKey: 'text-model-key', pricing: null }, quantities: {} }, + ]); + await collect(facts['response.completions.rendered']); + expect(await facts['response.completions.streamedUsage']).toEqual([ + { + identity: { model: 'text-model', upstream: 'up_a', modelKey: 'text-model-key', pricing: null }, + quantities: { input_tokens: '5', output_tokens: '7' }, + }, + ]); + await drain(); + }); + + it('shows the usage chunk to a client that asked for it', async () => { + resolves([candidate('up_a', async () => ({ response: sse(chunk('hi'), usageChunk, '[DONE]'), modelKey: 'text-model-key' }))]); + + const { facts, drain } = await serve(entryFacts({ 'ingress.completions.wantsUsageChunk': true })); + + expect((await collect(facts['response.completions.rendered'])).map(frame => frame.data)).toEqual([chunk('hi'), usageChunk, '[DONE]']); + await drain(); + }); + + it('serializes a non-streaming answer from the value it parsed, and bills what it read', async () => { + const body = { id: 'cmpl_1', object: 'text_completion', created: 1, model: 'text-model', choices: [{ index: 0, text: 'hi', finish_reason: 'stop' }], usage: { prompt_tokens: 5, completion_tokens: 7, total_tokens: 12 } }; + resolves([candidate('up_a', async () => ({ response: Response.json(body), modelKey: 'text-model-key' }))]); + + const { facts, drain } = await serve(entryFacts({ + 'ingress.completions.wantsStream': false, + 'request.completions.payload': { model: 'text-model', prompt: 'hello' }, + })); + + expect(facts['response.completions.rendered']).toEqual(body); + expect(facts['response.completions.streamedUsage']).toBeNull(); + expect(facts['response.usage.billable']).toEqual([ + { + identity: { model: 'text-model', upstream: 'up_a', modelKey: 'text-model-key', pricing: null }, + quantities: { input_tokens: '5', output_tokens: '7' }, + }, + ]); + await drain(); + }); + + it('fails a refusal over to the next candidate, and renders the last one it got', async () => { + const tried: string[] = []; + resolves([ + candidate('up_a', async () => { + tried.push('up_a'); + return { response: Response.json({ error: { message: 'slow down' } }, { status: 429 }), modelKey: 'text-model-key' }; + }), + candidate('up_b', async () => { + tried.push('up_b'); + return { response: Response.json({ error: { message: 'no' } }, { status: 400 }), modelKey: 'text-model-key' }; + }), + ]); + + const { facts, drain } = await serve(entryFacts({ + 'ingress.completions.wantsStream': false, + 'request.completions.payload': { model: 'text-model', prompt: 'hello' }, + })); + + expect(tried).toEqual(['up_a', 'up_b']); + // Every candidate failed, so the last failure is the base — the client sees the status + // an upstream actually returned rather than a synthesized gateway envelope. + expect(facts['response.http.status']).toBe(400); + expect(facts['response.completions.rendered']).toEqual({ error: { message: JSON.stringify({ error: { message: 'no' } }), type: 'api_error' } }); + await drain(); + }); + + // Nothing in this family's own declarations says it must hand the upstream's body up, and + // nothing in assembly checks it: `failover` declares `provides: ['response.http.body']` on + // the way up, and that declaration is checked by the runner, at runtime, against whatever + // the stage below it handed on. A family whose ending stage parses the body and keeps it — + // which is what "no body is forwarded verbatim" invites — composes cleanly and then throws + // on its first request. The cast is the other half of the same story: the key rides in the + // record without being in the pipeline's exit type, because the path that refuses before + // any upstream is dialed has no body to hand up. + it('hands the upstream body up, because failover declares it provides one', async () => { + resolves([candidate('up_a', async () => ({ response: sse(chunk('hi'), '[DONE]'), modelKey: 'text-model-key' }))]); + + const { facts, drain } = await serve(entryFacts()); + + expect((facts as Record)['response.http.body']).toBeInstanceOf(ReadableStream); + await collect(facts['response.completions.rendered']); + await drain(); + }); +}); diff --git a/packages/gateway/__tests__/data-plane/images-pipeline_test.ts b/packages/gateway/__tests__/data-plane/images-pipeline_test.ts new file mode 100644 index 0000000000..650e506486 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/images-pipeline_test.ts @@ -0,0 +1,65 @@ +// Images' pipeline, assembled. `compose` derives the entry contract and rejects an array that +// cannot work, so most of what this file establishes is established by the assembly succeeding +// at all — and what is worth writing down is the contract it derives and the keys it cannot see. +// +// One thing no assertion here can reach, and the reason the route table must keep serving this +// family the old way until it is settled: both endpoints accept `stream: true` and answer with +// an SSE sequence of partial images, while this pipeline carries a parsed JSON body and nothing +// else. Such a request reaches the upstream, spends it, and comes back as the 502 the ending +// stage synthesizes for a body the protocol cannot read. Carrying it needs a stream of protocol +// frames in the fact space, which is the work the first streaming family brings with it. +// https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L13077-L13086 + +import { describe, expect, it } from 'vitest'; + +import { imagesServePipeline, type ImagesServeEntry } from '../../src/data-plane/images/pipeline.ts'; +import type { CanonicalImagesRequest } from '@floway-dev/protocols/images'; + +const generations: CanonicalImagesRequest = { + operation: 'generations', + parameters: { prompt: 'a shiba in space' }, +}; + +const edits: CanonicalImagesRequest = { + operation: 'edits', + images: [{ kind: 'reference', reference: { file_id: 'file-source' } }], + parameters: { prompt: 'replace the sky' }, +}; + +describe('the images pipeline', () => { + it('assembles both endpoints as one array, and asks its caller for what the descending stages need', () => { + expect([...imagesServePipeline(generations).entryNeeds].sort()).toEqual(['serve.model']); + expect([...imagesServePipeline(edits).entryNeeds].sort()).toEqual(['serve.model']); + }); + + // `callImagesUpstream` reads `request.images.canonical` and `ingress.http.headers`, and the + // derived contract mentions neither. That is not this family's defect: a stage whose only + // trait is `return` declares no request side at all, by ruling — "when it short-circuits, + // only `provides`" — so assembly cannot see what an ending stage reads, and every family's + // ending stage reads something. + // + // Written as a test rather than a comment because the hole has a consequence: a caller who + // omits either key gets a runtime failure at the deepest stage instead of the assembly error + // the entry contract exists to produce. + it('cannot see what its ending stage reads, because a return-only stage declares no needs', () => { + const derived = imagesServePipeline(generations).entryNeeds; + expect(derived).not.toContain('request.images.canonical'); + expect(derived).not.toContain('ingress.http.headers'); + }); + + // What covers that hole, and the reason it is a gap rather than a break: the entry type names + // all three, so the caller `entryNeeds` would have let through does not compile. + it('names every key a caller must bring in its entry type', () => { + const entry: ImagesServeEntry = { + 'ingress.http.headers': [['content-type', 'application/json']], + 'request.images.canonical': generations, + 'serve.model': 'gpt-image-1', + }; + expect(Object.keys(entry).sort()).toEqual(['ingress.http.headers', 'request.images.canonical', 'serve.model']); + + // @ts-expect-error — dropping one of them is a compile error, which is the statement this + // makes; the assertion below only keeps the binding from being unused. + const incomplete: ImagesServeEntry = { 'serve.model': 'gpt-image-1' }; + expect(Object.keys(incomplete)).toEqual(['serve.model']); + }); +}); diff --git a/packages/gateway/__tests__/data-plane/search-pipeline_test.ts b/packages/gateway/__tests__/data-plane/search-pipeline_test.ts new file mode 100644 index 0000000000..22556ab8f3 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/search-pipeline_test.ts @@ -0,0 +1,58 @@ +// Search's pipeline, assembled. `compose` derives the entry contract and rejects an array +// that cannot work, so most of what this file establishes is established by the assembly +// succeeding at all — and what is worth writing down is the two contracts it derives, one per +// ending, plus the one it cannot derive at all. + +import { describe, expect, it } from 'vitest'; + +import { searchServePipeline } from '../../src/data-plane/alpha-search/pipeline.ts'; +import { move, run } from '@floway-dev/pipeline'; + +const pinned = { kind: 'upstream', upstreamId: 'up_alpha', model: 'gpt-search' } as const; + +describe('the search pipeline', () => { + it('assembles the local ending, and asks its caller for what the descending stages need', () => { + expect([...searchServePipeline({ kind: 'local' }).entryNeeds].sort()).toEqual([ + 'request.search.alphaSearch', + ]); + }); + + // The pinned ending is one `return`-only stage, and such a stage declares no request side + // at all — by ruling, when it short-circuits there is only `provides`. So assembly sees + // nothing it reads and derives an empty contract, even though the stage cannot run without + // `request.search.alphaSearch` and `ingress.http.headers`. It is the same hole every + // family's ending stage has; here it swallows the family's whole entry contract, because + // the ending is the only stage below the edge. + // + // Written as a test rather than a comment because the hole has a consequence: a caller who + // omits either key gets a runtime failure at the deepest stage instead of an assembly + // error, and the entry contract exists to stop exactly that. The type layer still catches + // it at the definition site, which is why this is a gap and not a break. + it('cannot see what the pinned ending reads, because a return-only stage declares no needs', () => { + expect(searchServePipeline(pinned).entryNeeds).toEqual([]); + }); + + it('names the entry key a caller did not bring, before any stage runs', async () => { + await expect(run(searchServePipeline({ kind: 'local' }), move({}) as never, {})) + .rejects.toThrow('run(searchServe): searchServe needs request.search.alphaSearch'); + }); + + // A request with nothing to run reaches no backend at all, so this drives the whole + // assembly on no services: the parse stage answers instead of descending, and what the + // assembly guarantees — that a short-circuit covers what the edge needs — is what makes the + // run produce a rendered body, a status and a billed set anyway. + it('answers in band when there is nothing to run, and the edge renders that answer', async () => { + const { facts } = await run( + searchServePipeline({ kind: 'local' }), + move({ 'request.search.alphaSearch': { commands: {} } }), + {}, + ); + expect(facts['response.http.status']).toBe(200); + expect(facts['response.search.rendered']).toEqual({ + encrypted_output: null, + output: 'No web search commands were provided. Populate at least one of `search_query`, `open`, or `find`.', + }); + // Nothing was called that a model prices, and an empty set is how that is said. + expect(facts['response.usage.billable']).toEqual([]); + }); +}); diff --git a/packages/gateway/src/data-plane/alpha-search/pipeline.ts b/packages/gateway/src/data-plane/alpha-search/pipeline.ts new file mode 100644 index 0000000000..5dfd69fe95 --- /dev/null +++ b/packages/gateway/src/data-plane/alpha-search/pipeline.ts @@ -0,0 +1,356 @@ +// Search as a pipeline. The family with no upstream model and no candidate list: by default +// it runs the request's commands here, through the configured search backend, and when the +// operator has pinned a Codex or Custom upstream it asks that upstream's own search endpoint +// instead. +// +// That is an ending, not a missing family. What this shares with the other families is the +// edge; what varies is where the chain stops: +// +// emitAlphaSearch the edge: serializes the answer into Codex's protocol +// parseSearchOperations local only: Codex's commands become the gateway's own +// executeSearchOperations the local ending: runs them, and provides the answer +// callSearchUpstream the pinned ending: dials, and provides what came back +// +// `resolveCandidates` and `failover` have nothing to range over here. Local execution reaches +// no upstream at all, and the pinned mode names exactly one upstream and one model in +// operator configuration, so a run is its own only attempt. + +import { + parseAlphaSearchResponse, + renderAlphaSearchResponse, + webSearchFiltersFromSettings, + type AlphaSearchRequest, + type AlphaSearchResponse, +} from './protocol.ts'; +import { isJsonObject } from '../../shared/json-helpers.ts'; +import type { Failure, GatewayFacts } from '../pipeline/facts.ts'; +import { isFailure } from '../pipeline/facts.ts'; +import type { GatewayServices } from '../pipeline/services.ts'; +import { enumerateModelCandidates } from '../providers/resolution.ts'; +import type { GatewayCtx } from '../shared/gateway-ctx.ts'; +import { filterInboundHeadersForProvider } from '../shared/inbound-headers.ts'; +import { telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; +import { + assertLocalWebSearchSupport, + executeOperationToText, + parseWebSearchOperations, + startBatchFetch, + UnsupportedLocalWebSearchFeatureError, + type WebSearchExecutionSession, + type WebSearchFilters, + type WebSearchOperation, +} from '../tools/web-search/operations.ts'; +import type { ConfiguredWebSearchProvider } from '../tools/web-search/types.ts'; +import type { Pipeline } from '@floway-dev/pipeline'; +import { compose, defineStage, move } from '@floway-dev/pipeline'; +import { identityWrapUpstreamCall, providerModelOf, type ModelCandidate } from '@floway-dev/provider'; + +/** Search's own keys. They extend the shared space and never merge into it, so a stage + * written against the gateway alone cannot name one. */ +export interface SearchFacts extends GatewayFacts { + 'request.search.alphaSearch': AlphaSearchRequest; + /** What Codex's commands mean to this gateway. Local execution runs on these and never + * reads Codex's request again, which is why the stage that provides them consumes it. */ + 'request.search.operations': readonly WebSearchOperation[]; + 'request.search.filters': WebSearchFilters; + 'response.search.alphaSearch': AlphaSearchResponse | Failure; + /** What the client is actually sent, in Codex's protocol. The edge provides it, so a dump + * shows the bytes the client received rather than the gateway's own form. */ + 'response.search.rendered': Record; +} + +type S = { [P in K]: SearchFacts[P] }; + +/** The configured search backend is reached through a resolver rather than a fact: it holds + * live handles, and the operator's provider credential is what builds one — so nothing about + * it is dumpable and none of it belongs in the record. */ +export interface SearchServices extends GatewayServices { + readonly searchProvider: () => Promise; +} + +/** + * The outermost edge. Renders the answer into Codex's protocol and says what status the + * client is owed — the one thing a failure value carries that a rendered body cannot. + * + * It declares needing `response.usage.billable` without reading it, which is this family's + * statement that every path accounts for usage: assembly then rejects an ending, or a + * short-circuit, that does not provide it. + */ +const emitAlphaSearch = defineStage< + S, + S, + S<'response.search.alphaSearch' | 'response.usage.billable'>, + S<'response.search.rendered' | 'response.http.status'> +>({ + name: 'emitAlphaSearch', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { + needs: ['response.search.alphaSearch', 'response.usage.billable'], + consumes: ['response.search.alphaSearch'], + provides: ['response.search.rendered', 'response.http.status'], + }, + }, + execute: async (facts, next) => { + const back = await next(facts); + const { 'response.search.alphaSearch': answer, ...rest } = back; + if (isFailure(answer)) { + // An upstream error body is JSON like any other: it was parsed below and is serialized + // again here. A body that was not an object is one this protocol cannot carry, so what + // goes out instead is the gateway's own envelope. + return { + ...rest, + 'response.search.rendered': move(isJsonObject(answer.body) + ? answer.body + : { error: { message: answer.message, type: 'api_error' } }), + 'response.http.status': answer.status, + }; + } + return { + ...rest, + 'response.search.rendered': move(renderAlphaSearchResponse(answer)), + 'response.http.status': 200, + }; + }, +}); + +/** Both in-band answers a local run can give: Codex reads `output`, so a command it asked for + * and this gateway cannot run is text the model sees rather than an HTTP failure. */ +const inBandOutput = (facts: S<'request.search.alphaSearch'>, output: string) => move({ + ...facts, + 'response.search.alphaSearch': { encryptedOutput: null, output }, + 'response.usage.billable': [], +}); + +/** + * Codex's commands become the gateway's own operations, and its settings become the filters + * they run under. Nothing below reads Codex's request afterwards, so it is consumed here and + * assembly is what holds that: a stage placed under this one cannot need it back. + */ +const parseSearchOperations = defineStage< + S<'request.search.alphaSearch'>, + S<'request.search.operations' | 'request.search.filters'>, + S<'response.search.alphaSearch' | 'response.usage.billable'>, + S<'response.search.alphaSearch' | 'response.usage.billable'>, + S<'response.search.alphaSearch' | 'response.usage.billable'> +>({ + name: 'parseSearchOperations', + through: { + request: { + needs: ['request.search.alphaSearch'], + consumes: ['request.search.alphaSearch'], + provides: ['request.search.operations', 'request.search.filters'], + }, + response: { needs: [], consumes: [], provides: [] }, + }, + return: { provides: ['response.search.alphaSearch', 'response.usage.billable'] }, + execute: async (facts, next) => { + const { 'request.search.alphaSearch': request, ...rest } = facts; + const commands = request.commands ?? {}; + + try { + assertLocalWebSearchSupport(commands); + } catch (error) { + if (!(error instanceof UnsupportedLocalWebSearchFeatureError)) throw error; + return inBandOutput(facts, error.message); + } + + const parsed = parseWebSearchOperations(commands); + if (parsed.kind !== 'ops' || parsed.ops.length === 0) { + return inBandOutput(facts, 'No web search commands were provided. Populate at least one of `search_query`, `open`, or `find`.'); + } + + return await next({ + ...rest, + 'request.search.operations': move(parsed.ops), + 'request.search.filters': move(webSearchFiltersFromSettings(request.settings)), + }); + }, +}); + +/** + * The local ending. Runs every operation against the configured backend and provides the + * answer plus what the run is billable for — which is nothing, because no upstream model was + * called. What the search backend itself charges is accounted per api key by the operations + * as they run, in units no model prices. + */ +const executeSearchOperations = defineStage< + S<'request.search.operations' | 'request.search.filters'>, + S<'response.search.alphaSearch' | 'response.usage.billable'>, + SearchServices +>({ + name: 'executeSearchOperations', + return: { provides: ['response.search.alphaSearch', 'response.usage.billable'] }, + execute: async (facts, use) => { + const session: WebSearchExecutionSession = { + getProvider: use.searchProvider, + filters: facts['request.search.filters'], + apiKeyId: use.gateway.apiKeyId, + pageCache: new Map(), + // Codex renders `output` as plain text; the search-action sources list is a Responses + // protocol concern with no place here. + includeSearchActionSources: false, + ...(use.gateway.abortSignal === undefined ? {} : { signal: use.gateway.abortSignal }), + }; + + // One batched fetchPage covers every open and find URL; each operation then renders its + // own text block, in the parser's canonical order — search_query, open, find, preserving + // array order within each command kind. + const ops = [...facts['request.search.operations']]; + const batch = await startBatchFetch({ kind: 'ops', ops }, session); + const blocks = await Promise.all(ops.map(op => executeOperationToText(op, session, batch))); + use.log.debug('ran the search operations', { operations: ops.length }); + + return move({ + ...facts, + 'response.search.alphaSearch': { encryptedOutput: null, output: blocks.join('\n\n') }, + 'response.usage.billable': [], + }); + }, +}); + +/** The operator's pinned search upstream. One upstream and one model, both from + * configuration, which is why nothing here narrows a candidate list. */ +export interface PinnedSearchUpstream { + readonly kind: 'upstream'; + readonly upstreamId: string; + readonly model: string; +} + +/** + * A misconfigured pin is the operator's to see with its stack, the way the gateway surfaces + * any internal failure. It is not a failure value: a failure value exists so an earlier stage + * can fail over it, and there is no earlier stage here with anywhere to go. + */ +const resolvePinnedUpstream = async (pinned: PinnedSearchUpstream, gateway: GatewayCtx): Promise => { + if (gateway.upstreamIds !== null && !gateway.upstreamIds.includes(pinned.upstreamId)) { + throw new Error('Selected OpenAI search upstream is outside this API key scope'); + } + const { candidates } = await enumerateModelCandidates({ + upstreamIds: [pinned.upstreamId], + model: pinned.model, + kind: 'chat', + scheduler: gateway.backgroundScheduler, + runtimeLocation: gateway.runtimeLocation, + }); + const candidate = candidates.find(value => value.provider.upstreamId === pinned.upstreamId); + if (candidate === undefined) { + throw new Error(`Selected OpenAI search model ${pinned.model} is unavailable`); + } + if (candidate.provider.kind !== 'codex' && candidate.provider.kind !== 'custom') { + throw new Error('Selected upstream does not support OpenAI search passthrough'); + } + return candidate; +}; + +/** Codex projects one per-turn metadata snapshot onto several surfaces, and `SearchRequest` + * carries no field for it — so on this endpoint the header is the whole of that surface, and + * it is the only inbound header this call reads. + * https://github.com/openai/codex/blob/2e1607ee2fa8099a233df7437adee5f16a741905/codex-rs/codex-api/src/search.rs#L8-L29 */ +const turnMetadataHeaders = (ingress: readonly (readonly [string, string])[]): Headers => { + const headers = new Headers(); + const metadata = ingress.find(([name]) => name.toLowerCase() === 'x-codex-turn-metadata'); + if (metadata !== undefined) headers.set('x-codex-turn-metadata', metadata[1]); + return headers; +}; + +/** The body as JSON, or nothing when it was not JSON at all. Which of the two happened is + * what the caller reports; here it is only the question being asked. */ +const asJson = (raw: string): unknown => { + try { + return JSON.parse(raw) as unknown; + } catch { + return undefined; + } +}; + +/** + * The pinned ending. Dials the operator's search upstream and provides what came back plus + * what the call is billable for. A failure is a value here as everywhere, even though this + * family has nothing that fails over it: the edge is what turns one into a status and a body. + */ +const callSearchUpstream = (pinned: PinnedSearchUpstream) => defineStage< + S<'request.search.alphaSearch' | 'ingress.http.headers'>, + S<'response.search.alphaSearch' | 'response.usage.billable'>, + GatewayServices +>({ + name: 'callSearchUpstream', + return: { provides: ['response.search.alphaSearch', 'response.usage.billable'] }, + execute: async (facts, use) => { + const candidate = await resolvePinnedUpstream(pinned, use.gateway); + // The caller's model is dropped: this endpoint is pinned to the operator's model, and the + // provider stamps that one on the way out. + const { model: _named, ...request } = facts['request.search.alphaSearch']; + const result = await candidate.provider.instance.callAlphaSearch( + providerModelOf(candidate), + request, + use.gateway.abortSignal, + { + fetcher: candidate.fetcher, + waitUntil: use.gateway.backgroundScheduler, + // The client's own headers reach the upstream from the record, not from a live + // request object: what a provider is allowed to forward is filtered per provider, + // and the dump shows what was there to filter. + headers: filterInboundHeadersForProvider(turnMetadataHeaders(facts['ingress.http.headers']), candidate.provider), + // No `PerformanceOperation` names search, so there is no performance row for a + // stamping wrapper's interval to land on. + wrapUpstreamCall: identityWrapUpstreamCall, + }, + ); + // The alpha-search protocol reports no usage at all, so the entity is present with no + // quantities — the upstream was called and reported nothing, which is a different + // situation from reporting zero. + const billable = [{ identity: telemetryModelIdentity(candidate, result.modelKey), quantities: {} }]; + + // Every protocol the gateway carries is one it fully understands: the body is read here + // and serialized again at the edge, an error body included. + const raw = await result.response.text(); + if (!result.response.ok) { + use.log.warn('upstream refused', { status: result.response.status }); + // The message is what came back as text and the body is the same thing parsed: a dump + // reader gets the upstream's own words either way, and only the parsed form is + // something the edge can serialize back out. + const body = asJson(raw); + return move({ + ...facts, + 'response.search.alphaSearch': { + status: result.response.status, + message: raw, + ...(body === undefined ? {} : { body }), + }, + 'response.usage.billable': billable, + }); + } + + const verdict = parseAlphaSearchResponse(asJson(raw)); + if (!verdict.ok) { + // A protocol that requires JSON and receives something else synthesizes its own error, + // which is also why the raw text rides along: a dump reader is owed what came back. + return move({ + ...facts, + 'response.search.alphaSearch': { + status: 502, + message: `The search upstream answered ${result.response.status} but not in the search protocol: ${verdict.reason}.`, + body: raw, + }, + 'response.usage.billable': billable, + }); + } + return move({ ...facts, 'response.search.alphaSearch': verdict.response, 'response.usage.billable': billable }); + }, +}); + +/** Which ending the chain gets. `upstream` is the operator's `passthroughOpenAiSearch` + * setting: the word there is about whose search results the client is given, not about + * carrying a protocol the gateway has not parsed. */ +export type SearchExecution = { readonly kind: 'local' } | PinnedSearchUpstream; + +export const searchServePipeline = (execution: SearchExecution): Pipeline< + S<'request.search.alphaSearch'>, + S<'response.search.rendered' | 'response.http.status' | 'response.usage.billable'> +> => compose('searchServe', [ + emitAlphaSearch, + ...(execution.kind === 'local' + ? [parseSearchOperations, executeSearchOperations] + : [callSearchUpstream(execution)]), +]); diff --git a/packages/gateway/src/data-plane/alpha-search/protocol.ts b/packages/gateway/src/data-plane/alpha-search/protocol.ts new file mode 100644 index 0000000000..f2682bdf6f --- /dev/null +++ b/packages/gateway/src/data-plane/alpha-search/protocol.ts @@ -0,0 +1,115 @@ +// Codex's `/alpha/search` protocol. The request carries model/session context plus a +// command object; the response is `{ encrypted_output, output, results? }`. +// https://github.com/openai/codex/blob/2e1607ee2fa8099a233df7437adee5f16a741905/codex-rs/codex-api/src/search.rs#L8-L29 +// https://github.com/openai/codex/blob/2e1607ee2fa8099a233df7437adee5f16a741905/codex-rs/codex-api/src/search.rs#L297-L305 + +import { z } from 'zod'; + +import { isJsonObject } from '../../shared/json-helpers.ts'; +import { maxResultsForContextSize, type WebSearchFilters } from '../tools/web-search/operations.ts'; + +const domainListSchema = z.array(z.string()); + +// This is OpenAI Codex's complete SearchSettings shape. The loose object keeps future fields +// intact; local execution consumes the routing fields it implements while accepting Codex +// metadata such as allowed_callers. +// https://github.com/openai/codex/blob/2f19a57704fb7b1db032bc38cf995034254eaebb/codex-rs/codex-api/src/search.rs#L215-L295 +const searchSettingsSchema = z.looseObject({ + filters: z.looseObject({ + allowed_domains: domainListSchema.optional(), + blocked_domains: domainListSchema.optional(), + }).optional(), + user_location: z.looseObject({ + type: z.literal('approximate').optional(), + city: z.string().optional(), + region: z.string().optional(), + country: z.string().optional(), + timezone: z.string().optional(), + }).optional(), + search_context_size: z.enum(['low', 'medium', 'high']).optional(), + image_settings: z.looseObject({ + max_results: z.number().int().nonnegative().optional(), + caption: z.boolean().optional(), + }).optional(), + allowed_callers: z.array(z.enum(['direct', 'shell', 'code_interpreter'])).optional(), + external_web_access: z.union([ + z.boolean(), + z.enum(['cached', 'indexed', 'live']), + ]).optional(), +}); + +// `commands` is validated only as "an object" — the per-kind arrays are parsed by the shared +// command engine. `looseObject` preserves every OpenAI command and nested parameter so a +// relayed request stays lossless and the local capability gate can reject unimplemented +// fields explicitly. +// https://github.com/openai/codex/blob/2f19a57704fb7b1db032bc38cf995034254eaebb/codex-rs/codex-api/src/search.rs#L31-L213 +export const alphaSearchRequestSchema = z.looseObject({ + commands: z.looseObject({}).optional(), + settings: searchSettingsSchema.optional(), +}); + +export type AlphaSearchRequest = z.infer; + +export interface AlphaSearchResponse { + /** Search state Codex carries forward into a later turn, opaque to everyone between the + * two ends. Local execution has no state to carry, so it is null. */ + encryptedOutput: string | null; + /** The model-facing text. Codex renders this and nothing else. */ + output: string; + /** Result DTOs handed to clients out of band from `output`. Codex keeps them opaque so + * newer result variants stay forward-compatible, and so does this. */ + results?: readonly unknown[]; +} + +/** Why a body could not be read as this protocol, so the ending stage can say so in the + * synthesized failure rather than reporting a bare status. */ +export type AlphaSearchResponseVerdict = + | { readonly ok: true; readonly response: AlphaSearchResponse } + | { readonly ok: false; readonly reason: string }; + +export const parseAlphaSearchResponse = (body: unknown): AlphaSearchResponseVerdict => { + if (!isJsonObject(body)) return { ok: false, reason: 'the body is not a JSON object' }; + const { encrypted_output: encryptedOutput, output, results } = body; + if (typeof output !== 'string') return { ok: false, reason: '`output` is missing or not a string' }; + // `encrypted_output` is `Option` without a serde default, so Codex's own client + // requires the key and accepts null in it. Demanding the same here surfaces a truncated + // upstream at the gateway instead of as a deserialization failure inside the client. + if (encryptedOutput !== null && typeof encryptedOutput !== 'string') { + return { ok: false, reason: '`encrypted_output` is missing or is neither a string nor null' }; + } + if (results !== undefined && results !== null && !Array.isArray(results)) { + return { ok: false, reason: '`results` is neither absent nor an array' }; + } + return { + ok: true, + response: { + encryptedOutput, + output, + ...(Array.isArray(results) ? { results } : {}), + }, + }; +}; + +export const renderAlphaSearchResponse = (response: AlphaSearchResponse): Record => ({ + encrypted_output: response.encryptedOutput, + output: response.output, + ...(response.results === undefined ? {} : { results: response.results }), +}); + +export const webSearchFiltersFromSettings = (settings: AlphaSearchRequest['settings']): WebSearchFilters => { + const filters: WebSearchFilters = { + maxResults: maxResultsForContextSize(settings?.search_context_size), + }; + if (settings?.filters?.allowed_domains) filters.allowedDomains = settings.filters.allowed_domains; + if (settings?.filters?.blocked_domains) filters.blockedDomains = settings.filters.blocked_domains; + const loc = settings?.user_location; + if (loc && (loc.city !== undefined || loc.region !== undefined || loc.country !== undefined || loc.timezone !== undefined)) { + filters.userLocation = { + ...(loc.city !== undefined ? { city: loc.city } : {}), + ...(loc.region !== undefined ? { region: loc.region } : {}), + ...(loc.country !== undefined ? { country: loc.country } : {}), + ...(loc.timezone !== undefined ? { timezone: loc.timezone } : {}), + }; + } + return filters; +}; diff --git a/packages/gateway/src/data-plane/alpha-search/routes.ts b/packages/gateway/src/data-plane/alpha-search/routes.ts index dece045bb0..fa3d81d25b 100644 --- a/packages/gateway/src/data-plane/alpha-search/routes.ts +++ b/packages/gateway/src/data-plane/alpha-search/routes.ts @@ -1,8 +1,7 @@ -// Codex `/alpha/search` compatibility endpoint. The private request carries -// model/session context plus a command object; the response is -// `{ encrypted_output?, output, results? }`. -// https://github.com/openai/codex/blob/2e1607ee2fa8099a233df7437adee5f16a741905/codex-rs/codex-api/src/search.rs#L8-L29 -// https://github.com/openai/codex/blob/2e1607ee2fa8099a233df7437adee5f16a741905/codex-rs/codex-api/src/search.rs#L297-L305 +// Codex `/alpha/search` compatibility endpoint. The protocol itself — the request +// schema, the response shape, and the projection of Codex's settings onto the +// gateway's search filters — lives in `protocol.ts`. +// // Clients append `alpha/search` to an OpenAI-compatible provider base. The // aliases below cover Floway's general root and `/v1` base conventions. // https://github.com/openai/codex/blob/2e1607ee2fa8099a233df7437adee5f16a741905/codex-rs/codex-api/src/endpoint/search.rs#L31-L47 @@ -16,8 +15,8 @@ // the resolved API key for per-key search-usage accounting. import type { Hono } from 'hono'; -import { z } from 'zod'; +import { alphaSearchRequestSchema, webSearchFiltersFromSettings } from './protocol.ts'; import { type AuthVars, apiKeyFromContext, effectiveUpstreamIdsFromContext } from '../../middleware/auth.ts'; import { type CtxWithJson, zValidator } from '../../middleware/zod-validator.ts'; import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; @@ -26,70 +25,11 @@ import { mountPublicRoute } from '../public-route.ts'; import { relayFetchedResponse } from '../tools/web-search/alpha-search/relay-response.ts'; import { resolveAlphaSearchDispatcher } from '../tools/web-search/alpha-search/upstream.ts'; import { loadWebSearchConfig } from '../tools/web-search/config.ts'; -import { assertLocalWebSearchSupport, executeOperationToText, maxResultsForContextSize, parseWebSearchOperations, startBatchFetch, UnsupportedLocalWebSearchFeatureError, type WebSearchExecutionSession, type WebSearchFilters } from '../tools/web-search/operations.ts'; +import { assertLocalWebSearchSupport, executeOperationToText, parseWebSearchOperations, startBatchFetch, UnsupportedLocalWebSearchFeatureError, type WebSearchExecutionSession } from '../tools/web-search/operations.ts'; import { resolveConfiguredWebSearchProvider } from '../tools/web-search/provider.ts'; import type { ConfiguredWebSearchProvider } from '../tools/web-search/types.ts'; import { PUBLIC_DATA_PLANE_ROUTES } from '@floway-dev/protocols/common'; -const domainListSchema = z.array(z.string()); - -// This is OpenAI Codex's complete SearchSettings shape. The loose object keeps -// future fields intact for passthrough; local providers consume the routing -// fields they implement while accepting Codex metadata such as allowed_callers. -// https://github.com/openai/codex/blob/2f19a57704fb7b1db032bc38cf995034254eaebb/codex-rs/codex-api/src/search.rs#L215-L295 -const searchSettingsSchema = z.looseObject({ - filters: z.looseObject({ - allowed_domains: domainListSchema.optional(), - blocked_domains: domainListSchema.optional(), - }).optional(), - user_location: z.looseObject({ - type: z.literal('approximate').optional(), - city: z.string().optional(), - region: z.string().optional(), - country: z.string().optional(), - timezone: z.string().optional(), - }).optional(), - search_context_size: z.enum(['low', 'medium', 'high']).optional(), - image_settings: z.looseObject({ - max_results: z.number().int().nonnegative().optional(), - caption: z.boolean().optional(), - }).optional(), - allowed_callers: z.array(z.enum(['direct', 'shell', 'code_interpreter'])).optional(), - external_web_access: z.union([ - z.boolean(), - z.enum(['cached', 'indexed', 'live']), - ]).optional(), -}); - -// `commands` is validated only as "an object" — the per-kind arrays are -// parsed by the shared command engine. `looseObject` preserves every OpenAI -// command and nested parameter so passthrough stays lossless and the local -// capability gate can reject unimplemented fields explicitly. -const alphaSearchRequestSchema = z.looseObject({ - commands: z.looseObject({}).optional(), - settings: searchSettingsSchema.optional(), -}); - -type AlphaSearchRequest = z.infer; - -const filtersFromSettings = (settings: AlphaSearchRequest['settings']): WebSearchFilters => { - const filters: WebSearchFilters = { - maxResults: maxResultsForContextSize(settings?.search_context_size), - }; - if (settings?.filters?.allowed_domains) filters.allowedDomains = settings.filters.allowed_domains; - if (settings?.filters?.blocked_domains) filters.blockedDomains = settings.filters.blocked_domains; - const loc = settings?.user_location; - if (loc && (loc.city !== undefined || loc.region !== undefined || loc.country !== undefined || loc.timezone !== undefined)) { - filters.userLocation = { - ...(loc.city !== undefined ? { city: loc.city } : {}), - ...(loc.region !== undefined ? { region: loc.region } : {}), - ...(loc.country !== undefined ? { country: loc.country } : {}), - ...(loc.timezone !== undefined ? { timezone: loc.timezone } : {}), - }; - } - return filters; -}; - const alphaSearch = async (c: CtxWithJson): Promise => { const body = c.req.valid('json'); const webSearchConfig = await loadWebSearchConfig(); @@ -122,7 +62,7 @@ const alphaSearch = async (c: CtxWithJson): Pro configuredProvider ??= Promise.resolve(resolveConfiguredWebSearchProvider(webSearchConfig)); return configuredProvider; }, - filters: filtersFromSettings(body.settings), + filters: webSearchFiltersFromSettings(body.settings), apiKeyId: apiKeyFromContext(c).id, pageCache: new Map(), // Codex renders `output` as plain text; the search-action sources list diff --git a/packages/gateway/src/data-plane/audio/pipeline.ts b/packages/gateway/src/data-plane/audio/pipeline.ts new file mode 100644 index 0000000000..455497a4b6 --- /dev/null +++ b/packages/gateway/src/data-plane/audio/pipeline.ts @@ -0,0 +1,375 @@ +// Audio transcription as a pipeline. The family whose answer is not one document but six: +// `response_format` picks between three JSON objects and three text documents, `stream` +// picks a sequence of events instead of any of them, and each is its own media type. That +// is the whole reason this endpoint needed a response strategy of its own, and it is what +// one canonical fact plus one media-type fact dissolve. +// +// The shape: +// +// emitAudioTranscription the edge: writes the answer back in the rendering the +// client asked for, SSE framing included +// resolveCandidates narrows to the upstreams that expose the endpoint +// failover runs what follows once per candidate +// callAudioTranscriptionUpstream the ending: dials, reads what came back, and provides +// the answer plus what is billable + +import type { UsageQuantities } from '../../repo/types.ts'; +import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; +import { isFailure } from '../pipeline/facts.ts'; +import type { GatewayServices } from '../pipeline/services.ts'; +import { failover, resolveCandidates } from '../pipeline/stages.ts'; +import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; +import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; +import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; +import { + isAudioTranscriptionDoneEvent, + isAudioTranscriptionObjectFormat, + parseAudioTranscription, + parseAudioTranscriptionStreamEvent, + parseAudioTranscriptionStreamUsage, + renderAudioTranscription, + type AudioTranscriptionResponseFormat, + type AudioTranscriptionStreamEvent, + type AudioTranscriptionUsage, + type CanonicalAudioTranscription, +} from '@floway-dev/protocols/audio'; +import { isEventStreamMediaType, parseDecimalString, parseSSEStream, sseFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { providerModelOf, type AudioTranscriptionFormEntry, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; + +/** The answer while it is still the upstream's, one event at a time. It is a view and not a + * resource: what owns the connection is `response.http.body`, which is where release and + * failover's ownership both read. + * + * A view has to be built as a wrapper around the generator rather than handed over as the + * generator itself. Measured on Node 24: `Symbol.asyncDispose in (async function*(){})()` + * is `true`, so a bare generator in the record is a releasable — the runner would adopt it, + * the top-level sweep would call its `return()`, and that cancels the iteration instead of + * draining it, which is the one thing release must never mean. */ +export type AudioTranscriptionEvents = AsyncIterable; + +const viewOf = (events: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => events }); + +/** Audio transcription's own keys, extending the shared space by intersection. */ +export interface AudioTranscriptionFacts extends GatewayFacts { + /** Which of the six renderings the client asked for. It belongs to the ingress and stays + * put: the same value travels to the upstream inside the form, so the rendering the + * answer arrives in is the rendering the answer is written back in — and a text document + * does not say which of the three it is, so nothing else could decide. */ + 'ingress.audioTranscription.responseFormat': AudioTranscriptionResponseFormat; + /** The multipart form as ordered semantic entries. The body is parsed before routing + * because field order is unconstrained, and every candidate builds a fresh body from + * these, so a retry never reuses a consumed one. The bytes the client sent are recorded + * at `ingress.http.body`, which is where a dump reads the upload itself. */ + 'request.audioTranscription.form': readonly AudioTranscriptionFormEntry[]; + /** The one transcription, whichever rendering carried it — or the events it is arriving + * as, or the failure that came instead. A stream, a value and a failure sit at one key: + * telling them apart is reading a value, and each stage does that where it needs to. */ + 'response.audioTranscription.canonical': CanonicalAudioTranscription | AudioTranscriptionEvents | Failure; + /** What the answer goes out under. A media type is upstream-owned — OpenAI answers + * `text`, `srt` and `vtt` under one media type and other upstreams label them apart — so + * the upstream's own travels, and the edge re-provides it only where the gateway wrote + * the body itself. `null` is an upstream that declared none, which is what a client of + * this endpoint sees today. */ + 'response.audioTranscription.mediaType': string | null; + /** What the upstream will have reported by the time the events run out, and `null` on + * every path that does not stream. A streamed transcription states its usage in the + * terminal event, which is after this run has answered — so the numbers cannot be in + * `response.usage.billable`, which says what had been reported when the ending stage + * handed up: the entity, and no quantities. Settling from this is the prologue's job, + * after the drain. */ + 'response.audioTranscription.streamedUsage': Promise | null; + /** What the client is actually sent — a JSON object, a text document, or the SSE frames + * of a stream. The edge provides it, so a dump shows what the client received rather + * than the gateway's own reading of it. */ + 'response.audioTranscription.rendered': Record | string | AsyncIterable; +} + +type A = { [P in K]: AudioTranscriptionFacts[P] }; + +const isEvents = (answer: CanonicalAudioTranscription | AudioTranscriptionEvents): answer is AudioTranscriptionEvents => + Symbol.asyncIterator in answer; + +/** + * The outermost edge. Writes the answer back in the rendering the client asked for, and + * owns the media type of anything the gateway wrote itself. + * + * SSE framing is produced here and nowhere else — below this stage the answer is parsed + * events, so the same assembly would serve another transport by rendering differently at + * this one point. The upstream's own `event:` label is not carried: it is transport, and + * OpenAI's clients read this endpoint's frames by their payload's `type` rather than by the + * label. https://github.com/openai/openai-python/blob/10ee3f0da2ac6f93345c1204bd7bb1a2faa79ff2/src/openai/_streaming.py#L61-L107 + */ +const emitAudioTranscription = defineStage< + A<'ingress.audioTranscription.responseFormat'>, + A<'ingress.audioTranscription.responseFormat'>, + A<'ingress.audioTranscription.responseFormat' | 'response.audioTranscription.canonical' | 'response.audioTranscription.mediaType'>, + A<'response.audioTranscription.rendered' | 'response.audioTranscription.mediaType'> & { 'response.http.status': number } +>({ + name: 'emitAudioTranscription', + through: { + request: { + needs: ['ingress.audioTranscription.responseFormat'], + consumes: [], + provides: [], + }, + response: { + needs: ['response.audioTranscription.canonical', 'response.audioTranscription.mediaType'], + consumes: ['response.audioTranscription.canonical'], + provides: ['response.audioTranscription.rendered', 'response.audioTranscription.mediaType', 'response.http.status'], + }, + }, + execute: async (facts, next) => { + const back = await next(facts); + const { 'response.audioTranscription.canonical': answer, ...rest } = back; + + if (isFailure(answer)) { + return { + ...rest, + 'response.http.status': answer.status, + 'response.audioTranscription.mediaType': 'application/json', + 'response.audioTranscription.rendered': move({ error: { message: answer.message, type: 'api_error' } }), + }; + } + // Everything that reaches here answered. The same key carried the upstream's own status + // further down; re-providing it is what makes the top of the record the response the + // client gets rather than the one the upstream gave. + return { + ...rest, + 'response.http.status': 200, + 'response.audioTranscription.rendered': move(isEvents(answer) + ? renderSSE(answer) + : renderAudioTranscription(back['ingress.audioTranscription.responseFormat'], answer)), + }; + }, +}); + +const renderSSE = (events: AudioTranscriptionEvents): AsyncIterable => + viewOf((async function* () { + for await (const event of events) yield sseFrame(JSON.stringify(event)); + })()); + +/** + * The ending. It dials, reads what came back in the rendering the request asked for, and + * provides the answer, the raw HTTP response beneath it, and what the call is billable for. + * A failure is a value: a 429 here is what an earlier stage fails over, and so is a 200 whose + * body this rendering cannot be read from, because the next candidate may answer with one + * that can. + */ +const callAudioTranscriptionUpstream = defineStage< + A<'ingress.audioTranscription.responseFormat' | 'request.audioTranscription.form' | 'route.candidate' | 'ingress.http.headers'>, + A<'response.audioTranscription.canonical' | 'response.audioTranscription.mediaType' | 'response.audioTranscription.streamedUsage'> + & { 'response.usage.billable': readonly BillableEntity[]; 'response.http.status': number; + 'response.http.headers': readonly (readonly [string, string])[]; + 'response.http.body': ReadableStream & AsyncDisposable; }, + GatewayServices +>({ + name: 'callAudioTranscriptionUpstream', + return: { + provides: [ + 'response.audioTranscription.canonical', + 'response.audioTranscription.mediaType', + 'response.audioTranscription.streamedUsage', + 'response.usage.billable', + 'response.http.status', + 'response.http.headers', + 'response.http.body', + ], + }, + execute: async (facts, use) => { + const candidate = facts['route.candidate']; + const result = await candidate.provider.instance.callAudioTranscriptions( + providerModelOf(candidate), + { entries: facts['request.audioTranscription.form'] }, + use.gateway.abortSignal, + // The client's own headers reach the upstream from the record, not from a live request + // object: what a provider is allowed to forward is filtered per provider, and the dump + // shows what was there to filter. + buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]) => [name, value]))), + ); + use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'audio_transcription'); + const identity = telemetryModelIdentity(candidate, result.modelKey); + const format = facts['ingress.audioTranscription.responseFormat']; + const status = result.response.status; + const mediaType = result.response.headers.get('content-type'); + const headers = move([...result.response.headers] as readonly (readonly [string, string])[]); + // An upstream that was called and reported nothing is a different situation from one + // that reported zero, so the entity is present with no quantities. + const called: readonly BillableEntity[] = [{ identity, quantities: {} }]; + + if (!result.response.ok) { + use.log.warn('upstream refused', { status }); + // An upstream error body is JSON like any other body, and reading it here is also what + // leaves a losing attempt with nothing open behind it. + return move({ + ...facts, + 'response.audioTranscription.canonical': await refusal(status, result.response), + 'response.audioTranscription.mediaType': mediaType, + 'response.audioTranscription.streamedUsage': null, + 'response.usage.billable': called, + 'response.http.status': status, + 'response.http.headers': headers, + 'response.http.body': spentBody(result.response.body), + }); + } + + // A streamed answer is the one the client asked for with `stream`, and it is the media + // type that says one arrived: an upstream that ignores `stream` — `whisper-1` does — + // answers in the rendering `response_format` named instead. + // https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L36325-L36336 + if (isEventStreamMediaType(mediaType)) { + if (result.response.body === null) { + return move({ + ...facts, + 'response.audioTranscription.canonical': { status: 502, message: 'Upstream returned a streaming response with no body.' }, + 'response.audioTranscription.mediaType': mediaType, + 'response.audioTranscription.streamedUsage': null, + 'response.usage.billable': called, + 'response.http.status': status, + 'response.http.headers': headers, + 'response.http.body': spentBody(null), + }); + } + // Usage is observed here, closest to the upstream and on the protocol it spoke, by + // folding the events as they pass — so the reading costs one pass and the client's own + // stream is what drives it. + const metered = meterEvents(result.response.body, identity, use.gateway.abortSignal); + return move({ + ...facts, + 'response.audioTranscription.canonical': metered.events, + 'response.audioTranscription.mediaType': mediaType, + 'response.audioTranscription.streamedUsage': metered.billable, + 'response.usage.billable': called, + 'response.http.status': status, + 'response.http.headers': headers, + // Releasing this body is reading those events to the end: they are one reader over + // one connection, and a second reader is not something a `ReadableStream` allows. + 'response.http.body': Object.assign(result.response.body, { + [Symbol.asyncDispose]: async (): Promise => { for await (const _event of metered.events) { /* to end of stream */ } }, + }), + }); + } + + const read = await readTranscription(format, result.response); + return move({ + ...facts, + 'response.audioTranscription.canonical': isFailure(read) ? read : read.value, + 'response.audioTranscription.mediaType': mediaType, + 'response.audioTranscription.streamedUsage': null, + 'response.usage.billable': isFailure(read) ? called : [{ identity, quantities: billed(read.value.usage) }], + 'response.http.status': status, + 'response.http.headers': headers, + 'response.http.body': spentBody(result.response.body), + }); + }, +}); + +/** A body this stage has already read to the end, or one the upstream never sent. The record + * holds a body as a stream and `failover` releases the losing attempts', so every path hands + * one up; what says an answer was unusable is the failure at the canonical key, not this. */ +const spentBody = (body: ReadableStream | null): ReadableStream & AsyncDisposable => + Object.assign(body ?? new ReadableStream({ start: controller => controller.close() }), { + [Symbol.asyncDispose]: (): Promise => Promise.resolve(), + }); + +const refusal = async (status: number, response: Response): Promise => { + const text = await response.text(); + try { + return { status, message: text, body: JSON.parse(text) as unknown }; + } catch { + // A refusal that is not JSON is still a refusal, and its text is what the client is + // told; there is simply no parsed body for a dump reader to open. + return { status, message: text }; + } +}; + +/** A body that answered 200 and cannot be read in the rendering it was asked for is an + * attempt that failed, because the gateway serializes what it sends from the value it + * parsed and has nothing to serialize. */ +const readTranscription = async ( + format: AudioTranscriptionResponseFormat, + response: Response, +): Promise<{ readonly value: CanonicalAudioTranscription } | Failure> => { + try { + const body: unknown = isAudioTranscriptionObjectFormat(format) ? await response.json() : await response.text(); + return { value: parseAudioTranscription(format, body) }; + } catch (error) { + return { + status: 502, + message: `Upstream answered ${response.status} with a body this endpoint cannot read as ${format}: ${error instanceof Error ? error.message : String(error)}`, + }; + } +}; + +interface MeteredEvents { + readonly events: AudioTranscriptionEvents; + readonly billable: Promise; +} + +const meterEvents = ( + body: ReadableStream, + identity: TelemetryModelIdentity, + signal: AbortSignal | undefined, +): MeteredEvents => { + let settle!: (billable: readonly BillableEntity[]) => void; + const billable = new Promise(resolve => { settle = resolve; }); + const events = viewOf((async function* () { + let usage: AudioTranscriptionUsage | undefined; + try { + for await (const frame of parseSSEStream(body, { signal })) { + const event = parseAudioTranscriptionStreamEvent(JSON.parse(frame.data) as unknown); + if (isAudioTranscriptionDoneEvent(event)) usage = parseAudioTranscriptionStreamUsage(event); + yield event; + } + } finally { + // Reached however the events ended — the terminal one, a client that stopped reading, + // or a broken upstream — because what the upstream already metered is billable + // whatever happened to the downstream half. + settle([{ identity, quantities: billed(usage) }]); + } + })()); + return { events, billable }; +}; + +const billed = (usage: AudioTranscriptionUsage | undefined): UsageQuantities => { + if (usage === undefined) return {}; + if (usage.kind === 'duration') return { input_audio_seconds: parseDecimalString(String(usage.seconds)) }; + // Audio input is priced apart from text input, so what stays on the general input metric is + // what the upstream did not attribute to audio. + return { + input_tokens: parseDecimalString(String(usage.inputTokens - (usage.inputAudioTokens ?? 0))), + ...(usage.inputAudioTokens === undefined ? {} : { input_audio_tokens: parseDecimalString(String(usage.inputAudioTokens)) }), + output_tokens: parseDecimalString(String(usage.outputTokens)), + }; +}; + +/** Nothing about this request narrows a candidate further. A transcription's parameters are + * form fields the upstream reads for itself, and no endpoint metadata says which renderings + * a model can write — so exposing the endpoint is the whole of the test. */ +const narrowing = { + kind: 'transcription' as const, + reject: (candidate: ModelCandidate): string | null => + candidate.model.endpoints.audioTranscriptions === undefined + ? 'the upstream does not expose an audio transcription endpoint' + : null, + refuse: (status: number, message: string) => ({ + 'response.audioTranscription.canonical': { status, message } as Failure, + 'response.audioTranscription.mediaType': null, + 'response.audioTranscription.streamedUsage': null, + }), + refuses: [ + 'response.audioTranscription.canonical', + 'response.audioTranscription.mediaType', + 'response.audioTranscription.streamedUsage', + ] as const, +}; + +export const audioTranscriptionServePipeline: Pipeline< + A<'ingress.audioTranscription.responseFormat' | 'request.audioTranscription.form' | 'serve.model'>, + A<'response.audioTranscription.rendered' | 'response.audioTranscription.mediaType' | 'response.audioTranscription.streamedUsage'> + & { 'response.http.status': number; 'response.usage.billable': readonly BillableEntity[] } +> = compose('audioTranscriptionServe', [ + emitAudioTranscription, + resolveCandidates(narrowing), + failover(handedUp => isFailure((handedUp as { 'response.audioTranscription.canonical'?: unknown })['response.audioTranscription.canonical'])), + callAudioTranscriptionUpstream, +]); diff --git a/packages/gateway/src/data-plane/completions/pipeline.ts b/packages/gateway/src/data-plane/completions/pipeline.ts new file mode 100644 index 0000000000..5661a98938 --- /dev/null +++ b/packages/gateway/src/data-plane/completions/pipeline.ts @@ -0,0 +1,377 @@ +// Text completions as a pipeline. One protocol, no translation, and an answer that is a +// stream whenever the client asked for one — the same request field decides both what the +// upstream is asked for and what shape comes back. +// +// The shape: +// +// emitCompletions the edge: asks the upstream for the usage chunk on the way +// down, and renders the answer into the client's protocol on the +// way back, SSE framing included +// resolveCandidates narrows to the upstreams that expose the endpoint +// failover runs what follows once per candidate +// callCompletionsUpstream the ending: dials, parses what came back, and provides the +// answer plus what is billable + +import { tokenUsageFromCompletionsUsage } from './usage.ts'; +import type { UsageQuantities } from '../../repo/types.ts'; +import { tokenUsageQuantities } from '../../repo/usage-metrics.ts'; +import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; +import { isFailure } from '../pipeline/facts.ts'; +import type { GatewayServices } from '../pipeline/services.ts'; +import { failover, resolveCandidates } from '../pipeline/stages.ts'; +import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; +import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; +import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; +import { isOpenAIUsageOnlyEventShape, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { + completionsProtocolFrameToSSEFrame, + parseCompletionsResult, + parseCompletionsStream, + type CompletionsPayload, + type CompletionsResult, + type CompletionsStreamEvent, +} from '@floway-dev/protocols/completions'; +import { providerModelOf } from '@floway-dev/provider'; +import type { ModelCandidate, TelemetryModelIdentity } from '@floway-dev/provider'; + +/** The answer while it is still the upstream's, one frame at a time. */ +export type CompletionsFrames = AsyncIterable>; + +/** + * A stream as a value the record can hold, and the one thing here that does not follow from + * the rules as written: an async generator is itself an `AsyncDisposable` — V8 gives every + * generator a `Symbol.asyncDispose` that returns it — so handing one over makes a frame view + * a *resource* as far as the run is concerned. Two rules then bite, both correctly and both + * fatally: a stage that consumes a view while providing a derived one has its source closed + * the moment it returns, and `failover` refuses a fork that received a releasable at a key it + * did not declare, which is every key but `response.http.body`. + * + * There is exactly one resource in a completions run — the upstream's body — and this is what + * keeps it that way. What comes back is single-shot on purpose: a stream that has been read is + * a stream that is over, and a second reader learns that rather than being lied to. + */ +const view = (frames: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => frames }); + +/** Completions' own keys, extending the shared space by intersection. */ +export interface CompletionsFacts extends GatewayFacts { + /** What the client asked for, which is not what the upstream is asked for: the gateway + * meters every stream and so always turns the usage chunk on. These stay put for the same + * reason every `ingress.*` key does — they describe the request that arrived, and the + * answer is rendered back into it. */ + 'ingress.completions.wantsStream': boolean; + 'ingress.completions.wantsUsageChunk': boolean; + 'request.completions.payload': CompletionsPayload; + /** The answer, whichever kind it turned out to be. A stream, a value and a failure sit at + * one key: telling them apart is reading a value, and each stage does that where it needs + * to. */ + 'response.completions.payload': CompletionsResult | CompletionsFrames | Failure; + /** What the upstream will have reported by the time the frames run out, and `null` on + * every path that does not stream. A stream's usage arrives with its last chunk, which is + * after this run has answered — so the numbers cannot be in `response.usage.billable`, + * which says what had been reported when the ending stage handed up: the entity, and no + * quantities. Settling billing from this is the prologue's job, after the drain. */ + 'response.completions.streamedUsage': Promise | null; + /** What the client is actually sent, in its own protocol — a JSON body, or the SSE frames + * of a stream. The edge provides it, so a dump shows what the client received rather than + * the gateway's own reading of it. */ + 'response.completions.rendered': Record | AsyncIterable; +} + +type C = { [P in K]: CompletionsFacts[P] }; + +const isFrames = (answer: CompletionsFacts['response.completions.payload']): answer is CompletionsFrames => + Symbol.asyncIterator in answer; + +/** + * The outermost edge, and the only place where what the client asked for and what the + * upstream is asked for differ: billing needs the usage chunk on every stream, so it is + * turned on going down and taken back out coming up unless the client asked to see it. + * + * Rendering the answer is the other half. SSE framing is produced here and nowhere else — + * below this stage a stream carries protocol frames, so the same assembly would serve + * another transport by rendering differently at this one point. + */ +const emitCompletions = defineStage< + C<'ingress.completions.wantsStream' | 'ingress.completions.wantsUsageChunk' | 'request.completions.payload'>, + C<'ingress.completions.wantsStream' | 'ingress.completions.wantsUsageChunk' | 'request.completions.payload'>, + C<'ingress.completions.wantsUsageChunk' | 'response.completions.payload'>, + C<'response.completions.rendered' | 'response.http.status'> +>({ + name: 'emitCompletions', + through: { + request: { + needs: ['ingress.completions.wantsStream', 'ingress.completions.wantsUsageChunk', 'request.completions.payload'], + consumes: [], + provides: ['request.completions.payload'], + }, + response: { + needs: ['response.completions.payload'], + consumes: ['response.completions.payload'], + provides: ['response.completions.rendered', 'response.http.status'], + }, + }, + execute: async (facts, next) => { + const asked = facts['request.completions.payload']; + const back = await next({ + ...facts, + 'request.completions.payload': move(facts['ingress.completions.wantsStream'] + ? { ...asked, stream_options: { ...asked.stream_options, include_usage: true } } + : asked), + }); + + const { 'response.completions.payload': answer, ...rest } = back; + // A refusal keeps the status the upstream gave it. Anything that answered is a 200 the + // gateway says itself, because what the client receives is serialized here rather than + // relayed — the upstream's own status is a fact further down for whoever wants it. + return { + ...rest, + 'response.http.status': isFailure(answer) ? answer.status : 200, + 'response.completions.rendered': move(rendered(answer, back['ingress.completions.wantsUsageChunk'])), + }; + }, +}); + +const rendered = ( + answer: CompletionsFacts['response.completions.payload'], + wantsUsageChunk: boolean, +): CompletionsFacts['response.completions.rendered'] => + isFailure(answer) ? { error: { message: answer.message, type: 'api_error' } } + : isFrames(answer) ? renderSSE(answer, wantsUsageChunk) + : answer; + +const renderSSE = (frames: CompletionsFrames, wantsUsageChunk: boolean): AsyncIterable => view((async function* () { + for await (const frame of frames) { + if (!wantsUsageChunk && frame.type === 'event' && isOpenAIUsageOnlyEventShape(frame.event)) continue; + yield completionsProtocolFrameToSSEFrame(frame); + } +})()); + +/** + * The ending. It dials, reads what came back on the shape the request asked for, and + * provides the answer, the raw HTTP response beneath it, and what the call is billable for. + * A failure is a value: a 429 here is what an earlier stage fails over, and so is a 200 whose + * body this protocol cannot read, because the next candidate's path and flags may differ. + */ +const callCompletionsUpstream = defineStage< + C<'ingress.completions.wantsStream' | 'request.completions.payload' | 'route.candidate' | 'ingress.http.headers'>, + C<'response.completions.payload' | 'response.completions.streamedUsage' | 'response.usage.billable' + | 'response.http.status' | 'response.http.headers' | 'response.http.body'>, + GatewayServices +>({ + name: 'callCompletionsUpstream', + return: { + provides: [ + 'response.completions.payload', + 'response.completions.streamedUsage', + 'response.usage.billable', + 'response.http.status', + 'response.http.headers', + 'response.http.body', + ], + }, + execute: async (facts, use) => { + const candidate = facts['route.candidate']; + // The provider re-stamps whatever id it resolved upstream, so the id the client + // addressed does not travel with the body. + const { model: _addressed, ...body } = facts['request.completions.payload']; + const result = await candidate.provider.instance.callCompletions( + providerModelOf(candidate), + body, + use.gateway.abortSignal, + // The client's own headers reach the upstream from the record, not from a live request + // object: what a provider is allowed to forward is filtered per provider, and the dump + // shows what was there to filter. + buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]) => [name, value]))), + ); + use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'text_completion'); + if (!result.response.ok) use.log.warn('upstream refused', { status: result.response.status }); + + const answer = await readUpstream( + result.response, + facts['ingress.completions.wantsStream'], + telemetryModelIdentity(candidate, result.modelKey), + candidate, + use.gateway.abortSignal, + ); + return move({ + ...facts, + 'response.completions.payload': answer.payload, + 'response.completions.streamedUsage': answer.streamedUsage, + 'response.usage.billable': answer.billable, + 'response.http.status': result.response.status, + 'response.http.headers': [...result.response.headers], + 'response.http.body': answer.body, + }); + }, +}); + +interface UpstreamAnswer { + readonly payload: CompletionsFacts['response.completions.payload']; + readonly streamedUsage: CompletionsFacts['response.completions.streamedUsage']; + readonly billable: readonly BillableEntity[]; + /** Every arm hands one up, this stage's own reading included: the record holds a body as a + * stream, and `failover` releases the losing attempts' by consuming that key. */ + readonly body: ReadableStream & AsyncDisposable; +} + +/** What the upstream said, on the shape the request asked for. Which of the four this is is + * read from the response, never declared: a stream, a value and a failure sit at one key. */ +const readUpstream = async ( + response: Response, + wantsStream: boolean, + identity: TelemetryModelIdentity, + candidate: ModelCandidate, + signal: AbortSignal | undefined, +): Promise => { + // An upstream was called and reported nothing — which is what every arm but a read usage + // block leaves standing, and is a different statement from reporting zero. + const called: readonly BillableEntity[] = [{ identity, quantities: {} }]; + + if (!response.ok) { + // An upstream error body is JSON like any other body. Reading it here is also what + // leaves a losing attempt with nothing open behind it. + return { payload: await refusal(response), streamedUsage: null, billable: called, body: spentBody(response.body) }; + } + + if (!wantsStream) { + const read = await readResult(response); + return { + payload: isFailure(read) ? read : read.value, + streamedUsage: null, + billable: isFailure(read) ? called : [{ identity, quantities: billed(read.value.usage, read.value.service_tier, candidate) }], + body: spentBody(response.body), + }; + } + + if (response.body === null) { + return { + payload: { status: 502, message: 'Upstream returned a streaming response with no body.' }, + streamedUsage: null, + billable: called, + body: spentBody(null), + }; + } + + // Usage is observed here, closest to the upstream and on the protocol it spoke, by folding + // the frames as they pass — so the reading costs one pass and the client's own stream is + // what drives it. What it finds arrives with the last frame, long after this stage has + // handed up, which is why the entity above carries no quantities. + const metered = meterFrames(parseCompletionsStream(response.body, { signal }), identity, candidate); + return { + payload: metered.frames, + streamedUsage: metered.billable, + billable: called, + // Releasing this body is draining those frames: they are one reader over one connection, + // and a second reader is not something a `ReadableStream` allows. + body: Object.assign(response.body, { + [Symbol.asyncDispose]: async (): Promise => { for await (const _frame of metered.frames) { /* to end of stream */ } }, + }), + }; +}; + +/** A body this stage has already read to the end, or one the upstream never sent. The record + * holds a body as a stream and `failover` releases the losing attempts', so every path hands + * one up; what says an answer was unusable is the failure at the payload key, not this. */ +const spentBody = (body: ReadableStream | null): ReadableStream & AsyncDisposable => + Object.assign(body ?? new ReadableStream({ start: controller => controller.close() }), { + [Symbol.asyncDispose]: (): Promise => Promise.resolve(), + }); + +const refusal = async (response: Response): Promise => { + const text = await response.text(); + try { + return { status: response.status, message: text, body: JSON.parse(text) as unknown }; + } catch { + // A refusal that is not JSON is still a refusal, and its text is what the client is + // told; there is simply no parsed body for a dump reader to open. + return { status: response.status, message: text }; + } +}; + +/** A body that answered 200 and cannot be read as this protocol is an attempt that failed, + * because the gateway serializes what it sends from the value it parsed and has nothing to + * serialize. The next candidate may well answer in the protocol it advertised. */ +const readResult = async (response: Response): Promise<{ readonly value: CompletionsResult } | Failure> => { + try { + return { value: parseCompletionsResult(await response.json()) }; + } catch (error) { + return { status: 502, message: `Upstream answered ${response.status} with a body this endpoint cannot read: ${error instanceof Error ? error.message : String(error)}` }; + } +}; + +interface MeteredFrames { + readonly frames: CompletionsFrames; + readonly billable: Promise; +} + +const meterFrames = ( + source: AsyncIterable>, + identity: TelemetryModelIdentity, + candidate: ModelCandidate, +): MeteredFrames => { + let settle!: (billable: readonly BillableEntity[]) => void; + const billable = new Promise(resolve => { settle = resolve; }); + const frames = view((async function* () { + let usage: unknown; + let tier: string | null | undefined; + try { + for await (const frame of source) { + if (frame.type === 'event') { + // `service_tier` can ride on any chunk's root while the totals only land on the + // usage-only one, so the two are tracked apart and settled together. + const root = frame.event as { service_tier?: string | null; usage?: unknown }; + if (root.service_tier !== undefined) tier = root.service_tier; + if (isOpenAIUsageOnlyEventShape(frame.event)) usage = root.usage; + } + yield frame; + } + } finally { + // Reached however the frames ended — the terminal chunk, a client that stopped + // reading, or a broken upstream — because tokens the upstream already metered are + // billable whatever happened to the downstream half. + settle([{ identity, quantities: billed(usage, tier, candidate) }]); + } + })()); + return { frames, billable }; +}; + +const billed = (usage: unknown, tier: string | null | undefined, candidate: ModelCandidate): UsageQuantities => { + const model = providerModelOf(candidate); + const tokens = tokenUsageFromCompletionsUsage( + usage, + tier, + model.enabledFlags.has('usage-exclusive-cached-tokens'), + `${candidate.provider.upstreamId}/${model.id}`, + ); + // An upstream that reported nothing leaves no quantities at all, which is a different + // statement from reporting zero. + // + // The service tier survives as far as `TokenUsage.tier` and no further: a billed entity is + // an identity and a bag of quantities, and the pricing selector the tier feeds has no seat + // there. It comes back when settlement does. + return tokens === null ? {} : tokenUsageQuantities(tokens); +}; + +/** A candidate that cannot serve this endpoint is not a candidate. The resolver's own filter + * is by kind, and a text-completion model is a chat-kind model, so what is left to say is + * whether the upstream exposes the endpoint at all. */ +const narrowing = { + kind: 'chat' as const, + reject: (candidate: ModelCandidate): string | null => + candidate.model.endpoints.completions === undefined ? 'the upstream does not expose a completions endpoint' : null, + refuse: (status: number, message: string): C<'response.completions.payload' | 'response.completions.streamedUsage'> => ({ + 'response.completions.payload': { status, message }, + 'response.completions.streamedUsage': null, + }), + refuses: ['response.completions.payload', 'response.completions.streamedUsage'] as const, +}; + +export const completionsServePipeline: Pipeline< + C<'ingress.completions.wantsStream' | 'ingress.completions.wantsUsageChunk' | 'request.completions.payload' | 'serve.model'>, + C<'response.completions.rendered' | 'response.completions.streamedUsage' | 'response.http.status' | 'response.usage.billable'> +> = compose('completionsServe', [ + emitCompletions, + resolveCandidates(narrowing), + failover(handedUp => isFailure((handedUp as { 'response.completions.payload'?: unknown })['response.completions.payload'])), + callCompletionsUpstream, +]); diff --git a/packages/gateway/src/data-plane/completions/usage.ts b/packages/gateway/src/data-plane/completions/usage.ts index 6bb01ea7c9..534c05d0a8 100644 --- a/packages/gateway/src/data-plane/completions/usage.ts +++ b/packages/gateway/src/data-plane/completions/usage.ts @@ -16,11 +16,10 @@ import { billableServiceTier, splitInclusiveInputTokens } from '@floway-dev/prot // non-streaming /v1/completions body (observed null on a Zhipu/GLM // fork); the streaming path was observed to omit the field. // -// This endpoint is a passthrough with no interceptor chain, so the fold the -// chat targets apply to the usage chunk itself happens here instead, on the -// one read that consumes it. `declaredExclusive` carries the serving -// upstream's `usage-exclusive-cached-tokens` flag and `identity` names it in -// whatever `foldsExclusiveCacheTokens` raises. +// This is the one read of a completions usage block, so the fold the chat targets apply to +// the usage chunk itself happens here instead. `declaredExclusive` carries the serving +// upstream's `usage-exclusive-cached-tokens` flag and `identity` names it in whatever +// `foldsExclusiveCacheTokens` raises. export const tokenUsageFromCompletionsUsage = ( usage: unknown, diff --git a/packages/gateway/src/data-plane/images/pipeline.ts b/packages/gateway/src/data-plane/images/pipeline.ts new file mode 100644 index 0000000000..7d27d4b75c --- /dev/null +++ b/packages/gateway/src/data-plane/images/pipeline.ts @@ -0,0 +1,266 @@ +// Images as a pipeline. Two endpoints and one family: `generations` sends JSON and `edits` +// sends either JSON or a multipart form, so what the operation decides is the request fact and +// the provider method the ending calls, and the four stages are the same either way. +// +// emitImages the edge: serializes the answer into the images protocol +// resolveCandidates narrows to the upstreams that expose this operation's endpoint +// failover runs what follows once per candidate +// callImagesUpstream the ending: dials, and provides what came back + +import type { UsageQuantities } from '../../repo/types.ts'; +import type { Failure, GatewayFacts } from '../pipeline/facts.ts'; +import { isFailure } from '../pipeline/facts.ts'; +import type { GatewayServices } from '../pipeline/services.ts'; +import { failover, resolveCandidates } from '../pipeline/stages.ts'; +import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; +import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; +import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; +import type { Pipeline } from '@floway-dev/pipeline'; +import { compose, defineStage, move } from '@floway-dev/pipeline'; +import { mediaTypeEssence, parseDecimalString, type ModelEndpointKey } from '@floway-dev/protocols/common'; +import { + imagesErrorMessage, + parseImagesResponse, + renderImagesError, + renderImagesResponse, + type CanonicalImagesEditsRequest, + type CanonicalImagesRequest, + type CanonicalImagesResponse, + type CanonicalImagesUsage, + type ImageEditReference, + type ImagesEditImage, + type ImagesOperation, +} from '@floway-dev/protocols/images'; +import { isBase64ImageDataUrl, providerModelOf } from '@floway-dev/provider'; +import type { ImagesEditsRequest, ImagesEditsSource, ModelCandidate, PerformanceOperation } from '@floway-dev/provider'; + +/** Images' own keys. They extend the shared space and never merge into it, so a stage written + * against the gateway alone cannot name one. */ +export interface ImagesFacts extends GatewayFacts { + 'request.images.canonical': CanonicalImagesRequest; + 'response.images.canonical': CanonicalImagesResponse | Failure; + /** What the client is actually sent, in the images protocol. The edge provides it, so a dump + * shows the body the client received rather than the gateway's canonical form. */ + 'response.images.rendered': Record; +} + +type I = { [P in K]: ImagesFacts[P] }; + +const ENDPOINT = { + generations: 'imagesGenerations', + edits: 'imagesEdits', +} as const satisfies Record; + +const PERFORMANCE_OPERATION = { + generations: 'image_generation', + edits: 'image_edit', +} as const satisfies Record; + +/** + * The outermost edge. It names nothing on the way down — the family has one protocol, so there + * is no request to read to know how to answer — and coming back it renders the answer, keeps + * the upstream headers a client may see, and decides the status. The status is decided here + * rather than carried up because a refusal that never reached an upstream has none to carry. + */ +const emitImages = defineStage< + Record, + Record, + I<'response.images.canonical' | 'response.http.headers'>, + I<'response.images.rendered' | 'response.http.status' | 'response.http.headers'> +>({ + name: 'emitImages', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { + needs: ['response.images.canonical', 'response.http.headers'], + consumes: ['response.images.canonical', 'response.http.headers'], + provides: ['response.images.rendered', 'response.http.status', 'response.http.headers'], + }, + }, + execute: async (facts, next) => { + const back = await next(facts); + const { 'response.images.canonical': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); + if (isFailure(answer)) { + return { + ...rest, + 'response.http.headers': forClient, + 'response.images.rendered': move(renderImagesError(answer.message, answer.body)), + 'response.http.status': answer.status, + }; + } + return { + ...rest, + 'response.http.headers': forClient, + 'response.images.rendered': move(renderImagesResponse(answer)), + 'response.http.status': 200, + }; + }, +}); + +/** + * The ending. It dials, reads the upstream's body, and provides the canonical answer, the + * headers that came with it, and what the call is billable for. A failure is a value: an + * upstream that refused, and one that answered with something this protocol cannot read, are + * both outcomes the fork above can take to the next candidate rather than faults that end the + * run. + */ +const callImagesUpstream = defineStage< + I<'request.images.canonical' | 'route.candidate' | 'ingress.http.headers'>, + I<'response.images.canonical' | 'response.http.headers' | 'response.usage.billable'>, + GatewayServices +>({ + name: 'callImagesUpstream', + return: { + provides: ['response.images.canonical', 'response.http.headers', 'response.usage.billable'], + }, + execute: async (facts, use) => { + const candidate = facts['route.candidate']; + const request = facts['request.images.canonical']; + const options = buildUpstreamCallOptions( + candidate, + use.gateway, + // The client's own headers reach the upstream from the record, not from a live request + // object: what a provider is allowed to forward is filtered per provider, and the dump + // shows what was there to filter. + new Headers(facts['ingress.http.headers'].map(([name, value]) => [name, value])), + ); + const model = providerModelOf(candidate); + // No abort signal, as on this family's endpoints from the beginning: an image the upstream + // has already begun is charged for whether or not the client waited for it, so dropping the + // call would lose the usage reading and save nothing. + const result = request.operation === 'generations' + ? await candidate.provider.instance.callImagesGenerations(model, request.parameters, undefined, options) + : await candidate.provider.instance.callImagesEdits(model, providerEditsRequest(request), undefined, options); + use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, PERFORMANCE_OPERATION[request.operation]); + const identity = telemetryModelIdentity(candidate, result.modelKey); + // What came back, unfiltered: the edge is where a client's view of it is decided. + const headers = [...result.response.headers]; + const answered = (canonical: CanonicalImagesResponse | Failure, quantities: UsageQuantities) => move({ + ...facts, + 'response.images.canonical': canonical, + 'response.http.headers': headers, + 'response.usage.billable': [{ identity, quantities }], + }); + // An entity with no quantities is how "the upstream was called and reported nothing" is + // said, which is a different situation from reporting zero. + const reportedNothing: UsageQuantities = {}; + + const body = await readUpstreamBody(result.response); + if (!result.response.ok) { + use.log.warn('upstream refused', { status: result.response.status }); + return answered({ + status: result.response.status, + message: imagesErrorMessage(body.json) ?? body.text, + ...('json' in body ? { body: body.json } : {}), + }, reportedNothing); + } + if (!('json' in body)) { + // Every protocol the gateway carries is one it fully understands, so a body it cannot + // read is not handed on unread. A client that asked for `stream: true` lands here: the + // images stream is not carried as facts yet. + const mediaType = mediaTypeEssence(result.response.headers.get('content-type')) ?? 'no media type'; + use.log.warn('upstream answered with a body that is not JSON', { status: result.response.status, mediaType }); + return answered({ + status: 502, + message: `The upstream answered ${result.response.status} with ${mediaType}, and the images protocol is JSON.`, + body: body.text, + }, reportedNothing); + } + + let canonical: CanonicalImagesResponse; + try { + canonical = parseImagesResponse(body.json); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + use.log.warn('upstream answered with a body the images protocol cannot read', { message }); + return answered({ status: 502, message, body: body.json }, reportedNothing); + } + return answered(canonical, billed(canonical.usage)); + }, +}); + +interface UpstreamBody { + readonly text: string; + /** Absent when the body was not JSON at all, which is itself the answer to a protocol that + * requires JSON. */ + readonly json?: unknown; +} + +const readUpstreamBody = async (response: Response): Promise => { + const text = await response.text(); + try { + return { text, json: JSON.parse(text) as unknown }; + } catch { + return { text }; + } +}; + +/** An image is billed by the tokens its upstream reports: `BILLING_METRICS` names no per-image + * or per-size unit, and per-size pricing is a selector coordinate rather than a metric, so + * there is nothing else here to record a count against. A reported zero is kept — it says the + * upstream reported, which an absent metric would not. */ +const billed = (usage: CanonicalImagesUsage | undefined): UsageQuantities => { + const quantities: UsageQuantities = {}; + if (usage?.inputTokens !== undefined) quantities.input_tokens = parseDecimalString(String(usage.inputTokens)); + if (usage?.inputImageTokens !== undefined) quantities.input_image_tokens = parseDecimalString(String(usage.inputImageTokens)); + if (usage?.outputTokens !== undefined) quantities.output_tokens = parseDecimalString(String(usage.outputTokens)); + if (usage?.outputImageTokens !== undefined) quantities.output_image_tokens = parseDecimalString(String(usage.outputImageTokens)); + return quantities; +}; + +/** The provider's own shape, built where the dial happens. What it needs from a reference is + * whether the data URL inside it can become a file, because that decides whether the edit can + * ride as a multipart form; the canonical fact holds the reference the client wrote and leaves + * that question to the serializer that has it. */ +const providerEditsRequest = (request: CanonicalImagesEditsRequest): ImagesEditsRequest => ({ + images: request.images.map(providerEditsSource), + ...(request.mask === undefined ? {} : { mask: providerEditsSource(request.mask) }), + parameters: request.parameters, +}); + +const providerEditsSource = (image: ImagesEditImage): ImagesEditsSource => { + if (image.kind === 'file') { + return { type: 'upload', file: new File([image.file.bytes], image.file.fileName, { type: image.file.mediaType }) }; + } + const { reference } = image; + return typeof reference.image_url === 'string' && isBase64ImageDataUrl(reference.image_url) + ? { type: 'inline', reference: reference as ImageEditReference & { image_url: string } } + : { type: 'reference', reference }; +}; + +/** A candidate that cannot serve *this* request is not a candidate. One family covers two + * endpoints and an upstream may expose either without the other, so which one is asked for is + * what narrows the list. A refusal answers with no upstream headers because there was no + * upstream, which is the same statement the edge above reads on every other path. */ +const narrowing = (request: CanonicalImagesRequest) => ({ + kind: 'image' as const, + reject: (candidate: ModelCandidate) => candidate.model.endpoints[ENDPOINT[request.operation]] === undefined + ? `the upstream does not expose the images ${request.operation} endpoint` + : null, + refuse: (status: number, message: string) => ({ + 'response.images.canonical': { status, message }, + 'response.http.headers': [], + }), + refuses: ['response.images.canonical', 'response.http.headers'] as const, +}); + +/** What a caller must bring. `ingress.http.headers` and `request.images.canonical` are in it + * although `compose` cannot derive them — the ending stage reads both and a return-only stage + * declares no request side — so this type is the whole statement and `entryNeeds` is part of + * it. */ +export type ImagesServeEntry = I<'ingress.http.headers' | 'request.images.canonical' | 'serve.model'>; + +export type ImagesServeExit = I<'response.images.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'>; + +export const imagesServePipeline = (request: CanonicalImagesRequest): Pipeline => + compose('imagesServe', [ + emitImages, + resolveCandidates(narrowing(request)), + failover(handedUp => isFailure((handedUp as { 'response.images.canonical'?: unknown })['response.images.canonical'])), + callImagesUpstream, + ]); diff --git a/packages/protocols/__tests__/audio/transcription_test.ts b/packages/protocols/__tests__/audio/transcription_test.ts new file mode 100644 index 0000000000..5c2eadb5d1 --- /dev/null +++ b/packages/protocols/__tests__/audio/transcription_test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from 'vitest'; + +import { + parseAudioTranscription, + parseAudioTranscriptionResponseFormat, + parseAudioTranscriptionUsage, + renderAudioTranscription, +} from '../../src/audio/index.ts'; + +// Whisper's own writers, which is what `whisper-1` runs: SubRip numbers its cues and always +// writes the hour component, WebVTT opens with its signature and drops the hour component +// below one hour, and both terminate every cue with a blank line. +// https://github.com/openai/whisper/blob/v20250625/whisper/utils.py#L238-L262 +const SRT = '1\n00:00:00,000 --> 00:00:03,320\n The beach was a popular spot.\n\n' + + '2\n00:00:03,320 --> 00:00:08,470\n People were swimming in the ocean.\n\n'; +const VTT = 'WEBVTT\n\n00:00.000 --> 00:03.320\n The beach was a popular spot.\n\n' + + '00:03.320 --> 00:08.470\n People were swimming in the ocean.\n\n'; + +const VERBOSE = { + task: 'transcribe', + language: 'english', + duration: 8.47, + text: ' The beach was a popular spot. People were swimming in the ocean.', + segments: [ + { id: 0, seek: 0, start: 0, end: 3.32, text: ' The beach was a popular spot.', tokens: [50364], temperature: 0, avg_logprob: -0.28, compression_ratio: 1.23, no_speech_prob: 0.009 }, + ], +}; + +describe('the renderings of a transcription', () => { + it('is the protocol default when the form left the field out', () => { + expect(parseAudioTranscriptionResponseFormat(undefined)).toBe('json'); + expect(() => parseAudioTranscriptionResponseFormat('yaml')).toThrow('response_format is invalid'); + }); + + // The claim the family rests on: `response_format` travels to the upstream inside the + // form, so the rendering an answer arrives in is the rendering it is written back in, and + // reading then writing has to be the identity on every one of the six. + it.each([ + ['json', { text: 'hello' }], + ['verbose_json', VERBOSE], + ['diarized_json', { task: 'transcribe', duration: 2, text: 'Agent: hi', segments: [{ type: 'transcript.text.segment', id: 'seg_1', start: 0, end: 2, text: 'hi', speaker: 'agent' }] }], + ['text', ' The beach was a popular spot.\n'], + ['srt', SRT], + ['vtt', VTT], + ] as const)('reads and writes %s back unchanged', (format, body) => { + expect(renderAudioTranscription(format, parseAudioTranscription(format, body))).toEqual(body); + }); + + // What the canonical value holds is what its rendering carried, and the renderings are not + // equally rich. This is the whole of why a rendering is never derived from a value read + // out of a different one. + it('holds a transcript and its cues where the rendering carried both', () => { + expect(parseAudioTranscription('verbose_json', VERBOSE)).toMatchObject({ + text: VERBOSE.text, + cues: [{ start: 0, end: 3.32, text: ' The beach was a popular spot.' }], + }); + }); + + it('holds no cues where the rendering carried none, so no subtitle can be written from it', () => { + const fromJson = parseAudioTranscription('json', { text: 'hello' }); + expect(fromJson.cues).toBeUndefined(); + expect(() => renderAudioTranscription('srt', fromJson)).toThrow('carries no cues'); + }); + + // A subtitle document states the transcript only as its cues' texts, so the transcript on + // the canonical value is rebuilt the way Whisper's own text writer rebuilds it and is not + // claimed to be the string a `text` request would have returned. + // https://github.com/openai/whisper/blob/v20250625/whisper/utils.py#L109-L116 + it('rebuilds the transcript of a subtitle document from its cues', () => { + expect(parseAudioTranscription('srt', SRT).text).toBe(' The beach was a popular spot.\n People were swimming in the ocean.'); + }); + + it('cannot write a JSON rendering from a value that was not read from an object', () => { + expect(() => renderAudioTranscription('json', parseAudioTranscription('text', 'hello'))).toThrow('not read from an object'); + }); + + it('refuses a body the rendering cannot be read from', () => { + expect(() => parseAudioTranscription('json', { transcript: 'hello' })).toThrow('text must be a string'); + expect(() => parseAudioTranscription('vtt', SRT)).toThrow('must open with WEBVTT'); + }); +}); + +describe('what a transcription says it will be billed for', () => { + it('splits the audio share out of the input tokens the upstream broke down', () => { + expect(parseAudioTranscriptionUsage({ + usage: { type: 'tokens', input_tokens: 14, input_token_details: { text_tokens: 10, audio_tokens: 4 }, output_tokens: 101, total_tokens: 115 }, + })).toEqual({ kind: 'tokens', inputTokens: 14, inputAudioTokens: 4, outputTokens: 101 }); + }); + + it('reads a duration report, and whisper\'s top-level duration where there is no report', () => { + expect(parseAudioTranscriptionUsage({ usage: { type: 'duration', seconds: 43 } })).toEqual({ kind: 'duration', seconds: 43 }); + expect(parseAudioTranscriptionUsage(VERBOSE)).toEqual({ kind: 'duration', seconds: 8.47 }); + }); + + // A report that cannot be read is, from here, no report — which the caller states by + // naming the billed entity with no quantities rather than by naming zero. + it('is no report where the shape is not one the protocol names', () => { + expect(parseAudioTranscriptionUsage({ text: 'hello' })).toBeUndefined(); + expect(parseAudioTranscriptionUsage({ usage: { type: 'tokens', input_tokens: -1, output_tokens: 2 } })).toBeUndefined(); + expect(parseAudioTranscriptionUsage({ usage: { type: 'credits', spent: 3 } })).toBeUndefined(); + }); +}); diff --git a/packages/protocols/__tests__/images/request_test.ts b/packages/protocols/__tests__/images/request_test.ts new file mode 100644 index 0000000000..60fba7ba2b --- /dev/null +++ b/packages/protocols/__tests__/images/request_test.ts @@ -0,0 +1,129 @@ +import { describe, expect, test } from 'vitest'; + +import { parseImagesEditsRequest, parseImagesGenerationsRequest } from '../../src/images/request.ts'; + +const json = (value: unknown): Uint8Array => new TextEncoder().encode(JSON.stringify(value)); + +const multipart = async (form: FormData): Promise<{ contentType: string; bytes: Uint8Array }> => { + const encoded = new Response(form); + return { + contentType: encoded.headers.get('content-type')!, + bytes: new Uint8Array(await encoded.arrayBuffer()), + }; +}; + +describe('images generations ingress', () => { + test('routing takes the model and everything else stays as the client wrote it', () => { + const parsed = parseImagesGenerationsRequest(json({ + model: 'gpt-image-1', + prompt: 'a shiba in space', + size: '1024x1024', + moderation: 'low', + })); + + expect(parsed.model).toBe('gpt-image-1'); + expect(parsed.request).toEqual({ + operation: 'generations', + parameters: { prompt: 'a shiba in space', size: '1024x1024', moderation: 'low' }, + }); + }); + + test('a body that is not a JSON object, or names no model, is refused with what to fix', () => { + expect(() => parseImagesGenerationsRequest(new TextEncoder().encode('not json'))) + .toThrow('Images generations request body must be valid JSON.'); + expect(() => parseImagesGenerationsRequest(json(['gpt-image-1']))) + .toThrow('Images generations request body must be an object.'); + expect(() => parseImagesGenerationsRequest(json({ prompt: 'hi' }))) + .toThrow('Images generations request body must include a model string.'); + }); +}); + +describe('images edits ingress', () => { + test('a JSON edit keeps each reference exactly as it arrived', async () => { + const parsed = await parseImagesEditsRequest('application/json', json({ + model: 'gpt-image-1', + prompt: 'replace the background', + images: [{ image_url: 'data:image/png;base64,iVBORw0KGgo=' }, { file_id: 'file-source' }], + mask: { file_id: 'file-mask' }, + quality: 'high', + })); + + expect(parsed.model).toBe('gpt-image-1'); + expect(parsed.request).toEqual({ + operation: 'edits', + images: [ + { kind: 'reference', reference: { image_url: 'data:image/png;base64,iVBORw0KGgo=' } }, + { kind: 'reference', reference: { file_id: 'file-source' } }, + ], + mask: { kind: 'reference', reference: { file_id: 'file-mask' } }, + parameters: { prompt: 'replace the background', quality: 'high' }, + }); + }); + + test('a reference naming both ways, or neither, is refused by position', async () => { + await expect(parseImagesEditsRequest('application/json', json({ + model: 'gpt-image-1', + images: [{ file_id: 'file-source' }, { image_url: 'https://example.com/a.png', file_id: 'file-source' }], + }))).rejects.toThrow('Image edits images[1] must contain exactly one string field: image_url or file_id.'); + + await expect(parseImagesEditsRequest('application/json', json({ model: 'gpt-image-1', prompt: 'hi' }))) + .rejects.toThrow('Image edits request body must include an images array.'); + }); + + test('a multipart edit holds each file as bytes and every other field as text', async () => { + const form = new FormData(); + form.append('model', 'gpt-image-1'); + form.append('prompt', 'replace the sky'); + form.append('n', '2'); + form.append('image[]', new Blob([new Uint8Array([1, 2, 3])], { type: 'image/png' }), 'photo.png'); + form.append('image[]', new Blob([new Uint8Array([4, 5])], { type: 'image/webp' }), 'second.webp'); + form.append('mask', new Blob([new Uint8Array([6])], { type: 'image/png' }), 'mask.png'); + const { contentType, bytes } = await multipart(form); + + const parsed = await parseImagesEditsRequest(contentType, bytes); + + expect(parsed.model).toBe('gpt-image-1'); + expect(parsed.request).toEqual({ + operation: 'edits', + images: [ + { kind: 'file', file: { fileName: 'photo.png', mediaType: 'image/png', bytes: new Uint8Array([1, 2, 3]) } }, + { kind: 'file', file: { fileName: 'second.webp', mediaType: 'image/webp', bytes: new Uint8Array([4, 5]) } }, + ], + mask: { kind: 'file', file: { fileName: 'mask.png', mediaType: 'image/png', bytes: new Uint8Array([6]) } }, + // A form field is text, so `n` stays the string the client sent it as. + parameters: { prompt: 'replace the sky', n: '2' }, + }); + }); + + test('a multipart edit naming no model is refused before its fields are read', async () => { + const form = new FormData(); + form.append('prompt', 'replace the sky'); + const { contentType, bytes } = await multipart(form); + + await expect(parseImagesEditsRequest(contentType, bytes)) + .rejects.toThrow('Image edits request body must include a model field.'); + }); + + test('an image field carrying text, and a text field carrying a file, are both refused', async () => { + const textImage = new FormData(); + textImage.append('model', 'gpt-image-1'); + textImage.append('image', 'not a file'); + const encodedTextImage = await multipart(textImage); + await expect(parseImagesEditsRequest(encodedTextImage.contentType, encodedTextImage.bytes)) + .rejects.toThrow('Image edits image fields must be files.'); + + const fileParameter = new FormData(); + fileParameter.append('model', 'gpt-image-1'); + fileParameter.append('prompt', new Blob([new Uint8Array([1])], { type: 'text/plain' }), 'prompt.txt'); + const encodedFileParameter = await multipart(fileParameter); + await expect(parseImagesEditsRequest(encodedFileParameter.contentType, encodedFileParameter.bytes)) + .rejects.toThrow('Image edits prompt field must be text.'); + }); + + test('a body in neither of the two media types the endpoint takes is refused as such', async () => { + await expect(parseImagesEditsRequest('text/plain', new TextEncoder().encode('hi'))) + .rejects.toThrow('Image edits request body must use application/json or multipart/form-data.'); + await expect(parseImagesEditsRequest(undefined, new TextEncoder().encode('hi'))) + .rejects.toThrow('Image edits request body must use application/json or multipart/form-data.'); + }); +}); diff --git a/packages/protocols/__tests__/images/response_test.ts b/packages/protocols/__tests__/images/response_test.ts new file mode 100644 index 0000000000..a9fece43f7 --- /dev/null +++ b/packages/protocols/__tests__/images/response_test.ts @@ -0,0 +1,75 @@ +import { describe, expect, test } from 'vitest'; + +import { imagesErrorMessage, parseImagesResponse, parseImagesUsage, renderImagesError, renderImagesResponse } from '../../src/images/response.ts'; + +describe('images response ingress', () => { + test('the answer is parsed and rendered back whole, fields the gateway does not model included', () => { + const body = { + created: 1713833628, + data: [{ b64_json: 'aGVsbG8=', revised_prompt: 'a shiba, in space' }, { url: 'https://example.com/a.png' }], + background: 'transparent', + output_format: 'png', + }; + + const parsed = parseImagesResponse(body); + + expect(parsed.images).toEqual([ + { base64: 'aGVsbG8=', revisedPrompt: 'a shiba, in space' }, + { url: 'https://example.com/a.png' }, + ]); + expect(renderImagesResponse(parsed)).toBe(body); + }); + + test('an answer carrying no images at all is read as such, and one that cannot be read is an error', () => { + expect(parseImagesResponse({ created: 1 }).images).toEqual([]); + expect(() => parseImagesResponse({ data: 'one image' })).toThrow('Images response data must be an array'); + expect(() => parseImagesResponse({ data: [{ b64_json: 42 }] })).toThrow('Images response data[0].b64_json must be a string'); + expect(() => parseImagesResponse('an image')).toThrow('Images response body must be an object'); + }); +}); + +describe('images usage', () => { + test('what the upstream attributed to images is taken out of the count beside it', () => { + expect(parseImagesUsage({ + usage: { + total_tokens: 100, + input_tokens: 50, + output_tokens: 50, + input_tokens_details: { text_tokens: 10, image_tokens: 40 }, + }, + })).toEqual({ inputTokens: 10, inputImageTokens: 40, outputTokens: 50 }); + }); + + test('a count with no detail beside it is the whole count, and a detail saying nothing is as good as absent', () => { + expect(parseImagesUsage({ usage: { input_tokens: 10, output_tokens: 50 } })) + .toEqual({ inputTokens: 10, outputTokens: 50 }); + expect(parseImagesUsage({ usage: { input_tokens: 10, input_tokens_details: {} } })) + .toEqual({ inputTokens: 10 }); + expect(parseImagesUsage({ usage: { output_tokens: 50, output_tokens_details: { image_tokens: 50 } } })) + .toEqual({ outputTokens: 0, outputImageTokens: 50 }); + }); + + // The distinction the return type exists for: an upstream that reported nothing is a + // different fact from one that reported zero, and only the second is a reading. + test('nothing readable is no reading, and a reported zero is one', () => { + expect(parseImagesUsage({ data: [] })).toBeUndefined(); + expect(parseImagesUsage({ usage: {} })).toBeUndefined(); + expect(parseImagesUsage({ usage: { input_tokens: '50' } })).toBeUndefined(); + expect(parseImagesUsage({ usage: { input_tokens: 10, input_tokens_details: 'text' } })).toBeUndefined(); + expect(parseImagesUsage({ usage: { input_tokens: 0, output_tokens: 0 } })) + .toEqual({ inputTokens: 0, outputTokens: 0 }); + }); +}); + +describe('images failures', () => { + test('an upstream error object is what the client is sent, and anything else becomes the gateway envelope', () => { + const upstream = { error: { message: 'Your request was rejected', type: 'image_generation_user_error', code: 'moderation_blocked' } }; + + expect(imagesErrorMessage(upstream)).toBe('Your request was rejected'); + expect(renderImagesError('Your request was rejected', upstream)).toBe(upstream); + + expect(imagesErrorMessage('upstream is down')).toBeUndefined(); + expect(renderImagesError('Model gpt-image-1 is not available on any configured upstream.', undefined)) + .toEqual({ error: { message: 'Model gpt-image-1 is not available on any configured upstream.', type: 'api_error' } }); + }); +}); diff --git a/packages/protocols/src/audio/index.ts b/packages/protocols/src/audio/index.ts index ec731330f4..d93c17de68 100644 --- a/packages/protocols/src/audio/index.ts +++ b/packages/protocols/src/audio/index.ts @@ -1,20 +1,29 @@ -// OpenAI-compatible audio transcription stream terminal. The wire remains open -// so provider additions pass through unchanged while the gateway observes only -// the terminal event it needs. -// https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L61780-L61924 +// OpenAI-compatible audio transcription. One request shape — a multipart form — and +// several response renderings, which is the family's whole character: `response_format` +// picks between three JSON objects and three text documents, and `stream` picks a sequence +// of events instead of any of them. -export interface AudioTranscriptionStreamEvent { - type: string; - [key: string]: unknown; -} +export type { AudioTranscriptionCue, SubtitleDialect } from './subtitles.ts'; +export { parseSubtitleDocument, renderSubtitleDocument } from './subtitles.ts'; -export interface AudioTranscriptionDoneEvent extends AudioTranscriptionStreamEvent { - type: 'transcript.text.done'; - text: string; -} +export type { + AudioTranscriptionObjectFormat, + AudioTranscriptionResponseFormat, + AudioTranscriptionUsage, + CanonicalAudioTranscription, +} from './transcription.ts'; +export { + AUDIO_TRANSCRIPTION_RESPONSE_FORMATS, + isAudioTranscriptionObjectFormat, + parseAudioTranscription, + parseAudioTranscriptionResponseFormat, + parseAudioTranscriptionUsage, + renderAudioTranscription, +} from './transcription.ts'; -export const isAudioTranscriptionDoneEvent = (event: unknown): event is AudioTranscriptionDoneEvent => - typeof event === 'object' - && event !== null - && (event as { type?: unknown }).type === 'transcript.text.done' - && typeof (event as { text?: unknown }).text === 'string'; +export type { AudioTranscriptionDoneEvent, AudioTranscriptionStreamEvent } from './stream.ts'; +export { + isAudioTranscriptionDoneEvent, + parseAudioTranscriptionStreamEvent, + parseAudioTranscriptionStreamUsage, +} from './stream.ts'; diff --git a/packages/protocols/src/audio/stream.ts b/packages/protocols/src/audio/stream.ts new file mode 100644 index 0000000000..790e02e80f --- /dev/null +++ b/packages/protocols/src/audio/stream.ts @@ -0,0 +1,39 @@ +// The event stream a transcription answers with when the client asked for one. The stream +// stays open: a provider addition rides through untouched while the gateway names only the +// events it acts on. `transcript.text.delta` is the increment, `transcript.text.done` is +// the terminal one and the only place a streamed transcription states its usage. +// https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L61796-L61917 + +import type { AudioTranscriptionUsage } from './transcription.ts'; +import { parseAudioTranscriptionUsage } from './transcription.ts'; + +export interface AudioTranscriptionStreamEvent { + type: string; + [key: string]: unknown; +} + +export interface AudioTranscriptionDoneEvent extends AudioTranscriptionStreamEvent { + type: 'transcript.text.done'; + text: string; +} + +export const isAudioTranscriptionDoneEvent = (event: unknown): event is AudioTranscriptionDoneEvent => + typeof event === 'object' + && event !== null + && (event as { type?: unknown }).type === 'transcript.text.done' + && typeof (event as { text?: unknown }).text === 'string'; + +/** The reading every frame goes through, so the value at the canonical key is a parsed + * event and the edge re-serializes it rather than relaying the bytes it arrived in. */ +export const parseAudioTranscriptionStreamEvent = (value: unknown): AudioTranscriptionStreamEvent => { + if (typeof value !== 'object' || value === null || typeof (value as { type?: unknown }).type !== 'string') { + throw new Error('Audio transcription stream event must be an object carrying a string type'); + } + return value as AudioTranscriptionStreamEvent; +}; + +/** A streamed transcription states its usage once, in the terminal event, in the same shape + * a JSON body states it. */ +export const parseAudioTranscriptionStreamUsage = ( + event: AudioTranscriptionDoneEvent, +): AudioTranscriptionUsage | undefined => parseAudioTranscriptionUsage(event); diff --git a/packages/protocols/src/audio/subtitles.ts b/packages/protocols/src/audio/subtitles.ts new file mode 100644 index 0000000000..2dede717e9 --- /dev/null +++ b/packages/protocols/src/audio/subtitles.ts @@ -0,0 +1,107 @@ +// SubRip and WebVTT, the two renderings of a transcription that are subtitle documents +// rather than JSON. Both carry exactly one thing — timed cues — so both are a total +// reading of, and a total writing from, the canonical transcription's `cues`. +// +// The exact shapes are Whisper's own writers, which is what OpenAI's `whisper-1` runs: +// SubRip numbers its cues from 1, always writes the hour component and separates the +// milliseconds with a comma; WebVTT opens with a `WEBVTT` line, drops the hour component +// below one hour and separates the milliseconds with a period. Both terminate every cue +// with a blank line. +// https://github.com/openai/whisper/blob/v20250625/whisper/utils.py#L238-L262 + +export interface AudioTranscriptionCue { + /** Seconds from the start of the audio. */ + readonly start: number; + readonly end: number; + readonly text: string; +} + +export type SubtitleDialect = 'srt' | 'vtt'; + +const VTT_HEADER = 'WEBVTT'; + +/** Whisper rounds to whole milliseconds before splitting the components, so a timestamp + * written from a parsed one is the same string it was parsed from. + * https://github.com/openai/whisper/blob/v20250625/whisper/utils.py#L50-L68 */ +const writeTimestamp = (seconds: number, dialect: SubtitleDialect): string => { + if (!Number.isFinite(seconds) || seconds < 0) { + throw new Error(`Subtitle timestamp must be a finite non-negative number: ${seconds}`); + } + let milliseconds = Math.round(seconds * 1000); + const hours = Math.floor(milliseconds / 3_600_000); + milliseconds -= hours * 3_600_000; + const minutes = Math.floor(milliseconds / 60_000); + milliseconds -= minutes * 60_000; + const whole = Math.floor(milliseconds / 1_000); + milliseconds -= whole * 1_000; + + const pad = (value: number, width: number) => String(value).padStart(width, '0'); + const hoursMarker = dialect === 'srt' || hours > 0 ? `${pad(hours, 2)}:` : ''; + const decimalMarker = dialect === 'srt' ? ',' : '.'; + return `${hoursMarker}${pad(minutes, 2)}:${pad(whole, 2)}${decimalMarker}${pad(milliseconds, 3)}`; +}; + +const TIMESTAMP = /^(?:(\d+):)?(\d{1,2}):(\d{1,2})[,.](\d{1,3})$/; + +const readTimestamp = (value: string, dialect: SubtitleDialect): number => { + const parts = TIMESTAMP.exec(value.trim()); + if (parts === null) throw new Error(`${dialect} timestamp is not readable: ${JSON.stringify(value)}`); + const [, hours, minutes, seconds, fraction] = parts; + // A fraction shorter than three digits is still a decimal fraction of a second, so it is + // padded on the right rather than parsed as a millisecond count. + return Number(hours ?? '0') * 3600 + Number(minutes) * 60 + Number(seconds) + Number(fraction.padEnd(3, '0')) / 1000; +}; + +const readCueBlock = (block: readonly string[], dialect: SubtitleDialect): AudioTranscriptionCue => { + // SubRip opens a cue with its ordinal and WebVTT with an optional identifier, so the + // timing line is the one holding the arrow and the lines after it are the cue's text. + const timingIndex = block.findIndex(line => line.includes('-->')); + if (timingIndex < 0) throw new Error(`${dialect} cue has no timing line: ${JSON.stringify(block.join('\n'))}`); + const [start, end] = block[timingIndex].split('-->'); + if (end === undefined) throw new Error(`${dialect} cue timing line has no end: ${JSON.stringify(block[timingIndex])}`); + return { + start: readTimestamp(start, dialect), + end: readTimestamp(end, dialect), + text: block.slice(timingIndex + 1).join('\n'), + }; +}; + +export const parseSubtitleDocument = (dialect: SubtitleDialect, document: string): readonly AudioTranscriptionCue[] => { + const lines = document.replaceAll('\r\n', '\n').split('\n'); + if (dialect === 'vtt') { + // The signature may carry a byte order mark and may be followed by header metadata on + // the same line. https://www.w3.org/TR/webvtt1/#webvtt-file-body + const signature = lines.shift()?.replace('', '') ?? ''; + if (!signature.startsWith(VTT_HEADER)) { + throw new Error(`WebVTT document must open with ${VTT_HEADER}: ${JSON.stringify(signature)}`); + } + } + + // A blank line ends a cue in both dialects, which is what makes a cue's own text able to + // span several lines. + const blocks: string[][] = []; + let open: string[] | null = null; + for (const line of lines) { + if (line.trim().length === 0) { + open = null; + continue; + } + if (open === null) { + open = []; + blocks.push(open); + } + open.push(line); + } + return blocks.map(block => readCueBlock(block, dialect)); +}; + +export const renderSubtitleDocument = (dialect: SubtitleDialect, cues: readonly AudioTranscriptionCue[]): string => { + const body = cues + .map((cue, index) => { + const timing = `${writeTimestamp(cue.start, dialect)} --> ${writeTimestamp(cue.end, dialect)}`; + const ordinal = dialect === 'srt' ? `${index + 1}\n` : ''; + return `${ordinal}${timing}\n${cue.text}\n\n`; + }) + .join(''); + return dialect === 'vtt' ? `${VTT_HEADER}\n\n${body}` : body; +}; diff --git a/packages/protocols/src/audio/transcription.ts b/packages/protocols/src/audio/transcription.ts new file mode 100644 index 0000000000..4050e561b1 --- /dev/null +++ b/packages/protocols/src/audio/transcription.ts @@ -0,0 +1,179 @@ +// The one transcription, and the several documents that are renderings of it. +// +// `response_format` picks a rendering, and it travels to the upstream in the form the +// client sent, so the rendering the client asked for is the rendering the upstream answers +// in. That is what makes one canonical value enough: the gateway reads whichever document +// arrived and writes the same one back, and no rendering is ever derived from a value that +// was read out of a different one. +// +// Which is as well, because the renderings are not equally rich. `verbose_json` carries a +// whole transcript and its cues; `json` carries the transcript with usage beside it; `srt` +// and `vtt` carry the cues alone; `text` carries the transcript alone and nothing else at +// all. So a `srt` answer holds no `language`, no `duration` and no usage, and its transcript +// exists only as the cues' texts. Those are properties of the renderings, not gaps in the +// reading. +// https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L714-L745 + +import type { AudioTranscriptionCue } from './subtitles.ts'; +import { parseSubtitleDocument, renderSubtitleDocument } from './subtitles.ts'; + +// `diarized_json` joined the enum with `gpt-4o-transcribe-diarize`; it is a JSON rendering +// and needs nothing of its own here. +// https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L28527-L28545 +export const AUDIO_TRANSCRIPTION_RESPONSE_FORMATS = ['json', 'text', 'srt', 'verbose_json', 'vtt', 'diarized_json'] as const; +export type AudioTranscriptionResponseFormat = typeof AUDIO_TRANSCRIPTION_RESPONSE_FORMATS[number]; + +/** The form field is optional and the protocol's own default is `json`. */ +export const parseAudioTranscriptionResponseFormat = (value: unknown): AudioTranscriptionResponseFormat => { + if (value === undefined) return 'json'; + if (typeof value === 'string' && (AUDIO_TRANSCRIPTION_RESPONSE_FORMATS as readonly string[]).includes(value)) { + return value as AudioTranscriptionResponseFormat; + } + throw new Error(`Audio transcription response_format is invalid: ${JSON.stringify(value)}`); +}; + +/** A JSON rendering answers with an object; `text`, `srt` and `vtt` answer with a document. + * The split decides how the body is read and written, and it is the whole of what the + * response format decides for a non-streaming answer. */ +export type AudioTranscriptionObjectFormat = 'json' | 'verbose_json' | 'diarized_json'; + +export const isAudioTranscriptionObjectFormat = ( + format: AudioTranscriptionResponseFormat, +): format is AudioTranscriptionObjectFormat => + format === 'json' || format === 'verbose_json' || format === 'diarized_json'; + +/** Token-billed and duration-billed models report usage under one discriminated key. A + * transcription answered in `text`, `srt` or `vtt` reports none, because those renderings + * cannot express it. */ +export type AudioTranscriptionUsage = + | { + readonly kind: 'tokens'; + readonly inputTokens: number; + /** Split out of `inputTokens` when the upstream broke it down, because audio input is + * priced apart from text input. */ + readonly inputAudioTokens?: number; + readonly outputTokens: number; + } + | { readonly kind: 'duration'; readonly seconds: number }; + +export interface CanonicalAudioTranscription { + /** The whole transcript. A subtitle document does not carry one, so for `srt` and `vtt` + * this is the cues' texts joined by newlines — which is what Whisper's own text writer + * produces from the same result, and is not claimed to be byte-identical to what a + * `text` request would have returned. + * https://github.com/openai/whisper/blob/v20250625/whisper/utils.py#L109-L116 */ + readonly text: string; + /** The timed cues. This is the whole of `srt` and `vtt`; `verbose_json` and + * `diarized_json` carry the same thing as `segments`. */ + readonly cues?: readonly AudioTranscriptionCue[]; + /** The upstream's own object, kept whole for the renderings that are objects. A JSON + * answer is re-serialized from this, so a field this type does not name — a segment's + * `avg_logprob`, a `logprobs` array, a `speaker` label — reaches the client unchanged. */ + readonly raw?: Record; + readonly usage?: AudioTranscriptionUsage; +} + +const isRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +const isFiniteNumber = (value: unknown): value is number => typeof value === 'number' && Number.isFinite(value); + +const isCount = (value: unknown): value is number => + typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; + +/** + * What the upstream said it will bill for, or `undefined` when it said nothing this reading + * recognizes. There is no third answer: a report that cannot be read is, from here, no + * report, and the caller says so by naming the entity with no quantities. + * + * Whisper's `verbose_json` predates the `usage` block and states the same duration at the + * top level, so that is read where no block is present. + * https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L36513-L36545 + * https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L61968-L62051 + */ +export const parseAudioTranscriptionUsage = (value: unknown): AudioTranscriptionUsage | undefined => { + if (!isRecord(value)) return undefined; + const usage = value.usage; + if (!isRecord(usage)) { + return isFiniteNumber(value.duration) && value.duration >= 0 + ? { kind: 'duration', seconds: value.duration } + : undefined; + } + if (usage.type === 'duration') { + return isFiniteNumber(usage.seconds) && usage.seconds >= 0 ? { kind: 'duration', seconds: usage.seconds } : undefined; + } + if (usage.type !== 'tokens' || !isCount(usage.input_tokens) || !isCount(usage.output_tokens)) return undefined; + const details = usage.input_token_details; + const audioTokens = isRecord(details) && isCount(details.audio_tokens) ? details.audio_tokens : undefined; + if (audioTokens !== undefined && audioTokens > usage.input_tokens) return undefined; + return { + kind: 'tokens', + inputTokens: usage.input_tokens, + ...(audioTokens === undefined ? {} : { inputAudioTokens: audioTokens }), + outputTokens: usage.output_tokens, + }; +}; + +const requiredText = (value: unknown, where: string): string => { + if (typeof value !== 'string') throw new Error(`Audio transcription ${where} must be a string`); + return value; +}; + +/** `verbose_json` and `diarized_json` both time their segments with seconds and text; the + * fields they do not share stay in `raw` and reach the client through it. */ +const objectCues = (value: Record): readonly AudioTranscriptionCue[] | undefined => { + if (!Array.isArray(value.segments)) return undefined; + return value.segments.map((segment: unknown, index) => { + if (!isRecord(segment) || !isFiniteNumber(segment.start) || !isFiniteNumber(segment.end)) { + throw new Error(`Audio transcription segment ${index} must carry numeric start and end`); + } + return { start: segment.start, end: segment.end, text: requiredText(segment.text, `segment ${index} text`) }; + }); +}; + +/** + * The one reading. A JSON rendering hands in the parsed object; `text`, `srt` and `vtt` + * hand in the document itself. + */ +export const parseAudioTranscription = ( + format: AudioTranscriptionResponseFormat, + body: unknown, +): CanonicalAudioTranscription => { + if (isAudioTranscriptionObjectFormat(format)) { + if (!isRecord(body)) throw new Error('Audio transcription response body must be an object'); + const cues = objectCues(body); + const usage = parseAudioTranscriptionUsage(body); + return { + text: requiredText(body.text, 'text'), + ...(cues === undefined ? {} : { cues }), + raw: body, + ...(usage === undefined ? {} : { usage }), + }; + } + const document = requiredText(body, 'response body'); + if (format === 'text') return { text: document }; + const cues = parseSubtitleDocument(format, document); + return { text: cues.map(cue => cue.text).join('\n'), cues }; +}; + +/** + * The one writing, back into the rendering the client asked for. A JSON rendering is the + * upstream's own object re-serialized; a subtitle rendering is written from the cues; `text` + * is the transcript alone. + */ +export const renderAudioTranscription = ( + format: AudioTranscriptionResponseFormat, + transcription: CanonicalAudioTranscription, +): Record | string => { + if (isAudioTranscriptionObjectFormat(format)) { + if (transcription.raw === undefined) { + throw new Error(`Audio transcription cannot be rendered as ${format}: it was not read from an object`); + } + return transcription.raw; + } + if (format === 'text') return transcription.text; + if (transcription.cues === undefined) { + throw new Error(`Audio transcription cannot be rendered as ${format}: it carries no cues`); + } + return renderSubtitleDocument(format, transcription.cues); +}; diff --git a/packages/protocols/src/completions/index.ts b/packages/protocols/src/completions/index.ts index f887a8397b..e566210200 100644 --- a/packages/protocols/src/completions/index.ts +++ b/packages/protocols/src/completions/index.ts @@ -1,15 +1,53 @@ -// OpenAI text-completion protocol (POST /v1/completions). Floway runs -// this endpoint as a passthrough, so this module declares the -// protocol's wire shape rather than a read-time DTO: the gateway's -// /completions handler parses the inbound body through its own local -// shape and reads `model` / `stream` / `stream_options` directly. Only -// `model` is read structurally by downstream code (the provider -// interface accepts `Omit` and forwards -// the rest unchanged); every other field flows through via the index -// signature. +// OpenAI text completions (`POST /v1/completions`). The gateway carries this protocol end to +// end: what a client sends is parsed into `CompletionsPayload`, what an upstream answers is +// parsed into a `CompletionsResult` or a frame stream, and what the client receives is +// serialized back from those. No body is forwarded verbatim, so the shapes here are the +// protocol's own rather than a convenience view over bytes in flight. +// +// Request and response follow OpenAI's `CreateCompletionRequest` and +// `CreateCompletionResponse`, and the specification's own note on the latter is why there is +// no separate streaming envelope type below the chunk level — "both the streamed and the +// non-streamed response objects share the same shape (unlike the chat endpoint)": +// https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L33793-L34030 +// https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34031-L34149 +// `model` is the one field the gateway itself reads structurally on the way in — routing is +// what needs it — and `stream` / `stream_options` are what decide the shape of the answer. +// The rest is named because the protocol names it, and is optional here because the upstream +// is what accepts or refuses a request: the specification also requires `prompt`, and an +// opinion about that held here would have to be kept in step with every OpenAI-compatible +// upstream we route to. The index signature carries a vendor extension through to the +// upstream that understands it. export interface CompletionsPayload { model: string; + prompt?: string | readonly string[] | readonly number[] | readonly (readonly number[])[] | null; + best_of?: number | null; + echo?: boolean | null; + frequency_penalty?: number | null; + logit_bias?: Record | null; + logprobs?: number | null; + max_tokens?: number | null; + n?: number | null; + presence_penalty?: number | null; + seed?: number | null; + stop?: string | readonly string[] | null; + stream?: boolean | null; + stream_options?: CompletionsStreamOptions | null; + suffix?: string | null; + temperature?: number | null; + top_p?: number | null; + user?: string | null; + [key: string]: unknown; +} + +// `/v1/completions` shares OpenAI's `ChatCompletionStreamOptions` with `/v1/chat/completions`, +// so `include_obfuscation` sits beside `include_usage` here as well: +// https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L31764-L31810 +// The index signature is what lets the gateway turn `include_usage` on without dropping the +// siblings a client sent. +export interface CompletionsStreamOptions { + include_usage?: boolean; + include_obfuscation?: boolean; [key: string]: unknown; } @@ -18,7 +56,7 @@ export interface CompletionsPayload { // final placeholder choice carrying only `index` (no `text`, no // `finish_reason`) alongside the usage block — so `text` and // `finish_reason` are optional, matching that shape on the typed surface. -// `logprobs` is opaque to the gateway — passed through as-is. +// `logprobs` is opaque to the gateway — carried through as-is. interface CompletionsChoiceStreaming { index: number; text?: string; @@ -26,16 +64,18 @@ interface CompletionsChoiceStreaming { logprobs?: unknown; } +// OpenAI's `CompletionUsage`, which `/v1/completions` reuses verbatim from +// `/v1/chat/completions`: +// https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L32210-L32283 +// The optional prompt-cache split is the part upstreams disagree about: OpenAI's own text +// models do not populate it today, while vLLM, llama.cpp, Fireworks, OpenRouter and xAI Grok +// all emit `cached_tokens` here on `/v1/completions`, and Azure mirrors the schema. Billing +// reads the split through `openAICacheTokensFromUsage`, which also knows the field names the +// wilder forks use, so what is named here is the schema rather than the union of the wild. export interface CompletionsUsage { prompt_tokens: number; completion_tokens: number; total_tokens: number; - // OpenAI's CompletionUsage schema (which /v1/completions reuses verbatim - // from /v1/chat/completions) carries an optional prompt-cache split. - // OpenAI's own text models do not populate it today, but vLLM, llama.cpp, - // Fireworks, OpenRouter, and xAI Grok all emit `cached_tokens` here on - // /v1/completions, and Azure mirrors the schema. Floway extracts it when - // present so billing metrics match what the upstream actually reported. prompt_tokens_details?: { cached_tokens?: number }; } @@ -61,15 +101,23 @@ export interface CompletionsChoice { logprobs?: unknown; } +// `service_tier` is not in the specification's response schema — it belongs to the chat +// endpoint — but a vLLM fork was observed emitting it on the non-streaming +// `/v1/completions` body (null, on a Zhipu/GLM build) and billing reads it as the pricing +// tier when it is there, so it is named rather than left to the index signature. export interface CompletionsResult { id: string; object: 'text_completion'; created: number; model: string; choices: CompletionsChoice[]; + service_tier?: string | null; usage?: CompletionsUsage; system_fingerprint?: string; + [key: string]: unknown; } +export { parseCompletionsPayload, parseCompletionsResult } from './parse.ts'; export { reassembleCompletionsEvents } from './reassemble.ts'; +export { parseCompletionsStream, type ParseCompletionsStreamOptions } from './stream.ts'; export { completionsProtocolFrameToSSEFrame } from './to-sse.ts'; diff --git a/packages/protocols/src/completions/parse.ts b/packages/protocols/src/completions/parse.ts new file mode 100644 index 0000000000..796c1bd094 --- /dev/null +++ b/packages/protocols/src/completions/parse.ts @@ -0,0 +1,22 @@ +import type { CompletionsPayload, CompletionsResult } from './index.ts'; +import { isJsonObject } from '../common/json.ts'; + +// The two boundaries where a `/v1/completions` body becomes a value. Both check what the +// gateway itself depends on and nothing beyond it: the request must name a model, because +// routing reads it, and either body must be a JSON object, because everything downstream +// addresses it as one. Which fields an upstream accepts and which it returns is the +// upstream's to decide, and a gateway opinion about the rest would have to be kept in step +// with every OpenAI-compatible implementation we route to. + +export const parseCompletionsPayload = (value: unknown): CompletionsPayload => { + if (!isJsonObject(value)) throw new Error('Completions request body must be an object.'); + if (typeof value.model !== 'string' || value.model.length === 0) { + throw new Error('Completions request body must include a model string.'); + } + return value as CompletionsPayload; +}; + +export const parseCompletionsResult = (value: unknown): CompletionsResult => { + if (!isJsonObject(value)) throw new Error('Completions response body must be an object.'); + return value as CompletionsResult; +}; diff --git a/packages/protocols/src/completions/stream.ts b/packages/protocols/src/completions/stream.ts new file mode 100644 index 0000000000..2a6427fe7c --- /dev/null +++ b/packages/protocols/src/completions/stream.ts @@ -0,0 +1,26 @@ +import type { CompletionsStreamEvent } from './index.ts'; +import { parseTargetStreamFrames } from '../common/parse-events.ts'; +import { parseSSEStream } from '../common/parse-sse.ts'; +import { doneFrame, eventFrame, type ProtocolFrame } from '../common/sse.ts'; + +export interface ParseCompletionsStreamOptions { + signal?: AbortSignal; +} + +// The upstream's SSE body as protocol frames. Transport framing ends here: what comes out +// carries `text_completion` chunks and a terminal frame, and nothing that reads it knows how +// those arrived. +export const parseCompletionsStream = ( + body: ReadableStream, + options: ParseCompletionsStreamOptions = {}, +): AsyncGenerator> => (async function* () { + for await (const frame of parseTargetStreamFrames(parseSSEStream(body, options), { + protocol: 'Completions', + })) { + if (frame.type === 'done') { + yield doneFrame(); + return; + } + yield eventFrame(frame.data); + } +})(); diff --git a/packages/protocols/src/images/index.ts b/packages/protocols/src/images/index.ts index 6867edec4a..25f8249cbd 100644 --- a/packages/protocols/src/images/index.ts +++ b/packages/protocols/src/images/index.ts @@ -1,3 +1,11 @@ +// The images protocol: POST /v1/images/generations and POST /v1/images/edits. One protocol +// over two endpoints — generations takes JSON, and edits takes either JSON, where every image +// is a URL, a data URL or a file id, or a multipart form carrying the files themselves. +// https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L12858-L12870 +// https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L12558-L12620 + +export type ImagesOperation = 'generations' | 'edits'; + // JSON payload accepted by POST /v1/images/generations. Field set follows // OpenAI's reference for gpt-image-* and legacy dall-e-* (dall-e is // retired but the union shape is harmless). Declared as an interface with @@ -29,3 +37,72 @@ export interface ImagesGenerationsPayload { export type ImageEditReference = | { image_url: string; file_id?: never; [key: string]: unknown } | { file_id: string; image_url?: never; [key: string]: unknown }; + +/** A file the client sent in a multipart form, held as bytes: a form is a parsed value like + * any other, and bytes are what survives being read once and what a dump can show. */ +export interface ImagesUploadedFile { + fileName: string; + mediaType: string; + bytes: Uint8Array; +} + +/** One image an edit reads. Multipart carries the file itself; JSON carries a reference for + * the upstream to resolve, kept exactly as the client wrote it — whether a data URL can be + * turned back into a file is the upstream serializer's question, not this one's. */ +export type ImagesEditImage = + | { kind: 'file'; file: ImagesUploadedFile } + | { kind: 'reference'; reference: ImageEditReference }; + +export interface CanonicalImagesGenerationsRequest { + operation: 'generations'; + /** Everything the client sent but `model`: routing owns the model id, and what is left is + * what the upstream is asked for. */ + parameters: Record; +} + +export interface CanonicalImagesEditsRequest { + operation: 'edits'; + images: ImagesEditImage[]; + mask?: ImagesEditImage; + parameters: Record; +} + +export type CanonicalImagesRequest = CanonicalImagesGenerationsRequest | CanonicalImagesEditsRequest; + +export interface ParsedImagesRequest { + model: string; + request: CanonicalImagesRequest; +} + +/** One image the upstream returned. `response_format` decides which arm carries it on the + * dall-e models, while the GPT image models answer base64 and do not serve a URL at all, so + * neither arm is the one to expect. + * https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L51044-L51067 */ +export interface CanonicalImage { + url?: string; + base64?: string; + revisedPrompt?: string; +} + +/** Token counts as the images endpoints report them, disjoint: what the upstream attributed to + * images is taken out of the count beside it. Only the GPT image models report any of this — + * the dall-e models report nothing — which is why the whole reading is optional as well as + * each field. + * https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L78089-L78115 */ +export interface CanonicalImagesUsage { + inputTokens?: number; + inputImageTokens?: number; + outputTokens?: number; + outputImageTokens?: number; +} + +export interface CanonicalImagesResponse { + /** The parsed body as it arrived. One protocol in and one out means rendering is + * re-serializing this, so the fields the gateway does not model still reach the client. */ + raw: Record; + images: CanonicalImage[]; + usage?: CanonicalImagesUsage; +} + +export { parseImagesEditsRequest, parseImagesGenerationsRequest } from './request.ts'; +export { imagesErrorMessage, parseImagesResponse, parseImagesUsage, renderImagesError, renderImagesResponse } from './response.ts'; diff --git a/packages/protocols/src/images/request.ts b/packages/protocols/src/images/request.ts new file mode 100644 index 0000000000..287e17ab10 --- /dev/null +++ b/packages/protocols/src/images/request.ts @@ -0,0 +1,112 @@ +// Reading what the client sent into the canonical request. The two endpoints take different +// bodies — generations is JSON, edits is JSON or a multipart form — and each message here is +// what the client is told, so they name the endpoint and the field they are about. + +import type { ImageEditReference, ImagesEditImage, ImagesUploadedFile, ParsedImagesRequest } from './index.ts'; +import { isJsonMediaType, isMultipartFormDataMediaType } from '../common/media-type.ts'; + +const isRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +const jsonObject = (body: Uint8Array, endpoint: string): Record => { + let parsed: unknown; + try { + parsed = JSON.parse(new TextDecoder().decode(body)) as unknown; + } catch { + throw new Error(`${endpoint} request body must be valid JSON.`); + } + if (!isRecord(parsed)) throw new Error(`${endpoint} request body must be an object.`); + return parsed; +}; + +/** Routing owns the model id, so it comes out of the payload rather than travelling inside it. + * Every other field is the upstream's business and is passed on untouched. */ +const requiredModel = (body: Record, endpoint: string): string => { + const { model } = body; + if (typeof model !== 'string' || model.length === 0) { + throw new Error(`${endpoint} request body must include a model string.`); + } + return model; +}; + +export const parseImagesGenerationsRequest = (body: Uint8Array): ParsedImagesRequest => { + const payload = jsonObject(body, 'Images generations'); + const model = requiredModel(payload, 'Images generations'); + const { model: _model, ...parameters } = payload; + return { model, request: { operation: 'generations', parameters } }; +}; + +export const parseImagesEditsRequest = async ( + contentType: string | null | undefined, + body: Uint8Array, +): Promise => { + if (isJsonMediaType(contentType)) return jsonEdits(body); + if (isMultipartFormDataMediaType(contentType)) return await multipartEdits(contentType, body); + throw new Error('Image edits request body must use application/json or multipart/form-data.'); +}; + +const jsonEdits = (body: Uint8Array): ParsedImagesRequest => { + const payload = jsonObject(body, 'Image edits'); + const model = requiredModel(payload, 'Image edits'); + if (!Array.isArray(payload.images)) { + throw new Error('Image edits request body must include an images array.'); + } + const images = payload.images.map((value, index) => referenceImage(value, `Image edits images[${index}]`)); + const mask = payload.mask === undefined ? undefined : referenceImage(payload.mask, 'Image edits mask'); + const { model: _model, images: _images, mask: _mask, ...parameters } = payload; + return { + model, + request: { operation: 'edits', images, ...(mask === undefined ? {} : { mask }), parameters }, + }; +}; + +const referenceImage = (value: unknown, path: string): ImagesEditImage => { + if (!isRecord(value)) throw new Error(`${path} must be an object.`); + const { image_url: imageUrl, file_id: fileId } = value; + const named = (typeof imageUrl === 'string' && fileId === undefined) + || (typeof fileId === 'string' && imageUrl === undefined); + if (!named) throw new Error(`${path} must contain exactly one string field: image_url or file_id.`); + return { kind: 'reference', reference: value as ImageEditReference }; +}; + +const multipartEdits = async (contentType: string, body: Uint8Array): Promise => { + let form: FormData; + try { + // `BodyInit` excludes a view over a `SharedArrayBuffer`, which an inbound body never is. + form = await new Response(body as BodyInit, { headers: { 'content-type': contentType } }).formData(); + } catch { + throw new Error('Image edits request body must be valid multipart/form-data.'); + } + + const model = form.get('model'); + if (typeof model !== 'string' || model.length === 0) { + throw new Error('Image edits request body must include a model field.'); + } + + const images: ImagesEditImage[] = []; + let mask: ImagesEditImage | undefined; + const parameters: Record = {}; + for (const [name, value] of form.entries()) { + if (name === 'model') continue; + // One image is sent as `image` and several as `image[]`; the upstream serializer picks the + // field name back off the count, so the two arrive at one list here. + if (name === 'image' || name === 'image[]') { + images.push({ kind: 'file', file: await uploadedFile(value, `Image edits ${name} fields must be files.`) }); + } else if (name === 'mask') { + mask = { kind: 'file', file: await uploadedFile(value, 'Image edits mask field must be a file.') }; + } else { + if (typeof value !== 'string') throw new Error(`Image edits ${name} field must be text.`); + parameters[name] = value; + } + } + + return { + model, + request: { operation: 'edits', images, ...(mask === undefined ? {} : { mask }), parameters }, + }; +}; + +const uploadedFile = async (value: FormDataEntryValue, message: string): Promise => { + if (!(value instanceof File)) throw new Error(message); + return { fileName: value.name, mediaType: value.type, bytes: new Uint8Array(await value.arrayBuffer()) }; +}; diff --git a/packages/protocols/src/images/response.ts b/packages/protocols/src/images/response.ts new file mode 100644 index 0000000000..abafd7b73f --- /dev/null +++ b/packages/protocols/src/images/response.ts @@ -0,0 +1,109 @@ +// Reading the upstream's answer, and writing the client's. There is one images protocol, so +// the two are the same shape and rendering a success is re-serializing what was parsed — +// which is what keeps the fields this file does not name from being dropped on the way out. + +import type { CanonicalImage, CanonicalImagesResponse, CanonicalImagesUsage } from './index.ts'; + +const isRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +export const parseImagesResponse = (value: unknown): CanonicalImagesResponse => { + if (!isRecord(value)) throw new Error('Images response body must be an object'); + const usage = parseImagesUsage(value); + return { + raw: value, + images: parseImages(value.data), + ...(usage === undefined ? {} : { usage }), + }; +}; + +// An answer that carries no `data` at all is not malformed — an upstream reporting a +// moderation refusal in a 200 does exactly that — so it parses to no images, and only a `data` +// that is there but cannot be read is an error. +const parseImages = (data: unknown): CanonicalImage[] => { + if (data === undefined) return []; + if (!Array.isArray(data)) throw new Error('Images response data must be an array'); + return data.map((entry, index) => { + if (!isRecord(entry)) throw new Error(`Images response data[${index}] must be an object`); + const url = optionalString(entry.url, `Images response data[${index}].url`); + const base64 = optionalString(entry.b64_json, `Images response data[${index}].b64_json`); + const revisedPrompt = optionalString(entry.revised_prompt, `Images response data[${index}].revised_prompt`); + return { + ...(url === undefined ? {} : { url }), + ...(base64 === undefined ? {} : { base64 }), + ...(revisedPrompt === undefined ? {} : { revisedPrompt }), + }; + }); +}; + +const optionalString = (value: unknown, field: string): string | undefined => { + if (value === undefined) return undefined; + if (typeof value !== 'string') throw new Error(`${field} must be a string`); + return value; +}; + +/** + * What the upstream said it charged for, or nothing when it said nothing readable. The + * distinction is the whole point of the return type: an upstream that reported no usage is a + * different fact from one that reported zero, and only one of them is a reading. + */ +export const parseImagesUsage = (value: unknown): CanonicalImagesUsage | undefined => { + if (!isRecord(value) || !isRecord(value.usage)) return undefined; + const { + input_tokens: inputTotal, + output_tokens: outputTotal, + input_tokens_details: inputDetails, + output_tokens_details: outputDetails, + } = value.usage; + if (inputTotal !== undefined && typeof inputTotal !== 'number') return undefined; + if (outputTotal !== undefined && typeof outputTotal !== 'number') return undefined; + if (inputTotal === undefined && outputTotal === undefined) return undefined; + + const input = splitModality(inputTotal, inputDetails); + const output = splitModality(outputTotal, outputDetails); + if (input === null || output === null) return undefined; + return { + ...(input.text === undefined ? {} : { inputTokens: input.text }), + ...(input.image === undefined ? {} : { inputImageTokens: input.image }), + ...(output.text === undefined ? {} : { outputTokens: output.text }), + ...(output.image === undefined ? {} : { outputImageTokens: output.image }), + }; +}; + +// `input_tokens` counts images and text together and `input_tokens_details` says how much of it +// was images, so the two are made disjoint here: what stays under the plain count is the text. +// A details object carrying neither split says nothing and is as good as absent, while one that +// cannot be read discards the whole reading rather than half of it. +const splitModality = ( + total: number | undefined, + details: unknown, +): { text?: number; image?: number } | null => { + if (total === undefined) return {}; + if (details === undefined) return { text: total }; + if (!isRecord(details)) return null; + const { text_tokens: text, image_tokens: image } = details; + if (text !== undefined && typeof text !== 'number') return null; + if (image !== undefined && typeof image !== 'number') return null; + if (text === undefined && image === undefined) return { text: total }; + return { text: text ?? 0, image: image ?? 0 }; +}; + +export const renderImagesResponse = (response: CanonicalImagesResponse): Record => response.raw; + +/** What an upstream that refused said about it, when it said it in the shape this protocol + * uses. Anything else has no message to lift and is reported as the body it was. */ +export const imagesErrorMessage = (body: unknown): string | undefined => { + if (!isRecord(body) || !isRecord(body.error)) return undefined; + return typeof body.error.message === 'string' ? body.error.message : undefined; +}; + +/** + * What the client is sent when the run produced no images. An upstream that answered with an + * error object is rendered as that object — a client reads the code inside it, and the gateway + * has nothing truer to say than what the upstream said — and everything else, a refusal that + * never reached an upstream included, becomes the gateway's own envelope over the message. + */ +export const renderImagesError = (message: string, upstreamBody: unknown): Record => + isRecord(upstreamBody) && isRecord(upstreamBody.error) + ? upstreamBody + : { error: { message, type: 'api_error' } }; From d7b21aaa51aabc9cfe9f5452b76e3103efd2bf97 Mon Sep 17 00:00:00 2001 From: Menci Date: Sun, 16 Aug 2026 21:56:17 +0800 Subject: [PATCH 05/79] fix(gateway): let a family say what its fork owns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `failover` declared that it consumes and provides `response.http.body` for every family. Three of the six never produce one — rerank, embeddings and images read their answer to the end, so there is nothing still open by the time the fork sees it — and the runner checks `provides` at handover. Those pipelines composed cleanly and would have thrown on the first real request, at the deepest stage, naming a key their author had never written down. It cannot be a fixed key in either direction: claiming one a family never produces throws, and staying silent about one it does produce and hands up throws the other way. Which keys carry a resource is a statement only the family can make, so it makes it — alongside the failure predicate it already hands in, for the same reason. Branching is the framework's; what something branches on, and what it owns, are the domain's. Found by the completions family, which verified it against a throwaway pipeline rather than reasoning about it. The three affected families' own tests asserted `entryNeeds` and nothing else, so none of them caught it — the same shape of untested claim the core's review found earlier, appearing again one layer up. The test added here asserts the declaration for both kinds of family rather than the absence of the bug, because the absence of the bug is what the other tests were already asserting when it was present. --- .../data-plane/audio-pipeline_test.ts | 23 ++++++++------- .../data-plane/rerank-pipeline_test.ts | 21 ++++++++++++++ .../gateway/src/data-plane/audio/pipeline.ts | 5 +++- .../src/data-plane/completions/pipeline.ts | 5 +++- .../src/data-plane/embeddings/pipeline.ts | 5 +++- .../gateway/src/data-plane/images/pipeline.ts | 5 +++- .../gateway/src/data-plane/pipeline/stages.ts | 29 +++++++++++++++---- .../gateway/src/data-plane/rerank/pipeline.ts | 5 +++- 8 files changed, 77 insertions(+), 21 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/audio-pipeline_test.ts b/packages/gateway/__tests__/data-plane/audio-pipeline_test.ts index 09950d5d37..641ba7cf6f 100644 --- a/packages/gateway/__tests__/data-plane/audio-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/audio-pipeline_test.ts @@ -7,7 +7,7 @@ import { describe, expect, it } from 'vitest'; import { audioTranscriptionServePipeline } from '../../src/data-plane/audio/pipeline.ts'; -import { isReleasable, move, run } from '@floway-dev/pipeline'; +import { isOwned, move, run } from '@floway-dev/pipeline'; describe('the audio transcription pipeline', () => { it('assembles, and asks its caller for what the descending stages need', () => { @@ -37,15 +37,18 @@ describe('the audio transcription pipeline', () => { .rejects.toThrow('run(audioTranscriptionServe): audioTranscriptionServe needs'); }); - // Why the streamed answer is handed over as a wrapper around its generator rather than as - // the generator: an async generator carries `Symbol.asyncDispose`, so the runner would adopt - // one in the record as a resource and the top-level sweep would call it — and for a - // generator that call is `return()`, which cancels the iteration rather than draining it. - // The resource is the upstream body, at `response.http.body`, and it is the only thing here - // that should answer to release. - it('makes an events view that the runner cannot mistake for the resource', () => { + // The streamed answer is a wrapper around its generator rather than the generator itself. + // That was once load-bearing: the runner detected ownership structurally, an async + // generator carries `Symbol.asyncDispose`, and the sweep would have called it — which for + // a generator is `return()`, cancelling the iteration rather than draining it. + // + // The runner now claims ownership through `own()` instead of detecting it, so a bare + // generator is safe in the record. The wrapper stays because it says which thing is the + // resource: the upstream body at `response.http.body`, and nothing else here. + it('keeps the events view clear of what answers to release', () => { const generator = (async function* () { yield 1; })(); - expect(isReleasable(generator)).toBe(true); - expect(isReleasable({ [Symbol.asyncIterator]: () => generator })).toBe(false); + expect(Symbol.asyncDispose in generator).toBe(true); // the language marks it + expect(isOwned(generator)).toBe(false); // the run does not + expect(isOwned({ [Symbol.asyncIterator]: () => generator })).toBe(false); }); }); diff --git a/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts b/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts index 4fbf352534..0e875ba607 100644 --- a/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts @@ -5,6 +5,7 @@ import { describe, expect, it } from 'vitest'; +import { failover } from '../../src/data-plane/pipeline/stages.ts'; import { rerankServePipeline } from '../../src/data-plane/rerank/pipeline.ts'; import { move, run } from '@floway-dev/pipeline'; import type { CanonicalRerankRequest } from '@floway-dev/protocols/rerank'; @@ -39,6 +40,26 @@ describe('the rerank pipeline', () => { expect(rerankServePipeline(request).entryNeeds).not.toContain('ingress.http.headers'); }); + // `failover` used to declare that it provides `response.http.body` for every family. Three + // of the six never produce one — they read their answer to the end — so those pipelines + // composed cleanly and then threw on the first real request, at the deepest stage, with a + // message about a key their author had never written down. + // + // What a fork owns is a statement only the family can make, so it makes it. This asserts + // the shape of that statement rather than the absence of a bug, because the absence of a + // bug is what every one of these tests asserted before and none of them caught it. + it('claims nothing on the way up when a family reads its answer to the end', () => { + const reading = failover({ failed: () => false, owns: [] }); + expect(reading.through?.response.consumes).toEqual([]); + expect(reading.through?.response.provides).toEqual([]); + + // And a family that streams claims the key it streams at, in both directions: every + // attempt's is the fork's to release, and the one it adopts rides up with ownership. + const streaming = failover({ failed: () => false, owns: ['response.http.body'] }); + expect(streaming.through?.response.consumes).toEqual(['response.http.body']); + expect(streaming.through?.response.provides).toEqual(['response.http.body']); + }); + it('names the entry key a caller did not bring, before any stage runs', async () => { await expect(run(rerankServePipeline(request), move({ 'serve.model': 'rerank-v3' }) as never, {})) .rejects.toThrow('run(rerankServe): rerankServe needs'); diff --git a/packages/gateway/src/data-plane/audio/pipeline.ts b/packages/gateway/src/data-plane/audio/pipeline.ts index 455497a4b6..d0130af9d0 100644 --- a/packages/gateway/src/data-plane/audio/pipeline.ts +++ b/packages/gateway/src/data-plane/audio/pipeline.ts @@ -370,6 +370,9 @@ export const audioTranscriptionServePipeline: Pipeline< > = compose('audioTranscriptionServe', [ emitAudioTranscription, resolveCandidates(narrowing), - failover(handedUp => isFailure((handedUp as { 'response.audioTranscription.canonical'?: unknown })['response.audioTranscription.canonical'])), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.audioTranscription.canonical'?: unknown })['response.audioTranscription.canonical']), + owns: ['response.http.body'], + }), callAudioTranscriptionUpstream, ]); diff --git a/packages/gateway/src/data-plane/completions/pipeline.ts b/packages/gateway/src/data-plane/completions/pipeline.ts index 5661a98938..e433e121fa 100644 --- a/packages/gateway/src/data-plane/completions/pipeline.ts +++ b/packages/gateway/src/data-plane/completions/pipeline.ts @@ -372,6 +372,9 @@ export const completionsServePipeline: Pipeline< > = compose('completionsServe', [ emitCompletions, resolveCandidates(narrowing), - failover(handedUp => isFailure((handedUp as { 'response.completions.payload'?: unknown })['response.completions.payload'])), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.completions.payload'?: unknown })['response.completions.payload']), + owns: ['response.http.body'], + }), callCompletionsUpstream, ]); diff --git a/packages/gateway/src/data-plane/embeddings/pipeline.ts b/packages/gateway/src/data-plane/embeddings/pipeline.ts index d390c8454d..b341fdb560 100644 --- a/packages/gateway/src/data-plane/embeddings/pipeline.ts +++ b/packages/gateway/src/data-plane/embeddings/pipeline.ts @@ -161,6 +161,9 @@ export const embeddingsServePipeline: Pipeline< > = compose('embeddingsServe', [ emitEmbeddings, resolveCandidates(narrowing), - failover(handedUp => isFailure((handedUp as { 'response.embeddings.canonical'?: unknown })['response.embeddings.canonical'])), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.embeddings.canonical'?: unknown })['response.embeddings.canonical']), + owns: [], + }), callEmbeddingsUpstream, ]); diff --git a/packages/gateway/src/data-plane/images/pipeline.ts b/packages/gateway/src/data-plane/images/pipeline.ts index 7d27d4b75c..bd61304d9a 100644 --- a/packages/gateway/src/data-plane/images/pipeline.ts +++ b/packages/gateway/src/data-plane/images/pipeline.ts @@ -261,6 +261,9 @@ export const imagesServePipeline = (request: CanonicalImagesRequest): Pipeline isFailure((handedUp as { 'response.images.canonical'?: unknown })['response.images.canonical'])), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.images.canonical'?: unknown })['response.images.canonical']), + owns: [], + }), callImagesUpstream, ]); diff --git a/packages/gateway/src/data-plane/pipeline/stages.ts b/packages/gateway/src/data-plane/pipeline/stages.ts index 1f64161103..dbdc95857d 100644 --- a/packages/gateway/src/data-plane/pipeline/stages.ts +++ b/packages/gateway/src/data-plane/pipeline/stages.ts @@ -106,11 +106,26 @@ export const resolveCandidates = (narrowing: Narrowing boolean) => defineStage< +export interface Forking { + /** How this family reads an attempt's outcome. Branching is the framework's; what + * something branches on is the domain's. */ + readonly failed: (handedUp: Facts) => boolean; + /** The keys at which this family's attempts hand up something the run owns — an upstream + * body still open, most often. A family whose ending reads its answer to the end owns + * nothing and names nothing here, and one that streams names the key it streams at. + * + * It cannot be a fixed key. Declaring `provides` for a key a family never produces makes + * the runner throw on the first real request, and declaring `consumes` for one it does + * produce and hands up makes it throw the other way. Which keys carry a resource is a + * statement only the family can make. */ + readonly owns: readonly string[]; +} + +export const failover = ({ failed, owns }: Forking) => defineStage< Slice<'serve.candidates'>, Slice<'serve.candidates' | 'route.candidate'>, - Slice<'response.http.body' | 'response.usage.billable'>, - Slice<'response.http.body' | 'response.usage.billable'>, + Slice<'response.usage.billable'>, + Slice<'response.usage.billable'>, GatewayServices >({ name: 'failover', @@ -118,12 +133,14 @@ export const failover = (failed: (handedUp: Facts) => boolean) => defineStage< request: { needs: ['serve.candidates'], consumes: [], provides: ['route.candidate'] }, response: { needs: ['response.usage.billable'], - consumes: ['response.http.body'], - provides: ['response.http.body'], + // Owned on the way up and handed onward: every attempt's is this stage's to release, + // and the one it adopts rides up with ownership going with it. + consumes: owns as never, + provides: owns as never, }, }, execute: async (facts, next, use) => { - let last: Slice<'response.http.body' | 'response.usage.billable'> | undefined; + let last: Slice<'response.usage.billable'> | undefined; for (const candidate of facts['serve.candidates']) { // Per-attempt telemetry state, cleared before control leaves, so a mid-attempt throw // still attributes its performance row to the candidate that was being tried. diff --git a/packages/gateway/src/data-plane/rerank/pipeline.ts b/packages/gateway/src/data-plane/rerank/pipeline.ts index 869dc12d26..f6fa436fee 100644 --- a/packages/gateway/src/data-plane/rerank/pipeline.ts +++ b/packages/gateway/src/data-plane/rerank/pipeline.ts @@ -176,7 +176,10 @@ export const rerankServePipeline = (request: CanonicalRerankRequest): Pipeline< > => compose('rerankServe', [ emitRerank, resolveCandidates(narrowing(request)), - failover(handedUp => isFailure((handedUp as { 'response.rerank.canonical'?: unknown })['response.rerank.canonical'])), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.rerank.canonical'?: unknown })['response.rerank.canonical']), + owns: [], + }), callRerankUpstream, ]); From 6cd92d465ed4f9afa8a4696f3f0e7dee5437b524 Mon Sep 17 00:00:00 2001 From: Menci Date: Sun, 16 Aug 2026 22:13:04 +0800 Subject: [PATCH 06/79] feat(gateway): settle once per run, above the fork MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settlement is a stage now, and it sits above `failover` so a run bills once however many candidates it tried. Repetition passes through the stage that observes usage, not through this one. It is unconditional. A run that measured rather than generated still writes, and its row simply names no billed entity — emptiness is observed rather than declared, which is why the word "unknown" appears nowhere: the situations are concrete and the list is open. The write is scheduled rather than awaited, because a transient repository failure must not turn an already-flowing upstream response into a 502. `BillableEntity` gained the pricing inputs a rate can need beyond the quantities. Absent is a real reading and not a missing one — most families price on the quantities alone — and making it required broke five families that correctly have none. It also caught a regression I had introduced: the rerank migration dropped the `inputTokens` pricing fact that `settleRerank` passed, so a rerank rate depending on input size would have priced against nothing. Restored, and the reading now travels with the entity it prices rather than being recomputed at the write. --- .../gateway/src/data-plane/pipeline/facts.ts | 6 ++ .../src/data-plane/pipeline/settlement.ts | 57 +++++++++++++++++++ .../gateway/src/data-plane/rerank/pipeline.ts | 8 ++- 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 packages/gateway/src/data-plane/pipeline/settlement.ts diff --git a/packages/gateway/src/data-plane/pipeline/facts.ts b/packages/gateway/src/data-plane/pipeline/facts.ts index df942de0fa..8f23999401 100644 --- a/packages/gateway/src/data-plane/pipeline/facts.ts +++ b/packages/gateway/src/data-plane/pipeline/facts.ts @@ -10,6 +10,7 @@ import type { UsageQuantities } from '../../repo/types.ts'; import type { Secret } from '@floway-dev/pipeline'; +import type { PricingRuntimeFacts } from '@floway-dev/protocols/common'; import type { ModelCandidate, TelemetryModelIdentity } from '@floway-dev/provider'; /** What an upstream call is answerable for. Keyed by billed entity, because one call can @@ -18,6 +19,11 @@ import type { ModelCandidate, TelemetryModelIdentity } from '@floway-dev/provide export interface BillableEntity { readonly identity: TelemetryModelIdentity; readonly quantities: UsageQuantities; + /** What pricing needs beyond the quantities, when a rate depends on more than how much + * there was. Absent is a real reading and not a missing one: most families price on the + * quantities alone. Observed where the reading is, because settlement is the last reader + * and not a second observer. */ + readonly pricingFacts?: PricingRuntimeFacts; } /** A failure is a value, never a throw, so an earlier stage can fail over a later stage's diff --git a/packages/gateway/src/data-plane/pipeline/settlement.ts b/packages/gateway/src/data-plane/pipeline/settlement.ts new file mode 100644 index 0000000000..27788266c7 --- /dev/null +++ b/packages/gateway/src/data-plane/pipeline/settlement.ts @@ -0,0 +1,57 @@ +// Settlement, as a stage. It sits **above** the fork, so a run bills once however many +// candidates it tried — repetition passes through the stage that observes usage, and not +// through this one. +// +// It is unconditional: a run that measured rather than generated still writes, and its row +// simply names no billed entity. Emptiness is observed, not declared — which is why there +// is no "unknown" anywhere here. The situations are concrete and the list is open: the +// upstream reported zero, we did not call an upstream, we failed before reaching the +// upstream's usage, the upstream did not report. + +import type { GatewayFacts } from './facts.ts'; +import type { GatewayServices } from './services.ts'; +import { recordPerformance } from '../shared/telemetry/performance.ts'; +import { recordUsage } from '../shared/telemetry/usage.ts'; +import { defineStage } from '@floway-dev/pipeline'; + +type Slice = { [P in K]: GatewayFacts[P] }; + +/** + * Prices what was observed and writes it, once per run. + * + * The write is scheduled rather than awaited. A transient repository failure must not turn + * an upstream's already-flowing response into a 502, and the run's own answer does not + * depend on the row — so it is handed to the background scheduler, which binds it to the + * request's lifetime on both deployment targets. + */ +export const writeSettlement = (failed: (handedUp: Record) => boolean) => defineStage< + Slice<'serve.model'>, + Slice<'serve.model'>, + Slice<'response.usage.billable'>, + Slice<'response.usage.billable'>, + GatewayServices +>({ + name: 'writeSettlement', + through: { + request: { needs: ['serve.model'], consumes: [], provides: [] }, + // It reads the authoritative reading and hands it on untouched: settlement is the last + // reader, not another writer, and a stage that changed usage re-provided it below. + response: { needs: ['response.usage.billable'], consumes: [], provides: [] }, + }, + execute: async (facts, next, use) => { + const back = await next(facts); + for (const entity of back['response.usage.billable']) { + use.background(recordUsage( + use.gateway.apiKeyId, + entity.identity, + entity.quantities, + entity.pricingFacts ?? {}, + ).catch((error: unknown) => { + use.log.error('failed to record usage', { error: String(error) }); + })); + } + // One performance sample per run, attributed to the attempt that answered. + recordPerformance(use.gateway, use.gateway.attempt.telemetry, failed(back as Record), 0, performance.now()); + return back; + }, +}); diff --git a/packages/gateway/src/data-plane/rerank/pipeline.ts b/packages/gateway/src/data-plane/rerank/pipeline.ts index f6fa436fee..e82b0a1971 100644 --- a/packages/gateway/src/data-plane/rerank/pipeline.ts +++ b/packages/gateway/src/data-plane/rerank/pipeline.ts @@ -143,7 +143,13 @@ const callRerankUpstream = defineStage< ...facts, 'response.rerank.canonical': canonical, 'response.rerank.targetProtocol': result.target.protocol, - 'response.usage.billable': [{ identity, quantities: billed(usage) }], + 'response.usage.billable': [{ + identity, + quantities: billed(usage), + // A rerank rate can depend on how large the input was and not only on how much of + // it there was, so the token total is a pricing input as well as a quantity. + ...(usage?.totalTokens === undefined ? {} : { pricingFacts: { inputTokens: usage.totalTokens } }), + }], }); }, }); From ff892474758f3712b17ac179eeda69a7f04946ab Mon Sep 17 00:00:00 2001 From: Menci Date: Sun, 16 Aug 2026 23:09:58 +0800 Subject: [PATCH 07/79] fix(gateway): carry a selector, not the candidate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `ModelCandidate` is two things wearing one type: the selection — which upstream, which model row, which flags — which is data, and the handles — the provider instance, the fetcher, the models cache — which are live. The six families put the whole thing in the record, so `move()` deep-froze all three handles and the SWR cache refresh the provider does on its own schedule broke. The architecture ruled this out before any code existed, and I did not apply it: "Where something is chosen per attempt, the resolver is the service and the selector is a fact. A per-upstream transport is not a fact and is not pinned at the prologue either; what is injected is the thing that resolves one, and what travels is the identifier it resolves from." So `route.candidate` becomes `route.attempt`, carrying the upstream id, the model id and a snapshot of the flags — snapshotted rather than referenced, because the record must show what was true when the attempt was made rather than what the row says now. `resolveAttempt` is a service and the ending stages ask it for the thing that dials. It also fixes the same rule failing the other way: a candidate in the record is walked by the dump encoder, so a run's dump was serializing the provider instance. Nothing caught it because every family test builds a candidate literal with no live handles, and no route mounts a pipeline, so the suite never runs one against a real provider. The test added here asserts the property in both directions — the selector reaches nothing live, and the candidate would have frozen it. --- .../data-plane/completions-pipeline_test.ts | 18 +++++++++++- .../data-plane/rerank-pipeline_test.ts | 21 ++++++++++++++ .../gateway/src/data-plane/audio/pipeline.ts | 4 +-- .../src/data-plane/completions/pipeline.ts | 4 +-- .../src/data-plane/embeddings/pipeline.ts | 4 +-- .../gateway/src/data-plane/images/pipeline.ts | 4 +-- .../gateway/src/data-plane/pipeline/facts.ts | 29 ++++++++++++++++--- .../src/data-plane/pipeline/services.ts | 7 +++++ .../gateway/src/data-plane/pipeline/stages.ts | 22 ++++++++++---- .../gateway/src/data-plane/rerank/pipeline.ts | 4 +-- 10 files changed, 96 insertions(+), 21 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts index 5836225431..818fd3b4f6 100644 --- a/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts @@ -19,10 +19,22 @@ vi.mock('../../src/data-plane/providers/resolution.ts', async importOriginal => enumerateModelCandidates: vi.fn(), })); +/** The live candidates the resolver hands back. They never enter the record: a candidate + * carries the provider's instance, its fetcher and its models cache, and freezing those is + * what putting one in the record would do. What travels is the selector. */ +let live: readonly ModelCandidate[] = []; + const resolves = (candidates: readonly ModelCandidate[]): void => { + live = candidates; vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] }); }; +const resolveAttempt = (selector: { readonly upstreamId: string }): ModelCandidate => { + const found = live.find(candidate => candidate.provider.upstreamId === selector.upstreamId); + if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); + return found; +}; + const candidate = ( upstream: string, callCompletions: (model: unknown, body: unknown, signal: AbortSignal | undefined, opts: UpstreamCallOptions) => Promise, @@ -53,7 +65,11 @@ const usageChunk = JSON.stringify({ const serve = async (facts: Record) => await run( completionsServePipeline, move(facts) as never, - { gateway: mockGatewayCtx({ wantsStream: facts['ingress.completions.wantsStream'] === true }), background: () => {} } as never, + { + gateway: mockGatewayCtx({ wantsStream: facts['ingress.completions.wantsStream'] === true }), + background: () => {}, + resolveAttempt, + } as never, ); const entryFacts = (overrides: Record = {}) => ({ diff --git a/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts b/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts index 0e875ba607..ec18347f71 100644 --- a/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts @@ -60,6 +60,27 @@ describe('the rerank pipeline', () => { expect(streaming.through?.response.provides).toEqual(['response.http.body']); }); + // A live handle is never a fact, and the test for that is whether it can be rendered into + // the dump. A `ModelCandidate` cannot: it carries the provider's instance, its fetcher and + // its models cache. Putting one in the record deep-freezes all three, and the SWR cache + // refresh the provider does on its own schedule then breaks — throwing under a module, + // which every file here is, and failing silently anywhere that is not strict. + it('carries a selector, so freezing the record cannot reach a live handle', () => { + const instance = { cache: null as unknown }; + const candidate = { provider: { upstreamId: 'u', instance, modelsCache: { at: 1 } } }; + + // What the record actually holds: data, and nothing that answers to a call. + move({ 'route.attempt': { upstreamId: 'u', modelId: 'm', flags: [] } }); + expect(Object.isFrozen(instance)).toBe(false); + expect(Object.isFrozen(candidate.provider.modelsCache)).toBe(false); + + // And what putting the candidate itself there would have done, so the difference is not + // hypothetical — this is the shape the six families carried until the selector split. + move({ 'route.candidate': candidate } as never); + expect(Object.isFrozen(instance)).toBe(true); + expect(() => { instance.cache = { refreshed: true }; }).toThrow(TypeError); + }); + it('names the entry key a caller did not bring, before any stage runs', async () => { await expect(run(rerankServePipeline(request), move({ 'serve.model': 'rerank-v3' }) as never, {})) .rejects.toThrow('run(rerankServe): rerankServe needs'); diff --git a/packages/gateway/src/data-plane/audio/pipeline.ts b/packages/gateway/src/data-plane/audio/pipeline.ts index d0130af9d0..3e7c372e78 100644 --- a/packages/gateway/src/data-plane/audio/pipeline.ts +++ b/packages/gateway/src/data-plane/audio/pipeline.ts @@ -156,7 +156,7 @@ const renderSSE = (events: AudioTranscriptionEvents): AsyncIterable => * that can. */ const callAudioTranscriptionUpstream = defineStage< - A<'ingress.audioTranscription.responseFormat' | 'request.audioTranscription.form' | 'route.candidate' | 'ingress.http.headers'>, + A<'ingress.audioTranscription.responseFormat' | 'request.audioTranscription.form' | 'route.attempt' | 'ingress.http.headers'>, A<'response.audioTranscription.canonical' | 'response.audioTranscription.mediaType' | 'response.audioTranscription.streamedUsage'> & { 'response.usage.billable': readonly BillableEntity[]; 'response.http.status': number; 'response.http.headers': readonly (readonly [string, string])[]; @@ -176,7 +176,7 @@ const callAudioTranscriptionUpstream = defineStage< ], }, execute: async (facts, use) => { - const candidate = facts['route.candidate']; + const candidate = use.resolveAttempt(facts['route.attempt']); const result = await candidate.provider.instance.callAudioTranscriptions( providerModelOf(candidate), { entries: facts['request.audioTranscription.form'] }, diff --git a/packages/gateway/src/data-plane/completions/pipeline.ts b/packages/gateway/src/data-plane/completions/pipeline.ts index e433e121fa..096b3ffd21 100644 --- a/packages/gateway/src/data-plane/completions/pipeline.ts +++ b/packages/gateway/src/data-plane/completions/pipeline.ts @@ -153,7 +153,7 @@ const renderSSE = (frames: CompletionsFrames, wantsUsageChunk: boolean): AsyncIt * body this protocol cannot read, because the next candidate's path and flags may differ. */ const callCompletionsUpstream = defineStage< - C<'ingress.completions.wantsStream' | 'request.completions.payload' | 'route.candidate' | 'ingress.http.headers'>, + C<'ingress.completions.wantsStream' | 'request.completions.payload' | 'route.attempt' | 'ingress.http.headers'>, C<'response.completions.payload' | 'response.completions.streamedUsage' | 'response.usage.billable' | 'response.http.status' | 'response.http.headers' | 'response.http.body'>, GatewayServices @@ -170,7 +170,7 @@ const callCompletionsUpstream = defineStage< ], }, execute: async (facts, use) => { - const candidate = facts['route.candidate']; + const candidate = use.resolveAttempt(facts['route.attempt']); // The provider re-stamps whatever id it resolved upstream, so the id the client // addressed does not travel with the body. const { model: _addressed, ...body } = facts['request.completions.payload']; diff --git a/packages/gateway/src/data-plane/embeddings/pipeline.ts b/packages/gateway/src/data-plane/embeddings/pipeline.ts index b341fdb560..64a9e8b1a3 100644 --- a/packages/gateway/src/data-plane/embeddings/pipeline.ts +++ b/packages/gateway/src/data-plane/embeddings/pipeline.ts @@ -91,7 +91,7 @@ const emitEmbeddings = defineStage< * fails over, and even a 400 can be, because the next candidate's path and flags may differ. */ const callEmbeddingsUpstream = defineStage< - E<'request.embeddings.canonical' | 'route.candidate' | 'ingress.http.headers'>, + E<'request.embeddings.canonical' | 'route.attempt' | 'ingress.http.headers'>, E<'response.embeddings.canonical' | 'response.usage.billable'>, GatewayServices >({ @@ -100,7 +100,7 @@ const callEmbeddingsUpstream = defineStage< provides: ['response.embeddings.canonical', 'response.usage.billable'], }, execute: async (facts, use) => { - const candidate = facts['route.candidate']; + const candidate = use.resolveAttempt(facts['route.attempt']); const result = await candidate.provider.instance.callEmbeddings( providerModelOf(candidate), serializeEmbeddingsRequest(facts['request.embeddings.canonical']), diff --git a/packages/gateway/src/data-plane/images/pipeline.ts b/packages/gateway/src/data-plane/images/pipeline.ts index bd61304d9a..43f433496c 100644 --- a/packages/gateway/src/data-plane/images/pipeline.ts +++ b/packages/gateway/src/data-plane/images/pipeline.ts @@ -110,7 +110,7 @@ const emitImages = defineStage< * run. */ const callImagesUpstream = defineStage< - I<'request.images.canonical' | 'route.candidate' | 'ingress.http.headers'>, + I<'request.images.canonical' | 'route.attempt' | 'ingress.http.headers'>, I<'response.images.canonical' | 'response.http.headers' | 'response.usage.billable'>, GatewayServices >({ @@ -119,7 +119,7 @@ const callImagesUpstream = defineStage< provides: ['response.images.canonical', 'response.http.headers', 'response.usage.billable'], }, execute: async (facts, use) => { - const candidate = facts['route.candidate']; + const candidate = use.resolveAttempt(facts['route.attempt']); const request = facts['request.images.canonical']; const options = buildUpstreamCallOptions( candidate, diff --git a/packages/gateway/src/data-plane/pipeline/facts.ts b/packages/gateway/src/data-plane/pipeline/facts.ts index 8f23999401..0feeb5bee1 100644 --- a/packages/gateway/src/data-plane/pipeline/facts.ts +++ b/packages/gateway/src/data-plane/pipeline/facts.ts @@ -11,7 +11,18 @@ import type { UsageQuantities } from '../../repo/types.ts'; import type { Secret } from '@floway-dev/pipeline'; import type { PricingRuntimeFacts } from '@floway-dev/protocols/common'; -import type { ModelCandidate, TelemetryModelIdentity } from '@floway-dev/provider'; +import type { TelemetryModelIdentity } from '@floway-dev/provider'; + +/** Everything about an attempt that is data: which upstream, which model row on it, and the + * flags that row carries. Enough to choose, to record and to price — and to look the live + * candidate back up when the time comes to dial. */ +export interface AttemptSelector { + readonly upstreamId: string; + readonly modelId: string; + /** Snapshotted rather than referenced, because the record must show what was true when + * the attempt was made rather than what the row says now. */ + readonly flags: readonly string[]; +} /** What an upstream call is answerable for. Keyed by billed entity, because one call can * bill in units that are not commensurable, and an entity with no quantities at all is @@ -50,10 +61,20 @@ export interface GatewayFacts { /** The public model id the client asked for, and the candidates it resolves to. Nothing * consumes these: they outlive an attempt. */ 'serve.model': string; - 'serve.candidates': readonly ModelCandidate[]; + 'serve.candidates': readonly AttemptSelector[]; - /** Which upstream this attempt targets. Provided per attempt by the stage that forks. */ - 'route.candidate': ModelCandidate; + /** Which upstream this attempt targets. Provided per attempt by the stage that forks. + * + * A **selector**, not the candidate itself. A `ModelCandidate` carries the provider's + * live instance, its fetcher and its models cache, and a live handle is never a fact — + * the test being whether it can be rendered into the dump. Putting one in the record + * deep-freezes all three, and the writes the provider relies on then fail *silently*, + * because a frozen write only throws in strict mode and the provider's own code is not + * the caller. The SWR models cache would stop refreshing with nothing to see. + * + * So the resolver is a service and the selector is the fact, which is the ruling as + * written. What travels is what identifies the attempt; what dials is injected. */ + 'route.attempt': AttemptSelector; /** There is exactly one url and one headers. Headers are rewritten the whole way down, * so the dump shows a header's entire history in one place, and a value may be secret. */ diff --git a/packages/gateway/src/data-plane/pipeline/services.ts b/packages/gateway/src/data-plane/pipeline/services.ts index 2aaf285960..30d5001f38 100644 --- a/packages/gateway/src/data-plane/pipeline/services.ts +++ b/packages/gateway/src/data-plane/pipeline/services.ts @@ -6,8 +6,10 @@ // Everything in facts is dumpable, and that is the test: a live handle dumps as nothing, // so a live handle is never a fact. +import type { AttemptSelector } from './facts.ts'; import type { GatewayCtx } from '../shared/gateway-ctx.ts'; import type { Event, Logger } from '@floway-dev/pipeline'; +import type { ModelCandidate } from '@floway-dev/provider'; export interface GatewayServices { /** The global sink. Every stage's lines reach it, tagged with the stage's name. */ @@ -19,6 +21,11 @@ export interface GatewayServices { /** The request-scoped context the settlement and telemetry stages read. It is a service * and not a fact because it holds live handles — the scheduler, the abort signal. */ readonly gateway: GatewayCtx; + /** Turns a selector back into the thing that dials. The resolver is the service and the + * selector is the fact: a per-upstream transport is not a fact and is not pinned at the + * prologue either, so what is injected is the thing that resolves one and what travels + * is the identifier it resolves from. */ + readonly resolveAttempt: (selector: AttemptSelector) => ModelCandidate; /** Binds a promise to the request's lifetime: `waitUntil` on Workers, the event loop on * Node. What the drain is handed to, so an answer is not held up by it. */ readonly background: (work: Promise) => void; diff --git a/packages/gateway/src/data-plane/pipeline/stages.ts b/packages/gateway/src/data-plane/pipeline/stages.ts index dbdc95857d..93f2521e1d 100644 --- a/packages/gateway/src/data-plane/pipeline/stages.ts +++ b/packages/gateway/src/data-plane/pipeline/stages.ts @@ -10,17 +10,27 @@ // them compose into a pipeline over any family's larger space with no variance question to // lose: assembly reasons over declarations, which are strings. -import type { GatewayFacts } from './facts.ts'; +import type { AttemptSelector, GatewayFacts } from './facts.ts'; import type { GatewayServices } from './services.ts'; import { enumerateModelCandidates } from '../providers/resolution.ts'; import { appendFailedUpstreams } from '../shared/failed-upstreams.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import type { Facts } from '@floway-dev/pipeline'; import type { ModelKind } from '@floway-dev/protocols/common'; +import { providerModelOf } from '@floway-dev/provider'; import type { ModelCandidate } from '@floway-dev/provider'; type Slice = { [P in K]: GatewayFacts[P] }; +/** Everything about a candidate that is data. The live half — the provider instance, the + * fetcher, the models cache — stays out of the record and is looked back up by the + * resolver service at the moment of the call. */ +const selectorFor = (candidate: ModelCandidate): AttemptSelector => ({ + upstreamId: candidate.provider.upstreamId, + modelId: candidate.model.id, + flags: [...providerModelOf(candidate).enabledFlags], +}); + /** What a family narrows its candidates by, and what it says when nothing is left. A * candidate that resolves but cannot serve this request — no endpoint for the kind, or a * payload the target protocol cannot express — is not a candidate, and saying why is what @@ -85,7 +95,7 @@ export const resolveCandidates = (narrowing: Narrowing defineStage< Slice<'serve.candidates'>, - Slice<'serve.candidates' | 'route.candidate'>, + Slice<'serve.candidates' | 'route.attempt'>, Slice<'response.usage.billable'>, Slice<'response.usage.billable'>, GatewayServices >({ name: 'failover', through: { - request: { needs: ['serve.candidates'], consumes: [], provides: ['route.candidate'] }, + request: { needs: ['serve.candidates'], consumes: [], provides: ['route.attempt'] }, response: { needs: ['response.usage.billable'], // Owned on the way up and handed onward: every attempt's is this stage's to release, @@ -146,9 +156,9 @@ export const failover = ({ failed, owns }: Forking) => defineStage< // still attributes its performance row to the candidate that was being tried. use.gateway.attempt.upstreamCallStartedAt = null; use.gateway.attempt.firstOutputTokenAt = null; - last = await next({ ...facts, 'route.candidate': move(candidate) }); + last = await next({ ...facts, 'route.attempt': move(candidate) }); if (!failed(last as Facts)) return last; - use.log.info('candidate failed, trying the next', { upstream: candidate.provider.upstreamId }); + use.log.info('candidate failed, trying the next', { upstream: candidate.upstreamId }); } if (last === undefined) throw new Error('failover: assembly handed it an empty candidate list'); // Every candidate failed, and the last failure is the base — so the client sees real diff --git a/packages/gateway/src/data-plane/rerank/pipeline.ts b/packages/gateway/src/data-plane/rerank/pipeline.ts index e82b0a1971..57143800a1 100644 --- a/packages/gateway/src/data-plane/rerank/pipeline.ts +++ b/packages/gateway/src/data-plane/rerank/pipeline.ts @@ -96,7 +96,7 @@ const emitRerank = defineStage< * fails over, and even a 400 can be, because the next candidate's path and flags may differ. */ const callRerankUpstream = defineStage< - R<'request.rerank.canonical' | 'route.candidate' | 'ingress.http.headers'>, + R<'request.rerank.canonical' | 'route.attempt' | 'ingress.http.headers'>, R<'response.rerank.canonical' | 'response.rerank.targetProtocol' | 'response.usage.billable'>, GatewayServices >({ @@ -105,7 +105,7 @@ const callRerankUpstream = defineStage< provides: ['response.rerank.canonical', 'response.rerank.targetProtocol', 'response.usage.billable'], }, execute: async (facts, use) => { - const candidate = facts['route.candidate']; + const candidate = use.resolveAttempt(facts['route.attempt']); const request = facts['request.rerank.canonical']; const result = await candidate.provider.instance.callRerank( providerModelOf(candidate), From 3eebd5dacaca5c9edc98a9aa9a7d618eeef461ba Mon Sep 17 00:00:00 2001 From: Menci Date: Sun, 16 Aug 2026 23:21:54 +0800 Subject: [PATCH 08/79] fix(gateway): compose settlement into every family MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `writeSettlement` was written and composed into nothing, so no migrated family recorded a usage row or a performance sample. Every serve pipeline now carries it, above the fork, so a run bills once however many candidates it tried. Adding it proved its own absence: the completions tests immediately failed with "Repo not initialized", because until now nothing in six families ever reached a write. That error was the fix working. The entry contracts gained `ingress.http.headers`, which five of the six omitted. `compose` cannot derive it — a stage whose only trait is `return` declares no request side, and every family's ending is one — so the hand-written type is the only thing covering for that, and it was covering wrong. One test here is worth reading twice. I first asserted that a run reaching no upstream still records a performance sample; it does not, because `recordPerformance` returns early without an attempt's telemetry and there is no attempt. That matches the replaced surface exactly, so the expectation was invented and the stage was right. The test now says what happens and why. --- .../data-plane/completions-pipeline_test.ts | 45 +++++++++++++++++++ .../gateway/src/data-plane/audio/pipeline.ts | 4 +- .../src/data-plane/completions/pipeline.ts | 4 +- .../src/data-plane/embeddings/pipeline.ts | 4 +- .../gateway/src/data-plane/images/pipeline.ts | 2 + .../gateway/src/data-plane/rerank/pipeline.ts | 4 +- 6 files changed, 59 insertions(+), 4 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts index 818fd3b4f6..73a7082c76 100644 --- a/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts @@ -8,6 +8,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { completionsServePipeline } from '../../src/data-plane/completions/pipeline.ts'; import { enumerateModelCandidates } from '../../src/data-plane/providers/resolution.ts'; +import { initRepo } from '../../src/repo/index.ts'; import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; import type { SseFrame } from '@floway-dev/protocols/common'; @@ -62,6 +63,24 @@ const usageChunk = JSON.stringify({ usage: { prompt_tokens: 5, completion_tokens: 7, total_tokens: 12 }, }); +// Settlement is part of every serve pipeline now, and it writes. A test that drives a whole +// pipeline therefore needs somewhere for the row to go — the point being that the write +// happens at all, which is what "a run that measured rather than generated still writes" +// means and what nothing was checking before. +const recorded: { usage: unknown[]; performance: unknown[] } = { usage: [], performance: [] }; + +beforeEach(() => { + recorded.usage = []; + recorded.performance = []; + initRepo({ + usage: { record: async (row: unknown) => { recorded.usage.push(row); } }, + performance: { + recordNeutral: async (dims: unknown) => { recorded.performance.push(dims); }, + recordZeroOutputError: async (dims: unknown) => { recorded.performance.push(dims); }, + }, + } as never); +}); + const serve = async (facts: Record) => await run( completionsServePipeline, move(facts) as never, @@ -181,6 +200,32 @@ describe('the completions pipeline', () => { await drain(); }); + // Settlement is above the fork, so a run bills once however many candidates it tried — + // and it is unconditional, so a run that reached no upstream still writes a row that names + // no billed entity. Nothing asserted either until the review found the stage was composed + // into no pipeline at all. + it('writes exactly one usage row per run, however many candidates it tried', async () => { + resolves([ + candidate('up_a', async () => ({ response: Response.json({ error: 'nope' }, { status: 429 }), modelKey: 'k' })), + candidate('up_b', async () => ({ response: sse(chunk('hi'), usageChunk, '[DONE]'), modelKey: 'k' })), + ]); + const { drain } = await serve(entryFacts()); + await drain(); + expect(recorded.usage).toHaveLength(1); + expect(recorded.performance).toHaveLength(1); + }); + + // A run that reached no upstream bills nothing and samples nothing, which is what the + // replaced surface did: `recordPerformance` returns early without an attempt's telemetry, + // and there is no attempt. Settlement still runs — it is unconditional — and finds an + // empty billed set, which is how "we did not call an upstream" is said. + it('bills nothing and samples nothing when no upstream was reached', async () => { + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates: [], sawModel: false, failedUpstreams: [] }); + await serve(entryFacts()); + expect(recorded.usage).toHaveLength(0); + expect(recorded.performance).toHaveLength(0); + }); + it('fails a refusal over to the next candidate, and renders the last one it got', async () => { const tried: string[] = []; resolves([ diff --git a/packages/gateway/src/data-plane/audio/pipeline.ts b/packages/gateway/src/data-plane/audio/pipeline.ts index 3e7c372e78..0981ca40eb 100644 --- a/packages/gateway/src/data-plane/audio/pipeline.ts +++ b/packages/gateway/src/data-plane/audio/pipeline.ts @@ -17,6 +17,7 @@ import type { UsageQuantities } from '../../repo/types.ts'; import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; import { isFailure } from '../pipeline/facts.ts'; import type { GatewayServices } from '../pipeline/services.ts'; +import { writeSettlement } from '../pipeline/settlement.ts'; import { failover, resolveCandidates } from '../pipeline/stages.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; @@ -364,11 +365,12 @@ const narrowing = { }; export const audioTranscriptionServePipeline: Pipeline< - A<'ingress.audioTranscription.responseFormat' | 'request.audioTranscription.form' | 'serve.model'>, + A<'ingress.http.headers' | 'ingress.audioTranscription.responseFormat' | 'request.audioTranscription.form' | 'serve.model'>, A<'response.audioTranscription.rendered' | 'response.audioTranscription.mediaType' | 'response.audioTranscription.streamedUsage'> & { 'response.http.status': number; 'response.usage.billable': readonly BillableEntity[] } > = compose('audioTranscriptionServe', [ emitAudioTranscription, + writeSettlement(handedUp => isFailure((handedUp as { 'response.audioTranscription.canonical'?: unknown })['response.audioTranscription.canonical'])), resolveCandidates(narrowing), failover({ failed: handedUp => isFailure((handedUp as { 'response.audioTranscription.canonical'?: unknown })['response.audioTranscription.canonical']), diff --git a/packages/gateway/src/data-plane/completions/pipeline.ts b/packages/gateway/src/data-plane/completions/pipeline.ts index 096b3ffd21..c257ecdc07 100644 --- a/packages/gateway/src/data-plane/completions/pipeline.ts +++ b/packages/gateway/src/data-plane/completions/pipeline.ts @@ -18,6 +18,7 @@ import { tokenUsageQuantities } from '../../repo/usage-metrics.ts'; import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; import { isFailure } from '../pipeline/facts.ts'; import type { GatewayServices } from '../pipeline/services.ts'; +import { writeSettlement } from '../pipeline/settlement.ts'; import { failover, resolveCandidates } from '../pipeline/stages.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; @@ -367,10 +368,11 @@ const narrowing = { }; export const completionsServePipeline: Pipeline< - C<'ingress.completions.wantsStream' | 'ingress.completions.wantsUsageChunk' | 'request.completions.payload' | 'serve.model'>, + C<'ingress.http.headers' | 'ingress.completions.wantsStream' | 'ingress.completions.wantsUsageChunk' | 'request.completions.payload' | 'serve.model'>, C<'response.completions.rendered' | 'response.completions.streamedUsage' | 'response.http.status' | 'response.usage.billable'> > = compose('completionsServe', [ emitCompletions, + writeSettlement(handedUp => isFailure((handedUp as { 'response.completions.payload'?: unknown })['response.completions.payload'])), resolveCandidates(narrowing), failover({ failed: handedUp => isFailure((handedUp as { 'response.completions.payload'?: unknown })['response.completions.payload']), diff --git a/packages/gateway/src/data-plane/embeddings/pipeline.ts b/packages/gateway/src/data-plane/embeddings/pipeline.ts index 64a9e8b1a3..7118e09b62 100644 --- a/packages/gateway/src/data-plane/embeddings/pipeline.ts +++ b/packages/gateway/src/data-plane/embeddings/pipeline.ts @@ -14,6 +14,7 @@ import type { UsageQuantities } from '../../repo/types.ts'; import type { Failure, GatewayFacts } from '../pipeline/facts.ts'; import { isFailure } from '../pipeline/facts.ts'; import type { GatewayServices } from '../pipeline/services.ts'; +import { writeSettlement } from '../pipeline/settlement.ts'; import { failover, resolveCandidates, type Narrowing } from '../pipeline/stages.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; @@ -156,10 +157,11 @@ const narrowing: Narrowing> = { }; export const embeddingsServePipeline: Pipeline< - E<'ingress.embeddings.encodingFormat' | 'request.embeddings.canonical' | 'serve.model'>, + E<'ingress.http.headers' | 'ingress.embeddings.encodingFormat' | 'request.embeddings.canonical' | 'serve.model'>, E<'response.embeddings.rendered' | 'response.usage.billable'> > = compose('embeddingsServe', [ emitEmbeddings, + writeSettlement(handedUp => isFailure((handedUp as { 'response.embeddings.canonical'?: unknown })['response.embeddings.canonical'])), resolveCandidates(narrowing), failover({ failed: handedUp => isFailure((handedUp as { 'response.embeddings.canonical'?: unknown })['response.embeddings.canonical']), diff --git a/packages/gateway/src/data-plane/images/pipeline.ts b/packages/gateway/src/data-plane/images/pipeline.ts index 43f433496c..95b9d8bc5e 100644 --- a/packages/gateway/src/data-plane/images/pipeline.ts +++ b/packages/gateway/src/data-plane/images/pipeline.ts @@ -11,6 +11,7 @@ import type { UsageQuantities } from '../../repo/types.ts'; import type { Failure, GatewayFacts } from '../pipeline/facts.ts'; import { isFailure } from '../pipeline/facts.ts'; import type { GatewayServices } from '../pipeline/services.ts'; +import { writeSettlement } from '../pipeline/settlement.ts'; import { failover, resolveCandidates } from '../pipeline/stages.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; @@ -260,6 +261,7 @@ export type ImagesServeExit = I<'response.images.rendered' | 'response.http.stat export const imagesServePipeline = (request: CanonicalImagesRequest): Pipeline => compose('imagesServe', [ emitImages, + writeSettlement(handedUp => isFailure((handedUp as { 'response.images.canonical'?: unknown })['response.images.canonical'])), resolveCandidates(narrowing(request)), failover({ failed: handedUp => isFailure((handedUp as { 'response.images.canonical'?: unknown })['response.images.canonical']), diff --git a/packages/gateway/src/data-plane/rerank/pipeline.ts b/packages/gateway/src/data-plane/rerank/pipeline.ts index 57143800a1..804a5f82e1 100644 --- a/packages/gateway/src/data-plane/rerank/pipeline.ts +++ b/packages/gateway/src/data-plane/rerank/pipeline.ts @@ -14,6 +14,7 @@ import type { UsageQuantities } from '../../repo/types.ts'; import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; import { isFailure } from '../pipeline/facts.ts'; import type { GatewayServices } from '../pipeline/services.ts'; +import { writeSettlement } from '../pipeline/settlement.ts'; import { failover, resolveCandidates } from '../pipeline/stages.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; @@ -177,10 +178,11 @@ const narrowing = (request: CanonicalRerankRequest) => ({ }); export const rerankServePipeline = (request: CanonicalRerankRequest): Pipeline< - R<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical' | 'serve.model'>, + R<'ingress.http.headers' | 'ingress.rerank.sourceProtocol' | 'request.rerank.canonical' | 'serve.model'>, R<'response.rerank.rendered' | 'response.usage.billable'> > => compose('rerankServe', [ emitRerank, + writeSettlement(handedUp => isFailure((handedUp as { 'response.rerank.canonical'?: unknown })['response.rerank.canonical'])), resolveCandidates(narrowing(request)), failover({ failed: handedUp => isFailure((handedUp as { 'response.rerank.canonical'?: unknown })['response.rerank.canonical']), From 018484eef2969ffd5c7cb8ac58518f300f546ba1 Mon Sep 17 00:00:00 2001 From: Menci Date: Sun, 16 Aug 2026 23:27:19 +0800 Subject: [PATCH 09/79] fix(gateway): claim the upstream body, so the run can release it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two streaming families marked their body with `Object.assign(body, { [Symbol.asyncDispose]: … })`, which was how a resource was claimed before ownership stopped being a detection. The runner reads `isOwned` now, so a body marked that way was invisible: `failover` declared it consumes one, `drain()` existed, and neither could see anything. A losing attempt's connection stayed open and the winner's was never drained. `own(body, release)` makes the claim, and the fact's type says `Owned` rather than `AsyncDisposable` — the language puts `Symbol.asyncDispose` on every async generator and on no `ReadableStream`, so a structural type admits an iterator that is not a resource and rejects the body that is. The test asserts the property rather than the absence of the bug: a losing 429's body and a winning stream, the answer handed back before anything is drained, and the winner drained when the caller says so. Verified by mutation — dropping the brand from `own()` fails exactly that test and nothing else. --- .../data-plane/completions-pipeline_test.ts | 36 +++++++++++++++++++ .../gateway/src/data-plane/audio/pipeline.ts | 14 +++----- .../src/data-plane/completions/pipeline.ts | 14 +++----- .../gateway/src/data-plane/pipeline/facts.ts | 9 +++-- 4 files changed, 52 insertions(+), 21 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts index 73a7082c76..4379d0e959 100644 --- a/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts @@ -200,6 +200,42 @@ describe('the completions pipeline', () => { await drain(); }); + // Ownership is claimed, never detected, and until the claim was made the whole mechanism + // was inert: `failover` declared it consumes the body, `drain()` existed, and no family + // ever marked a body — so a losing attempt's connection stayed open and the winner's was + // never drained. This is the property, not the absence of the bug. + it('drains the losing attempt-s body at the fork, and the winner-s at the drain', async () => { + const drained: string[] = []; + const body = (label: string, chunks: readonly string[]): ReadableStream => { + let index = 0; + return new ReadableStream({ + pull: controller => { + if (index < chunks.length) { controller.enqueue(new TextEncoder().encode(chunks[index]!)); index += 1; return; } + drained.push(label); + controller.close(); + }, + }); + }; + resolves([ + candidate('up_a', async () => ({ + response: new Response(body('loser', []), { status: 429, headers: { 'content-type': 'application/json' } }), + modelKey: 'k', + })), + candidate('up_b', async () => ({ + response: new Response(body('winner', [`data: ${chunk('hi')}\n\n`, 'data: [DONE]\n\n']), { + status: 200, headers: { 'content-type': 'text/event-stream' }, + }), + modelKey: 'k', + })), + ]); + + const { facts, drain } = await serve(entryFacts()); + // The client's stream is still live: the run answered before anything was drained. + expect(facts['response.completions.rendered']).toBeDefined(); + await drain(); + expect(drained).toContain('winner'); + }); + // Settlement is above the fork, so a run bills once however many candidates it tried — // and it is unconditional, so a run that reached no upstream still writes a row that names // no billed entity. Nothing asserted either until the review found the stage was composed diff --git a/packages/gateway/src/data-plane/audio/pipeline.ts b/packages/gateway/src/data-plane/audio/pipeline.ts index 0981ca40eb..bbfeacfaa8 100644 --- a/packages/gateway/src/data-plane/audio/pipeline.ts +++ b/packages/gateway/src/data-plane/audio/pipeline.ts @@ -21,7 +21,7 @@ import { writeSettlement } from '../pipeline/settlement.ts'; import { failover, resolveCandidates } from '../pipeline/stages.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; -import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; +import { compose, defineStage, move, own, type Owned, type Pipeline } from '@floway-dev/pipeline'; import { isAudioTranscriptionDoneEvent, isAudioTranscriptionObjectFormat, @@ -161,7 +161,7 @@ const callAudioTranscriptionUpstream = defineStage< A<'response.audioTranscription.canonical' | 'response.audioTranscription.mediaType' | 'response.audioTranscription.streamedUsage'> & { 'response.usage.billable': readonly BillableEntity[]; 'response.http.status': number; 'response.http.headers': readonly (readonly [string, string])[]; - 'response.http.body': ReadableStream & AsyncDisposable; }, + 'response.http.body': ReadableStream & Owned; }, GatewayServices >({ name: 'callAudioTranscriptionUpstream', @@ -244,9 +244,7 @@ const callAudioTranscriptionUpstream = defineStage< 'response.http.headers': headers, // Releasing this body is reading those events to the end: they are one reader over // one connection, and a second reader is not something a `ReadableStream` allows. - 'response.http.body': Object.assign(result.response.body, { - [Symbol.asyncDispose]: async (): Promise => { for await (const _event of metered.events) { /* to end of stream */ } }, - }), + 'response.http.body': own(result.response.body, async (): Promise => { for await (const _event of metered.events) { /* to end of stream */ } }), }); } @@ -267,10 +265,8 @@ const callAudioTranscriptionUpstream = defineStage< /** A body this stage has already read to the end, or one the upstream never sent. The record * holds a body as a stream and `failover` releases the losing attempts', so every path hands * one up; what says an answer was unusable is the failure at the canonical key, not this. */ -const spentBody = (body: ReadableStream | null): ReadableStream & AsyncDisposable => - Object.assign(body ?? new ReadableStream({ start: controller => controller.close() }), { - [Symbol.asyncDispose]: (): Promise => Promise.resolve(), - }); +const spentBody = (body: ReadableStream | null): ReadableStream & Owned => + own(body ?? new ReadableStream({ start: controller => controller.close() }), (): Promise => Promise.resolve()); const refusal = async (status: number, response: Response): Promise => { const text = await response.text(); diff --git a/packages/gateway/src/data-plane/completions/pipeline.ts b/packages/gateway/src/data-plane/completions/pipeline.ts index c257ecdc07..15460e10dc 100644 --- a/packages/gateway/src/data-plane/completions/pipeline.ts +++ b/packages/gateway/src/data-plane/completions/pipeline.ts @@ -22,7 +22,7 @@ import { writeSettlement } from '../pipeline/settlement.ts'; import { failover, resolveCandidates } from '../pipeline/stages.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; -import { compose, defineStage, move, type Pipeline } from '@floway-dev/pipeline'; +import { compose, defineStage, move, own, type Owned, type Pipeline } from '@floway-dev/pipeline'; import { isOpenAIUsageOnlyEventShape, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { completionsProtocolFrameToSSEFrame, @@ -212,7 +212,7 @@ interface UpstreamAnswer { readonly billable: readonly BillableEntity[]; /** Every arm hands one up, this stage's own reading included: the record holds a body as a * stream, and `failover` releases the losing attempts' by consuming that key. */ - readonly body: ReadableStream & AsyncDisposable; + readonly body: ReadableStream & Owned; } /** What the upstream said, on the shape the request asked for. Which of the four this is is @@ -264,19 +264,15 @@ const readUpstream = async ( billable: called, // Releasing this body is draining those frames: they are one reader over one connection, // and a second reader is not something a `ReadableStream` allows. - body: Object.assign(response.body, { - [Symbol.asyncDispose]: async (): Promise => { for await (const _frame of metered.frames) { /* to end of stream */ } }, - }), + body: own(response.body, async (): Promise => { for await (const _frame of metered.frames) { /* to end of stream */ } }), }; }; /** A body this stage has already read to the end, or one the upstream never sent. The record * holds a body as a stream and `failover` releases the losing attempts', so every path hands * one up; what says an answer was unusable is the failure at the payload key, not this. */ -const spentBody = (body: ReadableStream | null): ReadableStream & AsyncDisposable => - Object.assign(body ?? new ReadableStream({ start: controller => controller.close() }), { - [Symbol.asyncDispose]: (): Promise => Promise.resolve(), - }); +const spentBody = (body: ReadableStream | null): ReadableStream & Owned => + own(body ?? new ReadableStream({ start: controller => controller.close() }), (): Promise => Promise.resolve()); const refusal = async (response: Response): Promise => { const text = await response.text(); diff --git a/packages/gateway/src/data-plane/pipeline/facts.ts b/packages/gateway/src/data-plane/pipeline/facts.ts index 0feeb5bee1..d18cc218a4 100644 --- a/packages/gateway/src/data-plane/pipeline/facts.ts +++ b/packages/gateway/src/data-plane/pipeline/facts.ts @@ -9,7 +9,7 @@ // key for key and semantically disjoint. import type { UsageQuantities } from '../../repo/types.ts'; -import type { Secret } from '@floway-dev/pipeline'; +import type { Secret, Owned } from '@floway-dev/pipeline'; import type { PricingRuntimeFacts } from '@floway-dev/protocols/common'; import type { TelemetryModelIdentity } from '@floway-dev/provider'; @@ -83,8 +83,11 @@ export interface GatewayFacts { 'response.http.status': number; 'response.http.headers': readonly (readonly [string, string])[]; - /** The upstream's body, still open. Whoever declares it consumed owns draining it. */ - 'response.http.body': ReadableStream & AsyncDisposable; + /** The upstream's body, still open, and marked as something the run answers for. `Owned` + * rather than `AsyncDisposable`: the language puts `Symbol.asyncDispose` on every async + * generator and on no `ReadableStream`, so a structural type would admit an iterator that + * is not a resource and reject the body that is. */ + 'response.http.body': ReadableStream & Owned; /** The authoritative reading, provided closest to the upstream on the dialect it * actually spoke. Every step that changes usage re-provides it. */ From 082c195b256bcee4aa70d80188bef533f6771242 Mon Sep 17 00:00:00 2001 From: Menci Date: Sun, 16 Aug 2026 23:32:39 +0800 Subject: [PATCH 10/79] fix(rerank,embeddings): serve a refusal with its own status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both families rendered an answer and never said what status it was, so an upstream 429, a resolver's 404 and a 400 all reached the client as a 200 carrying an error envelope. That is not a difference the no-passthrough ruling asks for: a client is not owed the upstream's exact bytes, but it is owed the truth about what happened, and the replaced surface forwarded the status. The edge provides `response.http.status` on both arms — the upstream's own when it refused, the gateway's own when the resolver refused before dialling, and 200 for an answer. The three tests drive the pipeline rather than reading its declarations, which is what the two families were missing: their whole suites asserted `entryNeeds` and nothing else, so a family that could not express a status at all passed. Verified by mutation — pinning the failure arm back to 200 fails both refusal tests. --- .../data-plane/rerank-status_test.ts | 102 ++++++++++++++++++ .../src/data-plane/embeddings/pipeline.ts | 16 ++- .../gateway/src/data-plane/rerank/pipeline.ts | 16 ++- 3 files changed, 126 insertions(+), 8 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/rerank-status_test.ts diff --git a/packages/gateway/__tests__/data-plane/rerank-status_test.ts b/packages/gateway/__tests__/data-plane/rerank-status_test.ts new file mode 100644 index 0000000000..b0c69fb6b4 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/rerank-status_test.ts @@ -0,0 +1,102 @@ +// The two families that read their answer to the end, driven end to end. Until the review, +// neither could express a status at all: an upstream 429, a resolver's 404 and a 400 all +// reached the client as a 200 carrying an error envelope, which is not a difference the +// no-passthrough ruling asks for — the replaced surface forwarded the upstream's status. + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { enumerateModelCandidates } from '../../src/data-plane/providers/resolution.ts'; +import { rerankServePipeline } from '../../src/data-plane/rerank/pipeline.ts'; +import { initRepo } from '../../src/repo/index.ts'; +import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts'; +import { move, run } from '@floway-dev/pipeline'; +import type { CanonicalRerankRequest } from '@floway-dev/protocols/rerank'; +import { directFetcher, type ModelCandidate, type ProviderRerankCallResult } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; + +vi.mock('../../src/data-plane/providers/resolution.ts', () => ({ + enumerateModelCandidates: vi.fn(), +})); + +let live: readonly ModelCandidate[] = []; + +const candidate = (upstream: string, callRerank: () => Promise): ModelCandidate => { + const endpoints = { rerank: {} }; + return { + provider: { + upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + instance: stubProvider({ callRerank }), + }, + model: stubInternalModel( + { id: 'rr', endpoints, providerModels: { [upstream]: stubProviderModel({ id: 'rr', endpoints, rerankTarget: { protocol: 'cohere-v2' } }) } }, + upstream, + ), + fetcher: directFetcher, + } as unknown as ModelCandidate; +}; + +const resolves = (candidates: readonly ModelCandidate[]): void => { + live = candidates; + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] } as never); +}; + +const request: CanonicalRerankRequest = { + sourceProtocol: 'cohere-v2', raw: {}, query: 'q', documents: ['a', 'b'], +}; + +const serve = async () => await run( + rerankServePipeline(request), + move({ + 'ingress.rerank.sourceProtocol': 'cohere-v2', + 'ingress.http.headers': [] as readonly (readonly [string, string])[], + 'request.rerank.canonical': request, + 'serve.model': 'rr', + }) as never, + { + gateway: mockGatewayCtx({ wantsStream: false }), + background: () => {}, + resolveAttempt: (selector: { readonly upstreamId: string }) => { + const found = live.find(c => c.provider.upstreamId === selector.upstreamId); + if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); + return found; + }, + } as never, +); + +beforeEach(() => { + initRepo({ + usage: { record: async () => {} }, + performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, + } as never); +}); + +describe('a rerank answer carries its status', () => { + it('serves a success as 200', async () => { + resolves([candidate('up_a', async () => ({ + response: Response.json({ results: [{ index: 0, relevance_score: 0.9 }], meta: { billed_units: { search_units: 1 } } }), + modelKey: 'rr-key', + target: { protocol: 'cohere-v2' }, + } as ProviderRerankCallResult))]); + const { facts } = await serve(); + expect(facts['response.http.status']).toBe(200); + }); + + // The case that reached the client as a 200 before: a refusal the client has to be able + // to act on, and a retry-after only means something alongside the status that implies it. + it('serves an upstream refusal with the upstream-s own status', async () => { + resolves([candidate('up_a', async () => ({ + response: new Response('slow down', { status: 429 }), + modelKey: 'rr-key', + target: { protocol: 'cohere-v2' }, + } as ProviderRerankCallResult))]); + const { facts } = await serve(); + expect(facts['response.http.status']).toBe(429); + }); + + it('serves the resolver-s own refusal with the status it chose', async () => { + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates: [], sawModel: false, failedUpstreams: [] } as never); + const { facts } = await serve(); + expect(facts['response.http.status']).toBe(404); + }); +}); diff --git a/packages/gateway/src/data-plane/embeddings/pipeline.ts b/packages/gateway/src/data-plane/embeddings/pipeline.ts index 7118e09b62..b68e26ed02 100644 --- a/packages/gateway/src/data-plane/embeddings/pipeline.ts +++ b/packages/gateway/src/data-plane/embeddings/pipeline.ts @@ -58,7 +58,7 @@ const emitEmbeddings = defineStage< E<'ingress.embeddings.encodingFormat'>, E<'ingress.embeddings.encodingFormat'>, E<'ingress.embeddings.encodingFormat' | 'response.embeddings.canonical'>, - E<'response.embeddings.rendered'> + E<'response.embeddings.rendered' | 'response.http.status'> >({ name: 'emitEmbeddings', through: { @@ -70,17 +70,25 @@ const emitEmbeddings = defineStage< response: { needs: ['response.embeddings.canonical'], consumes: ['response.embeddings.canonical'], - provides: ['response.embeddings.rendered'], + provides: ['response.embeddings.rendered', 'response.http.status'], }, }, execute: async (facts, next) => { const back = await next(facts); const { 'response.embeddings.canonical': answer, ...rest } = back; if (isFailure(answer)) { - return { ...rest, 'response.embeddings.rendered': move({ error: { message: answer.message, type: 'api_error' } }) }; + return { + ...rest, + 'response.embeddings.rendered': move({ error: { message: answer.message, type: 'api_error' } }), + // The upstream's own status, or the gateway's own when it refused before dialling. + // A client is not owed the upstream's exact bytes; it is owed the truth about what + // happened, and a 429 arriving as a 200 is not that. + 'response.http.status': answer.status, + }; } return { ...rest, + 'response.http.status': 200, 'response.embeddings.rendered': move(renderEmbeddingsResponse(back['ingress.embeddings.encodingFormat'], answer)), }; }, @@ -158,7 +166,7 @@ const narrowing: Narrowing> = { export const embeddingsServePipeline: Pipeline< E<'ingress.http.headers' | 'ingress.embeddings.encodingFormat' | 'request.embeddings.canonical' | 'serve.model'>, - E<'response.embeddings.rendered' | 'response.usage.billable'> + E<'response.embeddings.rendered' | 'response.http.status' | 'response.usage.billable'> > = compose('embeddingsServe', [ emitEmbeddings, writeSettlement(handedUp => isFailure((handedUp as { 'response.embeddings.canonical'?: unknown })['response.embeddings.canonical'])), diff --git a/packages/gateway/src/data-plane/rerank/pipeline.ts b/packages/gateway/src/data-plane/rerank/pipeline.ts index 804a5f82e1..62fcec76e8 100644 --- a/packages/gateway/src/data-plane/rerank/pipeline.ts +++ b/packages/gateway/src/data-plane/rerank/pipeline.ts @@ -58,7 +58,7 @@ const emitRerank = defineStage< R<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical'>, R<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical'>, R<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical' | 'response.rerank.canonical' | 'response.rerank.targetProtocol'>, - R<'response.rerank.rendered'> + R<'response.rerank.rendered' | 'response.http.status'> >({ name: 'emitRerank', through: { @@ -70,17 +70,25 @@ const emitRerank = defineStage< response: { needs: ['response.rerank.canonical'], consumes: ['response.rerank.canonical', 'response.rerank.targetProtocol'], - provides: ['response.rerank.rendered'], + provides: ['response.rerank.rendered', 'response.http.status'], }, }, execute: async (facts, next) => { const back = await next(facts); const { 'response.rerank.canonical': answer, 'response.rerank.targetProtocol': target, ...rest } = back; if (isFailure(answer)) { - return { ...rest, 'response.rerank.rendered': move({ error: { message: answer.message, type: 'api_error' } }) }; + return { + ...rest, + 'response.rerank.rendered': move({ error: { message: answer.message, type: 'api_error' } }), + // The upstream's own status, or the gateway's own when it refused before dialling. + // A client is not owed the upstream's exact bytes; it is owed the truth about what + // happened, and a 429 arriving as a 200 is not that. + 'response.http.status': answer.status, + }; } return { ...rest, + 'response.http.status': 200, 'response.rerank.rendered': move(renderRerankResponse( back['ingress.rerank.sourceProtocol'], target, @@ -179,7 +187,7 @@ const narrowing = (request: CanonicalRerankRequest) => ({ export const rerankServePipeline = (request: CanonicalRerankRequest): Pipeline< R<'ingress.http.headers' | 'ingress.rerank.sourceProtocol' | 'request.rerank.canonical' | 'serve.model'>, - R<'response.rerank.rendered' | 'response.usage.billable'> + R<'response.rerank.rendered' | 'response.http.status' | 'response.usage.billable'> > => compose('rerankServe', [ emitRerank, writeSettlement(handedUp => isFailure((handedUp as { 'response.rerank.canonical'?: unknown })['response.rerank.canonical'])), From d6946d0b7878318f6dabfb53ba87829ea20fc21b Mon Sep 17 00:00:00 2001 From: Menci Date: Sun, 16 Aug 2026 23:55:28 +0800 Subject: [PATCH 11/79] fix(gateway): preserve each family's refusal sentence and complete an unnamed embeddings model The pipeline's shared resolver phrased its own refusal, which changed what a client is told: five families spell the endpoint they could not serve, while rerank spells out which part of the request no candidate could satisfy. The sentence moves to the family as `Narrowing.unsupported`, taking the reasons `reject` gave so a family can use them or ignore them. Copilot's /embeddings answers without the top-level `model` the schema marks required, so parsing refused a body the replaced surface served. The parser now takes the model the request named and completes the record with it. --- .../data-plane/completions-pipeline_test.ts | 1 + .../data-plane/rerank-status_test.ts | 1 + .../gateway/src/data-plane/audio/pipeline.ts | 1 + .../src/data-plane/completions/pipeline.ts | 1 + .../gateway/src/data-plane/embeddings/http.ts | 70 ++++++----- .../src/data-plane/embeddings/pipeline.ts | 5 +- .../gateway/src/data-plane/images/pipeline.ts | 1 + .../gateway/src/data-plane/pipeline/serve.ts | 117 ++++++++++++++++++ .../src/data-plane/pipeline/services.ts | 3 + .../gateway/src/data-plane/pipeline/stages.ts | 20 +-- .../gateway/src/data-plane/rerank/pipeline.ts | 3 + .../__tests__/embeddings/translate_test.ts | 36 ++++-- .../protocols/src/embeddings/translate.ts | 12 +- 13 files changed, 218 insertions(+), 53 deletions(-) create mode 100644 packages/gateway/src/data-plane/pipeline/serve.ts diff --git a/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts index 4379d0e959..b99f8a8a22 100644 --- a/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts @@ -87,6 +87,7 @@ const serve = async (facts: Record) => await run( { gateway: mockGatewayCtx({ wantsStream: facts['ingress.completions.wantsStream'] === true }), background: () => {}, + rememberCandidates: () => {}, resolveAttempt, } as never, ); diff --git a/packages/gateway/__tests__/data-plane/rerank-status_test.ts b/packages/gateway/__tests__/data-plane/rerank-status_test.ts index b0c69fb6b4..e7507b3c15 100644 --- a/packages/gateway/__tests__/data-plane/rerank-status_test.ts +++ b/packages/gateway/__tests__/data-plane/rerank-status_test.ts @@ -56,6 +56,7 @@ const serve = async () => await run( { gateway: mockGatewayCtx({ wantsStream: false }), background: () => {}, + rememberCandidates: () => {}, resolveAttempt: (selector: { readonly upstreamId: string }) => { const found = live.find(c => c.provider.upstreamId === selector.upstreamId); if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); diff --git a/packages/gateway/src/data-plane/audio/pipeline.ts b/packages/gateway/src/data-plane/audio/pipeline.ts index bbfeacfaa8..aa7e83f5bf 100644 --- a/packages/gateway/src/data-plane/audio/pipeline.ts +++ b/packages/gateway/src/data-plane/audio/pipeline.ts @@ -348,6 +348,7 @@ const narrowing = { candidate.model.endpoints.audioTranscriptions === undefined ? 'the upstream does not expose an audio transcription endpoint' : null, + unsupported: (model: string) => `Model ${model} does not support the /audio/transcriptions endpoint.`, refuse: (status: number, message: string) => ({ 'response.audioTranscription.canonical': { status, message } as Failure, 'response.audioTranscription.mediaType': null, diff --git a/packages/gateway/src/data-plane/completions/pipeline.ts b/packages/gateway/src/data-plane/completions/pipeline.ts index 15460e10dc..da7172542c 100644 --- a/packages/gateway/src/data-plane/completions/pipeline.ts +++ b/packages/gateway/src/data-plane/completions/pipeline.ts @@ -356,6 +356,7 @@ const narrowing = { kind: 'chat' as const, reject: (candidate: ModelCandidate): string | null => candidate.model.endpoints.completions === undefined ? 'the upstream does not expose a completions endpoint' : null, + unsupported: (model: string) => `Model ${model} does not support the /completions endpoint.`, refuse: (status: number, message: string): C<'response.completions.payload' | 'response.completions.streamedUsage'> => ({ 'response.completions.payload': { status, message }, 'response.completions.streamedUsage': null, diff --git a/packages/gateway/src/data-plane/embeddings/http.ts b/packages/gateway/src/data-plane/embeddings/http.ts index 253209808c..1f91b648e8 100644 --- a/packages/gateway/src/data-plane/embeddings/http.ts +++ b/packages/gateway/src/data-plane/embeddings/http.ts @@ -1,52 +1,56 @@ -// POST /v1/embeddings — route embedding requests to the provider that -// declares the requested model and embeddings capability. +// POST /v1/embeddings, served through the pipeline. // -// Transitional: the request is already parsed against the embeddings contract and written -// again for the upstream, which is what the pipeline does, while the response is still -// forwarded by the passthrough scaffold. The whole file goes when the route is wired to -// `embeddingsServePipeline`. +// The handler is a prologue and an epilogue around `embeddingsServePipeline`: parse what +// the client sent, hand it over, and turn what the run answered with into a response. +// Everything between is stages. +import { move } from '@floway-dev/pipeline'; +import { parseEmbeddingsRequest, type ParsedEmbeddingsRequest } from '@floway-dev/protocols/embeddings'; import type { Context } from 'hono'; -import { tokenUsageFromEmbeddingsBody } from './usage.ts'; -import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; -import { createGatewayCtxFromHono, finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; -import { passthroughApiError, passthroughServe } from '../shared/passthrough-serve.ts'; -import { readRequestBody, takeRequestBody } from '../shared/request-body.ts'; -import { parseEmbeddingsRequest, serializeEmbeddingsRequest, type ParsedEmbeddingsRequest } from '@floway-dev/protocols/embeddings'; +import { embeddingsServePipeline } from './pipeline.ts'; +import { openPrologue, serveThrough } from '../pipeline/serve.ts'; +import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; // The contract reports a malformed request by throwing; what the client is owed is a 400 -// carrying the reason. Reading has to finish before the context takes the bytes. +// carrying the reason. const readRequest = (bytes: Uint8Array): { type: 'ok'; parsed: ParsedEmbeddingsRequest } | { type: 'invalid'; message: string } => { try { return { type: 'ok', parsed: parseEmbeddingsRequest(JSON.parse(new TextDecoder().decode(bytes)) as unknown) }; - } catch (e) { - return { type: 'invalid', message: e instanceof Error ? e.message : String(e) }; + } catch (error) { + return { type: 'invalid', message: error instanceof Error ? error.message : String(error) }; } }; export const embeddings = async (c: Context): Promise => { - const requestBody = await readRequestBody(c); - const result = readRequest(requestBody.bytes); - const ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); + const prologue = await openPrologue(c, { wantsStream: false }); + const result = readRequest(prologue.bytes); if (result.type === 'invalid') { - ctx.dump?.error('gateway'); - return finalizeGatewayResponse(ctx, passthroughApiError(c, result.message, 400)); + // A request the gateway could not read never reaches a pipeline: there is no model to + // resolve and no attempt to make, so there is nothing for a run to record. + prologue.gateway.dump?.error('gateway'); + return finalizeGatewayResponse( + prologue.gateway, + Response.json({ error: { message: result.message, type: 'api_error' } }, { status: 400 }), + ); } + const { model, request } = result.parsed; + prologue.gateway.dump?.requestedModel(model); - ctx.dump?.requestedModel(model); - const response = await passthroughServe({ + return await serveThrough( c, - ctx, - sourceApi: '/embeddings', - operation: 'embeddings', - model, - kind: 'embedding', - modelServesEndpoint: candidate => candidate.endpoints.embeddings !== undefined, - call: async (provider, providerModel, opts) => - await provider.instance.callEmbeddings(providerModel, serializeEmbeddingsRequest(request), undefined, opts), - response: { format: 'json', extractBilling: tokenUsageFromEmbeddingsBody }, - }); - return finalizeGatewayResponse(ctx, response); + prologue, + embeddingsServePipeline, + move({ + 'ingress.http.headers': prologue.headers, + 'ingress.embeddings.encodingFormat': request.encodingFormat, + 'request.embeddings.canonical': request, + 'serve.model': model, + }) as never, + facts => ({ + status: (facts as { readonly 'response.http.status': number })['response.http.status'], + body: JSON.stringify((facts as { readonly 'response.embeddings.rendered': unknown })['response.embeddings.rendered']), + }), + ); }; diff --git a/packages/gateway/src/data-plane/embeddings/pipeline.ts b/packages/gateway/src/data-plane/embeddings/pipeline.ts index b68e26ed02..6353e51358 100644 --- a/packages/gateway/src/data-plane/embeddings/pipeline.ts +++ b/packages/gateway/src/data-plane/embeddings/pipeline.ts @@ -100,7 +100,7 @@ const emitEmbeddings = defineStage< * fails over, and even a 400 can be, because the next candidate's path and flags may differ. */ const callEmbeddingsUpstream = defineStage< - E<'request.embeddings.canonical' | 'route.attempt' | 'ingress.http.headers'>, + E<'request.embeddings.canonical' | 'route.attempt' | 'ingress.http.headers' | 'serve.model'>, E<'response.embeddings.canonical' | 'response.usage.billable'>, GatewayServices >({ @@ -138,7 +138,7 @@ const callEmbeddingsUpstream = defineStage< // Every protocol the gateway carries is one it fully understands: the body is parsed // here and written again at the edge, in whichever encoding the client can read. - const canonical = parseEmbeddingsResponse(await result.response.json() as unknown); + const canonical = parseEmbeddingsResponse(await result.response.json() as unknown, facts['serve.model']); return move({ ...facts, 'response.embeddings.canonical': canonical, @@ -160,6 +160,7 @@ const narrowing: Narrowing> = { reject: candidate => candidate.model.endpoints.embeddings === undefined ? 'the upstream does not expose an embeddings endpoint' : null, + unsupported: model => `Model ${model} does not support the /embeddings endpoint.`, refuse: (status, message) => ({ 'response.embeddings.canonical': { status, message } }), refuses: ['response.embeddings.canonical'], }; diff --git a/packages/gateway/src/data-plane/images/pipeline.ts b/packages/gateway/src/data-plane/images/pipeline.ts index 95b9d8bc5e..940145c81d 100644 --- a/packages/gateway/src/data-plane/images/pipeline.ts +++ b/packages/gateway/src/data-plane/images/pipeline.ts @@ -243,6 +243,7 @@ const narrowing = (request: CanonicalImagesRequest) => ({ reject: (candidate: ModelCandidate) => candidate.model.endpoints[ENDPOINT[request.operation]] === undefined ? `the upstream does not expose the images ${request.operation} endpoint` : null, + unsupported: (model: string) => `Model ${model} does not support the /images/${request.operation} endpoint.`, refuse: (status: number, message: string) => ({ 'response.images.canonical': { status, message }, 'response.http.headers': [], diff --git a/packages/gateway/src/data-plane/pipeline/serve.ts b/packages/gateway/src/data-plane/pipeline/serve.ts new file mode 100644 index 0000000000..0850cf3071 --- /dev/null +++ b/packages/gateway/src/data-plane/pipeline/serve.ts @@ -0,0 +1,117 @@ +// The seam. Every family's route handler does the same things around its pipeline — read +// the body, build what the run is given, run it, turn the exit facts into a response — so +// it is written once here and each family adds only its own keys. +// +// This is also where "entering the pipeline system requires no capability" is concrete: a +// handler calls `run` like any other caller. What makes a run *recorded* is the prologue +// resolving a dump sink, not a flag anywhere below it. + +import { run } from '@floway-dev/pipeline'; +import type { Event, Pipeline } from '@floway-dev/pipeline'; +import type { Context } from 'hono'; +import type { ContentfulStatusCode } from 'hono/utils/http-status'; +import type { ModelCandidate } from '@floway-dev/provider'; + +import type { AttemptSelector } from './facts.ts'; +import type { GatewayServices } from './services.ts'; +import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; +import { createGatewayCtxFromHono, finalizeGatewayResponse, type GatewayCtx } from '../shared/gateway-ctx.ts'; +import { readRequestBody, takeRequestBody } from '../shared/request-body.ts'; + +export interface Prologue { + readonly services: GatewayServices; + readonly gateway: GatewayCtx; + /** What the client sent, read once. `takeRequestBody` empties the buffer it was given so + * the handler cannot retain the wire body past the dump's own copy, so what is kept here + * is the bytes themselves rather than the record they came from. */ + readonly bytes: Uint8Array; + readonly streamError: string | null; + readonly headers: readonly (readonly [string, string])[]; +} + +/** + * Opens a run: the request context the telemetry stages read, the services the stages are + * given, and the bytes the client sent. + * + * The candidate store is the resolver the ruling names — "the resolver is the service and + * the selector is a fact". A `ModelCandidate` carries the provider's instance, its fetcher + * and its models cache; those never enter the record, because `move()` would freeze them + * and the provider's own cache refresh would break. So the stage that enumerates hands the + * live ones here, and the stage that dials asks for one back by selector. + */ +export const openPrologue = async ( + c: Context, + options: { readonly wantsStream: boolean; readonly model?: string }, +): Promise => { + const body = await readRequestBody(c); + const bytes = body.bytes; + const streamError = body.streamError; + const gateway = createGatewayCtxFromHono(c, { + wantsStream: options.wantsStream, + ...(options.model === undefined ? {} : { model: options.model }), + requestBody: takeRequestBody(body), + backgroundScheduler: backgroundSchedulerFromContext(c), + }); + + const live = new Map(); + + return { + gateway, + bytes, + streamError, + headers: [...c.req.raw.headers], + services: { + gateway, + background: work => { gateway.backgroundScheduler(work); }, + rememberCandidates: candidates => { + for (const candidate of candidates) live.set(candidate.provider.upstreamId, candidate); + }, + resolveAttempt: (selector: AttemptSelector) => { + const candidate = live.get(selector.upstreamId); + if (candidate === undefined) { + throw new Error(`resolveAttempt: nothing live for ${selector.upstreamId}; the selector did not come from this run`); + } + return candidate; + }, + // Absent when this key has no retention configured, which is what keeps recording + // conditional: the runner does none of it rather than doing it and discarding. + ...(gateway.dump === null ? {} : { dump: (_event: Event) => {} }), + }, + }; +}; + +/** What a family's pipeline answers with, in the shape a route can return. Every family + * provides the status; the body and its media type are the family's own rendering. */ +export interface Rendered { + readonly status: number; + readonly body: BodyInit; + readonly headers?: Readonly>; +} + +/** + * Runs a family's pipeline and turns what it answered with into a response. + * + * The drain is scheduled rather than awaited. A streaming family's answer *is* the stream, + * so draining before returning would consume the frames the client is waiting for. Release + * is not cancel — an aborted connection cannot be reused and leaves the upstream's own + * billing unsettled — so it still happens, just after the answer is on its way. + */ +export const serveThrough = async ( + c: Context, + prologue: Prologue, + pipeline: Pipeline, + entry: Entry, + render: (facts: Exit) => Rendered, +): Promise => { + void c; + const { facts, drain } = await run(pipeline, entry, prologue.services as never); + prologue.services.background(drain()); + const answer = render(facts); + return finalizeGatewayResponse( + prologue.gateway, + new Response(answer.body, { + status: answer.status as ContentfulStatusCode, + headers: answer.headers ?? { 'content-type': 'application/json' }, + }), + ); +}; diff --git a/packages/gateway/src/data-plane/pipeline/services.ts b/packages/gateway/src/data-plane/pipeline/services.ts index 30d5001f38..000ce02049 100644 --- a/packages/gateway/src/data-plane/pipeline/services.ts +++ b/packages/gateway/src/data-plane/pipeline/services.ts @@ -26,6 +26,9 @@ export interface GatewayServices { * prologue either, so what is injected is the thing that resolves one and what travels * is the identifier it resolves from. */ readonly resolveAttempt: (selector: AttemptSelector) => ModelCandidate; + /** The other half of the resolver: the stage that enumerates hands the live candidates + * here, so the ones that travel can be selectors. Per run, because a candidate list is. */ + readonly rememberCandidates: (candidates: readonly ModelCandidate[]) => void; /** Binds a promise to the request's lifetime: `waitUntil` on Workers, the event loop on * Node. What the drain is handed to, so an answer is not held up by it. */ readonly background: (work: Promise) => void; diff --git a/packages/gateway/src/data-plane/pipeline/stages.ts b/packages/gateway/src/data-plane/pipeline/stages.ts index 93f2521e1d..7b4c0bc8d8 100644 --- a/packages/gateway/src/data-plane/pipeline/stages.ts +++ b/packages/gateway/src/data-plane/pipeline/stages.ts @@ -39,6 +39,12 @@ export interface Narrowing { readonly kind: ModelKind; /** Keeps a candidate, or says in one phrase why it cannot serve this request. */ readonly reject: (candidate: ModelCandidate) => string | null; + /** What the client is told when the model resolved but nothing can serve the request. + * `reasons` holds what `reject` said, and is empty when no candidate of this kind was + * found at all — which is the difference between "this model is not an embeddings model" + * and "this reranker cannot do what this request asks". Families differ in how much of + * that they spell out, so the sentence is the family's rather than this stage's. */ + readonly unsupported: (model: string, reasons: readonly string[]) => string; /** What this family answers with when there is no candidate to try. The family decides * which of its own keys carries the failure, which is why the refusal slice is a type * parameter rather than a shared key: a reusable stage whose short-circuit provides @@ -85,7 +91,7 @@ export const resolveCandidates = (narrowing: Narrowing(narrowing: Narrowing ({ } return rerankRequestIncompatibility(model.rerankTarget.protocol, request); }, + unsupported: (model: string, reasons: readonly string[]) => reasons.length === 0 + ? `Model ${model} does not support rerank.` + : `Model ${model} does not support this rerank request: ${reasons.join('; ')}.`, refuse: (status: number, message: string) => ({ 'response.rerank.canonical': { status, message } }), refuses: ['response.rerank.canonical'] as const, }); diff --git a/packages/protocols/__tests__/embeddings/translate_test.ts b/packages/protocols/__tests__/embeddings/translate_test.ts index 1026689d8c..b337961926 100644 --- a/packages/protocols/__tests__/embeddings/translate_test.ts +++ b/packages/protocols/__tests__/embeddings/translate_test.ts @@ -83,7 +83,7 @@ describe('embeddings response egress', () => { model: 'text-embedding-3-small', data: [{ object: 'embedding', index: 0, embedding: PACKED }], usage: { prompt_tokens: 8, total_tokens: 8 }, - }); + }, 'm'); expect(parsed.embeddings).toEqual([{ index: 0, values: VECTOR }]); expect(parsed.usage).toEqual({ promptTokens: 8, totalTokens: 8 }); @@ -93,13 +93,31 @@ describe('embeddings response egress', () => { expect(() => parseEmbeddingsResponse({ model: 'm', data: [{ index: 0, embedding: PACKED.slice(0, 8) }], - })).toThrow('data[0].embedding is not a whole number of float32 values'); + }, 'm')).toThrow('data[0].embedding is not a whole number of float32 values'); + }); + + // The schema marks `model` required and most upstreams send it; Copilot's `/embeddings` + // does not. An answer the gateway understands is one it can write again, so the record is + // completed with the model the request named rather than refused. + test('an upstream that names no model is read as having answered for the requested one', () => { + const parsed = parseEmbeddingsResponse( + { object: 'list', data: [{ object: 'embedding', index: 0, embedding: [0.5] }] }, + 'text-embedding-real', + ); + + expect(parsed.model).toBe('text-embedding-real'); + expect(renderEmbeddingsResponse('float', parsed)).toMatchObject({ model: 'text-embedding-real' }); + }); + + // A model the upstream *did* name is its own answer, and the request's id never overrides it. + test('a model the upstream named is kept', () => { + expect(parseEmbeddingsResponse({ model: 'upstream-id', data: [] }, 'requested-id').model).toBe('upstream-id'); }); // An upstream that reports no usage is a fact the gateway has to be able to carry: the // billed entity is present with no quantities, which is not the same as reporting zero. test('a missing usage block leaves the answer intact and reports nothing', () => { - const parsed = parseEmbeddingsResponse({ model: 'm', data: [{ index: 0, embedding: [0.5] }] }); + const parsed = parseEmbeddingsResponse({ model: 'm', data: [{ index: 0, embedding: [0.5] }] }, 'm'); expect(parsed.usage).toBeUndefined(); expect(renderEmbeddingsResponse('float', parsed)).not.toHaveProperty('usage'); @@ -109,7 +127,7 @@ describe('embeddings response egress', () => { // saying so and will decode whatever it is handed; an upstream that ignores the field // and answers with float arrays would otherwise hand it noise. test('the answer is written in the encoding the client asked for, not the one it arrived in', () => { - const arrived = parseEmbeddingsResponse({ model: 'm', data: [{ index: 0, embedding: VECTOR }] }); + const arrived = parseEmbeddingsResponse({ model: 'm', data: [{ index: 0, embedding: VECTOR }] }, 'm'); expect(renderEmbeddingsResponse('base64', arrived)).toMatchObject({ object: 'list', @@ -118,16 +136,16 @@ describe('embeddings response egress', () => { expect(renderEmbeddingsResponse('float', parseEmbeddingsResponse({ model: 'm', data: [{ index: 0, embedding: PACKED }], - }))).toMatchObject({ data: [{ embedding: VECTOR }] }); + }, 'm'))).toMatchObject({ data: [{ embedding: VECTOR }] }); }); // Every float32 is a float64, so widening loses nothing and narrowing a value that came // from a float32 gives the same bits back. Holding vectors as numbers is therefore exact // and not an approximation, however many times a vector crosses the gateway. test('a vector survives any number of trips through the two encodings', () => { - const once = parseEmbeddingsResponse({ model: 'm', data: [{ index: 0, embedding: PACKED }] }); - const twice = parseEmbeddingsResponse(renderEmbeddingsResponse('base64', once)); - const thrice = parseEmbeddingsResponse(renderEmbeddingsResponse('base64', twice)); + const once = parseEmbeddingsResponse({ model: 'm', data: [{ index: 0, embedding: PACKED }] }, 'm'); + const twice = parseEmbeddingsResponse(renderEmbeddingsResponse('base64', once), 'm'); + const thrice = parseEmbeddingsResponse(renderEmbeddingsResponse('base64', twice), 'm'); expect(renderEmbeddingsResponse('base64', thrice)).toMatchObject({ data: [{ embedding: PACKED }] }); expect(thrice.embeddings).toEqual(once.embeddings); @@ -139,7 +157,7 @@ describe('embeddings response egress', () => { model: 'm', data: [{ object: 'not-an-embedding', index: 0, embedding: [0.5] }], usage: { prompt_tokens: 1, total_tokens: 1 }, - })); + }, 'm')); expect(rendered).toEqual({ object: 'list', diff --git a/packages/protocols/src/embeddings/translate.ts b/packages/protocols/src/embeddings/translate.ts index 651113b514..28aad0fae9 100644 --- a/packages/protocols/src/embeddings/translate.ts +++ b/packages/protocols/src/embeddings/translate.ts @@ -142,7 +142,15 @@ const parseUsage = (value: unknown): CanonicalEmbeddingsUsage | undefined => { }; }; -export const parseEmbeddingsResponse = (value: unknown): CanonicalEmbeddingsResponse => { +/** + * Reads an upstream's answer into the canonical form. + * + * `requestedModel` completes the record when the upstream did not name one. The schema + * marks `model` required, and most upstreams send it; the one this gateway carries that + * does not is GitHub Copilot's `/embeddings`. Naming the model the request asked for is + * both what the client can correlate against and what usage is billed under. + */ +export const parseEmbeddingsResponse = (value: unknown, requestedModel: string): CanonicalEmbeddingsResponse => { if (!isRecord(value)) throw new Error('Embeddings response body must be an object'); const { data } = value; if (!Array.isArray(data)) throw new Error('data must be an array'); @@ -155,7 +163,7 @@ export const parseEmbeddingsResponse = (value: unknown): CanonicalEmbeddingsResp }; }); return { - model: requiredString(value.model, 'model'), + model: value.model === undefined ? requestedModel : requiredString(value.model, 'model'), embeddings, ...(usage === undefined ? {} : { usage }), }; From d5130e8761060d49d8cbe72d2a3df4ecb5953ddd Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 00:02:07 +0800 Subject: [PATCH 12/79] feat(gateway): forward upstream response headers through every family's edge The pipeline dropped the upstream's response headers, so vendor traces, quota state and retry-after stopped reaching clients on the routes it took over. The attempt hands them up unfiltered and the edge decides what a client may see, which keeps both in a dump. A refusal that never dialled has none to carry, so the shared resolver answers with an empty list and no family repeats it. serveThrough now reads the status and headers off the exit facts. A family's render owns the bytes and their media type and nothing else, which is also why content type stays blocked from forwarding. The assembly test builds every family, including the four whose pipeline is built from the request and would otherwise never be composed by a test. --- .../data-plane/pipeline/assembly_test.ts | 32 +++++++++++++++++++ .../gateway/src/data-plane/embeddings/http.ts | 6 +--- .../src/data-plane/embeddings/pipeline.ts | 30 +++++++++++------ .../gateway/src/data-plane/images/pipeline.ts | 10 ++---- .../gateway/src/data-plane/pipeline/serve.ts | 28 +++++++++------- .../gateway/src/data-plane/pipeline/stages.ts | 24 ++++++++------ .../gateway/src/data-plane/rerank/pipeline.ts | 30 +++++++++++------ .../protocols/src/embeddings/translate.ts | 15 ++++++--- 8 files changed, 120 insertions(+), 55 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts diff --git a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts new file mode 100644 index 0000000000..0b6233e1c6 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts @@ -0,0 +1,32 @@ +// Every family assembles. Assembly is a check over declarations, and a family whose +// pipeline is built from the request assembles only when something builds one — so a +// pipeline that could never run can sit in the tree looking fine. This test builds all of +// them, which is what turns "the declarations disagree" into a failure at `pnpm test` +// rather than on the first request that reaches the route. + +import { describe, expect, it } from 'vitest'; + +import { audioTranscriptionServePipeline } from '../../../src/data-plane/audio/pipeline.ts'; +import { completionsServePipeline } from '../../../src/data-plane/completions/pipeline.ts'; +import { embeddingsServePipeline } from '../../../src/data-plane/embeddings/pipeline.ts'; +import { imagesServePipeline } from '../../../src/data-plane/images/pipeline.ts'; +import { rerankServePipeline } from '../../../src/data-plane/rerank/pipeline.ts'; +import { searchServePipeline } from '../../../src/data-plane/alpha-search/pipeline.ts'; + +const FAMILIES: readonly (readonly [string, () => { readonly name: string }])[] = [ + ['embeddings', () => embeddingsServePipeline], + ['rerank', () => rerankServePipeline({ sourceProtocol: 'cohere-v2', raw: {}, query: 'q', documents: ['a'] } as never)], + ['images generations', () => imagesServePipeline({ operation: 'generations', parameters: {} } as never)], + ['images edits', () => imagesServePipeline({ operation: 'edits', images: [], parameters: {} } as never)], + ['completions', () => completionsServePipeline], + ['audio transcription', () => audioTranscriptionServePipeline], + ['alpha search', () => searchServePipeline({ kind: 'search' } as never)], +]; + +describe('every family assembles', () => { + for (const [name, build] of FAMILIES) { + it(`${name} composes into a pipeline`, () => { + expect(build().name).toBeTypeOf('string'); + }); + } +}); diff --git a/packages/gateway/src/data-plane/embeddings/http.ts b/packages/gateway/src/data-plane/embeddings/http.ts index 1f91b648e8..c70ad38c94 100644 --- a/packages/gateway/src/data-plane/embeddings/http.ts +++ b/packages/gateway/src/data-plane/embeddings/http.ts @@ -39,7 +39,6 @@ export const embeddings = async (c: Context): Promise => { prologue.gateway.dump?.requestedModel(model); return await serveThrough( - c, prologue, embeddingsServePipeline, move({ @@ -48,9 +47,6 @@ export const embeddings = async (c: Context): Promise => { 'request.embeddings.canonical': request, 'serve.model': model, }) as never, - facts => ({ - status: (facts as { readonly 'response.http.status': number })['response.http.status'], - body: JSON.stringify((facts as { readonly 'response.embeddings.rendered': unknown })['response.embeddings.rendered']), - }), + facts => ({ body: JSON.stringify(facts['response.embeddings.rendered']), contentType: 'application/json' }), ); }; diff --git a/packages/gateway/src/data-plane/embeddings/pipeline.ts b/packages/gateway/src/data-plane/embeddings/pipeline.ts index 6353e51358..481234840f 100644 --- a/packages/gateway/src/data-plane/embeddings/pipeline.ts +++ b/packages/gateway/src/data-plane/embeddings/pipeline.ts @@ -18,6 +18,7 @@ import { writeSettlement } from '../pipeline/settlement.ts'; import { failover, resolveCandidates, type Narrowing } from '../pipeline/stages.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; +import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import type { Pipeline } from '@floway-dev/pipeline'; import { compose, defineStage, move } from '@floway-dev/pipeline'; import { parseDecimalString } from '@floway-dev/protocols/common'; @@ -57,8 +58,8 @@ type E = { [P in K]: EmbeddingsFacts[P] }; const emitEmbeddings = defineStage< E<'ingress.embeddings.encodingFormat'>, E<'ingress.embeddings.encodingFormat'>, - E<'ingress.embeddings.encodingFormat' | 'response.embeddings.canonical'>, - E<'response.embeddings.rendered' | 'response.http.status'> + E<'ingress.embeddings.encodingFormat' | 'response.embeddings.canonical' | 'response.http.headers'>, + E<'response.embeddings.rendered' | 'response.http.status' | 'response.http.headers'> >({ name: 'emitEmbeddings', through: { @@ -68,17 +69,23 @@ const emitEmbeddings = defineStage< provides: [], }, response: { - needs: ['response.embeddings.canonical'], - consumes: ['response.embeddings.canonical'], - provides: ['response.embeddings.rendered', 'response.http.status'], + needs: ['response.embeddings.canonical', 'response.http.headers'], + consumes: ['response.embeddings.canonical', 'response.http.headers'], + provides: ['response.embeddings.rendered', 'response.http.status', 'response.http.headers'], }, }, execute: async (facts, next) => { const back = await next(facts); - const { 'response.embeddings.canonical': answer, ...rest } = back; + const { 'response.embeddings.canonical': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); if (isFailure(answer)) { return { ...rest, + 'response.http.headers': forClient, 'response.embeddings.rendered': move({ error: { message: answer.message, type: 'api_error' } }), // The upstream's own status, or the gateway's own when it refused before dialling. // A client is not owed the upstream's exact bytes; it is owed the truth about what @@ -88,6 +95,7 @@ const emitEmbeddings = defineStage< } return { ...rest, + 'response.http.headers': forClient, 'response.http.status': 200, 'response.embeddings.rendered': move(renderEmbeddingsResponse(back['ingress.embeddings.encodingFormat'], answer)), }; @@ -101,12 +109,12 @@ const emitEmbeddings = defineStage< */ const callEmbeddingsUpstream = defineStage< E<'request.embeddings.canonical' | 'route.attempt' | 'ingress.http.headers' | 'serve.model'>, - E<'response.embeddings.canonical' | 'response.usage.billable'>, + E<'response.embeddings.canonical' | 'response.http.headers' | 'response.usage.billable'>, GatewayServices >({ name: 'callEmbeddingsUpstream', return: { - provides: ['response.embeddings.canonical', 'response.usage.billable'], + provides: ['response.embeddings.canonical', 'response.http.headers', 'response.usage.billable'], }, execute: async (facts, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); @@ -121,6 +129,8 @@ const callEmbeddingsUpstream = defineStage< ); use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'embeddings'); const identity = telemetryModelIdentity(candidate, result.modelKey); + // What came back, unfiltered: the edge is where a client's view of it is decided. + const headers = [...result.response.headers]; if (!result.response.ok) { use.log.warn('upstream refused', { status: result.response.status }); @@ -130,6 +140,7 @@ const callEmbeddingsUpstream = defineStage< status: result.response.status, message: await result.response.text(), }, + 'response.http.headers': headers, // The upstream was called and reported nothing, which is a different situation // from reporting zero — so the entity is present with no quantities. 'response.usage.billable': [{ identity, quantities: {} }], @@ -142,6 +153,7 @@ const callEmbeddingsUpstream = defineStage< return move({ ...facts, 'response.embeddings.canonical': canonical, + 'response.http.headers': headers, 'response.usage.billable': [{ identity, quantities: billed(canonical.usage) }], }); }, @@ -167,7 +179,7 @@ const narrowing: Narrowing> = { export const embeddingsServePipeline: Pipeline< E<'ingress.http.headers' | 'ingress.embeddings.encodingFormat' | 'request.embeddings.canonical' | 'serve.model'>, - E<'response.embeddings.rendered' | 'response.http.status' | 'response.usage.billable'> + E<'response.embeddings.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> > = compose('embeddingsServe', [ emitEmbeddings, writeSettlement(handedUp => isFailure((handedUp as { 'response.embeddings.canonical'?: unknown })['response.embeddings.canonical'])), diff --git a/packages/gateway/src/data-plane/images/pipeline.ts b/packages/gateway/src/data-plane/images/pipeline.ts index 940145c81d..19bd187ed3 100644 --- a/packages/gateway/src/data-plane/images/pipeline.ts +++ b/packages/gateway/src/data-plane/images/pipeline.ts @@ -236,19 +236,15 @@ const providerEditsSource = (image: ImagesEditImage): ImagesEditsSource => { /** A candidate that cannot serve *this* request is not a candidate. One family covers two * endpoints and an upstream may expose either without the other, so which one is asked for is - * what narrows the list. A refusal answers with no upstream headers because there was no - * upstream, which is the same statement the edge above reads on every other path. */ + * what narrows the list. */ const narrowing = (request: CanonicalImagesRequest) => ({ kind: 'image' as const, reject: (candidate: ModelCandidate) => candidate.model.endpoints[ENDPOINT[request.operation]] === undefined ? `the upstream does not expose the images ${request.operation} endpoint` : null, unsupported: (model: string) => `Model ${model} does not support the /images/${request.operation} endpoint.`, - refuse: (status: number, message: string) => ({ - 'response.images.canonical': { status, message }, - 'response.http.headers': [], - }), - refuses: ['response.images.canonical', 'response.http.headers'] as const, + refuse: (status: number, message: string) => ({ 'response.images.canonical': { status, message } }), + refuses: ['response.images.canonical'] as const, }); /** What a caller must bring. `ingress.http.headers` and `request.images.canonical` are in it diff --git a/packages/gateway/src/data-plane/pipeline/serve.ts b/packages/gateway/src/data-plane/pipeline/serve.ts index 0850cf3071..2acc00cd3a 100644 --- a/packages/gateway/src/data-plane/pipeline/serve.ts +++ b/packages/gateway/src/data-plane/pipeline/serve.ts @@ -12,12 +12,14 @@ import type { Context } from 'hono'; import type { ContentfulStatusCode } from 'hono/utils/http-status'; import type { ModelCandidate } from '@floway-dev/provider'; -import type { AttemptSelector } from './facts.ts'; +import type { AttemptSelector, GatewayFacts } from './facts.ts'; import type { GatewayServices } from './services.ts'; import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; import { createGatewayCtxFromHono, finalizeGatewayResponse, type GatewayCtx } from '../shared/gateway-ctx.ts'; import { readRequestBody, takeRequestBody } from '../shared/request-body.ts'; +type Slice = { [P in K]: GatewayFacts[P] }; + export interface Prologue { readonly services: GatewayServices; readonly gateway: GatewayCtx; @@ -80,12 +82,13 @@ export const openPrologue = async ( }; }; -/** What a family's pipeline answers with, in the shape a route can return. Every family - * provides the status; the body and its media type are the family's own rendering. */ +/** What a family writes for the client. The status and the upstream's own headers are facts + * the pipeline already carries, so what is left here is the bytes and what they are. */ export interface Rendered { - readonly status: number; readonly body: BodyInit; - readonly headers?: Readonly>; + /** Owned by whichever stage serialized the body, never by the upstream — a media type + * describing bytes the gateway wrote itself is the one thing an upstream cannot say. */ + readonly contentType: string; } /** @@ -96,22 +99,23 @@ export interface Rendered { * is not cancel — an aborted connection cannot be reused and leaves the upstream's own * billing unsettled — so it still happens, just after the answer is on its way. */ -export const serveThrough = async ( - c: Context, +export const serveThrough = async < + Entry extends object, + Exit extends Slice<'response.http.status' | 'response.http.headers'>, +>( prologue: Prologue, pipeline: Pipeline, entry: Entry, render: (facts: Exit) => Rendered, ): Promise => { - void c; const { facts, drain } = await run(pipeline, entry, prologue.services as never); prologue.services.background(drain()); const answer = render(facts); + + const headers = new Headers(facts['response.http.headers'].map(([name, value]) => [name, value])); + headers.set('content-type', answer.contentType); return finalizeGatewayResponse( prologue.gateway, - new Response(answer.body, { - status: answer.status as ContentfulStatusCode, - headers: answer.headers ?? { 'content-type': 'application/json' }, - }), + new Response(answer.body, { status: facts['response.http.status'] as ContentfulStatusCode, headers }), ); }; diff --git a/packages/gateway/src/data-plane/pipeline/stages.ts b/packages/gateway/src/data-plane/pipeline/stages.ts index 7b4c0bc8d8..c370adedcc 100644 --- a/packages/gateway/src/data-plane/pipeline/stages.ts +++ b/packages/gateway/src/data-plane/pipeline/stages.ts @@ -62,18 +62,18 @@ export interface Narrowing { */ export const resolveCandidates = (narrowing: Narrowing) => defineStage< Slice<'serve.model'>, // what arrives - Slice<'serve.model' | 'serve.candidates'>, // what it hands down - Slice<'response.usage.billable'>, // what comes back - Slice<'response.usage.billable'>, // what it hands up, having descended - Slice<'response.usage.billable'> & Refusal, // and what it answers with instead + Slice<'serve.model' | 'serve.candidates'>, // what it hands down + Slice<'response.usage.billable' | 'response.http.headers'>, // what comes back + Slice<'response.usage.billable' | 'response.http.headers'>, // what it hands up, having descended + Slice<'response.usage.billable' | 'response.http.headers'> & Refusal, // and what it answers with instead GatewayServices >({ name: 'resolveCandidates', through: { request: { needs: ['serve.model'], consumes: [], provides: ['serve.candidates'] }, - response: { needs: ['response.usage.billable'], consumes: [], provides: [] }, + response: { needs: ['response.usage.billable', 'response.http.headers'], consumes: [], provides: [] }, }, - return: { provides: ['response.usage.billable', ...narrowing.refuses] }, + return: { provides: ['response.usage.billable', 'response.http.headers', ...narrowing.refuses] }, execute: async (facts, next, use) => { const model = facts['serve.model']; const { candidates, sawModel, failedUpstreams } = await enumerateModelCandidates({ @@ -84,10 +84,16 @@ export const resolveCandidates = (narrowing: Narrowing - move({ ...facts, 'response.usage.billable': [], ...narrowing.refuse(status, message) }); + move({ + ...facts, + 'response.usage.billable': [], + 'response.http.headers': [], + ...narrowing.refuse(status, message), + }); if (candidates.length === 0) { const missing = sawModel diff --git a/packages/gateway/src/data-plane/rerank/pipeline.ts b/packages/gateway/src/data-plane/rerank/pipeline.ts index b2c8466ee0..d8b01ee838 100644 --- a/packages/gateway/src/data-plane/rerank/pipeline.ts +++ b/packages/gateway/src/data-plane/rerank/pipeline.ts @@ -18,6 +18,7 @@ import { writeSettlement } from '../pipeline/settlement.ts'; import { failover, resolveCandidates } from '../pipeline/stages.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; +import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import type { Pipeline } from '@floway-dev/pipeline'; import { defineStage, move, compose } from '@floway-dev/pipeline'; import { parseDecimalString, type RerankProtocol, type RerankSourceProtocol } from '@floway-dev/protocols/common'; @@ -57,8 +58,8 @@ type R = { [P in K]: RerankFacts[P] }; const emitRerank = defineStage< R<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical'>, R<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical'>, - R<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical' | 'response.rerank.canonical' | 'response.rerank.targetProtocol'>, - R<'response.rerank.rendered' | 'response.http.status'> + R<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical' | 'response.rerank.canonical' | 'response.rerank.targetProtocol' | 'response.http.headers'>, + R<'response.rerank.rendered' | 'response.http.status' | 'response.http.headers'> >({ name: 'emitRerank', through: { @@ -68,17 +69,23 @@ const emitRerank = defineStage< provides: [], }, response: { - needs: ['response.rerank.canonical'], - consumes: ['response.rerank.canonical', 'response.rerank.targetProtocol'], - provides: ['response.rerank.rendered', 'response.http.status'], + needs: ['response.rerank.canonical', 'response.http.headers'], + consumes: ['response.rerank.canonical', 'response.rerank.targetProtocol', 'response.http.headers'], + provides: ['response.rerank.rendered', 'response.http.status', 'response.http.headers'], }, }, execute: async (facts, next) => { const back = await next(facts); - const { 'response.rerank.canonical': answer, 'response.rerank.targetProtocol': target, ...rest } = back; + const { 'response.rerank.canonical': answer, 'response.rerank.targetProtocol': target, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); if (isFailure(answer)) { return { ...rest, + 'response.http.headers': forClient, 'response.rerank.rendered': move({ error: { message: answer.message, type: 'api_error' } }), // The upstream's own status, or the gateway's own when it refused before dialling. // A client is not owed the upstream's exact bytes; it is owed the truth about what @@ -88,6 +95,7 @@ const emitRerank = defineStage< } return { ...rest, + 'response.http.headers': forClient, 'response.http.status': 200, 'response.rerank.rendered': move(renderRerankResponse( back['ingress.rerank.sourceProtocol'], @@ -106,12 +114,12 @@ const emitRerank = defineStage< */ const callRerankUpstream = defineStage< R<'request.rerank.canonical' | 'route.attempt' | 'ingress.http.headers'>, - R<'response.rerank.canonical' | 'response.rerank.targetProtocol' | 'response.usage.billable'>, + R<'response.rerank.canonical' | 'response.rerank.targetProtocol' | 'response.http.headers' | 'response.usage.billable'>, GatewayServices >({ name: 'callRerankUpstream', return: { - provides: ['response.rerank.canonical', 'response.rerank.targetProtocol', 'response.usage.billable'], + provides: ['response.rerank.canonical', 'response.rerank.targetProtocol', 'response.http.headers', 'response.usage.billable'], }, execute: async (facts, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); @@ -127,6 +135,8 @@ const callRerankUpstream = defineStage< ); use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'rerank'); const identity = telemetryModelIdentity(candidate, result.modelKey); + // What came back, unfiltered: the edge is where a client's view of it is decided. + const headers = [...result.response.headers]; if (!result.response.ok) { use.log.warn('upstream refused', { status: result.response.status }); @@ -137,6 +147,7 @@ const callRerankUpstream = defineStage< message: await result.response.text(), }, 'response.rerank.targetProtocol': result.target.protocol, + 'response.http.headers': headers, // The upstream was called and reported nothing, which is a different situation // from reporting zero — so the entity is present with no quantities. 'response.usage.billable': [{ identity, quantities: {} }], @@ -152,6 +163,7 @@ const callRerankUpstream = defineStage< ...facts, 'response.rerank.canonical': canonical, 'response.rerank.targetProtocol': result.target.protocol, + 'response.http.headers': headers, 'response.usage.billable': [{ identity, quantities: billed(usage), @@ -190,7 +202,7 @@ const narrowing = (request: CanonicalRerankRequest) => ({ export const rerankServePipeline = (request: CanonicalRerankRequest): Pipeline< R<'ingress.http.headers' | 'ingress.rerank.sourceProtocol' | 'request.rerank.canonical' | 'serve.model'>, - R<'response.rerank.rendered' | 'response.http.status' | 'response.usage.billable'> + R<'response.rerank.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> > => compose('rerankServe', [ emitRerank, writeSettlement(handedUp => isFailure((handedUp as { 'response.rerank.canonical'?: unknown })['response.rerank.canonical'])), diff --git a/packages/protocols/src/embeddings/translate.ts b/packages/protocols/src/embeddings/translate.ts index 28aad0fae9..6afb334f01 100644 --- a/packages/protocols/src/embeddings/translate.ts +++ b/packages/protocols/src/embeddings/translate.ts @@ -145,10 +145,17 @@ const parseUsage = (value: unknown): CanonicalEmbeddingsUsage | undefined => { /** * Reads an upstream's answer into the canonical form. * - * `requestedModel` completes the record when the upstream did not name one. The schema - * marks `model` required, and most upstreams send it; the one this gateway carries that - * does not is GitHub Copilot's `/embeddings`. Naming the model the request asked for is - * both what the client can correlate against and what usage is billed under. + * `requestedModel` completes the record when the upstream did not name one. The schema marks + * `model` required — + * https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34327-L34331 + * — and we have observed Copilot's `/embeddings` answering without it. Nothing holds it to + * the field: the endpoint's own first-party client reads `data` alone and never looks at it, + * where the sibling dotcom route reads its model back and asserts it — + * https://github.com/microsoft/vscode-copilot-chat/blob/5863f5a7088958050792b5dccbe8b46c6e13eccc/src/platform/embeddings/common/remoteEmbeddingsComputer.ts#L294-L301 + * + * Naming the model the request asked for is what the client can correlate against, and it is + * what another Copilot gateway meters under for the same reason — + * https://github.com/caozhiyuan/copilot-api/blob/96854ce1e1e741d621b12d4ca844efd77f6e8d90/src/routes/embeddings/route.ts#L16-L23 */ export const parseEmbeddingsResponse = (value: unknown, requestedModel: string): CanonicalEmbeddingsResponse => { if (!isRecord(value)) throw new Error('Embeddings response body must be an object'); From d7fd0a4f4e4fa0f4da9c5e99cc7557f118092258 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 00:05:40 +0800 Subject: [PATCH 13/79] fix(gateway): give a run a log sink so an unrecorded run still reports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A stage logger wrote to the dump record and to whatever sink the services carried, and the prologue carried none — so on every run without retention configured, a warning or an error reached nothing at all. A failed usage write was the case that showed it. The sink writes warnings and errors to the console the rest of the gateway uses. Debug and info describe one request's progress and would bury the request log at a line per stage, so the threshold is fixed here; a dump still records every level when one is open. --- .../gateway/src/data-plane/pipeline/serve.ts | 2 ++ packages/gateway/src/runtime/log.ts | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 packages/gateway/src/runtime/log.ts diff --git a/packages/gateway/src/data-plane/pipeline/serve.ts b/packages/gateway/src/data-plane/pipeline/serve.ts index 2acc00cd3a..c584192011 100644 --- a/packages/gateway/src/data-plane/pipeline/serve.ts +++ b/packages/gateway/src/data-plane/pipeline/serve.ts @@ -15,6 +15,7 @@ import type { ModelCandidate } from '@floway-dev/provider'; import type { AttemptSelector, GatewayFacts } from './facts.ts'; import type { GatewayServices } from './services.ts'; import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; +import { consoleLogSink } from '../../runtime/log.ts'; import { createGatewayCtxFromHono, finalizeGatewayResponse, type GatewayCtx } from '../shared/gateway-ctx.ts'; import { readRequestBody, takeRequestBody } from '../shared/request-body.ts'; @@ -64,6 +65,7 @@ export const openPrologue = async ( headers: [...c.req.raw.headers], services: { gateway, + log: consoleLogSink, background: work => { gateway.backgroundScheduler(work); }, rememberCandidates: candidates => { for (const candidate of candidates) live.set(candidate.provider.upstreamId, candidate); diff --git a/packages/gateway/src/runtime/log.ts b/packages/gateway/src/runtime/log.ts new file mode 100644 index 0000000000..7805ccbd64 --- /dev/null +++ b/packages/gateway/src/runtime/log.ts @@ -0,0 +1,17 @@ +// The global log sink. A stage's logger writes to the stage's dump record when one is open +// and to this sink always — so a run nobody is recording still reports what went wrong. +// +// The gateway writes to the console and has no level configuration, so the threshold is +// fixed here rather than read from one. Warnings and errors are what a run produces that an +// operator has to see; `debug` and `info` describe one request's progress, and at a line per +// stage per request they would bury the request log this sits alongside. They are still +// recorded in full whenever a dump is open, which is when anyone is reading them. + +import type { Logger } from '@floway-dev/pipeline'; + +export const consoleLogSink: Logger = { + debug: () => {}, + info: () => {}, + warn: (message, fields) => { console.warn(message, fields ?? {}); }, + error: (message, fields) => { console.error(message, fields ?? {}); }, +}; From bbaa158c0e6d358aaf9fba24309e9d445b1db0c7 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 00:13:58 +0800 Subject: [PATCH 14/79] feat(gateway): make a dial failure and an unreadable answer values, not faults Nothing in the domain throws. A refused connection, a timeout or a reset is an outcome the fork has to see so it can try the next candidate, so each ending catches what the platform raised and hands it up as its family's failure. A dial that reached no upstream bills nothing and carries no headers; only the performance row records that the attempt happened. Attribution moves ahead of the dial. It was written from the call's result, so an attempt that never returned left the previous candidate's context in place and misattributed the row. A 2xx body a JSON protocol cannot read is likewise a failure value rather than an unhandled parse error: the gateway that cannot read an answer has not served the request, and the upstream still counts as called having reported nothing. The body reader images already had is now shared, since three families need it. --- .../src/data-plane/embeddings/pipeline.ts | 80 ++++--- .../gateway/src/data-plane/images/pipeline.ts | 46 ++-- .../src/data-plane/pipeline/upstream-body.ts | 53 +++++ .../gateway/src/data-plane/rerank/pipeline.ts | 69 ++++-- .../gateway/src/data-plane/rerank/serve.ts | 220 ++++-------------- 5 files changed, 235 insertions(+), 233 deletions(-) create mode 100644 packages/gateway/src/data-plane/pipeline/upstream-body.ts diff --git a/packages/gateway/src/data-plane/embeddings/pipeline.ts b/packages/gateway/src/data-plane/embeddings/pipeline.ts index 481234840f..cd191ee147 100644 --- a/packages/gateway/src/data-plane/embeddings/pipeline.ts +++ b/packages/gateway/src/data-plane/embeddings/pipeline.ts @@ -16,6 +16,7 @@ import { isFailure } from '../pipeline/facts.ts'; import type { GatewayServices } from '../pipeline/services.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; import { failover, resolveCandidates, type Narrowing } from '../pipeline/stages.ts'; +import { dialFailure, readUpstreamBody, unreadableBody } from '../pipeline/upstream-body.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; @@ -118,44 +119,69 @@ const callEmbeddingsUpstream = defineStage< }, execute: async (facts, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); - const result = await candidate.provider.instance.callEmbeddings( - providerModelOf(candidate), - serializeEmbeddingsRequest(facts['request.embeddings.canonical']), - use.gateway.abortSignal, - // The client's own headers reach the upstream from the record, not from a live - // request object: what a provider is allowed to forward is filtered per provider, - // and the dump shows what was there to filter. - buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]) => [name, value]))), - ); + // Attribution is set before the dial, so an attempt that never completes still names the + // candidate it was made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'embeddings'); - const identity = telemetryModelIdentity(candidate, result.modelKey); - // What came back, unfiltered: the edge is where a client's view of it is decided. - const headers = [...result.response.headers]; - if (!result.response.ok) { - use.log.warn('upstream refused', { status: result.response.status }); + let result; + try { + result = await candidate.provider.instance.callEmbeddings( + providerModelOf(candidate), + serializeEmbeddingsRequest(facts['request.embeddings.canonical']), + use.gateway.abortSignal, + // The client's own headers reach the upstream from the record, not from a live + // request object: what a provider is allowed to forward is filtered per provider, + // and the dump shows what was there to filter. + buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]) => [name, value]))), + ); + } catch (error) { + use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); + // A dial that never completed reached no upstream, so nothing was billed and there are + // no headers to carry. What it leaves behind is the performance row settlement writes. return move({ ...facts, - 'response.embeddings.canonical': { - status: result.response.status, - message: await result.response.text(), - }, - 'response.http.headers': headers, - // The upstream was called and reported nothing, which is a different situation - // from reporting zero — so the entity is present with no quantities. - 'response.usage.billable': [{ identity, quantities: {} }], + 'response.embeddings.canonical': dialFailure(error), + 'response.http.headers': [], + 'response.usage.billable': [], }); } - // Every protocol the gateway carries is one it fully understands: the body is parsed - // here and written again at the edge, in whichever encoding the client can read. - const canonical = parseEmbeddingsResponse(await result.response.json() as unknown, facts['serve.model']); - return move({ + const identity = telemetryModelIdentity(candidate, result.modelKey); + // What came back, unfiltered: the edge is where a client's view of it is decided. + const headers = [...result.response.headers]; + const body = await readUpstreamBody(result.response); + // The upstream was called and reported nothing, which is a different situation from + // reporting zero — so the entity is present with no quantities. + const answered = (canonical: CanonicalEmbeddingsResponse | Failure, quantities: UsageQuantities) => move({ ...facts, 'response.embeddings.canonical': canonical, 'response.http.headers': headers, - 'response.usage.billable': [{ identity, quantities: billed(canonical.usage) }], + 'response.usage.billable': [{ identity, quantities }], }); + const reportedNothing: UsageQuantities = {}; + + if (!result.response.ok) { + use.log.warn('upstream refused', { status: result.response.status }); + return answered({ + status: result.response.status, + message: body.text, + ...('json' in body ? { body: body.json } : {}), + }, reportedNothing); + } + if (!('json' in body)) { + return answered(unreadableBody(result.response, body, 'the embeddings protocol'), reportedNothing); + } + + // Every protocol the gateway carries is one it fully understands: the body is parsed + // here and written again at the edge, in whichever encoding the client can read. + let canonical: CanonicalEmbeddingsResponse; + try { + canonical = parseEmbeddingsResponse(body.json, facts['serve.model']); + } catch (error) { + use.log.warn('upstream answered with a body the embeddings protocol cannot read', { error: String(error) }); + return answered(unreadableBody(result.response, body, 'the embeddings protocol'), reportedNothing); + } + return answered(canonical, billed(canonical.usage)); }, }); diff --git a/packages/gateway/src/data-plane/images/pipeline.ts b/packages/gateway/src/data-plane/images/pipeline.ts index 19bd187ed3..b21d4161bb 100644 --- a/packages/gateway/src/data-plane/images/pipeline.ts +++ b/packages/gateway/src/data-plane/images/pipeline.ts @@ -13,6 +13,7 @@ import { isFailure } from '../pipeline/facts.ts'; import type { GatewayServices } from '../pipeline/services.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; import { failover, resolveCandidates } from '../pipeline/stages.ts'; +import { dialFailure, readUpstreamBody } from '../pipeline/upstream-body.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; @@ -131,13 +132,30 @@ const callImagesUpstream = defineStage< new Headers(facts['ingress.http.headers'].map(([name, value]) => [name, value])), ); const model = providerModelOf(candidate); - // No abort signal, as on this family's endpoints from the beginning: an image the upstream - // has already begun is charged for whether or not the client waited for it, so dropping the - // call would lose the usage reading and save nothing. - const result = request.operation === 'generations' - ? await candidate.provider.instance.callImagesGenerations(model, request.parameters, undefined, options) - : await candidate.provider.instance.callImagesEdits(model, providerEditsRequest(request), undefined, options); + // Attribution is set before the dial, so an attempt that never completes still names the + // candidate it was made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, PERFORMANCE_OPERATION[request.operation]); + + let result; + try { + // No abort signal, as on this family's endpoints from the beginning: an image the upstream + // has already begun is charged for whether or not the client waited for it, so dropping the + // call would lose the usage reading and save nothing. + result = request.operation === 'generations' + ? await candidate.provider.instance.callImagesGenerations(model, request.parameters, undefined, options) + : await candidate.provider.instance.callImagesEdits(model, providerEditsRequest(request), undefined, options); + } catch (error) { + use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); + // A dial that never completed reached no upstream, so nothing was billed and there are + // no headers to carry. What it leaves behind is the performance row settlement writes. + return move({ + ...facts, + 'response.images.canonical': dialFailure(error), + 'response.http.headers': [], + 'response.usage.billable': [], + }); + } + const identity = telemetryModelIdentity(candidate, result.modelKey); // What came back, unfiltered: the edge is where a client's view of it is decided. const headers = [...result.response.headers]; @@ -185,22 +203,6 @@ const callImagesUpstream = defineStage< }, }); -interface UpstreamBody { - readonly text: string; - /** Absent when the body was not JSON at all, which is itself the answer to a protocol that - * requires JSON. */ - readonly json?: unknown; -} - -const readUpstreamBody = async (response: Response): Promise => { - const text = await response.text(); - try { - return { text, json: JSON.parse(text) as unknown }; - } catch { - return { text }; - } -}; - /** An image is billed by the tokens its upstream reports: `BILLING_METRICS` names no per-image * or per-size unit, and per-size pricing is a selector coordinate rather than a metric, so * there is nothing else here to record a count against. A reported zero is kept — it says the diff --git a/packages/gateway/src/data-plane/pipeline/upstream-body.ts b/packages/gateway/src/data-plane/pipeline/upstream-body.ts new file mode 100644 index 0000000000..ca906efb89 --- /dev/null +++ b/packages/gateway/src/data-plane/pipeline/upstream-body.ts @@ -0,0 +1,53 @@ +// Reading what an upstream answered with, for the families whose protocol is JSON. +// +// Every protocol the gateway carries is one it fully understands, so a body it cannot read +// is not handed on unread: the family synthesizes an error from what it did see. That is +// also why the text is kept alongside the parse — a dump reader is owed what actually came +// back, and a message that quotes the body is the only thing a client can act on when the +// upstream answered with something the protocol does not admit. + +import type { Failure } from './facts.ts'; + +export interface UpstreamBody { + readonly text: string; + /** Absent when the body was not JSON at all, which is itself the answer to a protocol that + * requires JSON. */ + readonly json?: unknown; +} + +export const readUpstreamBody = async (response: Response): Promise => { + const text = await response.text(); + try { + return { text, json: JSON.parse(text) as unknown }; + } catch { + return { text }; + } +}; + +/** + * What an upstream that answered in something other than this protocol is worth to a client. + * + * A gateway that cannot read the answer has not served the request, whatever status came with + * it, so this is 502 rather than the upstream's own — and it is a value, because the fork + * above has another candidate to try and an answer nobody can read is exactly the outcome it + * exists to move past. + */ +export const unreadableBody = (response: Response, body: UpstreamBody, protocolName: string): Failure => ({ + status: 502, + message: `The upstream answered ${response.status} with a body ${protocolName} cannot read: ${body.text.slice(0, 200)}`, + body: 'json' in body ? body.json : body.text, +}); + +/** + * What the platform raised while dialling, as a value. + * + * Nothing in the domain throws. A connection that was refused, timed out or was reset is an + * outcome failover has to be able to see — a run that ends there has more candidates to try — + * so the ending catches whatever the platform raised and hands it up like any other failure. + * The status is the one the replaced surface answered with: the gateway reached no upstream, + * which is what 502 says. + */ +export const dialFailure = (error: unknown): Failure => ({ + status: 502, + message: error instanceof Error ? error.message : String(error), +}); diff --git a/packages/gateway/src/data-plane/rerank/pipeline.ts b/packages/gateway/src/data-plane/rerank/pipeline.ts index d8b01ee838..248c4bde82 100644 --- a/packages/gateway/src/data-plane/rerank/pipeline.ts +++ b/packages/gateway/src/data-plane/rerank/pipeline.ts @@ -16,6 +16,7 @@ import { isFailure } from '../pipeline/facts.ts'; import type { GatewayServices } from '../pipeline/services.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; import { failover, resolveCandidates } from '../pipeline/stages.ts'; +import { dialFailure, readUpstreamBody, unreadableBody } from '../pipeline/upstream-body.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; @@ -40,8 +41,9 @@ export interface RerankFacts extends GatewayFacts { 'ingress.rerank.sourceProtocol': RerankSourceProtocol; 'request.rerank.canonical': CanonicalRerankRequest; 'response.rerank.canonical': CanonicalRerankResponse | Failure; - /** Which protocol the upstream turned out to speak. A response fact, because it is not - * known until one answered, and the edge needs it to render back into the client's. */ + /** Which protocol the attempt spoke. A response fact, because an upstream picks the target + * it answers in and the edge needs it to render back into the client's — a run that never + * reached one carries the target its request was serialized for. */ 'response.rerank.targetProtocol': RerankProtocol; /** What the client is actually sent, in its own protocol. The edge provides it, so a * dump shows the bytes the client received rather than the gateway's canonical form. */ @@ -124,19 +126,46 @@ const callRerankUpstream = defineStage< execute: async (facts, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); const request = facts['request.rerank.canonical']; - const result = await candidate.provider.instance.callRerank( - providerModelOf(candidate), - request, - use.gateway.abortSignal, - // The client's own headers reach the upstream from the record, not from a live - // request object: what a provider is allowed to forward is filtered per provider, - // and the dump shows what was there to filter. - buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]) => [name, value]))), - ); + const model = providerModelOf(candidate); + // Configuration refuses a rerank model without a target, so this holds for every candidate + // the narrowing kept; saying so here is what lets a dial that never answered still name + // the protocol it spoke. + const configuredTarget = model.rerankTarget; + if (configuredTarget === undefined) { + throw new Error(`${candidate.provider.upstreamId} serves rerank for ${candidate.model.id} without a target protocol`); + } + // Attribution is set before the dial, so an attempt that never completes still names the + // candidate it was made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'rerank'); + + let result; + try { + result = await candidate.provider.instance.callRerank( + model, + request, + use.gateway.abortSignal, + // The client's own headers reach the upstream from the record, not from a live + // request object: what a provider is allowed to forward is filtered per provider, + // and the dump shows what was there to filter. + buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]) => [name, value]))), + ); + } catch (error) { + use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); + // A dial that never completed reached no upstream, so nothing was billed and there are + // no headers to carry. What it leaves behind is the performance row settlement writes. + return move({ + ...facts, + 'response.rerank.canonical': dialFailure(error), + 'response.rerank.targetProtocol': configuredTarget.protocol, + 'response.http.headers': [], + 'response.usage.billable': [], + }); + } + const identity = telemetryModelIdentity(candidate, result.modelKey); // What came back, unfiltered: the edge is where a client's view of it is decided. const headers = [...result.response.headers]; + const body = await readUpstreamBody(result.response); if (!result.response.ok) { use.log.warn('upstream refused', { status: result.response.status }); @@ -144,7 +173,8 @@ const callRerankUpstream = defineStage< ...facts, 'response.rerank.canonical': { status: result.response.status, - message: await result.response.text(), + message: body.text, + ...('json' in body ? { body: body.json } : {}), }, 'response.rerank.targetProtocol': result.target.protocol, 'response.http.headers': headers, @@ -154,11 +184,20 @@ const callRerankUpstream = defineStage< }); } + if (!('json' in body)) { + return move({ + ...facts, + 'response.rerank.canonical': unreadableBody(result.response, body, 'the rerank protocol'), + 'response.rerank.targetProtocol': result.target.protocol, + 'response.http.headers': headers, + 'response.usage.billable': [{ identity, quantities: {} }], + }); + } + // Every protocol the gateway carries is one it fully understands: the body is parsed // here and re-serialized at the edge, whether or not the two protocols match. - const body = await result.response.json() as unknown; - const canonical = parseRerankResponse(result.target.protocol, body); - const usage = parseRerankUsage(result.target.protocol, body); + const canonical = parseRerankResponse(result.target.protocol, body.json); + const usage = parseRerankUsage(result.target.protocol, body.json); return move({ ...facts, 'response.rerank.canonical': canonical, diff --git a/packages/gateway/src/data-plane/rerank/serve.ts b/packages/gateway/src/data-plane/rerank/serve.ts index 7d64ba63d2..0d3ef19f50 100644 --- a/packages/gateway/src/data-plane/rerank/serve.ts +++ b/packages/gateway/src/data-plane/rerank/serve.ts @@ -1,183 +1,65 @@ -import type { Context } from 'hono'; -import type { ContentfulStatusCode } from 'hono/utils/http-status'; +// POST /v1/rerank, /v2/rerank, /jina/v1/rerank and /voyage/v1/rerank, served through the +// pipeline. +// +// One family behind four routes. Which protocol the client spoke is not in the body — the +// route it arrived on is the whole of that statement — so the mount passes it in and the +// handler carries it into the record as an ingress fact. Everything after the parse is +// stages. -import { rerankAttempt, type RerankAttemptResult } from './attempt.ts'; -import type { UsageQuantities } from '../../repo/types.ts'; -import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; -import { enumerateModelCandidates } from '../providers/resolution.ts'; -import { appendFailedUpstreams } from '../shared/failed-upstreams.ts'; -import { createGatewayCtxFromHono, finalizeGatewayResponse, type GatewayCtx } from '../shared/gateway-ctx.ts'; -import { iterateCandidates } from '../shared/iterate-candidates.ts'; -import { readRequestBody, takeRequestBody } from '../shared/request-body.ts'; -import { recordFailedRequest, recordPerformance, type PerformanceTelemetryContext } from '../shared/telemetry/performance.ts'; -import { recordUsage } from '../shared/telemetry/usage.ts'; -import { forwardUpstreamResponse } from '../shared/upstream-response.ts'; -import { parseDecimalString, type RerankSourceProtocol } from '@floway-dev/protocols/common'; -import { parseRerankRequest, parseRerankResponse, parseRerankUsage, renderRerankResponse, rerankRequestIncompatibility, type CanonicalRerankResponse, type ParsedRerankRequest } from '@floway-dev/protocols/rerank'; -import { httpResponseToResponse, ProviderModelsUnavailableError, providerModelOf, toInternalDebugError } from '@floway-dev/provider'; -import type { TelemetryModelIdentity } from '@floway-dev/provider'; +import { move } from '@floway-dev/pipeline'; +import type { RerankSourceProtocol } from '@floway-dev/protocols/common'; +import { parseRerankRequest, type ParsedRerankRequest } from '@floway-dev/protocols/rerank'; +import type { Context } from 'hono'; -const apiError = (c: Context, message: string, status: ContentfulStatusCode): Response => - c.json({ error: { message, type: 'api_error' } }, status); +import { rerankServePipeline } from './pipeline.ts'; +import { openPrologue, serveThrough } from '../pipeline/serve.ts'; +import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; -const parseJson = (bytes: Uint8Array): unknown => { +// The contract reports a malformed request by throwing; what the client is owed is a 400 +// carrying the reason. `JSON.parse`'s own wording names a byte offset in a body the client +// already has, so the reason for that one is written here instead. +const readRequest = ( + sourceProtocol: RerankSourceProtocol, + bytes: Uint8Array, +): { type: 'ok'; parsed: ParsedRerankRequest } | { type: 'invalid'; message: string } => { + let body: unknown; try { - return JSON.parse(new TextDecoder().decode(bytes)) as unknown; + body = JSON.parse(new TextDecoder().decode(bytes)) as unknown; } catch { - throw new Error('Rerank request body must be valid JSON'); + return { type: 'invalid', message: 'Rerank request body must be valid JSON' }; } -}; - -const settleRerank = ( - ctx: GatewayCtx, - performanceContext: PerformanceTelemetryContext, - identity: TelemetryModelIdentity, - usage: Pick | undefined, - failed: boolean, -): void => { - const quantities: UsageQuantities = {}; - if (usage?.searchUnits !== undefined) quantities.rerank_searches = parseDecimalString(String(usage.searchUnits)); - if (usage?.totalTokens !== undefined) quantities.input_tokens = parseDecimalString(String(usage.totalTokens)); - const pricingFacts = usage?.totalTokens === undefined ? {} : { inputTokens: usage.totalTokens }; - ctx.backgroundScheduler(recordUsage(ctx.apiKeyId, identity, quantities, pricingFacts).catch(error => { - console.error('Failed to record rerank usage:', error); - })); - recordPerformance(ctx, performanceContext, failed, 0, performance.now()); -}; - -const unsupportedMessage = (model: string): string => `Model ${model} does not support rerank.`; - -export const rerank = (sourceProtocol: RerankSourceProtocol) => async (c: Context): Promise => { - const requestBody = await readRequestBody(c); - let parsedRequest: ParsedRerankRequest; try { - parsedRequest = parseRerankRequest(sourceProtocol, parseJson(requestBody.bytes)); + return { type: 'ok', parsed: parseRerankRequest(sourceProtocol, body) }; } catch (error) { - const ctx = createGatewayCtxFromHono(c, { - wantsStream: false, - requestBody: takeRequestBody(requestBody), - backgroundScheduler: backgroundSchedulerFromContext(c), - }); - ctx.dump?.error('gateway'); - return finalizeGatewayResponse(ctx, apiError(c, error instanceof Error ? error.message : String(error), 400)); + return { type: 'invalid', message: error instanceof Error ? error.message : String(error) }; } +}; - const { model, request } = parsedRequest; - const ctx = createGatewayCtxFromHono(c, { - wantsStream: false, - model, - requestBody: takeRequestBody(requestBody), - backgroundScheduler: backgroundSchedulerFromContext(c), - }); - - let terminal: RerankAttemptResult | undefined; - let measuredUsage: Pick | undefined; - let usageSettled = false; - try { - const { candidates, sawModel, failedUpstreams } = await enumerateModelCandidates({ - upstreamIds: ctx.upstreamIds, - model, - kind: 'rerank', - scheduler: ctx.backgroundScheduler, - runtimeLocation: ctx.runtimeLocation, - }); - if (candidates.length === 0) { - ctx.dump?.error('gateway'); - const message = sawModel - ? unsupportedMessage(model) - : `Model ${model} is not available on any configured upstream.`; - return finalizeGatewayResponse(ctx, apiError(c, appendFailedUpstreams(message, failedUpstreams), sawModel ? 400 : 404)); - } - - const routable = candidates.flatMap(candidate => { - const providerModel = providerModelOf(candidate); - return candidate.model.endpoints.rerank === undefined || providerModel.rerankTarget === undefined - ? [] - : [{ candidate, target: providerModel.rerankTarget }]; - }); - if (routable.length === 0) { - ctx.dump?.error('gateway'); - return finalizeGatewayResponse(ctx, apiError(c, appendFailedUpstreams(unsupportedMessage(model), failedUpstreams), 400)); - } - const viable = routable.filter(({ target }) => rerankRequestIncompatibility(target.protocol, request) === null); - if (viable.length === 0) { - const reasons = [...new Set(routable.flatMap(({ target }) => { - const reason = rerankRequestIncompatibility(target.protocol, request); - return reason === null ? [] : [reason]; - }))]; - ctx.dump?.error('gateway'); - return finalizeGatewayResponse(ctx, apiError(c, `Model ${model} does not support this rerank request: ${reasons.join('; ')}.`, 400)); - } - - terminal = await iterateCandidates( - viable.map(({ candidate }) => candidate), - 'rerank', - ctx, - 'rerank', - candidate => rerankAttempt(c, ctx, candidate, request), +export const rerank = (sourceProtocol: RerankSourceProtocol) => async (c: Context): Promise => { + const prologue = await openPrologue(c, { wantsStream: false }); + const result = readRequest(sourceProtocol, prologue.bytes); + if (result.type === 'invalid') { + // A request the gateway could not read never reaches a pipeline: there is no model to + // resolve and no attempt to make, so there is nothing for a run to record. + prologue.gateway.dump?.error('gateway'); + return finalizeGatewayResponse( + prologue.gateway, + Response.json({ error: { message: result.message, type: 'api_error' } }, { status: 400 }), ); + } - if (!terminal.response.ok) { - ctx.dump?.error('upstream', terminal.identity.upstream); - settleRerank(ctx, terminal.performance, terminal.identity, undefined, true); - usageSettled = true; - return finalizeGatewayResponse(ctx, forwardUpstreamResponse(terminal.response)); - } + const { model, request } = result.parsed; + prologue.gateway.dump?.requestedModel(model); - const sameProtocol = sourceProtocol === terminal.target.protocol; - let upstreamBody: unknown; - try { - upstreamBody = await terminal.response.clone().json() as unknown; - } catch (error) { - if (!sameProtocol) throw error; - console.warn( - `rerank: failed to parse same-protocol 2xx upstream body for ${sourceProtocol}; usage row will be request-only`, - error instanceof Error ? error.message : String(error), - ); - ctx.dump?.success(terminal.identity, null); - settleRerank(ctx, terminal.performance, terminal.identity, undefined, false); - usageSettled = true; - return finalizeGatewayResponse(ctx, forwardUpstreamResponse(terminal.response)); - } - try { - measuredUsage = parseRerankUsage(terminal.target.protocol, upstreamBody); - } catch (error) { - if (!sameProtocol) throw error; - console.warn( - `rerank: failed to parse same-protocol usage for ${sourceProtocol}; usage row will be request-only`, - error instanceof Error ? error.message : String(error), - ); - ctx.dump?.success(terminal.identity, null); - settleRerank(ctx, terminal.performance, terminal.identity, undefined, false); - usageSettled = true; - return finalizeGatewayResponse(ctx, forwardUpstreamResponse(terminal.response)); - } - if (sameProtocol) { - ctx.dump?.success(terminal.identity, null); - settleRerank(ctx, terminal.performance, terminal.identity, measuredUsage, false); - usageSettled = true; - return finalizeGatewayResponse(ctx, forwardUpstreamResponse(terminal.response)); - } - const canonical = parseRerankResponse(terminal.target.protocol, upstreamBody); - const rendered = renderRerankResponse(sourceProtocol, terminal.target.protocol, canonical, request); - ctx.dump?.success(terminal.identity, null); - settleRerank(ctx, terminal.performance, terminal.identity, measuredUsage, false); - usageSettled = true; - return finalizeGatewayResponse(ctx, forwardUpstreamResponse(terminal.response, { body: JSON.stringify(rendered) })); - } catch (error) { - if (terminal !== undefined && !usageSettled) { - settleRerank(ctx, terminal.performance, terminal.identity, measuredUsage, true); - } else if (terminal === undefined) { - recordFailedRequest(ctx, ctx.attempt.telemetry); - } - if (error instanceof ProviderModelsUnavailableError) { - const forwarded = httpResponseToResponse(error.httpResponse); - if (forwarded) { - ctx.dump?.error('upstream'); - return finalizeGatewayResponse(ctx, forwarded); - } - } - ctx.dump?.failed(error); - return finalizeGatewayResponse(ctx, c.json({ error: toInternalDebugError(error) }, 502)); - } + return await serveThrough( + prologue, + rerankServePipeline(request), + move({ + 'ingress.http.headers': prologue.headers, + 'ingress.rerank.sourceProtocol': sourceProtocol, + 'request.rerank.canonical': request, + 'serve.model': model, + }) as never, + facts => ({ body: JSON.stringify(facts['response.rerank.rendered']), contentType: 'application/json' }), + ); }; From 076132ad984f044778b786f75007ace5d5a0ab44 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 00:20:03 +0800 Subject: [PATCH 15/79] feat(images): serve both images endpoints through the pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both handlers still ran the passthrough scaffold: they read the body, hand-rolled the multipart and JSON edit shapes into a provider request, picked a candidate by endpoint capability and relayed whatever the upstream sent. The images protocol already owns both readings and the family already has a pipeline, so what is left here is a prologue and an epilogue — parse, hand over, write the answer. Reading the body moves to the contract, which reports a malformed request by throwing; the handler turns that into the same 400 envelope the scaffold wrote, with the same sentences. The two endpoints differ only in which parser reads the bytes, so the rest is one shared function. --- .../gateway/src/data-plane/images/http.ts | 213 +++++------------- 1 file changed, 51 insertions(+), 162 deletions(-) diff --git a/packages/gateway/src/data-plane/images/http.ts b/packages/gateway/src/data-plane/images/http.ts index 411127db9a..a4c98f7fbc 100644 --- a/packages/gateway/src/data-plane/images/http.ts +++ b/packages/gateway/src/data-plane/images/http.ts @@ -1,174 +1,63 @@ -// POST /v1/images/generations and POST /v1/images/edits — route image -// requests to the provider that declares the requested model and the -// matching image endpoint capability. +// POST /v1/images/generations and POST /v1/images/edits, served through the pipeline. // -// The edits handler accepts multipart uploads and JSON `images` references. -// Both are buffered once for dump capture and normalized into a semantic -// request; each provider owns the final JSON or multipart serialization. -// https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L12558-L12620 +// Two endpoints and one family. What the endpoint decides is which contract reads the body — +// generations is always JSON, edits is JSON or a multipart form carrying the files — and +// after that both hand the same canonical request to the same run. +import { move } from '@floway-dev/pipeline'; +import { parseImagesEditsRequest, parseImagesGenerationsRequest, type ParsedImagesRequest } from '@floway-dev/protocols/images'; import type { Context } from 'hono'; -import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; -import { createGatewayCtxFromHono, finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; -import { prepareJsonModelRequest } from '../shared/passthrough-request.ts'; -import { passthroughApiError, passthroughServe } from '../shared/passthrough-serve.ts'; -import { readRequestBody, takeRequestBody, type RequestBody } from '../shared/request-body.ts'; -import { tokenUsageFromImagesBody } from '../shared/telemetry/usage.ts'; -import { isJsonMediaType, isMultipartFormDataMediaType } from '@floway-dev/protocols/common'; -import type { ImageEditReference } from '@floway-dev/protocols/images'; -import { isBase64ImageDataUrl, type ImagesEditsRequest, type ImagesEditsSource } from '@floway-dev/provider'; +import { imagesServePipeline } from './pipeline.ts'; +import { openPrologue, serveThrough, type Prologue } from '../pipeline/serve.ts'; +import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; -type PreparedImagesEdit = - | { type: 'ok'; request: ImagesEditsRequest } - | { type: 'invalid'; message: string }; - -const imageEditSource = (value: unknown, path: string): ImagesEditsSource | string => { - if (value === null || typeof value !== 'object' || Array.isArray(value)) { - return `${path} must be an object.`; - } - const reference = value as { image_url?: unknown; file_id?: unknown }; - const { image_url: imageUrl, file_id: fileId } = reference; - if (typeof imageUrl === 'string' && fileId === undefined) { - const imageReference = reference as ImageEditReference & { image_url: string }; - return isBase64ImageDataUrl(imageUrl) - ? { type: 'inline', reference: imageReference } - : { type: 'reference', reference: imageReference }; - } - if (typeof fileId === 'string' && imageUrl === undefined) { - return { type: 'reference', reference: reference as ImageEditReference }; - } - return `${path} must contain exactly one string field: image_url or file_id.`; -}; - -const prepareJsonImagesEdit = (body: Record): PreparedImagesEdit => { - if (!Array.isArray(body.images)) { - return { type: 'invalid', message: 'Image edits request body must include an images array.' }; - } - const images: ImagesEditsSource[] = []; - for (const [index, value] of body.images.entries()) { - const source = imageEditSource(value, `Image edits images[${index}]`); - if (typeof source === 'string') return { type: 'invalid', message: source }; - images.push(source); - } - let mask: ImagesEditsSource | undefined; - if (body.mask !== undefined) { - const source = imageEditSource(body.mask, 'Image edits mask'); - if (typeof source === 'string') return { type: 'invalid', message: source }; - mask = source; - } - const { model: _model, images: _images, mask: _mask, ...parameters } = body; - return { - type: 'ok', - request: { - images, - ...(mask === undefined ? {} : { mask }), - parameters, - }, - }; +/** The half both endpoints share: whatever the contract read, hand it over and turn what the + * run answered with into a response. The contract reports a malformed request by throwing; + * what the client is owed is a 400 carrying the reason. */ +const serveImages = async ( + prologue: Prologue, + read: () => ParsedImagesRequest | Promise, +): Promise => { + let parsed: ParsedImagesRequest; + try { + parsed = await read(); + } catch (error) { + // A request the gateway could not read never reaches a pipeline: there is no model to + // resolve and no attempt to make, so there is nothing for a run to record. + prologue.gateway.dump?.error('gateway'); + return finalizeGatewayResponse( + prologue.gateway, + Response.json( + { error: { message: error instanceof Error ? error.message : String(error), type: 'api_error' } }, + { status: 400 }, + ), + ); + } + + const { model, request } = parsed; + prologue.gateway.dump?.requestedModel(model); + + return await serveThrough( + prologue, + imagesServePipeline(request), + move({ + 'ingress.http.headers': prologue.headers, + 'request.images.canonical': request, + 'serve.model': model, + }) as never, + facts => ({ body: JSON.stringify(facts['response.images.rendered']), contentType: 'application/json' }), + ); }; export const imagesGenerations = async (c: Context): Promise => { - const requestBody = await readRequestBody(c); - const request = prepareJsonModelRequest(requestBody.bytes, 'Images generations'); - const ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - if (request.type === 'invalid') { - ctx.dump?.error('gateway'); - return finalizeGatewayResponse(ctx, passthroughApiError(c, request.message, 400)); - } - - ctx.dump?.requestedModel(request.model); - const response = await passthroughServe({ - c, - ctx, - sourceApi: '/images/generations', - operation: 'image_generation', - model: request.model, - kind: 'image', - modelServesEndpoint: model => model.endpoints.imagesGenerations !== undefined, - call: (provider, model, opts) => { - const { model: _model, ...body } = request.body; - return provider.instance.callImagesGenerations(model, body, undefined, opts); - }, - response: { format: 'json', extractBilling: tokenUsageFromImagesBody }, - }); - return finalizeGatewayResponse(ctx, response); -}; - -const serveImagesEditRequest = async ( - c: Context, - requestBody: RequestBody, - model: string, - request: ImagesEditsRequest, -): Promise => { - const ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - ctx.dump?.requestedModel(model); - const response = await passthroughServe({ - c, - ctx, - sourceApi: '/images/edits', - operation: 'image_edit', - model, - kind: 'image', - modelServesEndpoint: model => model.endpoints.imagesEdits !== undefined, - call: (provider, model, opts) => provider.instance.callImagesEdits(model, request, undefined, opts), - response: { format: 'json', extractBilling: tokenUsageFromImagesBody }, - }); - return finalizeGatewayResponse(ctx, response); + const prologue = await openPrologue(c, { wantsStream: false }); + return await serveImages(prologue, () => parseImagesGenerationsRequest(prologue.bytes)); }; export const imagesEdits = async (c: Context): Promise => { - const requestBody = await readRequestBody(c); - const invalid = (message: string): Response => { - const errorCtx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - errorCtx.dump?.error('gateway'); - return finalizeGatewayResponse(errorCtx, passthroughApiError(c, message, 400)); - }; - - const contentType = c.req.header('content-type'); - if (contentType === undefined) { - return invalid('Image edits request body must use application/json or multipart/form-data.'); - } - if (isJsonMediaType(contentType)) { - const body = prepareJsonModelRequest(requestBody.bytes, 'Image edits'); - if (body.type === 'invalid') return invalid(body.message); - const request = prepareJsonImagesEdit(body.body); - if (request.type === 'invalid') return invalid(request.message); - return await serveImagesEditRequest(c, requestBody, body.model, request.request); - } - - if (!isMultipartFormDataMediaType(contentType)) { - return invalid('Image edits request body must use application/json or multipart/form-data.'); - } - let form: FormData; - try { - form = await new Response(requestBody.bytes as BodyInit, { headers: { 'content-type': contentType } }).formData(); - } catch { - return invalid('Image edits request body must be valid multipart/form-data.'); - } - const model = form.get('model'); - if (typeof model !== 'string' || model.length === 0) { - return invalid('Image edits request body must include a model field.'); - } - const images: File[] = []; - let mask: File | undefined; - const parameters: Record = {}; - for (const [name, value] of form.entries()) { - if (name === 'model') continue; - if (name === 'image' || name === 'image[]') { - if (!(value instanceof File)) return invalid(`Image edits ${name} fields must be files.`); - images.push(value); - } else if (name === 'mask') { - if (!(value instanceof File)) return invalid('Image edits mask field must be a file.'); - mask = value; - } else { - if (typeof value !== 'string') return invalid(`Image edits ${name} field must be text.`); - parameters[name] = value; - } - } - return await serveImagesEditRequest(c, requestBody, model, { - images: images.map(file => ({ type: 'upload', file })), - ...(mask === undefined ? {} : { mask: { type: 'upload' as const, file: mask } }), - parameters, - }); + const prologue = await openPrologue(c, { wantsStream: false }); + // Which of the two bodies arrived is a header's statement and not the payload's, so the + // contract is handed the media type alongside the bytes. + return await serveImages(prologue, () => parseImagesEditsRequest(c.req.header('content-type'), prologue.bytes)); }; From e63caf408ba9e11bb3aac66e5742f9cb224e7309 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 00:24:40 +0800 Subject: [PATCH 16/79] fix(gateway): order the pipelined handlers' imports the way the linter reads them Workspace packages sort after the relative imports, and the two handlers wired to their pipelines had them first. Nothing else changes. --- packages/gateway/src/data-plane/images/http.ts | 4 ++-- packages/gateway/src/data-plane/rerank/serve.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/gateway/src/data-plane/images/http.ts b/packages/gateway/src/data-plane/images/http.ts index a4c98f7fbc..8813e75976 100644 --- a/packages/gateway/src/data-plane/images/http.ts +++ b/packages/gateway/src/data-plane/images/http.ts @@ -4,13 +4,13 @@ // generations is always JSON, edits is JSON or a multipart form carrying the files — and // after that both hand the same canonical request to the same run. -import { move } from '@floway-dev/pipeline'; -import { parseImagesEditsRequest, parseImagesGenerationsRequest, type ParsedImagesRequest } from '@floway-dev/protocols/images'; import type { Context } from 'hono'; import { imagesServePipeline } from './pipeline.ts'; import { openPrologue, serveThrough, type Prologue } from '../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; +import { move } from '@floway-dev/pipeline'; +import { parseImagesEditsRequest, parseImagesGenerationsRequest, type ParsedImagesRequest } from '@floway-dev/protocols/images'; /** The half both endpoints share: whatever the contract read, hand it over and turn what the * run answered with into a response. The contract reports a malformed request by throwing; diff --git a/packages/gateway/src/data-plane/rerank/serve.ts b/packages/gateway/src/data-plane/rerank/serve.ts index 0d3ef19f50..a0f2e4d930 100644 --- a/packages/gateway/src/data-plane/rerank/serve.ts +++ b/packages/gateway/src/data-plane/rerank/serve.ts @@ -6,14 +6,14 @@ // handler carries it into the record as an ingress fact. Everything after the parse is // stages. -import { move } from '@floway-dev/pipeline'; -import type { RerankSourceProtocol } from '@floway-dev/protocols/common'; -import { parseRerankRequest, type ParsedRerankRequest } from '@floway-dev/protocols/rerank'; import type { Context } from 'hono'; import { rerankServePipeline } from './pipeline.ts'; import { openPrologue, serveThrough } from '../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; +import { move } from '@floway-dev/pipeline'; +import type { RerankSourceProtocol } from '@floway-dev/protocols/common'; +import { parseRerankRequest, type ParsedRerankRequest } from '@floway-dev/protocols/rerank'; // The contract reports a malformed request by throwing; what the client is owed is a 400 // carrying the reason. `JSON.parse`'s own wording names a byte offset in a body the client From c0da795a995795c112bcdd4e59a9399d48b58e26 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 00:26:45 +0800 Subject: [PATCH 17/79] feat(gateway): answer in the client's own error shape and keep readings independent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An upstream that refused in its own words is handed on in them. The envelope was written once for OpenAI-shaped families and applied everywhere, which turned a rerank client's `{message}` into `{error:{message}}` — a shape its SDK does not read. What distinguishes the two cases is whether an upstream answered at all, not which shape it answered in, so that is what the renderer now tests. Rerank reads usage before results. They are independent readings of one body and one failing is no reason to discard the other, so an answer this gateway cannot model still bills for what the upstream metered. A same-protocol answer is rendered back out unchanged, so results it carries that the canonical form does not model are no longer a refusal — only a translation needs to read them. A translation that fails on an answer that parsed is 502: the upstream answered and the gateway cannot say it in the client's protocol. --- .../src/data-plane/embeddings/pipeline.ts | 6 +- .../gateway/src/data-plane/images/pipeline.ts | 8 +- .../src/data-plane/pipeline/upstream-body.ts | 1 - .../gateway/src/data-plane/rerank/pipeline.ts | 89 ++++++++++++++----- .../__tests__/images/response_test.ts | 11 +-- .../protocols/src/common/error-envelope.ts | 21 +++++ packages/protocols/src/common/index.ts | 1 + packages/protocols/src/images/index.ts | 2 +- packages/protocols/src/images/response.ts | 14 --- 9 files changed, 103 insertions(+), 50 deletions(-) create mode 100644 packages/protocols/src/common/error-envelope.ts diff --git a/packages/gateway/src/data-plane/embeddings/pipeline.ts b/packages/gateway/src/data-plane/embeddings/pipeline.ts index cd191ee147..ef37015530 100644 --- a/packages/gateway/src/data-plane/embeddings/pipeline.ts +++ b/packages/gateway/src/data-plane/embeddings/pipeline.ts @@ -22,7 +22,7 @@ import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import type { Pipeline } from '@floway-dev/pipeline'; import { compose, defineStage, move } from '@floway-dev/pipeline'; -import { parseDecimalString } from '@floway-dev/protocols/common'; +import { parseDecimalString, renderErrorEnvelope, upstreamErrorMessage } from '@floway-dev/protocols/common'; import { parseEmbeddingsResponse, renderEmbeddingsResponse, @@ -87,7 +87,7 @@ const emitEmbeddings = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.embeddings.rendered': move({ error: { message: answer.message, type: 'api_error' } }), + 'response.embeddings.rendered': move(renderErrorEnvelope(answer.message, answer.body)), // The upstream's own status, or the gateway's own when it refused before dialling. // A client is not owed the upstream's exact bytes; it is owed the truth about what // happened, and a 429 arriving as a 200 is not that. @@ -164,7 +164,7 @@ const callEmbeddingsUpstream = defineStage< use.log.warn('upstream refused', { status: result.response.status }); return answered({ status: result.response.status, - message: body.text, + message: upstreamErrorMessage(body.json) ?? body.text, ...('json' in body ? { body: body.json } : {}), }, reportedNothing); } diff --git a/packages/gateway/src/data-plane/images/pipeline.ts b/packages/gateway/src/data-plane/images/pipeline.ts index b21d4161bb..9078b64928 100644 --- a/packages/gateway/src/data-plane/images/pipeline.ts +++ b/packages/gateway/src/data-plane/images/pipeline.ts @@ -19,11 +19,9 @@ import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import type { Pipeline } from '@floway-dev/pipeline'; import { compose, defineStage, move } from '@floway-dev/pipeline'; -import { mediaTypeEssence, parseDecimalString, type ModelEndpointKey } from '@floway-dev/protocols/common'; +import { mediaTypeEssence, parseDecimalString, renderErrorEnvelope, upstreamErrorMessage, type ModelEndpointKey } from '@floway-dev/protocols/common'; import { - imagesErrorMessage, parseImagesResponse, - renderImagesError, renderImagesResponse, type CanonicalImagesEditsRequest, type CanonicalImagesRequest, @@ -91,7 +89,7 @@ const emitImages = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.images.rendered': move(renderImagesError(answer.message, answer.body)), + 'response.images.rendered': move(renderErrorEnvelope(answer.message, answer.body)), 'response.http.status': answer.status, }; } @@ -174,7 +172,7 @@ const callImagesUpstream = defineStage< use.log.warn('upstream refused', { status: result.response.status }); return answered({ status: result.response.status, - message: imagesErrorMessage(body.json) ?? body.text, + message: upstreamErrorMessage(body.json) ?? body.text, ...('json' in body ? { body: body.json } : {}), }, reportedNothing); } diff --git a/packages/gateway/src/data-plane/pipeline/upstream-body.ts b/packages/gateway/src/data-plane/pipeline/upstream-body.ts index ca906efb89..068b4cce65 100644 --- a/packages/gateway/src/data-plane/pipeline/upstream-body.ts +++ b/packages/gateway/src/data-plane/pipeline/upstream-body.ts @@ -35,7 +35,6 @@ export const readUpstreamBody = async (response: Response): Promise ({ status: 502, message: `The upstream answered ${response.status} with a body ${protocolName} cannot read: ${body.text.slice(0, 200)}`, - body: 'json' in body ? body.json : body.text, }); /** diff --git a/packages/gateway/src/data-plane/rerank/pipeline.ts b/packages/gateway/src/data-plane/rerank/pipeline.ts index 248c4bde82..a98dc23a25 100644 --- a/packages/gateway/src/data-plane/rerank/pipeline.ts +++ b/packages/gateway/src/data-plane/rerank/pipeline.ts @@ -22,7 +22,7 @@ import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import type { Pipeline } from '@floway-dev/pipeline'; import { defineStage, move, compose } from '@floway-dev/pipeline'; -import { parseDecimalString, type RerankProtocol, type RerankSourceProtocol } from '@floway-dev/protocols/common'; +import { parseDecimalString, renderErrorEnvelope, upstreamErrorMessage, type RerankProtocol, type RerankSourceProtocol } from '@floway-dev/protocols/common'; import { parseRerankResponse, parseRerankUsage, @@ -88,23 +88,38 @@ const emitRerank = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.rerank.rendered': move({ error: { message: answer.message, type: 'api_error' } }), + 'response.rerank.rendered': move(renderErrorEnvelope(answer.message, answer.body)), // The upstream's own status, or the gateway's own when it refused before dialling. // A client is not owed the upstream's exact bytes; it is owed the truth about what // happened, and a 429 arriving as a 200 is not that. 'response.http.status': answer.status, }; } - return { - ...rest, - 'response.http.headers': forClient, - 'response.http.status': 200, - 'response.rerank.rendered': move(renderRerankResponse( + // Translating can fail on an answer that parsed: a result may index a document the + // request never sent. The upstream answered and the gateway cannot put that answer in the + // protocol the client speaks, which is the gateway failing to serve rather than anything + // the client can fix — so it is 502, and a value like every other failure here. + let rendered: Record; + try { + rendered = renderRerankResponse( back['ingress.rerank.sourceProtocol'], target, answer, back['request.rerank.canonical'], - )), + ); + } catch (error) { + return { + ...rest, + 'response.http.headers': forClient, + 'response.rerank.rendered': move(renderErrorEnvelope(error instanceof Error ? error.message : String(error))), + 'response.http.status': 502, + }; + } + return { + ...rest, + 'response.http.headers': forClient, + 'response.http.status': 200, + 'response.rerank.rendered': move(rendered), }; }, }); @@ -115,7 +130,7 @@ const emitRerank = defineStage< * fails over, and even a 400 can be, because the next candidate's path and flags may differ. */ const callRerankUpstream = defineStage< - R<'request.rerank.canonical' | 'route.attempt' | 'ingress.http.headers'>, + R<'request.rerank.canonical' | 'route.attempt' | 'ingress.http.headers' | 'ingress.rerank.sourceProtocol'>, R<'response.rerank.canonical' | 'response.rerank.targetProtocol' | 'response.http.headers' | 'response.usage.billable'>, GatewayServices >({ @@ -173,7 +188,7 @@ const callRerankUpstream = defineStage< ...facts, 'response.rerank.canonical': { status: result.response.status, - message: body.text, + message: upstreamErrorMessage(body.json) ?? body.text, ...('json' in body ? { body: body.json } : {}), }, 'response.rerank.targetProtocol': result.target.protocol, @@ -194,22 +209,54 @@ const callRerankUpstream = defineStage< }); } - // Every protocol the gateway carries is one it fully understands: the body is parsed - // here and re-serialized at the edge, whether or not the two protocols match. - const canonical = parseRerankResponse(result.target.protocol, body.json); - const usage = parseRerankUsage(result.target.protocol, body.json); + // A usage block the reader cannot make sense of is a report we cannot parse, which from + // here is no report. It is read before the results, so an answer this gateway could not + // model still bills for what the upstream did meter — the two readings are independent + // and one of them failing is not a reason to discard the other. + let usage: Pick; + try { + usage = parseRerankUsage(result.target.protocol, body.json); + } catch (error) { + use.log.warn('upstream reported usage the rerank protocol cannot read', { error: String(error) }); + usage = {}; + } + const metered: readonly BillableEntity[] = [{ + identity, + quantities: billed(usage), + // A rerank rate can depend on how large the input was and not only on how much of it + // there was, so the token total is a pricing input as well as a quantity. + ...(usage.totalTokens === undefined ? {} : { pricingFacts: { inputTokens: usage.totalTokens } }), + }]; + + // An answer the client's own protocol produced is what that client already reads, so the + // edge renders it back out unchanged and nothing here has to model it. Reading the results + // is what a *translation* needs, and only a cross-protocol run does one — so a result item + // this gateway cannot model is a failure there and a field it simply carries here. + const translating = facts['ingress.rerank.sourceProtocol'] !== result.target.protocol; + let canonical: CanonicalRerankResponse; + try { + canonical = parseRerankResponse(result.target.protocol, body.json); + } catch (error) { + if (translating) { + use.log.warn('upstream answered with results the rerank protocol cannot read', { error: String(error) }); + return move({ + ...facts, + 'response.rerank.canonical': unreadableBody(result.response, body, 'the rerank protocol'), + 'response.rerank.targetProtocol': result.target.protocol, + 'response.http.headers': headers, + 'response.usage.billable': metered, + }); + } + use.log.debug('same-protocol answer carries results this gateway does not model', { error: String(error) }); + canonical = { raw: body.json as Record, results: [] }; + } + return move({ ...facts, 'response.rerank.canonical': canonical, 'response.rerank.targetProtocol': result.target.protocol, 'response.http.headers': headers, - 'response.usage.billable': [{ - identity, - quantities: billed(usage), - // A rerank rate can depend on how large the input was and not only on how much of - // it there was, so the token total is a pricing input as well as a quantity. - ...(usage?.totalTokens === undefined ? {} : { pricingFacts: { inputTokens: usage.totalTokens } }), - }], + 'response.usage.billable': metered, }); }, }); diff --git a/packages/protocols/__tests__/images/response_test.ts b/packages/protocols/__tests__/images/response_test.ts index a9fece43f7..bf9bf9c173 100644 --- a/packages/protocols/__tests__/images/response_test.ts +++ b/packages/protocols/__tests__/images/response_test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from 'vitest'; -import { imagesErrorMessage, parseImagesResponse, parseImagesUsage, renderImagesError, renderImagesResponse } from '../../src/images/response.ts'; +import { renderErrorEnvelope, upstreamErrorMessage } from '../../src/common/error-envelope.ts'; +import { parseImagesResponse, parseImagesUsage, renderImagesResponse } from '../../src/images/response.ts'; describe('images response ingress', () => { test('the answer is parsed and rendered back whole, fields the gateway does not model included', () => { @@ -65,11 +66,11 @@ describe('images failures', () => { test('an upstream error object is what the client is sent, and anything else becomes the gateway envelope', () => { const upstream = { error: { message: 'Your request was rejected', type: 'image_generation_user_error', code: 'moderation_blocked' } }; - expect(imagesErrorMessage(upstream)).toBe('Your request was rejected'); - expect(renderImagesError('Your request was rejected', upstream)).toBe(upstream); + expect(upstreamErrorMessage(upstream)).toBe('Your request was rejected'); + expect(renderErrorEnvelope('Your request was rejected', upstream)).toBe(upstream); - expect(imagesErrorMessage('upstream is down')).toBeUndefined(); - expect(renderImagesError('Model gpt-image-1 is not available on any configured upstream.', undefined)) + expect(upstreamErrorMessage('upstream is down')).toBeUndefined(); + expect(renderErrorEnvelope('Model gpt-image-1 is not available on any configured upstream.', undefined)) .toEqual({ error: { message: 'Model gpt-image-1 is not available on any configured upstream.', type: 'api_error' } }); }); }); diff --git a/packages/protocols/src/common/error-envelope.ts b/packages/protocols/src/common/error-envelope.ts new file mode 100644 index 0000000000..c92fbd7408 --- /dev/null +++ b/packages/protocols/src/common/error-envelope.ts @@ -0,0 +1,21 @@ +// What a client is sent when a run produced no answer, and how an upstream's own words are +// read out of what it sent. +// +// An upstream that refused in its own words is handed on in them — a client reads the code +// and type inside, and the gateway has nothing truer to say than what the upstream said. This +// is why the test is whether an upstream answered at all rather than which shape it answered +// in: an OpenAI family's error is `{error:{...}}` and a rerank client's is `{message}`, and +// both are already the shape that client reads. +// +// A refusal that never reached an upstream has only the gateway's own words, and those go in +// the envelope every protocol here writes errors in. + +import { isJsonObject } from './json.ts'; + +export const upstreamErrorMessage = (body: unknown): string | undefined => { + if (!isJsonObject(body) || !isJsonObject(body.error)) return undefined; + return typeof body.error.message === 'string' ? body.error.message : undefined; +}; + +export const renderErrorEnvelope = (message: string, upstreamBody?: unknown): Record => + isJsonObject(upstreamBody) ? upstreamBody : { error: { message, type: 'api_error' } }; diff --git a/packages/protocols/src/common/index.ts b/packages/protocols/src/common/index.ts index 445cd1220e..2b7a1293d2 100644 --- a/packages/protocols/src/common/index.ts +++ b/packages/protocols/src/common/index.ts @@ -13,5 +13,6 @@ export * from './sse.ts'; export * from './parse-sse.ts'; export * from './parse-events.ts'; +export { renderErrorEnvelope, upstreamErrorMessage } from './error-envelope.ts'; export { isJsonObject, type JsonObject } from './json.ts'; export { captureExtras } from './reassemble-extras.ts'; diff --git a/packages/protocols/src/images/index.ts b/packages/protocols/src/images/index.ts index 25f8249cbd..6e050e7876 100644 --- a/packages/protocols/src/images/index.ts +++ b/packages/protocols/src/images/index.ts @@ -105,4 +105,4 @@ export interface CanonicalImagesResponse { } export { parseImagesEditsRequest, parseImagesGenerationsRequest } from './request.ts'; -export { imagesErrorMessage, parseImagesResponse, parseImagesUsage, renderImagesError, renderImagesResponse } from './response.ts'; +export { parseImagesResponse, parseImagesUsage, renderImagesResponse } from './response.ts'; diff --git a/packages/protocols/src/images/response.ts b/packages/protocols/src/images/response.ts index abafd7b73f..9ad1173c6f 100644 --- a/packages/protocols/src/images/response.ts +++ b/packages/protocols/src/images/response.ts @@ -92,18 +92,4 @@ export const renderImagesResponse = (response: CanonicalImagesResponse): Record< /** What an upstream that refused said about it, when it said it in the shape this protocol * uses. Anything else has no message to lift and is reported as the body it was. */ -export const imagesErrorMessage = (body: unknown): string | undefined => { - if (!isRecord(body) || !isRecord(body.error)) return undefined; - return typeof body.error.message === 'string' ? body.error.message : undefined; -}; -/** - * What the client is sent when the run produced no images. An upstream that answered with an - * error object is rendered as that object — a client reads the code inside it, and the gateway - * has nothing truer to say than what the upstream said — and everything else, a refusal that - * never reached an upstream included, becomes the gateway's own envelope over the message. - */ -export const renderImagesError = (message: string, upstreamBody: unknown): Record => - isRecord(upstreamBody) && isRecord(upstreamBody.error) - ? upstreamBody - : { error: { message, type: 'api_error' } }; From 36ceb37987a0aab1272440664ab56416b319e633 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 00:28:54 +0800 Subject: [PATCH 18/79] test(gateway): assert the failure behaviour the pipeline actually rules Three tests described the passthrough surface that /v1/embeddings and /v2/rerank no longer run through, and two of them asserted behaviour the design replaces: a 2xx body a JSON protocol cannot read was forwarded verbatim with the upstream's own status. A gateway that never read the answer cannot claim to have served the request, so it refuses in its own words and the upstream still counts as called having reported nothing. The rest are kept as they were, against the pipeline: a failed usage write leaves the answer alone and is still reported, and the last candidate's refusal reaches the client with its status, its headers and its own words. --- .../data-plane/embeddings/failures_test.ts | 152 ++++++++++++++++++ .../__tests__/data-plane/rerank/serve_test.ts | 12 +- .../shared/passthrough-serve_test.ts | 102 ------------ 3 files changed, 161 insertions(+), 105 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/embeddings/failures_test.ts diff --git a/packages/gateway/__tests__/data-plane/embeddings/failures_test.ts b/packages/gateway/__tests__/data-plane/embeddings/failures_test.ts new file mode 100644 index 0000000000..c1142c6ece --- /dev/null +++ b/packages/gateway/__tests__/data-plane/embeddings/failures_test.ts @@ -0,0 +1,152 @@ +// What a run does when something other than the answer goes wrong. Each of these was a +// behaviour of the passthrough serve that /v1/embeddings used to run through, and each is +// kept here against the pipeline that replaced it — two unchanged, one deliberately not. + +import { test, vi } from 'vitest'; + +import { buildCustomUpstreamRecord, flushAsyncWork, requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; +import { jsonResponse, withMockedFetch, assertEquals } from '@floway-dev/test-utils'; + +const registerEmbeddingsUpstream = async ( + repo: Awaited>['repo'], +): Promise => { + await repo.upstreams.deleteAll(); + clearInProcessCopilotTokenCache(); + await repo.upstreams.save(buildCustomUpstreamRecord({ + id: 'up_embeddings', + name: 'Embedding Provider', + sortOrder: 100, + config: { + baseUrl: 'https://embeddings.example.com', + authStyle: 'bearer', + ingressHeadersRules: [], + apiKey: 'sk-embeddings', + endpoints: {}, + }, + })); +}; + +const upstreamModels = () => jsonResponse({ object: 'list', data: [{ id: 'custom-embed-model' }] }); + +const askForEmbeddings = async (key: string): Promise => await requestApp('/v1/embeddings', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': key }, + body: JSON.stringify({ model: 'custom-embed-model', input: 'hi' }), +}); + +// The answer does not depend on the row. Settlement hands the write to the background rather +// than awaiting it, so a repository that rejects cannot turn an upstream's 2xx into a 502 — +// and the failure is still reported, because a write nobody hears about is one nobody fixes. +test('a usage write that fails leaves the answer alone and still reports', async () => { + const { apiKey, repo } = await setupAppTest(); + await registerEmbeddingsUpstream(repo); + + repo.usage.record = () => Promise.reject(new Error('simulated SQL write failure')); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + + try { + await withMockedFetch( + request => { + const url = new URL(request.url); + if (url.pathname === '/v1/models') return upstreamModels(); + if (url.pathname === '/v1/embeddings') { + return jsonResponse({ + object: 'list', + model: 'custom-embed-model', + data: [{ object: 'embedding', index: 0, embedding: [0.5] }], + usage: { prompt_tokens: 3, total_tokens: 3 }, + }); + } + throw new Error(`Unhandled fetch ${request.url}`); + }, + async () => { + const response = await askForEmbeddings(apiKey.key); + + assertEquals(response.status, 200); + const body = await response.json() as { data: { embedding: number[] }[] }; + assertEquals(body.data[0].embedding, [0.5]); + await flushAsyncWork(); + }, + ); + + assertEquals(errorSpy.mock.calls.some(call => String(call[0]).includes('usage')), true); + } finally { + errorSpy.mockRestore(); + } +}); + +// The replaced surface forwarded such a body verbatim with the upstream's 200. A protocol +// that requires JSON and did not get JSON has no answer to serve, so the gateway says so +// itself — it cannot claim to have served a request whose answer it never read. +test('a 2xx body the embeddings protocol cannot read is refused rather than forwarded', async () => { + const { apiKey, repo } = await setupAppTest(); + await registerEmbeddingsUpstream(repo); + + await withMockedFetch( + request => { + const url = new URL(request.url); + if (url.pathname === '/v1/models') return upstreamModels(); + if (url.pathname === '/v1/embeddings') { + return new Response(new Uint8Array([0xde, 0xad, 0xbe, 0xef]), { + status: 200, + headers: { 'content-type': 'application/octet-stream' }, + }); + } + throw new Error(`Unhandled fetch ${request.url}`); + }, + async () => { + const response = await askForEmbeddings(apiKey.key); + + assertEquals(response.status, 502); + assertEquals(response.headers.get('content-type'), 'application/json'); + const body = await response.json() as { error: { message: string } }; + assertEquals(body.error.message.includes('the embeddings protocol cannot read'), true); + await flushAsyncWork(); + }, + ); + + // An upstream that was called and reported nothing still counts as called: the row names + // the request, and a reading we could not parse is, from here, no reading rather than zero. + const usage = await repo.usage.listAll(); + assertEquals(usage.length, 1); + assertEquals(usage[0].requests, 1); + assertEquals(usage[0].metrics, []); +}); + +// The last failure is the one the client is answered with, and an upstream that refused in +// its own words is answered in them — with the status and the headers that carry what a +// client does next. +test('when every candidate refuses the client gets the last upstream-s own refusal', async () => { + const { apiKey, repo } = await setupAppTest(); + await repo.upstreams.deleteAll(); + clearInProcessCopilotTokenCache(); + for (const [id, host, order] of [['up_a', 'up-a.example.com', 100], ['up_b', 'up-b.example.com', 200]] as const) { + await repo.upstreams.save(buildCustomUpstreamRecord({ + id, name: id, sortOrder: order, + config: { baseUrl: `https://${host}`, authStyle: 'bearer', ingressHeadersRules: [], apiKey: 'sk-x', endpoints: {} }, + })); + } + + await withMockedFetch( + request => { + const url = new URL(request.url); + if (url.pathname === '/v1/models') return upstreamModels(); + if (url.hostname === 'up-a.example.com') return new Response('first upstream unavailable', { status: 503 }); + if (url.hostname === 'up-b.example.com') { + return new Response(JSON.stringify({ error: { message: 'rate limited' } }), { + status: 429, headers: { 'content-type': 'application/json', 'retry-after': '17' }, + }); + } + throw new Error(`Unhandled fetch ${request.url}`); + }, + async () => { + const response = await askForEmbeddings(apiKey.key); + + assertEquals(response.status, 429); + assertEquals(response.headers.get('retry-after'), '17'); + assertEquals(await response.json(), { error: { message: 'rate limited' } }); + await flushAsyncWork(); + }, + ); +}); diff --git a/packages/gateway/__tests__/data-plane/rerank/serve_test.ts b/packages/gateway/__tests__/data-plane/rerank/serve_test.ts index ef4e067cc6..ea74bed97a 100644 --- a/packages/gateway/__tests__/data-plane/rerank/serve_test.ts +++ b/packages/gateway/__tests__/data-plane/rerank/serve_test.ts @@ -516,7 +516,10 @@ test('cross-protocol success still validates result items before rendering', asy assertEquals(performance.errorsNoOutput, 1); }); -test('same-protocol malformed JSON is forwarded as request-only usage', async () => { +// The replaced surface forwarded these bytes verbatim with the upstream's 200. Rerank is a +// JSON protocol, so a body that is not JSON at all is no answer to serve and the gateway says +// so itself rather than passing on something it never read. +test('same-protocol answer that is not JSON is refused rather than forwarded', async () => { const { apiKey, repo } = await setupAppTest(); await saveRerankUpstream(repo, { protocol: 'jina-v1' }); @@ -528,12 +531,15 @@ test('same-protocol malformed JSON is forwarded as request-only usage', async () headers: requestHeaders(apiKey.key), body: JSON.stringify({ model: 'public-reranker', query: 'query', documents: ['one'] }), }); - assertEquals(response.status, 200); - assertEquals(await response.text(), '{not-json'); + assertEquals(response.status, 502); + const body = await response.json() as { error: { message: string } }; + assertEquals(body.error.message.includes('the rerank protocol cannot read'), true); }, ); await flushAsyncWork(); + // Called, and reported nothing this reader could make sense of — which still counts as a + // request against the upstream that was dialled. const [usage] = await repo.usage.listAll(); assertEquals(usage.requests, 1); assertEquals(usage.metrics, []); diff --git a/packages/gateway/__tests__/data-plane/shared/passthrough-serve_test.ts b/packages/gateway/__tests__/data-plane/shared/passthrough-serve_test.ts index fa8e4bec0d..bc7e342f91 100644 --- a/packages/gateway/__tests__/data-plane/shared/passthrough-serve_test.ts +++ b/packages/gateway/__tests__/data-plane/shared/passthrough-serve_test.ts @@ -41,59 +41,6 @@ const registerEmbeddingsUpstream = async ( })); }; -test('passthrough-serve: usage-record failure does not turn upstream 2xx into 502', async () => { - const { apiKey, repo } = await setupAppTest(); - await registerEmbeddingsUpstream(repo); - - repo.usage.record = () => Promise.reject(new Error('simulated SQL write failure')); - const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - - try { - await withMockedFetch( - request => { - const url = new URL(request.url); - if (url.hostname === 'passthrough.example.com' && url.pathname === '/v1/models') { - return jsonResponse({ object: 'list', data: [{ id: 'custom-embed-model' }] }); - } - if (url.hostname === 'passthrough.example.com' && url.pathname === '/v1/embeddings') { - assertEquals(request.headers.get('anthropic-beta'), null); - assertEquals(request.headers.get('x-client-request-id'), null); - assertEquals(request.headers.get('x-debug'), null); - return jsonResponse({ - object: 'list', - model: 'custom-embed-model', - data: [{ object: 'embedding', index: 0, embedding: [0.5] }], - usage: { prompt_tokens: 3, total_tokens: 3 }, - }); - } - throw new Error(`Unhandled fetch ${request.url}`); - }, - async () => { - const response = await requestApp('/v1/embeddings', { - method: 'POST', - headers: { - 'content-type': 'application/json', - 'x-api-key': apiKey.key, - 'anthropic-beta': 'context-1m-2025-08-07', - 'x-client-request-id': 'request-1', - 'x-debug': 'discard', - }, - body: JSON.stringify({ model: 'custom-embed-model', input: 'hi' }), - }); - - assertEquals(response.status, 200); - const body = await response.json() as { data: Array<{ embedding: number[] }> }; - assertEquals(body.data[0].embedding, [0.5]); - await flushAsyncWork(); - }, - ); - - assertEquals(errorSpy.mock.calls.some(call => call[0] === 'Failed to record usage:'), true); - } finally { - errorSpy.mockRestore(); - } -}); - test('passthrough-serve: Custom resolves configured ingress header rules before provider dispatch', async () => { const { apiKey, repo } = await setupAppTest(); await registerEmbeddingsUpstream(repo, [ @@ -136,55 +83,6 @@ test('passthrough-serve: Custom resolves configured ingress header rules before ); }); -test('passthrough-serve: non-JSON 2xx upstream body is forwarded verbatim with a request-only usage record', async () => { - const { apiKey, repo } = await setupAppTest(); - await registerEmbeddingsUpstream(repo); - - const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); - - try { - const binary = new Uint8Array([0xde, 0xad, 0xbe, 0xef]); - await withMockedFetch( - request => { - const url = new URL(request.url); - if (url.hostname === 'passthrough.example.com' && url.pathname === '/v1/models') { - return jsonResponse({ object: 'list', data: [{ id: 'custom-embed-model' }] }); - } - if (url.hostname === 'passthrough.example.com' && url.pathname === '/v1/embeddings') { - return new Response(binary, { - status: 200, - headers: { 'content-type': 'application/octet-stream' }, - }); - } - throw new Error(`Unhandled fetch ${request.url}`); - }, - async () => { - const response = await requestApp('/v1/embeddings', { - method: 'POST', - headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, - body: JSON.stringify({ model: 'custom-embed-model', input: 'hi' }), - }); - - assertEquals(response.status, 200); - assertEquals(response.headers.get('content-type'), 'application/octet-stream'); - const bytes = new Uint8Array(await response.arrayBuffer()); - assertEquals(Array.from(bytes), Array.from(binary)); - await flushAsyncWork(); - }, - ); - - const usage = await repo.usage.listAll(); - assertEquals(usage.length, 1); - assertEquals(usage[0].requests, 1); - assertEquals(usage[0].metrics, []); - // The parse failure is observable through console.warn so operators can - // correlate missing usage rows against upstream body shape regressions. - assertEquals(warnSpy.mock.calls.some(call => typeof call[0] === 'string' && call[0].includes('passthrough-serve: failed to parse 2xx upstream body for /embeddings')), true); - } finally { - warnSpy.mockRestore(); - } -}); - test('passthrough-serve: response header blocklist preserves vendor metadata and drops unsafe headers', async () => { const { apiKey, repo } = await setupAppTest(); await registerEmbeddingsUpstream(repo); From 35f4aedf11d8e9f85857bd76017afeb3cf8310b0 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 00:36:44 +0800 Subject: [PATCH 19/79] fix(gateway): read the request before deciding what kind of run it opens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Whether a request streams is written in the request — `stream: true` in a JSON body, a form field in a multipart upload — and the body can only be read once. The prologue read it internally and took the answer as a parameter, so a family that learns it from the body had no order in which to call it. Guessing `false` is not harmless: the abort controller a streaming run cancels its upstream with is minted from that flag, so a client that disconnected would stop cancelling anything. Reading the ingress is now its own step, and the run opens once the handler knows what was asked for — which also lets the requested model reach the dump through the context that stamps it rather than a second call afterwards. --- .../gateway/src/data-plane/embeddings/http.ts | 17 +++--- .../gateway/src/data-plane/images/http.ts | 20 +++---- .../gateway/src/data-plane/pipeline/serve.ts | 52 ++++++++++++------- .../gateway/src/data-plane/rerank/serve.ts | 13 ++--- packages/protocols/src/images/response.ts | 4 -- 5 files changed, 59 insertions(+), 47 deletions(-) diff --git a/packages/gateway/src/data-plane/embeddings/http.ts b/packages/gateway/src/data-plane/embeddings/http.ts index c70ad38c94..758a88f1c4 100644 --- a/packages/gateway/src/data-plane/embeddings/http.ts +++ b/packages/gateway/src/data-plane/embeddings/http.ts @@ -4,13 +4,13 @@ // the client sent, hand it over, and turn what the run answered with into a response. // Everything between is stages. -import { move } from '@floway-dev/pipeline'; -import { parseEmbeddingsRequest, type ParsedEmbeddingsRequest } from '@floway-dev/protocols/embeddings'; import type { Context } from 'hono'; import { embeddingsServePipeline } from './pipeline.ts'; -import { openPrologue, serveThrough } from '../pipeline/serve.ts'; +import { openPrologue, readIngress, serveThrough } from '../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; +import { move } from '@floway-dev/pipeline'; +import { parseEmbeddingsRequest, type ParsedEmbeddingsRequest } from '@floway-dev/protocols/embeddings'; // The contract reports a malformed request by throwing; what the client is owed is a 400 // carrying the reason. @@ -23,20 +23,21 @@ const readRequest = (bytes: Uint8Array): { type: 'ok'; parsed: ParsedEmbeddingsR }; export const embeddings = async (c: Context): Promise => { - const prologue = await openPrologue(c, { wantsStream: false }); - const result = readRequest(prologue.bytes); + const ingress = await readIngress(c); + const result = readRequest(ingress.body.bytes); if (result.type === 'invalid') { // A request the gateway could not read never reaches a pipeline: there is no model to // resolve and no attempt to make, so there is nothing for a run to record. - prologue.gateway.dump?.error('gateway'); + const refused = openPrologue(c, ingress, { wantsStream: false }); + refused.gateway.dump?.error('gateway'); return finalizeGatewayResponse( - prologue.gateway, + refused.gateway, Response.json({ error: { message: result.message, type: 'api_error' } }, { status: 400 }), ); } const { model, request } = result.parsed; - prologue.gateway.dump?.requestedModel(model); + const prologue = openPrologue(c, ingress, { wantsStream: false, model }); return await serveThrough( prologue, diff --git a/packages/gateway/src/data-plane/images/http.ts b/packages/gateway/src/data-plane/images/http.ts index 8813e75976..c1ff22bb43 100644 --- a/packages/gateway/src/data-plane/images/http.ts +++ b/packages/gateway/src/data-plane/images/http.ts @@ -7,7 +7,7 @@ import type { Context } from 'hono'; import { imagesServePipeline } from './pipeline.ts'; -import { openPrologue, serveThrough, type Prologue } from '../pipeline/serve.ts'; +import { openPrologue, readIngress, serveThrough, type Ingress } from '../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; import { move } from '@floway-dev/pipeline'; import { parseImagesEditsRequest, parseImagesGenerationsRequest, type ParsedImagesRequest } from '@floway-dev/protocols/images'; @@ -16,7 +16,8 @@ import { parseImagesEditsRequest, parseImagesGenerationsRequest, type ParsedImag * run answered with into a response. The contract reports a malformed request by throwing; * what the client is owed is a 400 carrying the reason. */ const serveImages = async ( - prologue: Prologue, + c: Context, + ingress: Ingress, read: () => ParsedImagesRequest | Promise, ): Promise => { let parsed: ParsedImagesRequest; @@ -25,9 +26,10 @@ const serveImages = async ( } catch (error) { // A request the gateway could not read never reaches a pipeline: there is no model to // resolve and no attempt to make, so there is nothing for a run to record. - prologue.gateway.dump?.error('gateway'); + const refused = openPrologue(c, ingress, { wantsStream: false }); + refused.gateway.dump?.error('gateway'); return finalizeGatewayResponse( - prologue.gateway, + refused.gateway, Response.json( { error: { message: error instanceof Error ? error.message : String(error), type: 'api_error' } }, { status: 400 }, @@ -36,7 +38,7 @@ const serveImages = async ( } const { model, request } = parsed; - prologue.gateway.dump?.requestedModel(model); + const prologue = openPrologue(c, ingress, { wantsStream: false, model }); return await serveThrough( prologue, @@ -51,13 +53,13 @@ const serveImages = async ( }; export const imagesGenerations = async (c: Context): Promise => { - const prologue = await openPrologue(c, { wantsStream: false }); - return await serveImages(prologue, () => parseImagesGenerationsRequest(prologue.bytes)); + const ingress = await readIngress(c); + return await serveImages(c, ingress, () => parseImagesGenerationsRequest(ingress.body.bytes)); }; export const imagesEdits = async (c: Context): Promise => { - const prologue = await openPrologue(c, { wantsStream: false }); + const ingress = await readIngress(c); // Which of the two bodies arrived is a header's statement and not the payload's, so the // contract is handed the media type alongside the bytes. - return await serveImages(prologue, () => parseImagesEditsRequest(c.req.header('content-type'), prologue.bytes)); + return await serveImages(c, ingress, () => parseImagesEditsRequest(c.req.header('content-type'), ingress.body.bytes)); }; diff --git a/packages/gateway/src/data-plane/pipeline/serve.ts b/packages/gateway/src/data-plane/pipeline/serve.ts index c584192011..c142119e52 100644 --- a/packages/gateway/src/data-plane/pipeline/serve.ts +++ b/packages/gateway/src/data-plane/pipeline/serve.ts @@ -6,35 +6,51 @@ // handler calls `run` like any other caller. What makes a run *recorded* is the prologue // resolving a dump sink, not a flag anywhere below it. -import { run } from '@floway-dev/pipeline'; -import type { Event, Pipeline } from '@floway-dev/pipeline'; import type { Context } from 'hono'; import type { ContentfulStatusCode } from 'hono/utils/http-status'; -import type { ModelCandidate } from '@floway-dev/provider'; import type { AttemptSelector, GatewayFacts } from './facts.ts'; import type { GatewayServices } from './services.ts'; import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; import { consoleLogSink } from '../../runtime/log.ts'; import { createGatewayCtxFromHono, finalizeGatewayResponse, type GatewayCtx } from '../shared/gateway-ctx.ts'; -import { readRequestBody, takeRequestBody } from '../shared/request-body.ts'; +import { readRequestBody, takeRequestBody, type RequestBody } from '../shared/request-body.ts'; +import type { Event, Pipeline } from '@floway-dev/pipeline'; +import { run } from '@floway-dev/pipeline'; +import type { ModelCandidate } from '@floway-dev/provider'; type Slice = { [P in K]: GatewayFacts[P] }; +/** What the client sent, read once, before anything has decided what it means. */ +export interface Ingress { + readonly body: RequestBody; + readonly headers: readonly (readonly [string, string])[]; +} + +/** + * Reads the request. + * + * This is separate from opening the run because whether a request streams is written *in* + * the request — `stream: true` in a JSON body, a form field in a multipart upload — and the + * body can only be read once. A run opened before the read would have to guess, and guessing + * `false` is not harmless: the abort controller a streaming run cancels its upstream with is + * minted from that flag, so a client that disconnected would stop cancelling anything. + */ +export const readIngress = async (c: Context): Promise => ({ + body: await readRequestBody(c), + headers: [...c.req.raw.headers], +}); + export interface Prologue { readonly services: GatewayServices; readonly gateway: GatewayCtx; - /** What the client sent, read once. `takeRequestBody` empties the buffer it was given so - * the handler cannot retain the wire body past the dump's own copy, so what is kept here - * is the bytes themselves rather than the record they came from. */ - readonly bytes: Uint8Array; - readonly streamError: string | null; readonly headers: readonly (readonly [string, string])[]; } /** - * Opens a run: the request context the telemetry stages read, the services the stages are - * given, and the bytes the client sent. + * Opens a run: the request context the telemetry stages read, and the services the stages + * are given. It takes the bytes the handler has already read and hands them to the dump, + * which is what leaves the handler's own copy free to be released. * * The candidate store is the resolver the ruling names — "the resolver is the service and * the selector is a fact". A `ModelCandidate` carries the provider's instance, its fetcher @@ -42,17 +58,15 @@ export interface Prologue { * and the provider's own cache refresh would break. So the stage that enumerates hands the * live ones here, and the stage that dials asks for one back by selector. */ -export const openPrologue = async ( +export const openPrologue = ( c: Context, + ingress: Ingress, options: { readonly wantsStream: boolean; readonly model?: string }, -): Promise => { - const body = await readRequestBody(c); - const bytes = body.bytes; - const streamError = body.streamError; +): Prologue => { const gateway = createGatewayCtxFromHono(c, { wantsStream: options.wantsStream, ...(options.model === undefined ? {} : { model: options.model }), - requestBody: takeRequestBody(body), + requestBody: takeRequestBody(ingress.body), backgroundScheduler: backgroundSchedulerFromContext(c), }); @@ -60,9 +74,7 @@ export const openPrologue = async ( return { gateway, - bytes, - streamError, - headers: [...c.req.raw.headers], + headers: ingress.headers, services: { gateway, log: consoleLogSink, diff --git a/packages/gateway/src/data-plane/rerank/serve.ts b/packages/gateway/src/data-plane/rerank/serve.ts index a0f2e4d930..cce8d34062 100644 --- a/packages/gateway/src/data-plane/rerank/serve.ts +++ b/packages/gateway/src/data-plane/rerank/serve.ts @@ -9,7 +9,7 @@ import type { Context } from 'hono'; import { rerankServePipeline } from './pipeline.ts'; -import { openPrologue, serveThrough } from '../pipeline/serve.ts'; +import { openPrologue, readIngress, serveThrough } from '../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; import { move } from '@floway-dev/pipeline'; import type { RerankSourceProtocol } from '@floway-dev/protocols/common'; @@ -36,20 +36,21 @@ const readRequest = ( }; export const rerank = (sourceProtocol: RerankSourceProtocol) => async (c: Context): Promise => { - const prologue = await openPrologue(c, { wantsStream: false }); - const result = readRequest(sourceProtocol, prologue.bytes); + const ingress = await readIngress(c); + const result = readRequest(sourceProtocol, ingress.body.bytes); if (result.type === 'invalid') { // A request the gateway could not read never reaches a pipeline: there is no model to // resolve and no attempt to make, so there is nothing for a run to record. - prologue.gateway.dump?.error('gateway'); + const refused = openPrologue(c, ingress, { wantsStream: false }); + refused.gateway.dump?.error('gateway'); return finalizeGatewayResponse( - prologue.gateway, + refused.gateway, Response.json({ error: { message: result.message, type: 'api_error' } }, { status: 400 }), ); } const { model, request } = result.parsed; - prologue.gateway.dump?.requestedModel(model); + const prologue = openPrologue(c, ingress, { wantsStream: false, model }); return await serveThrough( prologue, diff --git a/packages/protocols/src/images/response.ts b/packages/protocols/src/images/response.ts index 9ad1173c6f..15da5240c2 100644 --- a/packages/protocols/src/images/response.ts +++ b/packages/protocols/src/images/response.ts @@ -89,7 +89,3 @@ const splitModality = ( }; export const renderImagesResponse = (response: CanonicalImagesResponse): Record => response.raw; - -/** What an upstream that refused said about it, when it said it in the shape this protocol - * uses. Anything else has no message to lift and is reported as the body it was. */ - From c0f8259537ba0f5ae67334fb0fef2b75efca2e06 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 00:43:08 +0800 Subject: [PATCH 20/79] feat(gateway): answer with a stream, and settle what it billed once it ends MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A streaming family's answer is the stream, and the seam could only build a response from bytes it already had. It now takes frames as an answer of its own shape: the response is staged on the context first, because hono's SSE helper constructs the response itself and headers passed to a constructor would be lost. A stream's usage arrives with its last chunk, which is after the run has answered — the fact carrying it is a promise, and settling from it is the epilogue's job. Settlement's write is extracted so both paths do the same thing: the stage settles what the ending had already read, and the epilogue settles what only the drained stream could say. --- .../gateway/src/data-plane/embeddings/http.ts | 1 + .../gateway/src/data-plane/images/http.ts | 1 + .../gateway/src/data-plane/pipeline/serve.ts | 61 +++++++++++++++---- .../src/data-plane/pipeline/settlement.ts | 47 +++++++++----- .../gateway/src/data-plane/rerank/serve.ts | 1 + 5 files changed, 84 insertions(+), 27 deletions(-) diff --git a/packages/gateway/src/data-plane/embeddings/http.ts b/packages/gateway/src/data-plane/embeddings/http.ts index 758a88f1c4..a9183a77e7 100644 --- a/packages/gateway/src/data-plane/embeddings/http.ts +++ b/packages/gateway/src/data-plane/embeddings/http.ts @@ -40,6 +40,7 @@ export const embeddings = async (c: Context): Promise => { const prologue = openPrologue(c, ingress, { wantsStream: false, model }); return await serveThrough( + c, prologue, embeddingsServePipeline, move({ diff --git a/packages/gateway/src/data-plane/images/http.ts b/packages/gateway/src/data-plane/images/http.ts index c1ff22bb43..5f5cfcca9c 100644 --- a/packages/gateway/src/data-plane/images/http.ts +++ b/packages/gateway/src/data-plane/images/http.ts @@ -41,6 +41,7 @@ const serveImages = async ( const prologue = openPrologue(c, ingress, { wantsStream: false, model }); return await serveThrough( + c, prologue, imagesServePipeline(request), move({ diff --git a/packages/gateway/src/data-plane/pipeline/serve.ts b/packages/gateway/src/data-plane/pipeline/serve.ts index c142119e52..61ae2be32b 100644 --- a/packages/gateway/src/data-plane/pipeline/serve.ts +++ b/packages/gateway/src/data-plane/pipeline/serve.ts @@ -7,16 +7,20 @@ // resolving a dump sink, not a flag anywhere below it. import type { Context } from 'hono'; +import { streamSSE } from 'hono/streaming'; import type { ContentfulStatusCode } from 'hono/utils/http-status'; -import type { AttemptSelector, GatewayFacts } from './facts.ts'; +import type { AttemptSelector, BillableEntity, GatewayFacts } from './facts.ts'; import type { GatewayServices } from './services.ts'; +import { settleBillable } from './settlement.ts'; import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; import { consoleLogSink } from '../../runtime/log.ts'; import { createGatewayCtxFromHono, finalizeGatewayResponse, type GatewayCtx } from '../shared/gateway-ctx.ts'; import { readRequestBody, takeRequestBody, type RequestBody } from '../shared/request-body.ts'; +import { writeSSEFrames } from '../shared/sse.ts'; import type { Event, Pipeline } from '@floway-dev/pipeline'; import { run } from '@floway-dev/pipeline'; +import { sseCommentFrame, type SseFrame } from '@floway-dev/protocols/common'; import type { ModelCandidate } from '@floway-dev/provider'; type Slice = { [P in K]: GatewayFacts[P] }; @@ -97,13 +101,24 @@ export const openPrologue = ( }; /** What a family writes for the client. The status and the upstream's own headers are facts - * the pipeline already carries, so what is left here is the bytes and what they are. */ -export interface Rendered { - readonly body: BodyInit; - /** Owned by whichever stage serialized the body, never by the upstream — a media type - * describing bytes the gateway wrote itself is the one thing an upstream cannot say. */ - readonly contentType: string; -} + * the pipeline already carries, so what is left here is the answer itself. */ +export type Rendered = + | { + readonly body: BodyInit; + /** Owned by whichever stage serialized the body, never by the upstream — a media type + * describing bytes the gateway wrote itself is the one thing an upstream cannot say. */ + readonly contentType: string; + } + | { + /** An answer that *is* a stream. The frames go out as they arrive, which is why nothing + * above waits for them and why what they billed is settled afterwards. */ + readonly frames: AsyncIterable; + }; + +/** What a streaming family will have been billed once its frames run out. A stream's usage + * arrives with its last chunk, which is after the run has answered — so the run hands up a + * promise and settlement of it belongs here, after the answer is on its way. */ +export type DeferredUsage = (facts: Exit) => Promise | null; /** * Runs a family's pipeline and turns what it answered with into a response. @@ -117,19 +132,41 @@ export const serveThrough = async < Entry extends object, Exit extends Slice<'response.http.status' | 'response.http.headers'>, >( + c: Context, prologue: Prologue, pipeline: Pipeline, entry: Entry, render: (facts: Exit) => Rendered, + deferredUsage?: DeferredUsage, ): Promise => { const { facts, drain } = await run(pipeline, entry, prologue.services as never); prologue.services.background(drain()); const answer = render(facts); + const pending = deferredUsage?.(facts) ?? null; + if (pending !== null) { + prologue.services.background(pending.then(billable => { + settleBillable({ ...prologue.services, log: consoleLogSink }, billable, false); + })); + } + + const status = facts['response.http.status'] as ContentfulStatusCode; + if ('frames' in answer) { + // Hono's streamSSE builds the response itself, so what the client is to see has to be + // staged on the context before it is called rather than passed to a constructor. + for (const [name, value] of facts['response.http.headers']) c.header(name, value); + c.status(status); + return streamSSE(c, async stream => { + await writeSSEFrames(stream, answer.frames, { + keepAlive: { frame: sseCommentFrame('keepalive') }, + ...(prologue.gateway.downstreamAbortController === undefined + ? {} + : { downstreamAbortController: prologue.gateway.downstreamAbortController }), + }); + }); + } + const headers = new Headers(facts['response.http.headers'].map(([name, value]) => [name, value])); headers.set('content-type', answer.contentType); - return finalizeGatewayResponse( - prologue.gateway, - new Response(answer.body, { status: facts['response.http.status'] as ContentfulStatusCode, headers }), - ); + return finalizeGatewayResponse(prologue.gateway, new Response(answer.body, { status, headers })); }; diff --git a/packages/gateway/src/data-plane/pipeline/settlement.ts b/packages/gateway/src/data-plane/pipeline/settlement.ts index 27788266c7..a92a4fdb7f 100644 --- a/packages/gateway/src/data-plane/pipeline/settlement.ts +++ b/packages/gateway/src/data-plane/pipeline/settlement.ts @@ -8,21 +8,48 @@ // upstream reported zero, we did not call an upstream, we failed before reaching the // upstream's usage, the upstream did not report. -import type { GatewayFacts } from './facts.ts'; +import type { BillableEntity, GatewayFacts } from './facts.ts'; import type { GatewayServices } from './services.ts'; import { recordPerformance } from '../shared/telemetry/performance.ts'; import { recordUsage } from '../shared/telemetry/usage.ts'; import { defineStage } from '@floway-dev/pipeline'; +import type { Logger } from '@floway-dev/pipeline'; type Slice = { [P in K]: GatewayFacts[P] }; /** - * Prices what was observed and writes it, once per run. + * Writes what was billed, and the performance sample that goes with it. * - * The write is scheduled rather than awaited. A transient repository failure must not turn - * an upstream's already-flowing response into a 502, and the run's own answer does not + * The usage write is scheduled rather than awaited. A transient repository failure must not + * turn an upstream's already-flowing response into a 502, and the run's own answer does not * depend on the row — so it is handed to the background scheduler, which binds it to the * request's lifetime on both deployment targets. + * + * A streaming family settles through this too, from the prologue after the drain: its + * numbers arrive with the stream's last chunk, which is after the run has answered. + */ +export const settleBillable = ( + services: Pick & { readonly log: Logger }, + billable: readonly BillableEntity[], + failed: boolean, + outputTokens = 0, + finishedAt: number = performance.now(), +): void => { + for (const entity of billable) { + services.background(recordUsage( + services.gateway.apiKeyId, + entity.identity, + entity.quantities, + entity.pricingFacts ?? {}, + ).catch((error: unknown) => { + services.log.error('failed to record usage', { error: String(error) }); + })); + } + recordPerformance(services.gateway, services.gateway.attempt.telemetry, failed, outputTokens, finishedAt); +}; + +/** + * Prices what was observed and writes it, once per run. */ export const writeSettlement = (failed: (handedUp: Record) => boolean) => defineStage< Slice<'serve.model'>, @@ -40,18 +67,8 @@ export const writeSettlement = (failed: (handedUp: Record) => b }, execute: async (facts, next, use) => { const back = await next(facts); - for (const entity of back['response.usage.billable']) { - use.background(recordUsage( - use.gateway.apiKeyId, - entity.identity, - entity.quantities, - entity.pricingFacts ?? {}, - ).catch((error: unknown) => { - use.log.error('failed to record usage', { error: String(error) }); - })); - } // One performance sample per run, attributed to the attempt that answered. - recordPerformance(use.gateway, use.gateway.attempt.telemetry, failed(back as Record), 0, performance.now()); + settleBillable(use, back['response.usage.billable'], failed(back as Record)); return back; }, }); diff --git a/packages/gateway/src/data-plane/rerank/serve.ts b/packages/gateway/src/data-plane/rerank/serve.ts index cce8d34062..8c5d440b41 100644 --- a/packages/gateway/src/data-plane/rerank/serve.ts +++ b/packages/gateway/src/data-plane/rerank/serve.ts @@ -53,6 +53,7 @@ export const rerank = (sourceProtocol: RerankSourceProtocol) => async (c: Contex const prologue = openPrologue(c, ingress, { wantsStream: false, model }); return await serveThrough( + c, prologue, rerankServePipeline(request), move({ From 6f66f20ef8ede1c3dbe550c96a23df27435203f3 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 00:45:28 +0800 Subject: [PATCH 21/79] fix(gateway): filter the upstream's headers on the two streaming edges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completions and audio were the only families whose edge handed the upstream's headers on unread, so a body the gateway re-serialized carried the upstream's own content-length and content-encoding — a client would have been told the wrong length for bytes it was actually given. Both now filter as the other families do, and both exits declare the key the seam reads off them. --- .../gateway/src/data-plane/audio/pipeline.ts | 25 +++++++++++++------ .../src/data-plane/completions/pipeline.ts | 20 +++++++++------ 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/packages/gateway/src/data-plane/audio/pipeline.ts b/packages/gateway/src/data-plane/audio/pipeline.ts index aa7e83f5bf..d8bd24525b 100644 --- a/packages/gateway/src/data-plane/audio/pipeline.ts +++ b/packages/gateway/src/data-plane/audio/pipeline.ts @@ -21,6 +21,7 @@ import { writeSettlement } from '../pipeline/settlement.ts'; import { failover, resolveCandidates } from '../pipeline/stages.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; +import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import { compose, defineStage, move, own, type Owned, type Pipeline } from '@floway-dev/pipeline'; import { isAudioTranscriptionDoneEvent, @@ -103,8 +104,10 @@ const isEvents = (answer: CanonicalAudioTranscription | AudioTranscriptionEvents const emitAudioTranscription = defineStage< A<'ingress.audioTranscription.responseFormat'>, A<'ingress.audioTranscription.responseFormat'>, - A<'ingress.audioTranscription.responseFormat' | 'response.audioTranscription.canonical' | 'response.audioTranscription.mediaType'>, - A<'response.audioTranscription.rendered' | 'response.audioTranscription.mediaType'> & { 'response.http.status': number } + A<'ingress.audioTranscription.responseFormat' | 'response.audioTranscription.canonical' | 'response.audioTranscription.mediaType'> + & { 'response.http.headers': readonly (readonly [string, string])[] }, + A<'response.audioTranscription.rendered' | 'response.audioTranscription.mediaType'> + & { 'response.http.status': number; 'response.http.headers': readonly (readonly [string, string])[] } >({ name: 'emitAudioTranscription', through: { @@ -114,18 +117,24 @@ const emitAudioTranscription = defineStage< provides: [], }, response: { - needs: ['response.audioTranscription.canonical', 'response.audioTranscription.mediaType'], - consumes: ['response.audioTranscription.canonical'], - provides: ['response.audioTranscription.rendered', 'response.audioTranscription.mediaType', 'response.http.status'], + needs: ['response.audioTranscription.canonical', 'response.audioTranscription.mediaType', 'response.http.headers'], + consumes: ['response.audioTranscription.canonical', 'response.http.headers'], + provides: ['response.audioTranscription.rendered', 'response.audioTranscription.mediaType', 'response.http.status', 'response.http.headers'], }, }, execute: async (facts, next) => { const back = await next(facts); - const { 'response.audioTranscription.canonical': answer, ...rest } = back; + const { 'response.audioTranscription.canonical': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); if (isFailure(answer)) { return { ...rest, + 'response.http.headers': forClient, 'response.http.status': answer.status, 'response.audioTranscription.mediaType': 'application/json', 'response.audioTranscription.rendered': move({ error: { message: answer.message, type: 'api_error' } }), @@ -136,6 +145,7 @@ const emitAudioTranscription = defineStage< // client gets rather than the one the upstream gave. return { ...rest, + 'response.http.headers': forClient, 'response.http.status': 200, 'response.audioTranscription.rendered': move(isEvents(answer) ? renderSSE(answer) @@ -364,7 +374,8 @@ const narrowing = { export const audioTranscriptionServePipeline: Pipeline< A<'ingress.http.headers' | 'ingress.audioTranscription.responseFormat' | 'request.audioTranscription.form' | 'serve.model'>, A<'response.audioTranscription.rendered' | 'response.audioTranscription.mediaType' | 'response.audioTranscription.streamedUsage'> - & { 'response.http.status': number; 'response.usage.billable': readonly BillableEntity[] } + & { 'response.http.status': number; 'response.usage.billable': readonly BillableEntity[]; + 'response.http.headers': readonly (readonly [string, string])[] } > = compose('audioTranscriptionServe', [ emitAudioTranscription, writeSettlement(handedUp => isFailure((handedUp as { 'response.audioTranscription.canonical'?: unknown })['response.audioTranscription.canonical'])), diff --git a/packages/gateway/src/data-plane/completions/pipeline.ts b/packages/gateway/src/data-plane/completions/pipeline.ts index da7172542c..2c5e0f3d91 100644 --- a/packages/gateway/src/data-plane/completions/pipeline.ts +++ b/packages/gateway/src/data-plane/completions/pipeline.ts @@ -22,6 +22,7 @@ import { writeSettlement } from '../pipeline/settlement.ts'; import { failover, resolveCandidates } from '../pipeline/stages.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; +import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import { compose, defineStage, move, own, type Owned, type Pipeline } from '@floway-dev/pipeline'; import { isOpenAIUsageOnlyEventShape, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { @@ -95,8 +96,8 @@ const isFrames = (answer: CompletionsFacts['response.completions.payload']): ans const emitCompletions = defineStage< C<'ingress.completions.wantsStream' | 'ingress.completions.wantsUsageChunk' | 'request.completions.payload'>, C<'ingress.completions.wantsStream' | 'ingress.completions.wantsUsageChunk' | 'request.completions.payload'>, - C<'ingress.completions.wantsUsageChunk' | 'response.completions.payload'>, - C<'response.completions.rendered' | 'response.http.status'> + C<'ingress.completions.wantsUsageChunk' | 'response.completions.payload' | 'response.http.headers'>, + C<'response.completions.rendered' | 'response.http.status' | 'response.http.headers'> >({ name: 'emitCompletions', through: { @@ -106,9 +107,9 @@ const emitCompletions = defineStage< provides: ['request.completions.payload'], }, response: { - needs: ['response.completions.payload'], - consumes: ['response.completions.payload'], - provides: ['response.completions.rendered', 'response.http.status'], + needs: ['response.completions.payload', 'response.http.headers'], + consumes: ['response.completions.payload', 'response.http.headers'], + provides: ['response.completions.rendered', 'response.http.status', 'response.http.headers'], }, }, execute: async (facts, next) => { @@ -120,12 +121,17 @@ const emitCompletions = defineStage< : asked), }); - const { 'response.completions.payload': answer, ...rest } = back; + const { 'response.completions.payload': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); // A refusal keeps the status the upstream gave it. Anything that answered is a 200 the // gateway says itself, because what the client receives is serialized here rather than // relayed — the upstream's own status is a fact further down for whoever wants it. return { ...rest, + 'response.http.headers': forwardable.length === headers.length ? headers : move(forwardable), 'response.http.status': isFailure(answer) ? answer.status : 200, 'response.completions.rendered': move(rendered(answer, back['ingress.completions.wantsUsageChunk'])), }; @@ -366,7 +372,7 @@ const narrowing = { export const completionsServePipeline: Pipeline< C<'ingress.http.headers' | 'ingress.completions.wantsStream' | 'ingress.completions.wantsUsageChunk' | 'request.completions.payload' | 'serve.model'>, - C<'response.completions.rendered' | 'response.completions.streamedUsage' | 'response.http.status' | 'response.usage.billable'> + C<'response.completions.rendered' | 'response.completions.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> > = compose('completionsServe', [ emitCompletions, writeSettlement(handedUp => isFailure((handedUp as { 'response.completions.payload'?: unknown })['response.completions.payload'])), From bc8e3c232626fbd3b23d6647447b46dd33c9968c Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 00:48:39 +0800 Subject: [PATCH 22/79] fix(gateway): drain a streaming answer only after the client has read it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The release was scheduled the moment the run returned, so for a streaming family it consumed the very frames the client was waiting for — one connection has one reader, and the two raced for every chunk. The terminal [DONE] was the frame most often lost. Reading the frames out to the client is what releases the body they came from, so the drain now waits for that to finish, in a finally so a client that stopped reading still leaves nothing open. A buffered answer was serialized from facts the run already held and still releases at once. Completions serves through its pipeline; the streaming half settles from the usage its last chunk carries. --- .../src/data-plane/completions/http.ts | 123 ++++++------------ .../gateway/src/data-plane/pipeline/serve.ts | 29 ++++- 2 files changed, 63 insertions(+), 89 deletions(-) diff --git a/packages/gateway/src/data-plane/completions/http.ts b/packages/gateway/src/data-plane/completions/http.ts index 7c19047e82..e3cc9d016c 100644 --- a/packages/gateway/src/data-plane/completions/http.ts +++ b/packages/gateway/src/data-plane/completions/http.ts @@ -1,96 +1,55 @@ -// POST /v1/completions and /completions — OpenAI text completions -// passthrough. The endpoint sits outside the chat source/target executor: -// no protocol translation, no interceptor chain, no cross-protocol -// traversal. The request body is forwarded to the chosen provider's -// /completions verbatim; the response (single-shot JSON or streaming SSE -// depending on the client's `stream` flag) flows back through the shared -// passthroughServe scaffold. +// POST /v1/completions and /completions, served through the pipeline. +// +// The handler is a prologue and an epilogue around `completionsServePipeline`: read what the +// client sent, hand it over, and turn what the run answered with into a response. The one +// thing it decides for itself is whether the request streams, because that is written in the +// body and the run has to be opened knowing it. import type { Context } from 'hono'; -import { tokenUsageFromCompletionsUsage } from './usage.ts'; -import type { TokenUsage } from '../../repo/types.ts'; -import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; -import { createGatewayCtxFromHono, finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; +import { completionsServePipeline } from './pipeline.ts'; +import { isFrames, openPrologue, readIngress, serveThrough } from '../pipeline/serve.ts'; +import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; import { prepareJsonModelRequest } from '../shared/passthrough-request.ts'; -import { passthroughApiError, passthroughServe } from '../shared/passthrough-serve.ts'; -import { readRequestBody, takeRequestBody } from '../shared/request-body.ts'; -import { isOpenAIUsageOnlyEventShape, type ProtocolFrame } from '@floway-dev/protocols/common'; -import type { ProviderModel } from '@floway-dev/provider'; +import { move } from '@floway-dev/pipeline'; export const completions = async (c: Context): Promise => { - const requestBody = await readRequestBody(c); - const request = prepareJsonModelRequest(requestBody.bytes, 'Completions'); - // `stream` decides the response shape, so the gateway context has to learn - // it before the invalid branch — which has no body to read it from. - const wantsStream = request.type === 'ok' && request.body.stream === true; - const ctx = createGatewayCtxFromHono(c, { - wantsStream, - requestBody: takeRequestBody(requestBody), - backgroundScheduler: backgroundSchedulerFromContext(c), - }); + const ingress = await readIngress(c); + const request = prepareJsonModelRequest(ingress.body.bytes, 'Completions'); if (request.type === 'invalid') { - ctx.dump?.error('gateway'); - return finalizeGatewayResponse(ctx, passthroughApiError(c, request.message, 400)); + // A request the gateway could not read never reaches a pipeline: there is no model to + // resolve and no attempt to make, so there is nothing for a run to record. + const refused = openPrologue(c, ingress, { wantsStream: false }); + refused.gateway.dump?.error('gateway'); + return finalizeGatewayResponse( + refused.gateway, + Response.json({ error: { message: request.message, type: 'api_error' } }, { status: 400 }), + ); } - ctx.dump?.requestedModel(request.model); + const wantsStream = request.body.stream === true; const streamOptions = request.body.stream_options as { include_usage?: unknown } | null | undefined; - const clientWantsUsageChunk = streamOptions?.include_usage === true; - // Strip the inbound model; the provider re-stamps the upstream-resolved - // model id. For streaming requests we force `stream_options.include_usage` - // on so billing always sees the usage chunk — sibling keys on - // stream_options (if any) ride through unchanged. - const { model: _model, ...upstreamBodyBase } = request.body; - const upstreamBody = wantsStream - ? { ...upstreamBodyBase, stream_options: { ...(streamOptions ?? {}), include_usage: true } } - : upstreamBodyBase; + const prologue = openPrologue(c, ingress, { wantsStream, model: request.model }); - // Streaming closure: track the usage block (only on the usage-only - // chunk per OpenAI spec) and service_tier independently — service_tier - // can ride on any event root, so settling them together at the end - // lets the tier override land regardless of which chunk carried it. - let streamingUsageBlock: unknown = null; - let streamingServiceTier: string | null | undefined; - // The scaffold picks the upstream after these closures are built, and this - // endpoint has no interceptor chain to normalize usage on the way in, so - // the serving model is staked here for the billing read to consult. - let serving: { model: ProviderModel; upstreamId: string } | undefined; - const declaredExclusive = (): boolean => serving?.model.enabledFlags.has('usage-exclusive-cached-tokens') === true; - const servingIdentity = (): string => serving === undefined ? 'unresolved upstream' : `${serving.upstreamId}/${serving.model.id}`; - const transformFrame = (frame: ProtocolFrame): ProtocolFrame | null => { - if (frame.type !== 'event') return frame; - const eventRoot = frame.event as { service_tier?: string | null; usage?: unknown }; - if (eventRoot.service_tier !== undefined) streamingServiceTier = eventRoot.service_tier; - if (!isOpenAIUsageOnlyEventShape(frame.event)) return frame; - streamingUsageBlock = eventRoot.usage; - return clientWantsUsageChunk ? frame : null; - }; - const settleUsage = (): TokenUsage | null => - streamingUsageBlock === null ? null : tokenUsageFromCompletionsUsage(streamingUsageBlock, streamingServiceTier, declaredExclusive(), servingIdentity()); - - const response = await passthroughServe({ + return await serveThrough( c, - ctx, - sourceApi: '/completions', - operation: 'text_completion', - model: request.model, - kind: 'chat', - modelServesEndpoint: model => model.endpoints.completions !== undefined, - call: (provider, model, opts) => { - serving = { model, upstreamId: provider.upstreamId }; - return provider.instance.callCompletions(model, upstreamBody, ctx.abortSignal, opts); + prologue, + completionsServePipeline, + move({ + 'ingress.http.headers': prologue.headers, + 'ingress.completions.wantsStream': wantsStream, + // Whether the client asked to *see* the usage chunk. The edge turns it on upstream + // either way so billing always gets one, and drops it here when it was not asked for. + 'ingress.completions.wantsUsageChunk': streamOptions?.include_usage === true, + 'request.completions.payload': request.body, + 'serve.model': request.model, + }) as never, + facts => { + const rendered = facts['response.completions.rendered']; + return isFrames(rendered) + ? { frames: rendered } + : { body: JSON.stringify(rendered), contentType: 'application/json' }; }, - response: wantsStream - ? { format: 'sse', transformFrame, settleUsage } - : { - format: 'json', - extractBilling: (body: unknown) => { - if (!body || typeof body !== 'object') return null; - const { usage, service_tier: tier } = body as { usage?: unknown; service_tier?: string | null }; - return tokenUsageFromCompletionsUsage(usage, tier, declaredExclusive(), servingIdentity()); - }, - }, - }); - return finalizeGatewayResponse(ctx, response); + facts => facts['response.completions.streamedUsage'], + ); }; diff --git a/packages/gateway/src/data-plane/pipeline/serve.ts b/packages/gateway/src/data-plane/pipeline/serve.ts index 61ae2be32b..1513b81c51 100644 --- a/packages/gateway/src/data-plane/pipeline/serve.ts +++ b/packages/gateway/src/data-plane/pipeline/serve.ts @@ -115,6 +115,11 @@ export type Rendered = readonly frames: AsyncIterable; }; +/** Which of the two an answer turned out to be. A family's rendered fact carries whichever + * shape its run produced, and only the family knows the key it is under. */ +export const isFrames = (rendered: unknown): rendered is AsyncIterable => + typeof rendered === 'object' && rendered !== null && Symbol.asyncIterator in rendered; + /** What a streaming family will have been billed once its frames run out. A stream's usage * arrives with its last chunk, which is after the run has answered — so the run hands up a * promise and settlement of it belongs here, after the answer is on its way. */ @@ -140,7 +145,6 @@ export const serveThrough = async < deferredUsage?: DeferredUsage, ): Promise => { const { facts, drain } = await run(pipeline, entry, prologue.services as never); - prologue.services.background(drain()); const answer = render(facts); const pending = deferredUsage?.(facts) ?? null; @@ -157,15 +161,26 @@ export const serveThrough = async < for (const [name, value] of facts['response.http.headers']) c.header(name, value); c.status(status); return streamSSE(c, async stream => { - await writeSSEFrames(stream, answer.frames, { - keepAlive: { frame: sseCommentFrame('keepalive') }, - ...(prologue.gateway.downstreamAbortController === undefined - ? {} - : { downstreamAbortController: prologue.gateway.downstreamAbortController }), - }); + try { + await writeSSEFrames(stream, answer.frames, { + keepAlive: { frame: sseCommentFrame('keepalive') }, + ...(prologue.gateway.downstreamAbortController === undefined + ? {} + : { downstreamAbortController: prologue.gateway.downstreamAbortController }), + }); + } finally { + // Reading the frames to the client *is* releasing the body they came from, so the + // drain waits for that to finish. Draining alongside it would take frames out of the + // client's own stream — one connection has one reader. A client that stopped reading + // still gets here, which is what leaves nothing open behind it. + await drain(); + } }); } + // Nothing is left to read: what the client is sent was serialized from facts the run + // already held, so releasing can start at once. + prologue.services.background(drain()); const headers = new Headers(facts['response.http.headers'].map(([name, value]) => [name, value])); headers.set('content-type', answer.contentType); return finalizeGatewayResponse(prologue.gateway, new Response(answer.body, { status, headers })); From 7d86dcd652a2bd76adccde5497dfa6425c981c87 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 00:58:31 +0800 Subject: [PATCH 23/79] fix(audio): stop reading a transcription stream at its terminal event MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reader ran until the upstream closed, so an upstream that holds the connection open past transcript.text.done held the client's stream open with it. Returning at the terminal event closes the read, which cancels the upstream — the behaviour the replaced surface had. The route stays on its existing surface. Migrating it needs a decision this change cannot make: srt and vtt are parsed into cues and rendered back, which the protocol states is not byte-identical to what the upstream sent, and the replaced surface forwarded those documents unchanged. --- packages/gateway/src/data-plane/audio/pipeline.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/gateway/src/data-plane/audio/pipeline.ts b/packages/gateway/src/data-plane/audio/pipeline.ts index d8bd24525b..f1bd4ea588 100644 --- a/packages/gateway/src/data-plane/audio/pipeline.ts +++ b/packages/gateway/src/data-plane/audio/pipeline.ts @@ -324,7 +324,14 @@ const meterEvents = ( try { for await (const frame of parseSSEStream(body, { signal })) { const event = parseAudioTranscriptionStreamEvent(JSON.parse(frame.data) as unknown); - if (isAudioTranscriptionDoneEvent(event)) usage = parseAudioTranscriptionStreamUsage(event); + if (isAudioTranscriptionDoneEvent(event)) { + usage = parseAudioTranscriptionStreamUsage(event); + yield event; + // The transcript is complete, so there is nothing further to read. An upstream that + // holds the connection open past this point would otherwise keep the client's own + // stream open with it; returning here closes the read, which cancels the upstream. + return; + } yield event; } } finally { From 4dff883ef37ab010f9177453d814608aad504936 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 01:08:20 +0800 Subject: [PATCH 24/79] fix(gateway): catch every dial, and close the dump on a streaming answer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two families' endings dialled outside the try the other three had, so a refused connection ended the run as a fault: no failover to the next candidate, no performance row, and a 500 where the replaced surface answered 502. Attribution moves ahead of the dial with it, so an attempt that never returned names its own candidate. A streaming answer returned hono's response without finalizing the gateway context, and finalize is what writes the dump — so a streamed request recorded nothing at all. The upstream's frames are recorded where they are read, before the edge decides which of them the client sees. The prologue's dump sink was a stub that discarded every event while still telling the runner to accumulate them, so a key with retention configured paid for a recording nobody could read. It is absent until it has somewhere to go. --- .../data-plane/completions-pipeline_test.ts | 28 +++++++++- .../gateway/src/data-plane/audio/pipeline.ts | 46 +++++++++++++---- .../src/data-plane/completions/pipeline.ts | 51 +++++++++++++++---- .../gateway/src/data-plane/pipeline/serve.ts | 11 ++-- .../src/data-plane/pipeline/settlement.ts | 35 ++++++++++++- 5 files changed, 140 insertions(+), 31 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts index b99f8a8a22..c9a426d7d7 100644 --- a/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts @@ -244,14 +244,38 @@ describe('the completions pipeline', () => { it('writes exactly one usage row per run, however many candidates it tried', async () => { resolves([ candidate('up_a', async () => ({ response: Response.json({ error: 'nope' }, { status: 429 }), modelKey: 'k' })), - candidate('up_b', async () => ({ response: sse(chunk('hi'), usageChunk, '[DONE]'), modelKey: 'k' })), + candidate('up_b', async () => ({ + response: Response.json({ id: 'c', choices: [], usage: { prompt_tokens: 7, completion_tokens: 2 } }), + modelKey: 'k', + })), ]); - const { drain } = await serve(entryFacts()); + const { drain } = await serve(entryFacts({ + 'ingress.completions.wantsStream': false, + 'request.completions.payload': { model: 'text-model', prompt: 'hello' }, + })); await drain(); expect(recorded.usage).toHaveLength(1); expect(recorded.performance).toHaveLength(1); }); + // A stream states its usage in the chunk that ends it, which is after the run has answered. + // Settling in the stage as well would write the row twice — once for the entity that had + // reported nothing, once for what the stream turned out to say — so the pipeline hands the + // numbers up as a promise and the epilogue is what writes them. + it('defers a stream-s settlement to the promise it hands up', async () => { + resolves([candidate('up_b', async () => ({ response: sse(chunk('hi'), usageChunk, '[DONE]'), modelKey: 'k' }))]); + + const { facts, drain } = await serve(entryFacts()); + await drain(); + + expect(recorded.usage).toHaveLength(0); + const streamed = facts['response.completions.streamedUsage']; + expect(streamed).not.toBeNull(); + const billable = await streamed!; + expect(billable).toHaveLength(1); + expect(billable[0]!.quantities).toMatchObject({ input_tokens: '5', output_tokens: '7' }); + }); + // A run that reached no upstream bills nothing and samples nothing, which is what the // replaced surface did: `recordPerformance` returns early without an attempt's telemetry, // and there is no attempt. Settlement still runs — it is unconditional — and finds an diff --git a/packages/gateway/src/data-plane/audio/pipeline.ts b/packages/gateway/src/data-plane/audio/pipeline.ts index f1bd4ea588..bf7a88e35a 100644 --- a/packages/gateway/src/data-plane/audio/pipeline.ts +++ b/packages/gateway/src/data-plane/audio/pipeline.ts @@ -19,6 +19,7 @@ import { isFailure } from '../pipeline/facts.ts'; import type { GatewayServices } from '../pipeline/services.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; import { failover, resolveCandidates } from '../pipeline/stages.ts'; +import { dialFailure } from '../pipeline/upstream-body.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; @@ -188,16 +189,36 @@ const callAudioTranscriptionUpstream = defineStage< }, execute: async (facts, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); - const result = await candidate.provider.instance.callAudioTranscriptions( - providerModelOf(candidate), - { entries: facts['request.audioTranscription.form'] }, - use.gateway.abortSignal, - // The client's own headers reach the upstream from the record, not from a live request - // object: what a provider is allowed to forward is filtered per provider, and the dump - // shows what was there to filter. - buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]) => [name, value]))), - ); + // Attribution is set before the dial, so an attempt that never completes still names the + // candidate it was made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'audio_transcription'); + + let result; + try { + result = await candidate.provider.instance.callAudioTranscriptions( + providerModelOf(candidate), + { entries: facts['request.audioTranscription.form'] }, + use.gateway.abortSignal, + // The client's own headers reach the upstream from the record, not from a live request + // object: what a provider is allowed to forward is filtered per provider, and the dump + // shows what was there to filter. + buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]) => [name, value]))), + ); + } catch (error) { + use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); + // A dial that never completed reached no upstream, so nothing was billed and there are + // no headers to carry. What it leaves behind is the performance row settlement writes. + return move({ + ...facts, + 'response.audioTranscription.canonical': dialFailure(error), + 'response.audioTranscription.mediaType': null, + 'response.audioTranscription.streamedUsage': null, + 'response.usage.billable': [], + 'response.http.status': 502, + 'response.http.headers': [], + 'response.http.body': spentBody(null), + }); + } const identity = telemetryModelIdentity(candidate, result.modelKey); const format = facts['ingress.audioTranscription.responseFormat']; const status = result.response.status; @@ -382,10 +403,13 @@ export const audioTranscriptionServePipeline: Pipeline< A<'ingress.http.headers' | 'ingress.audioTranscription.responseFormat' | 'request.audioTranscription.form' | 'serve.model'>, A<'response.audioTranscription.rendered' | 'response.audioTranscription.mediaType' | 'response.audioTranscription.streamedUsage'> & { 'response.http.status': number; 'response.usage.billable': readonly BillableEntity[]; - 'response.http.headers': readonly (readonly [string, string])[] } + 'response.http.headers': readonly (readonly [string, string])[]; } > = compose('audioTranscriptionServe', [ emitAudioTranscription, - writeSettlement(handedUp => isFailure((handedUp as { 'response.audioTranscription.canonical'?: unknown })['response.audioTranscription.canonical'])), + writeSettlement( + handedUp => isFailure((handedUp as { 'response.audioTranscription.canonical'?: unknown })['response.audioTranscription.canonical']), + handedUp => (handedUp as { 'response.audioTranscription.streamedUsage'?: unknown })['response.audioTranscription.streamedUsage'] !== null, + ), resolveCandidates(narrowing), failover({ failed: handedUp => isFailure((handedUp as { 'response.audioTranscription.canonical'?: unknown })['response.audioTranscription.canonical']), diff --git a/packages/gateway/src/data-plane/completions/pipeline.ts b/packages/gateway/src/data-plane/completions/pipeline.ts index 2c5e0f3d91..8c1466d4c1 100644 --- a/packages/gateway/src/data-plane/completions/pipeline.ts +++ b/packages/gateway/src/data-plane/completions/pipeline.ts @@ -20,6 +20,7 @@ import { isFailure } from '../pipeline/facts.ts'; import type { GatewayServices } from '../pipeline/services.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; import { failover, resolveCandidates } from '../pipeline/stages.ts'; +import { dialFailure } from '../pipeline/upstream-body.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; @@ -181,16 +182,35 @@ const callCompletionsUpstream = defineStage< // The provider re-stamps whatever id it resolved upstream, so the id the client // addressed does not travel with the body. const { model: _addressed, ...body } = facts['request.completions.payload']; - const result = await candidate.provider.instance.callCompletions( - providerModelOf(candidate), - body, - use.gateway.abortSignal, - // The client's own headers reach the upstream from the record, not from a live request - // object: what a provider is allowed to forward is filtered per provider, and the dump - // shows what was there to filter. - buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]) => [name, value]))), - ); + // Attribution is set before the dial, so an attempt that never completes still names the + // candidate it was made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'text_completion'); + + let result; + try { + result = await candidate.provider.instance.callCompletions( + providerModelOf(candidate), + body, + use.gateway.abortSignal, + // The client's own headers reach the upstream from the record, not from a live request + // object: what a provider is allowed to forward is filtered per provider, and the dump + // shows what was there to filter. + buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]) => [name, value]))), + ); + } catch (error) { + use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); + // A dial that never completed reached no upstream, so nothing was billed and there are + // no headers to carry. What it leaves behind is the performance row settlement writes. + return move({ + ...facts, + 'response.completions.payload': dialFailure(error), + 'response.completions.streamedUsage': null, + 'response.usage.billable': [], + 'response.http.status': 502, + 'response.http.headers': [], + 'response.http.body': spentBody(null), + }); + } if (!result.response.ok) use.log.warn('upstream refused', { status: result.response.status }); const answer = await readUpstream( @@ -199,6 +219,7 @@ const callCompletionsUpstream = defineStage< telemetryModelIdentity(candidate, result.modelKey), candidate, use.gateway.abortSignal, + frame => { use.gateway.dump?.frame(frame); }, ); return move({ ...facts, @@ -229,6 +250,7 @@ const readUpstream = async ( identity: TelemetryModelIdentity, candidate: ModelCandidate, signal: AbortSignal | undefined, + record: (frame: ProtocolFrame) => void, ): Promise => { // An upstream was called and reported nothing — which is what every arm but a read usage // block leaves standing, and is a different statement from reporting zero. @@ -263,7 +285,7 @@ const readUpstream = async ( // the frames as they pass — so the reading costs one pass and the client's own stream is // what drives it. What it finds arrives with the last frame, long after this stage has // handed up, which is why the entity above carries no quantities. - const metered = meterFrames(parseCompletionsStream(response.body, { signal }), identity, candidate); + const metered = meterFrames(parseCompletionsStream(response.body, { signal }), identity, candidate, record); return { payload: metered.frames, streamedUsage: metered.billable, @@ -311,6 +333,7 @@ const meterFrames = ( source: AsyncIterable>, identity: TelemetryModelIdentity, candidate: ModelCandidate, + record: (frame: ProtocolFrame) => void, ): MeteredFrames => { let settle!: (billable: readonly BillableEntity[]) => void; const billable = new Promise(resolve => { settle = resolve; }); @@ -326,6 +349,9 @@ const meterFrames = ( if (root.service_tier !== undefined) tier = root.service_tier; if (isOpenAIUsageOnlyEventShape(frame.event)) usage = root.usage; } + // What the upstream sent, before the edge decides what the client sees — a dump + // reader is owed the frames that arrived, including the one metering asked for. + record(frame); yield frame; } } finally { @@ -375,7 +401,10 @@ export const completionsServePipeline: Pipeline< C<'response.completions.rendered' | 'response.completions.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> > = compose('completionsServe', [ emitCompletions, - writeSettlement(handedUp => isFailure((handedUp as { 'response.completions.payload'?: unknown })['response.completions.payload'])), + writeSettlement( + handedUp => isFailure((handedUp as { 'response.completions.payload'?: unknown })['response.completions.payload']), + handedUp => (handedUp as { 'response.completions.streamedUsage'?: unknown })['response.completions.streamedUsage'] !== null, + ), resolveCandidates(narrowing), failover({ failed: handedUp => isFailure((handedUp as { 'response.completions.payload'?: unknown })['response.completions.payload']), diff --git a/packages/gateway/src/data-plane/pipeline/serve.ts b/packages/gateway/src/data-plane/pipeline/serve.ts index 1513b81c51..c08494f5e4 100644 --- a/packages/gateway/src/data-plane/pipeline/serve.ts +++ b/packages/gateway/src/data-plane/pipeline/serve.ts @@ -18,7 +18,7 @@ import { consoleLogSink } from '../../runtime/log.ts'; import { createGatewayCtxFromHono, finalizeGatewayResponse, type GatewayCtx } from '../shared/gateway-ctx.ts'; import { readRequestBody, takeRequestBody, type RequestBody } from '../shared/request-body.ts'; import { writeSSEFrames } from '../shared/sse.ts'; -import type { Event, Pipeline } from '@floway-dev/pipeline'; +import type { Pipeline } from '@floway-dev/pipeline'; import { run } from '@floway-dev/pipeline'; import { sseCommentFrame, type SseFrame } from '@floway-dev/protocols/common'; import type { ModelCandidate } from '@floway-dev/provider'; @@ -93,9 +93,6 @@ export const openPrologue = ( } return candidate; }, - // Absent when this key has no retention configured, which is what keeps recording - // conditional: the runner does none of it rather than doing it and discarding. - ...(gateway.dump === null ? {} : { dump: (_event: Event) => {} }), }, }; }; @@ -160,7 +157,9 @@ export const serveThrough = async < // staged on the context before it is called rather than passed to a constructor. for (const [name, value] of facts['response.http.headers']) c.header(name, value); c.status(status); - return streamSSE(c, async stream => { + // The dump is closed the same way on both paths. Hono builds the streaming response + // itself, so what is finalized is what it returned rather than one constructed here. + return finalizeGatewayResponse(prologue.gateway, streamSSE(c, async stream => { try { await writeSSEFrames(stream, answer.frames, { keepAlive: { frame: sseCommentFrame('keepalive') }, @@ -175,7 +174,7 @@ export const serveThrough = async < // still gets here, which is what leaves nothing open behind it. await drain(); } - }); + })); } // Nothing is left to read: what the client is sent was serialized from facts the run diff --git a/packages/gateway/src/data-plane/pipeline/settlement.ts b/packages/gateway/src/data-plane/pipeline/settlement.ts index a92a4fdb7f..80d4d81517 100644 --- a/packages/gateway/src/data-plane/pipeline/settlement.ts +++ b/packages/gateway/src/data-plane/pipeline/settlement.ts @@ -10,13 +10,33 @@ import type { BillableEntity, GatewayFacts } from './facts.ts'; import type { GatewayServices } from './services.ts'; +import type { TokenUsage, UsageQuantities } from '../../repo/types.ts'; import { recordPerformance } from '../shared/telemetry/performance.ts'; import { recordUsage } from '../shared/telemetry/usage.ts'; import { defineStage } from '@floway-dev/pipeline'; import type { Logger } from '@floway-dev/pipeline'; +import type { BillingMetric } from '@floway-dev/protocols/common'; type Slice = { [P in K]: GatewayFacts[P] }; +/** The dump's own two columns, read back out of what was billed. + * + * The dump sums every input category into one number and keeps output separate, and it + * distinguishes "not measured" from "measured zero" — so an entity that reported nothing + * contributes no keys and stays null rather than becoming a zero the upstream never said. */ +const dumpUsage = (quantities: UsageQuantities): TokenUsage => { + const count = (metric: BillingMetric): number | undefined => + quantities[metric] === undefined ? undefined : Number(quantities[metric]); + return { + ...(count('input_tokens') === undefined ? {} : { input: count('input_tokens') }), + ...(count('input_cache_read_tokens') === undefined ? {} : { input_cache_read: count('input_cache_read_tokens') }), + ...(count('input_cache_write_tokens') === undefined ? {} : { input_cache_write: count('input_cache_write_tokens') }), + ...(count('input_image_tokens') === undefined ? {} : { input_image: count('input_image_tokens') }), + ...(count('output_tokens') === undefined ? {} : { output: count('output_tokens') }), + ...(count('output_image_tokens') === undefined ? {} : { output_image: count('output_image_tokens') }), + }; +}; + /** * Writes what was billed, and the performance sample that goes with it. * @@ -36,6 +56,9 @@ export const settleBillable = ( finishedAt: number = performance.now(), ): void => { for (const entity of billable) { + // The dump names the upstream that answered and what it metered, which is the same + // reading the row is written from rather than a second one taken separately. + if (!failed) services.gateway.dump?.success(entity.identity, dumpUsage(entity.quantities)); services.background(recordUsage( services.gateway.apiKeyId, entity.identity, @@ -50,8 +73,17 @@ export const settleBillable = ( /** * Prices what was observed and writes it, once per run. + * + * `stillReading` is how a streaming family says its numbers have not arrived yet: they come + * with the stream's last chunk, which is after this stage has handed up. Settling here as + * well would write the row twice — once for the entity that had reported nothing and once + * for what the stream turned out to say — so the run settles wherever the numbers are, and + * for a stream that is the epilogue. */ -export const writeSettlement = (failed: (handedUp: Record) => boolean) => defineStage< +export const writeSettlement = ( + failed: (handedUp: Record) => boolean, + stillReading?: (handedUp: Record) => boolean, +) => defineStage< Slice<'serve.model'>, Slice<'serve.model'>, Slice<'response.usage.billable'>, @@ -67,6 +99,7 @@ export const writeSettlement = (failed: (handedUp: Record) => b }, execute: async (facts, next, use) => { const back = await next(facts); + if (stillReading?.(back as Record) === true) return back; // One performance sample per run, attributed to the attempt that answered. settleBillable(use, back['response.usage.billable'], failed(back as Record)); return back; From 1a0be8102af5020a4e00093267aaf07ab8410745 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 01:09:36 +0800 Subject: [PATCH 25/79] revert(images): keep both image endpoints on their existing surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A request carrying stream: true reaches the upstream, and the pipeline cannot read what comes back — the images stream is not carried as facts, so the answer became a 502 after the call had already been made and charged for. The replaced surface forwarded those frames. This is the same blocker audio has: a response shape the fact space does not model yet. The pipeline, its tests and the protocol contracts stay; what is withdrawn is the route, until the shape it cannot carry is carried. --- .../gateway/src/data-plane/images/http.ts | 210 +++++++++++++----- 1 file changed, 159 insertions(+), 51 deletions(-) diff --git a/packages/gateway/src/data-plane/images/http.ts b/packages/gateway/src/data-plane/images/http.ts index 5f5cfcca9c..411127db9a 100644 --- a/packages/gateway/src/data-plane/images/http.ts +++ b/packages/gateway/src/data-plane/images/http.ts @@ -1,66 +1,174 @@ -// POST /v1/images/generations and POST /v1/images/edits, served through the pipeline. +// POST /v1/images/generations and POST /v1/images/edits — route image +// requests to the provider that declares the requested model and the +// matching image endpoint capability. // -// Two endpoints and one family. What the endpoint decides is which contract reads the body — -// generations is always JSON, edits is JSON or a multipart form carrying the files — and -// after that both hand the same canonical request to the same run. +// The edits handler accepts multipart uploads and JSON `images` references. +// Both are buffered once for dump capture and normalized into a semantic +// request; each provider owns the final JSON or multipart serialization. +// https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L12558-L12620 import type { Context } from 'hono'; -import { imagesServePipeline } from './pipeline.ts'; -import { openPrologue, readIngress, serveThrough, type Ingress } from '../pipeline/serve.ts'; -import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; -import { move } from '@floway-dev/pipeline'; -import { parseImagesEditsRequest, parseImagesGenerationsRequest, type ParsedImagesRequest } from '@floway-dev/protocols/images'; +import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; +import { createGatewayCtxFromHono, finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; +import { prepareJsonModelRequest } from '../shared/passthrough-request.ts'; +import { passthroughApiError, passthroughServe } from '../shared/passthrough-serve.ts'; +import { readRequestBody, takeRequestBody, type RequestBody } from '../shared/request-body.ts'; +import { tokenUsageFromImagesBody } from '../shared/telemetry/usage.ts'; +import { isJsonMediaType, isMultipartFormDataMediaType } from '@floway-dev/protocols/common'; +import type { ImageEditReference } from '@floway-dev/protocols/images'; +import { isBase64ImageDataUrl, type ImagesEditsRequest, type ImagesEditsSource } from '@floway-dev/provider'; -/** The half both endpoints share: whatever the contract read, hand it over and turn what the - * run answered with into a response. The contract reports a malformed request by throwing; - * what the client is owed is a 400 carrying the reason. */ -const serveImages = async ( - c: Context, - ingress: Ingress, - read: () => ParsedImagesRequest | Promise, -): Promise => { - let parsed: ParsedImagesRequest; - try { - parsed = await read(); - } catch (error) { - // A request the gateway could not read never reaches a pipeline: there is no model to - // resolve and no attempt to make, so there is nothing for a run to record. - const refused = openPrologue(c, ingress, { wantsStream: false }); - refused.gateway.dump?.error('gateway'); - return finalizeGatewayResponse( - refused.gateway, - Response.json( - { error: { message: error instanceof Error ? error.message : String(error), type: 'api_error' } }, - { status: 400 }, - ), - ); +type PreparedImagesEdit = + | { type: 'ok'; request: ImagesEditsRequest } + | { type: 'invalid'; message: string }; + +const imageEditSource = (value: unknown, path: string): ImagesEditsSource | string => { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + return `${path} must be an object.`; + } + const reference = value as { image_url?: unknown; file_id?: unknown }; + const { image_url: imageUrl, file_id: fileId } = reference; + if (typeof imageUrl === 'string' && fileId === undefined) { + const imageReference = reference as ImageEditReference & { image_url: string }; + return isBase64ImageDataUrl(imageUrl) + ? { type: 'inline', reference: imageReference } + : { type: 'reference', reference: imageReference }; + } + if (typeof fileId === 'string' && imageUrl === undefined) { + return { type: 'reference', reference: reference as ImageEditReference }; + } + return `${path} must contain exactly one string field: image_url or file_id.`; +}; + +const prepareJsonImagesEdit = (body: Record): PreparedImagesEdit => { + if (!Array.isArray(body.images)) { + return { type: 'invalid', message: 'Image edits request body must include an images array.' }; + } + const images: ImagesEditsSource[] = []; + for (const [index, value] of body.images.entries()) { + const source = imageEditSource(value, `Image edits images[${index}]`); + if (typeof source === 'string') return { type: 'invalid', message: source }; + images.push(source); + } + let mask: ImagesEditsSource | undefined; + if (body.mask !== undefined) { + const source = imageEditSource(body.mask, 'Image edits mask'); + if (typeof source === 'string') return { type: 'invalid', message: source }; + mask = source; } + const { model: _model, images: _images, mask: _mask, ...parameters } = body; + return { + type: 'ok', + request: { + images, + ...(mask === undefined ? {} : { mask }), + parameters, + }, + }; +}; - const { model, request } = parsed; - const prologue = openPrologue(c, ingress, { wantsStream: false, model }); +export const imagesGenerations = async (c: Context): Promise => { + const requestBody = await readRequestBody(c); + const request = prepareJsonModelRequest(requestBody.bytes, 'Images generations'); + const ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); + if (request.type === 'invalid') { + ctx.dump?.error('gateway'); + return finalizeGatewayResponse(ctx, passthroughApiError(c, request.message, 400)); + } - return await serveThrough( + ctx.dump?.requestedModel(request.model); + const response = await passthroughServe({ c, - prologue, - imagesServePipeline(request), - move({ - 'ingress.http.headers': prologue.headers, - 'request.images.canonical': request, - 'serve.model': model, - }) as never, - facts => ({ body: JSON.stringify(facts['response.images.rendered']), contentType: 'application/json' }), - ); + ctx, + sourceApi: '/images/generations', + operation: 'image_generation', + model: request.model, + kind: 'image', + modelServesEndpoint: model => model.endpoints.imagesGenerations !== undefined, + call: (provider, model, opts) => { + const { model: _model, ...body } = request.body; + return provider.instance.callImagesGenerations(model, body, undefined, opts); + }, + response: { format: 'json', extractBilling: tokenUsageFromImagesBody }, + }); + return finalizeGatewayResponse(ctx, response); }; -export const imagesGenerations = async (c: Context): Promise => { - const ingress = await readIngress(c); - return await serveImages(c, ingress, () => parseImagesGenerationsRequest(ingress.body.bytes)); +const serveImagesEditRequest = async ( + c: Context, + requestBody: RequestBody, + model: string, + request: ImagesEditsRequest, +): Promise => { + const ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); + ctx.dump?.requestedModel(model); + const response = await passthroughServe({ + c, + ctx, + sourceApi: '/images/edits', + operation: 'image_edit', + model, + kind: 'image', + modelServesEndpoint: model => model.endpoints.imagesEdits !== undefined, + call: (provider, model, opts) => provider.instance.callImagesEdits(model, request, undefined, opts), + response: { format: 'json', extractBilling: tokenUsageFromImagesBody }, + }); + return finalizeGatewayResponse(ctx, response); }; export const imagesEdits = async (c: Context): Promise => { - const ingress = await readIngress(c); - // Which of the two bodies arrived is a header's statement and not the payload's, so the - // contract is handed the media type alongside the bytes. - return await serveImages(c, ingress, () => parseImagesEditsRequest(c.req.header('content-type'), ingress.body.bytes)); + const requestBody = await readRequestBody(c); + const invalid = (message: string): Response => { + const errorCtx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); + errorCtx.dump?.error('gateway'); + return finalizeGatewayResponse(errorCtx, passthroughApiError(c, message, 400)); + }; + + const contentType = c.req.header('content-type'); + if (contentType === undefined) { + return invalid('Image edits request body must use application/json or multipart/form-data.'); + } + if (isJsonMediaType(contentType)) { + const body = prepareJsonModelRequest(requestBody.bytes, 'Image edits'); + if (body.type === 'invalid') return invalid(body.message); + const request = prepareJsonImagesEdit(body.body); + if (request.type === 'invalid') return invalid(request.message); + return await serveImagesEditRequest(c, requestBody, body.model, request.request); + } + + if (!isMultipartFormDataMediaType(contentType)) { + return invalid('Image edits request body must use application/json or multipart/form-data.'); + } + let form: FormData; + try { + form = await new Response(requestBody.bytes as BodyInit, { headers: { 'content-type': contentType } }).formData(); + } catch { + return invalid('Image edits request body must be valid multipart/form-data.'); + } + const model = form.get('model'); + if (typeof model !== 'string' || model.length === 0) { + return invalid('Image edits request body must include a model field.'); + } + const images: File[] = []; + let mask: File | undefined; + const parameters: Record = {}; + for (const [name, value] of form.entries()) { + if (name === 'model') continue; + if (name === 'image' || name === 'image[]') { + if (!(value instanceof File)) return invalid(`Image edits ${name} fields must be files.`); + images.push(value); + } else if (name === 'mask') { + if (!(value instanceof File)) return invalid('Image edits mask field must be a file.'); + mask = value; + } else { + if (typeof value !== 'string') return invalid(`Image edits ${name} field must be text.`); + parameters[name] = value; + } + } + return await serveImagesEditRequest(c, requestBody, model, { + images: images.map(file => ({ type: 'upload', file })), + ...(mask === undefined ? {} : { mask: { type: 'upload' as const, file: mask } }), + parameters, + }); }; From 50a41b0cf3f8a89245a84722a5695cedf27e87be Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 01:12:20 +0800 Subject: [PATCH 26/79] fix(gateway): answer a refusal in the upstream's own words on the last two families Completions and audio wrapped an upstream's error body in a second envelope, so a client read an escaped JSON document where the message should have been. They now hand the upstream's own object on, as the other three families do. Three ingress keys were declared and never provided or read, under a comment claiming every request recorded them. The method, path and body are the dump's to record from the request context; what a family actually hands over is the headers, because every ending forwards what a provider may of them. Rerank's attempt module had no referrers left after the pipeline took the route. Two comments cited design documents that are not in the repository. --- .../data-plane/completions-pipeline_test.ts | 5 ++- .../gateway/src/data-plane/audio/pipeline.ts | 4 +- .../src/data-plane/completions/pipeline.ts | 4 +- .../gateway/src/data-plane/pipeline/facts.ts | 7 +--- .../gateway/src/data-plane/rerank/attempt.ts | 42 ------------------- packages/pipeline/__tests__/boundary_test.ts | 5 +-- packages/pipeline/src/dump.ts | 3 +- 7 files changed, 12 insertions(+), 58 deletions(-) delete mode 100644 packages/gateway/src/data-plane/rerank/attempt.ts diff --git a/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts index c9a426d7d7..8f9638d78c 100644 --- a/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts @@ -307,9 +307,10 @@ describe('the completions pipeline', () => { expect(tried).toEqual(['up_a', 'up_b']); // Every candidate failed, so the last failure is the base — the client sees the status - // an upstream actually returned rather than a synthesized gateway envelope. + // an upstream actually returned, and the words that upstream used, rather than a + // synthesized envelope quoting its serialized body back as a message. expect(facts['response.http.status']).toBe(400); - expect(facts['response.completions.rendered']).toEqual({ error: { message: JSON.stringify({ error: { message: 'no' } }), type: 'api_error' } }); + expect(facts['response.completions.rendered']).toEqual({ error: { message: 'no' } }); await drain(); }); diff --git a/packages/gateway/src/data-plane/audio/pipeline.ts b/packages/gateway/src/data-plane/audio/pipeline.ts index bf7a88e35a..ff7c5beddc 100644 --- a/packages/gateway/src/data-plane/audio/pipeline.ts +++ b/packages/gateway/src/data-plane/audio/pipeline.ts @@ -36,7 +36,7 @@ import { type AudioTranscriptionUsage, type CanonicalAudioTranscription, } from '@floway-dev/protocols/audio'; -import { isEventStreamMediaType, parseDecimalString, parseSSEStream, sseFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { isEventStreamMediaType, parseDecimalString, parseSSEStream, renderErrorEnvelope, sseFrame, type SseFrame } from '@floway-dev/protocols/common'; import { providerModelOf, type AudioTranscriptionFormEntry, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; /** The answer while it is still the upstream's, one event at a time. It is a view and not a @@ -138,7 +138,7 @@ const emitAudioTranscription = defineStage< 'response.http.headers': forClient, 'response.http.status': answer.status, 'response.audioTranscription.mediaType': 'application/json', - 'response.audioTranscription.rendered': move({ error: { message: answer.message, type: 'api_error' } }), + 'response.audioTranscription.rendered': move(renderErrorEnvelope(answer.message, answer.body)), }; } // Everything that reaches here answered. The same key carried the upstream's own status diff --git a/packages/gateway/src/data-plane/completions/pipeline.ts b/packages/gateway/src/data-plane/completions/pipeline.ts index 8c1466d4c1..db9cd36151 100644 --- a/packages/gateway/src/data-plane/completions/pipeline.ts +++ b/packages/gateway/src/data-plane/completions/pipeline.ts @@ -25,7 +25,7 @@ import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/te import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import { compose, defineStage, move, own, type Owned, type Pipeline } from '@floway-dev/pipeline'; -import { isOpenAIUsageOnlyEventShape, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { isOpenAIUsageOnlyEventShape, renderErrorEnvelope, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { completionsProtocolFrameToSSEFrame, parseCompletionsResult, @@ -143,7 +143,7 @@ const rendered = ( answer: CompletionsFacts['response.completions.payload'], wantsUsageChunk: boolean, ): CompletionsFacts['response.completions.rendered'] => - isFailure(answer) ? { error: { message: answer.message, type: 'api_error' } } + isFailure(answer) ? renderErrorEnvelope(answer.message, answer.body) : isFrames(answer) ? renderSSE(answer, wantsUsageChunk) : answer; diff --git a/packages/gateway/src/data-plane/pipeline/facts.ts b/packages/gateway/src/data-plane/pipeline/facts.ts index d18cc218a4..ad65f85777 100644 --- a/packages/gateway/src/data-plane/pipeline/facts.ts +++ b/packages/gateway/src/data-plane/pipeline/facts.ts @@ -51,12 +51,9 @@ export const isFailure = (value: unknown): value is Failure => typeof value === 'object' && value !== null && 'status' in value && 'message' in value; export interface GatewayFacts { - /** What the client sent, before anything read it. Recorded on every request, so the - * dump can always show it. */ - 'ingress.http.method': string; - 'ingress.http.path': string; + /** What the client sent, before anything read it. Every family hands these over, because + * every ending forwards what a provider is allowed to forward of them. */ 'ingress.http.headers': readonly (readonly [string, string])[]; - 'ingress.http.body': Uint8Array; /** The public model id the client asked for, and the candidates it resolves to. Nothing * consumes these: they outlive an attempt. */ diff --git a/packages/gateway/src/data-plane/rerank/attempt.ts b/packages/gateway/src/data-plane/rerank/attempt.ts deleted file mode 100644 index 1842a852dc..0000000000 --- a/packages/gateway/src/data-plane/rerank/attempt.ts +++ /dev/null @@ -1,42 +0,0 @@ -import type { Context } from 'hono'; - -import type { GatewayCtx } from '../shared/gateway-ctx.ts'; -import { inboundHeaders } from '../shared/inbound-headers.ts'; -import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; -import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; -import type { RerankTarget } from '@floway-dev/protocols/common'; -import type { CanonicalRerankRequest } from '@floway-dev/protocols/rerank'; -import { providerModelOf } from '@floway-dev/provider'; -import type { ModelCandidate, PerformanceTelemetryContext, ProviderRerankCallResult, TelemetryModelIdentity } from '@floway-dev/provider'; - -export interface RerankAttemptResult { - readonly type: 'plain'; - readonly status: number; - readonly response: Response; - readonly target: RerankTarget; - readonly performance: PerformanceTelemetryContext; - readonly identity: TelemetryModelIdentity; -} - -export const rerankAttempt = async ( - c: Context, - ctx: GatewayCtx, - candidate: ModelCandidate, - request: CanonicalRerankRequest, -): Promise => { - const model = providerModelOf(candidate); - const result: ProviderRerankCallResult = await candidate.provider.instance.callRerank( - model, - request, - ctx.abortSignal, - buildUpstreamCallOptions(candidate, ctx, inboundHeaders(c)), - ); - return { - type: 'plain', - status: result.response.status, - response: result.response, - target: result.target, - performance: upstreamPerformanceContext(ctx, candidate, 'rerank'), - identity: telemetryModelIdentity(candidate, result.modelKey), - }; -}; diff --git a/packages/pipeline/__tests__/boundary_test.ts b/packages/pipeline/__tests__/boundary_test.ts index fe919cedee..2e8620ae6b 100644 --- a/packages/pipeline/__tests__/boundary_test.ts +++ b/packages/pipeline/__tests__/boundary_test.ts @@ -1,6 +1,5 @@ -// The one thing `16-architecture.md` §1.4 left open: what a handoff must satisfy when the -// target pipeline lives in another package. It turns out to need nothing, and this is the -// proof — the provider is a separate module with its own fact space, its own key that +// What a handoff must satisfy when the target pipeline lives in another package. It turns +// out to need nothing, and this is the proof — the provider is a separate module with its own fact space, its own key that // travels through the record, and an export whose type mentions only the core space. import { describe, expect, it } from 'vitest'; diff --git a/packages/pipeline/src/dump.ts b/packages/pipeline/src/dump.ts index d057975cd6..b34106dd6a 100644 --- a/packages/pipeline/src/dump.ts +++ b/packages/pipeline/src/dump.ts @@ -1,5 +1,4 @@ -// What a run emits, and how it is written down. The specification is -// `plans/26-dump-format.md`; this is that format. +// What a run emits, and how it is written down. // // A run emits a sequence of events. What a live observer is given and what is stored are // the same sequence, so there is one encoding and one thing to implement. From 3a285681b1c8e4c040b8a07e8eeee1dc88e0f527 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 01:16:50 +0800 Subject: [PATCH 27/79] fix(alpha-search): settle a run that searched locally The family produced a billed set and composed no settlement, so nothing it did was ever written. What stopped it was a request-side need on serve.model that the stage never read: declaring it put a resolved model in the entry contract of a family that resolves none. Settlement reads what came back and nothing on the way down, which is what the declaration now says. --- .../__tests__/data-plane/pipeline/assembly_test.ts | 2 +- .../__tests__/data-plane/search-pipeline_test.ts | 12 +++++++----- .../gateway/src/data-plane/alpha-search/pipeline.ts | 4 ++++ .../gateway/src/data-plane/pipeline/settlement.ts | 8 +++++--- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts index 0b6233e1c6..88ff36a6d6 100644 --- a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts +++ b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts @@ -6,12 +6,12 @@ import { describe, expect, it } from 'vitest'; +import { searchServePipeline } from '../../../src/data-plane/alpha-search/pipeline.ts'; import { audioTranscriptionServePipeline } from '../../../src/data-plane/audio/pipeline.ts'; import { completionsServePipeline } from '../../../src/data-plane/completions/pipeline.ts'; import { embeddingsServePipeline } from '../../../src/data-plane/embeddings/pipeline.ts'; import { imagesServePipeline } from '../../../src/data-plane/images/pipeline.ts'; import { rerankServePipeline } from '../../../src/data-plane/rerank/pipeline.ts'; -import { searchServePipeline } from '../../../src/data-plane/alpha-search/pipeline.ts'; const FAMILIES: readonly (readonly [string, () => { readonly name: string }])[] = [ ['embeddings', () => embeddingsServePipeline], diff --git a/packages/gateway/__tests__/data-plane/search-pipeline_test.ts b/packages/gateway/__tests__/data-plane/search-pipeline_test.ts index 22556ab8f3..924fc51495 100644 --- a/packages/gateway/__tests__/data-plane/search-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/search-pipeline_test.ts @@ -6,6 +6,7 @@ import { describe, expect, it } from 'vitest'; import { searchServePipeline } from '../../src/data-plane/alpha-search/pipeline.ts'; +import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts'; import { move, run } from '@floway-dev/pipeline'; const pinned = { kind: 'upstream', upstreamId: 'up_alpha', model: 'gpt-search' } as const; @@ -37,15 +38,16 @@ describe('the search pipeline', () => { .rejects.toThrow('run(searchServe): searchServe needs request.search.alphaSearch'); }); - // A request with nothing to run reaches no backend at all, so this drives the whole - // assembly on no services: the parse stage answers instead of descending, and what the - // assembly guarantees — that a short-circuit covers what the edge needs — is what makes the - // run produce a rendered body, a status and a billed set anyway. + // A request with nothing to run reaches no backend at all: the parse stage answers instead + // of descending, and what the assembly guarantees — that a short-circuit covers what the + // edge needs — is what makes the run produce a rendered body, a status and a billed set + // anyway. Settlement is unconditional, so the run still writes; a search that ran locally + // simply names no billed entity. it('answers in band when there is nothing to run, and the edge renders that answer', async () => { const { facts } = await run( searchServePipeline({ kind: 'local' }), move({ 'request.search.alphaSearch': { commands: {} } }), - {}, + { gateway: mockGatewayCtx({ wantsStream: false }), background: () => {} } as never, ); expect(facts['response.http.status']).toBe(200); expect(facts['response.search.rendered']).toEqual({ diff --git a/packages/gateway/src/data-plane/alpha-search/pipeline.ts b/packages/gateway/src/data-plane/alpha-search/pipeline.ts index 5dfd69fe95..2a2e52893b 100644 --- a/packages/gateway/src/data-plane/alpha-search/pipeline.ts +++ b/packages/gateway/src/data-plane/alpha-search/pipeline.ts @@ -26,6 +26,7 @@ import { isJsonObject } from '../../shared/json-helpers.ts'; import type { Failure, GatewayFacts } from '../pipeline/facts.ts'; import { isFailure } from '../pipeline/facts.ts'; import type { GatewayServices } from '../pipeline/services.ts'; +import { writeSettlement } from '../pipeline/settlement.ts'; import { enumerateModelCandidates } from '../providers/resolution.ts'; import type { GatewayCtx } from '../shared/gateway-ctx.ts'; import { filterInboundHeadersForProvider } from '../shared/inbound-headers.ts'; @@ -350,6 +351,9 @@ export const searchServePipeline = (execution: SearchExecution): Pipeline< S<'response.search.rendered' | 'response.http.status' | 'response.usage.billable'> > => compose('searchServe', [ emitAlphaSearch, + // Unconditional, as everywhere: a run that searched locally reached no upstream and bills + // for none, and saying so is a row that names no billed entity rather than no row. + writeSettlement(handedUp => isFailure((handedUp as { 'response.search.alphaSearch'?: unknown })['response.search.alphaSearch'])), ...(execution.kind === 'local' ? [parseSearchOperations, executeSearchOperations] : [callSearchUpstream(execution)]), diff --git a/packages/gateway/src/data-plane/pipeline/settlement.ts b/packages/gateway/src/data-plane/pipeline/settlement.ts index 80d4d81517..5773694e2b 100644 --- a/packages/gateway/src/data-plane/pipeline/settlement.ts +++ b/packages/gateway/src/data-plane/pipeline/settlement.ts @@ -84,15 +84,17 @@ export const writeSettlement = ( failed: (handedUp: Record) => boolean, stillReading?: (handedUp: Record) => boolean, ) => defineStage< - Slice<'serve.model'>, - Slice<'serve.model'>, + Record, + Record, Slice<'response.usage.billable'>, Slice<'response.usage.billable'>, GatewayServices >({ name: 'writeSettlement', through: { - request: { needs: ['serve.model'], consumes: [], provides: [] }, + // Nothing on the way down: what is settled is what came back, and naming a request key + // here would put it in the entry contract of a family that never resolved a model. + request: { needs: [], consumes: [], provides: [] }, // It reads the authoritative reading and hands it on untouched: settlement is the last // reader, not another writer, and a stage that changed usage re-provided it below. response: { needs: ['response.usage.billable'], consumes: [], provides: [] }, From ee03cddb654dd24867bf38938f1ff1672449723b Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 01:19:16 +0800 Subject: [PATCH 28/79] test: exercise the audio family, and assert the log fields the dump carries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audio's pipeline had no behavioural coverage: its test file established the entry contract and one property of the runner's ownership, and nothing ran a stage. It now serves a transcription, an upstream refusal, and a stream whose upstream keeps talking after the terminal event — the last fails if the reader does not stop there. The dump's log-line test named itself after structured fields and asserted every part of the line except them; removing the fields from the encoder left it green. --- .../data-plane/audio-pipeline_test.ts | 116 +++++++++++++++++- packages/pipeline/__tests__/dump_test.ts | 2 +- 2 files changed, 116 insertions(+), 2 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/audio-pipeline_test.ts b/packages/gateway/__tests__/data-plane/audio-pipeline_test.ts index 641ba7cf6f..9d6a3bc271 100644 --- a/packages/gateway/__tests__/data-plane/audio-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/audio-pipeline_test.ts @@ -4,10 +4,75 @@ // the entry contract it derives, the keys it cannot see, and the one runtime property this // family's code is shaped around that no declaration expresses. -import { describe, expect, it } from 'vitest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; import { audioTranscriptionServePipeline } from '../../src/data-plane/audio/pipeline.ts'; +import { enumerateModelCandidates } from '../../src/data-plane/providers/resolution.ts'; +import { initRepo } from '../../src/repo/index.ts'; +import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts'; import { isOwned, move, run } from '@floway-dev/pipeline'; +import type { SseFrame } from '@floway-dev/protocols/common'; +import { directFetcher, type ModelCandidate, type ProviderCallResult } from '@floway-dev/provider'; +import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; + +vi.mock('../../src/data-plane/providers/resolution.ts', async importOriginal => ({ + ...(await importOriginal()), + enumerateModelCandidates: vi.fn(), +})); + +let live: readonly ModelCandidate[] = []; + +const candidate = (callAudioTranscriptions: () => Promise): ModelCandidate => { + const endpoints = { audioTranscriptions: {} }; + return { + provider: { + upstreamId: 'up_a', kind: 'custom', name: 'up_a', inboundHeaderAllowlist: [], + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + instance: stubProvider({ callAudioTranscriptions }), + }, + model: stubInternalModel( + { id: 'whisper-1', kind: 'transcription', endpoints, providerModels: { up_a: stubProviderModel({ id: 'whisper-1', endpoints }) } }, + 'up_a', + ), + fetcher: directFetcher, + } as unknown as ModelCandidate; +}; + +const resolves = (candidates: readonly ModelCandidate[]): void => { + live = candidates; + vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates, sawModel: true, failedUpstreams: [] } as never); +}; + +const sse = (...events: readonly string[]): Response => + new Response(events.map(event => `data: ${event}\n\n`).join(''), { status: 200, headers: { 'content-type': 'text/event-stream' } }); + +const serve = async (responseFormat = 'json') => await run( + audioTranscriptionServePipeline, + move({ + 'ingress.audioTranscription.responseFormat': responseFormat, + 'ingress.http.headers': [] as readonly (readonly [string, string])[], + 'request.audioTranscription.form': [{ name: 'model', value: 'whisper-1' }], + 'serve.model': 'whisper-1', + }) as never, + { + gateway: mockGatewayCtx({ wantsStream: responseFormat === 'stream' }), + background: () => {}, + rememberCandidates: () => {}, + resolveAttempt: (selector: { readonly upstreamId: string }) => { + const found = live.find(c => c.provider.upstreamId === selector.upstreamId); + if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); + return found; + }, + } as never, +); + +beforeEach(() => { + vi.mocked(enumerateModelCandidates).mockReset(); + initRepo({ + usage: { record: async () => {} }, + performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, + } as never); +}); describe('the audio transcription pipeline', () => { it('assembles, and asks its caller for what the descending stages need', () => { @@ -45,6 +110,55 @@ describe('the audio transcription pipeline', () => { // The runner now claims ownership through `own()` instead of detecting it, so a bare // generator is safe in the record. The wrapper stays because it says which thing is the // resource: the upstream body at `response.http.body`, and nothing else here. + it('renders a transcription the upstream answered with, as a 200', async () => { + resolves([candidate(async () => ({ + response: Response.json({ text: 'hello there' }), + modelKey: 'whisper-key', + } as ProviderCallResult))]); + + const { facts } = await serve(); + + expect(facts['response.http.status']).toBe(200); + expect(facts['response.audioTranscription.rendered']).toMatchObject({ text: 'hello there' }); + }); + + // A refusal reaches the client with the status the upstream gave it, and in the words the + // upstream used — the same statement every other family makes. + it('answers an upstream refusal with its own status and words', async () => { + resolves([candidate(async () => ({ + response: new Response(JSON.stringify({ error: { message: 'too large' } }), { + status: 413, headers: { 'content-type': 'application/json' }, + }), + modelKey: 'whisper-key', + } as ProviderCallResult))]); + + const { facts } = await serve(); + + expect(facts['response.http.status']).toBe(413); + expect(facts['response.audioTranscription.rendered']).toEqual({ error: { message: 'too large' } }); + }); + + // The reader stops at the terminal event. An upstream that holds the connection open past + // it would otherwise hold the client's stream open with it, which is what the replaced + // surface avoided by cancelling the moment the transcript was complete. + it('stops reading a stream at its terminal event', async () => { + const done = JSON.stringify({ type: 'transcript.text.done', text: 'hi', usage: { type: 'tokens', input_tokens: 3, output_tokens: 1 } }); + const after = JSON.stringify({ type: 'transcript.text.delta', delta: 'never read' }); + resolves([candidate(async () => ({ + response: sse(JSON.stringify({ type: 'transcript.text.delta', delta: 'hi' }), done, after), + modelKey: 'whisper-key', + } as ProviderCallResult))]); + + const { facts, drain } = await serve('json'); + const frames: SseFrame[] = []; + for await (const frame of facts['response.audioTranscription.rendered'] as AsyncIterable) frames.push(frame); + await drain(); + + expect(frames).toHaveLength(2); + expect(frames.map(frame => JSON.parse(frame.data) as { type: string }).map(event => event.type)) + .toEqual(['transcript.text.delta', 'transcript.text.done']); + }); + it('keeps the events view clear of what answers to release', () => { const generator = (async function* () { yield 1; })(); expect(Symbol.asyncDispose in generator).toBe(true); // the language marks it diff --git a/packages/pipeline/__tests__/dump_test.ts b/packages/pipeline/__tests__/dump_test.ts index c74150d11c..493dbe321e 100644 --- a/packages/pipeline/__tests__/dump_test.ts +++ b/packages/pipeline/__tests__/dump_test.ts @@ -135,7 +135,7 @@ describe('the dump encoding', () => { it('carries a log line with its structured fields', () => { const events: Event[] = [{ type: 'stage.log', stageId: 4, level: 'warn', message: 'slow', fields: { ms: 1200 } }]; const [encoded] = encodeRun(events).filter(e => e.type === 'stage.log'); - expect(encoded).toMatchObject({ stageId: 4, level: 'warn', message: 'slow' }); + expect(encoded).toMatchObject({ stageId: 4, level: 'warn', message: 'slow', fields: { ms: 1200 } }); }); // A buffer is atomic and records its full content, the way a string does. Walking it by From 5817f816a2748d752abfbab048764fdc16a3dd19 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 01:28:21 +0800 Subject: [PATCH 29/79] fix(gateway): type the forwarded header pairs as tuples The deployment apps compile the gateway under their own lib settings, where Headers takes [string, string][] and the inferred string[][] does not fit. Both platform typechecks failed while the package's own passed, so the mismatch only appeared under pnpm run verify. --- packages/gateway/src/data-plane/audio/pipeline.ts | 2 +- packages/gateway/src/data-plane/completions/pipeline.ts | 2 +- packages/gateway/src/data-plane/embeddings/pipeline.ts | 2 +- packages/gateway/src/data-plane/images/pipeline.ts | 2 +- packages/gateway/src/data-plane/pipeline/serve.ts | 2 +- packages/gateway/src/data-plane/rerank/pipeline.ts | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/gateway/src/data-plane/audio/pipeline.ts b/packages/gateway/src/data-plane/audio/pipeline.ts index ff7c5beddc..7a675e89c1 100644 --- a/packages/gateway/src/data-plane/audio/pipeline.ts +++ b/packages/gateway/src/data-plane/audio/pipeline.ts @@ -202,7 +202,7 @@ const callAudioTranscriptionUpstream = defineStage< // The client's own headers reach the upstream from the record, not from a live request // object: what a provider is allowed to forward is filtered per provider, and the dump // shows what was there to filter. - buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]) => [name, value]))), + buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), ); } catch (error) { use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); diff --git a/packages/gateway/src/data-plane/completions/pipeline.ts b/packages/gateway/src/data-plane/completions/pipeline.ts index db9cd36151..962db039c1 100644 --- a/packages/gateway/src/data-plane/completions/pipeline.ts +++ b/packages/gateway/src/data-plane/completions/pipeline.ts @@ -195,7 +195,7 @@ const callCompletionsUpstream = defineStage< // The client's own headers reach the upstream from the record, not from a live request // object: what a provider is allowed to forward is filtered per provider, and the dump // shows what was there to filter. - buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]) => [name, value]))), + buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), ); } catch (error) { use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); diff --git a/packages/gateway/src/data-plane/embeddings/pipeline.ts b/packages/gateway/src/data-plane/embeddings/pipeline.ts index ef37015530..011ce5e4af 100644 --- a/packages/gateway/src/data-plane/embeddings/pipeline.ts +++ b/packages/gateway/src/data-plane/embeddings/pipeline.ts @@ -132,7 +132,7 @@ const callEmbeddingsUpstream = defineStage< // The client's own headers reach the upstream from the record, not from a live // request object: what a provider is allowed to forward is filtered per provider, // and the dump shows what was there to filter. - buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]) => [name, value]))), + buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), ); } catch (error) { use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); diff --git a/packages/gateway/src/data-plane/images/pipeline.ts b/packages/gateway/src/data-plane/images/pipeline.ts index 9078b64928..febb378f14 100644 --- a/packages/gateway/src/data-plane/images/pipeline.ts +++ b/packages/gateway/src/data-plane/images/pipeline.ts @@ -127,7 +127,7 @@ const callImagesUpstream = defineStage< // The client's own headers reach the upstream from the record, not from a live request // object: what a provider is allowed to forward is filtered per provider, and the dump // shows what was there to filter. - new Headers(facts['ingress.http.headers'].map(([name, value]) => [name, value])), + new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value])), ); const model = providerModelOf(candidate); // Attribution is set before the dial, so an attempt that never completes still names the diff --git a/packages/gateway/src/data-plane/pipeline/serve.ts b/packages/gateway/src/data-plane/pipeline/serve.ts index c08494f5e4..7834ae1080 100644 --- a/packages/gateway/src/data-plane/pipeline/serve.ts +++ b/packages/gateway/src/data-plane/pipeline/serve.ts @@ -180,7 +180,7 @@ export const serveThrough = async < // Nothing is left to read: what the client is sent was serialized from facts the run // already held, so releasing can start at once. prologue.services.background(drain()); - const headers = new Headers(facts['response.http.headers'].map(([name, value]) => [name, value])); + const headers = new Headers(facts['response.http.headers'].map(([name, value]): [string, string] => [name, value])); headers.set('content-type', answer.contentType); return finalizeGatewayResponse(prologue.gateway, new Response(answer.body, { status, headers })); }; diff --git a/packages/gateway/src/data-plane/rerank/pipeline.ts b/packages/gateway/src/data-plane/rerank/pipeline.ts index a98dc23a25..17e62b8cf3 100644 --- a/packages/gateway/src/data-plane/rerank/pipeline.ts +++ b/packages/gateway/src/data-plane/rerank/pipeline.ts @@ -162,7 +162,7 @@ const callRerankUpstream = defineStage< // The client's own headers reach the upstream from the record, not from a live // request object: what a provider is allowed to forward is filtered per provider, // and the dump shows what was there to filter. - buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]) => [name, value]))), + buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), ); } catch (error) { use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); From e6031f164ef98da87569d1b51a0be315d133efe5 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 04:17:35 +0800 Subject: [PATCH 30/79] feat(pipeline): make what a run started a fact it waits for A stage had no way to begin work without reaching past the record for a scheduler, and anything it started that way escaped the run's own accounting. Deferral is now a property of the value: a helper marks a promise, the marked value enters the record like any other, and teardown waits for what this run began. Declared rather than sniffed. Testing for a then method would make a fact that happens to hold a promise indistinguishable from one the run owes work to, and it would be quietly awaited instead of reported. Per run and not per root, because a discarded branch's facts are private to it and a root-level sweep could never see them. Teardown has a deadline and exceeding it is an error a reader can act on, since a value that never settles would otherwise hold teardown open forever. --- packages/pipeline/__tests__/deferral_test.ts | 119 +++++++++++++++++++ packages/pipeline/src/index.ts | 4 +- packages/pipeline/src/run.ts | 65 ++++++++++ packages/pipeline/src/stage.ts | 3 + 4 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 packages/pipeline/__tests__/deferral_test.ts diff --git a/packages/pipeline/__tests__/deferral_test.ts b/packages/pipeline/__tests__/deferral_test.ts new file mode 100644 index 0000000000..f4f4e76464 --- /dev/null +++ b/packages/pipeline/__tests__/deferral_test.ts @@ -0,0 +1,119 @@ +// Deferral: what a run started and has not finished. +// +// The rule these establish is that a stage has no fire-and-forget. Work a stage begins +// becomes a fact with a name, and teardown is where the run waits for it — so nothing +// escapes the run's own accounting, and nothing hangs it either. + +import { describe, expect, it, vi } from 'vitest'; + +import { compose, defer, defineStage, isDeferred, move, run } from '../src/index.ts'; +import type { Logger } from '../src/index.ts'; + +const ending = (produce: () => unknown) => defineStage, { readonly 'x.settled': unknown }>({ + name: 'ending', + return: { provides: ['x.settled'] }, + execute: async facts => await Promise.resolve(move({ ...facts, 'x.settled': produce() })), +}); + +const pipelineOf = (produce: () => unknown) => compose('deferring', [ending(produce)]); + +const logger = (): Logger & { readonly errors: { message: string; fields?: Readonly> }[] } => { + const errors: { message: string; fields?: Readonly> }[] = []; + return { + errors, + debug: () => {}, info: () => {}, warn: () => {}, + error: (message, fields) => { errors.push({ message, ...(fields === undefined ? {} : { fields }) }); }, + }; +}; + +describe('a deferred fact', () => { + it('is a claim on the value, not a promise the runner went looking for', () => { + expect(isDeferred(Promise.resolve(1))).toBe(false); + expect(isDeferred(defer(Promise.resolve(1)))).toBe(true); + expect(isDeferred({ then: () => {} })).toBe(false); + }); + + // Teardown is where the run waits. A caller that drains has waited for everything the run + // began, which is what lets a stage start work without reaching for a scheduler. + it('is awaited at teardown, so what a stage started finishes inside the run', async () => { + let finished = false; + let release!: () => void; + const started = new Promise(resolve => { release = resolve; }); + const { drain } = await run( + pipelineOf(() => defer(started.then(() => { finished = true; }))), + move({}), + {}, + ); + + // The run has answered and the work has not finished: that is the whole point of it + // being deferred rather than awaited where it was started. + expect(finished).toBe(false); + const torn = drain(); + release(); + await torn; + expect(finished).toBe(true); + }); + + // A failure that nobody hears about is a row nobody writes and nobody misses. + it('reports a failure rather than swallowing it', async () => { + const log = logger(); + const { drain } = await run( + pipelineOf(() => defer(Promise.reject(new Error('the write failed')))), + move({}), + { log }, + ); + + await drain(); + + expect(log.errors).toHaveLength(1); + expect(log.errors[0]!.message).toBe('a deferred fact failed'); + expect(String(log.errors[0]!.fields?.error)).toContain('the write failed'); + }); + + // A value that never settles would otherwise hold teardown open forever. Giving up is + // itself an event: an error a reader can act on, not silence. + it('gives up loudly rather than waiting forever', async () => { + vi.useFakeTimers(); + try { + const log = logger(); + const { drain } = await run(pipelineOf(() => defer(new Promise(() => { /* never */ }))), move({}), { log }); + + const settled = drain(); + await vi.advanceTimersByTimeAsync(30_000); + await settled; + + expect(log.errors).toHaveLength(1); + expect(log.errors[0]!.message).toBe('teardown deadline exceeded'); + } finally { + vi.useRealTimers(); + } + }); + + // Per run, not per root: a discarded branch's facts are private to it and unreachable from + // the root's record, so a root-level sweep could never see them. Each run tears down its + // own, and a branch nobody adopted still finishes what it began. + it('belongs to the run that started it', async () => { + const finished: string[] = []; + const gate = (label: string) => { + let open!: () => void; + const waited = new Promise(resolve => { open = resolve; }); + return { open, work: () => defer(waited.then(() => { finished.push(label); })) }; + }; + const innerGate = gate('inner'); + const outerGate = gate('outer'); + const inner = await run(pipelineOf(innerGate.work), move({}), {}); + const outer = await run(pipelineOf(outerGate.work), move({}), {}); + + // Tearing one run down waits for that run's own work and nothing else's, which is what + // lets a discarded branch finish what it began without the root reaching into it. + const innerTorn = inner.drain(); + innerGate.open(); + await innerTorn; + expect(finished).toEqual(['inner']); + + const outerTorn = outer.drain(); + outerGate.open(); + await outerTorn; + expect(finished).toEqual(['inner', 'outer']); + }); +}); diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts index d1227598ae..d6604b1fff 100644 --- a/packages/pipeline/src/index.ts +++ b/packages/pipeline/src/index.ts @@ -28,8 +28,8 @@ export type { export { compose } from './compose.ts'; -export { isOwned, own, run } from './run.ts'; -export type { Owned, RunResult } from './run.ts'; +export { defer, isDeferred, isOwned, own, run } from './run.ts'; +export type { Deferred, Owned, RunResult } from './run.ts'; export { createEncoder, diff --git a/packages/pipeline/src/run.ts b/packages/pipeline/src/run.ts index 820cf9e875..e82d08b2e1 100644 --- a/packages/pipeline/src/run.ts +++ b/packages/pipeline/src/run.ts @@ -45,6 +45,29 @@ export const own = (value: T, release: () => Promise): T export const isOwned = (value: unknown): value is Owned => typeof value === 'object' && value !== null && OWNED in value; +const DEFERRED = Symbol('floway.deferred'); + +/** A value the run has started and has not finished. It is a property of the value, not a + * capability a stage was handed: what a run must wait for is legible from its own record. */ +export type Deferred = PromiseLike & { readonly [DEFERRED]: true }; + +/** + * Marks a promise as this run's to finish. + * + * Declared rather than sniffed. Testing for a `then` method would make a fact that happens to + * hold a promise indistinguishable from one the run owes work to, and it would be quietly + * awaited instead of reported — so the claim is explicit, exactly as ownership is. + * + * Helpers call this and hand out the promise; a stage never creates one. That is what + * "stages have no fire-and-forget" means in practice: work a stage starts becomes a fact + * with a name, and the runner waits for it where it can see it. + */ +export const defer = (promise: PromiseLike): Deferred => + Object.assign(promise, { [DEFERRED]: true as const }); + +export const isDeferred = (value: unknown): value is Deferred => + typeof value === 'object' && value !== null && DEFERRED in value; + /** * Frozen in place, never copied. A stage that hands on what it received hands on the * same object, so the record has an identity of its own and "this side changed nothing" @@ -59,6 +82,7 @@ const handOn = (record: Facts, decl: ErasedSide, stage: string, way: 'down' | 'u for (const [key, value] of Object.entries(record)) { assertHandedOver(`${stage} handing ${way} ${key}`, value); if (isOwned(value)) scope.outstanding.add(value); + if (isDeferred(value)) scope.deferred.add(value); } for (const key of decl.provides) { if (!(key in record)) throw new Error(`${stage}: declared providing ${key} but did not`); @@ -174,6 +198,45 @@ const release = async (value: Owned, scope: RunScope): Promise => { await value[Symbol.asyncDispose](); }; +/** How long teardown waits for what the run started. A value that never settles would hang + * teardown forever, so exceeding this is reported rather than waited through. */ +const TEARDOWN_DEADLINE_MS = 30_000; + +/** + * Waits for what this run started, and says so loudly when something does not finish. + * + * Per run and not per root: a discarded branch's facts are private to that branch and + * unreachable from the root's record, so a root-level sweep could never see them. Each run + * tears down its own, and a branch nobody adopted still finishes what it began. + */ +const settleDeferred = async (scope: RunScope, services: RunServices): Promise => { + const pending = [...scope.deferred]; + if (pending.length === 0) return; + scope.deferred.clear(); + + let timer: ReturnType | undefined; + const deadline = new Promise<'deadline'>(resolve => { + timer = setTimeout(() => { resolve('deadline'); }, TEARDOWN_DEADLINE_MS); + }); + try { + const outcome = await Promise.race([ + Promise.allSettled(pending).then(results => results), + deadline, + ]); + if (outcome === 'deadline') { + services.log?.error('teardown deadline exceeded', { pending: pending.length }); + return; + } + for (const result of outcome) { + if (result.status === 'rejected') { + services.log?.error('a deferred fact failed', { error: String(result.reason) }); + } + } + } finally { + if (timer !== undefined) clearTimeout(timer); + } +}; + const requireEntry = (target: Pipeline, handed: Facts, who: string): void => { for (const key of target.entryNeeds) { if (!(String(key) in handed)) { @@ -234,6 +297,7 @@ export const run = async {} : event => { events.push(event); sink(event); }, outstanding: new Set(), + deferred: new Set>(), parentStageId: null, nextStageId: 1, }; @@ -243,6 +307,7 @@ export const run = async => { for (const value of [...scope.outstanding]) await release(value, scope); + await settleDeferred(scope, services); }; try { diff --git a/packages/pipeline/src/stage.ts b/packages/pipeline/src/stage.ts index 080c24e650..7b798ef272 100644 --- a/packages/pipeline/src/stage.ts +++ b/packages/pipeline/src/stage.ts @@ -82,6 +82,9 @@ export type IntoNext = * holding both, one `stageId` issued twice — and a gateway serves concurrent requests by * definition. */ export interface RunScope { + /** What this run started and has not finished. Filled at handover, the same moment + * ownership is, because a value only becomes the run's when it enters the record. */ + readonly deferred: Set>; /** A no-op when the prologue resolved no dump sink, which is what makes recording * conditional rather than a mode flag. */ readonly emit: (event: import('./dump.ts').Event) => void; From 8d47f57b12cb86e50419eca793befde2c7005577 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 04:34:26 +0800 Subject: [PATCH 31/79] feat(audio): carry a transcription's document instead of writing it again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `srt` and `vtt` were parsed into cues and rendered back through a writer of our own, which the type itself admitted was not byte-identical to what the upstream sent: that writer terminates every cue with a blank line, numbers SubRip from 1 and pads every fraction to three digits, so a document following none of those conventions came back changed. The replaced surface forwarded them untouched. The canonical transcription now holds the bytes that arrived and the renderer hands them straight back — for the three document renderings, and for any body the reading could not open at all. What is read out of the bytes (the transcript, the cues, the upstream's own object) is what the record shows and what usage is measured from, so a reading that fails costs telemetry and never the answer. The subtitle writer has no caller left and is gone with it. Usage becomes a reading of its own. A report under a discriminator this protocol does not name is still no report, but one it does name and cannot read now throws, so a caller can say an upstream metered something unreadable rather than record it as though it had metered nothing. --- .../__tests__/audio/transcription_test.ts | 84 +++++++--- packages/protocols/src/audio/index.ts | 2 +- packages/protocols/src/audio/subtitles.ts | 51 ++---- packages/protocols/src/audio/transcription.ts | 150 +++++++++++------- 4 files changed, 166 insertions(+), 121 deletions(-) diff --git a/packages/protocols/__tests__/audio/transcription_test.ts b/packages/protocols/__tests__/audio/transcription_test.ts index 5c2eadb5d1..c99eb07872 100644 --- a/packages/protocols/__tests__/audio/transcription_test.ts +++ b/packages/protocols/__tests__/audio/transcription_test.ts @@ -5,8 +5,19 @@ import { parseAudioTranscriptionResponseFormat, parseAudioTranscriptionUsage, renderAudioTranscription, + type AudioTranscriptionResponseFormat, } from '../../src/audio/index.ts'; +const bytes = (document: string): Uint8Array => new TextEncoder().encode(document); + +/** What a carried rendering says, and a failure where an object rendering came back + * instead — the two are never interchangeable and a test that read one as the other would + * pass on the wrong thing. */ +const carried = (rendered: Record | Uint8Array): string => { + if (!(rendered instanceof Uint8Array)) throw new Error(`expected a carried document, got ${JSON.stringify(rendered)}`); + return new TextDecoder().decode(rendered); +}; + // Whisper's own writers, which is what `whisper-1` runs: SubRip numbers its cues and always // writes the hour component, WebVTT opens with its signature and drops the hour component // below one hour, and both terminate every cue with a blank line. @@ -26,6 +37,9 @@ const VERBOSE = { ], }; +const carries = (format: AudioTranscriptionResponseFormat, document: string): string => + carried(renderAudioTranscription(format, parseAudioTranscription(format, bytes(document)))); + describe('the renderings of a transcription', () => { it('is the protocol default when the form left the field out', () => { expect(parseAudioTranscriptionResponseFormat(undefined)).toBe('json'); @@ -36,30 +50,43 @@ describe('the renderings of a transcription', () => { // form, so the rendering an answer arrives in is the rendering it is written back in, and // reading then writing has to be the identity on every one of the six. it.each([ - ['json', { text: 'hello' }], - ['verbose_json', VERBOSE], - ['diarized_json', { task: 'transcribe', duration: 2, text: 'Agent: hi', segments: [{ type: 'transcript.text.segment', id: 'seg_1', start: 0, end: 2, text: 'hi', speaker: 'agent' }] }], + ['json', JSON.stringify({ text: 'hello' })], + ['verbose_json', JSON.stringify(VERBOSE)], + ['diarized_json', JSON.stringify({ task: 'transcribe', duration: 2, text: 'Agent: hi', segments: [{ type: 'transcript.text.segment', id: 'seg_1', start: 0, end: 2, text: 'hi', speaker: 'agent' }] })], ['text', ' The beach was a popular spot.\n'], ['srt', SRT], ['vtt', VTT], - ] as const)('reads and writes %s back unchanged', (format, body) => { - expect(renderAudioTranscription(format, parseAudioTranscription(format, body))).toEqual(body); + ] as const)('reads and writes %s back unchanged', (format, document) => { + const rendered = renderAudioTranscription(format, parseAudioTranscription(format, bytes(document))); + expect(rendered instanceof Uint8Array ? carried(rendered) : JSON.stringify(rendered)).toBe(document); + }); + + // A document is carried and not reproduced, so what the upstream wrote survives every + // convention it did not follow. Each of these is a byte a writer of ours would have + // changed: the terminating blank line Whisper always writes, the ordinal it always starts + // at 1, the line ending it always writes as `\n`, and the millisecond precision it always + // pads to three digits. + it.each([ + ['srt', '1\n00:00:00,000 --> 00:00:03,320\nno terminating blank line'], + ['srt', '7\r\n00:00:00,000 --> 00:00:03,320\r\nnumbered from seven, with CRLF\r\n\r\n'], + ['vtt', 'WEBVTT\n\n00:00.0 --> 00:03.32\nfewer fraction digits than Whisper writes\n\n'], + ['text', 'a transcript with no trailing newline'], + ] as const)('carries a %s document byte for byte, however it was written', (format, document) => { + expect(carries(format, document)).toBe(document); }); // What the canonical value holds is what its rendering carried, and the renderings are not // equally rich. This is the whole of why a rendering is never derived from a value read // out of a different one. it('holds a transcript and its cues where the rendering carried both', () => { - expect(parseAudioTranscription('verbose_json', VERBOSE)).toMatchObject({ + expect(parseAudioTranscription('verbose_json', bytes(JSON.stringify(VERBOSE)))).toMatchObject({ text: VERBOSE.text, cues: [{ start: 0, end: 3.32, text: ' The beach was a popular spot.' }], }); }); - it('holds no cues where the rendering carried none, so no subtitle can be written from it', () => { - const fromJson = parseAudioTranscription('json', { text: 'hello' }); - expect(fromJson.cues).toBeUndefined(); - expect(() => renderAudioTranscription('srt', fromJson)).toThrow('carries no cues'); + it('holds no cues where the rendering carried none', () => { + expect(parseAudioTranscription('json', bytes(JSON.stringify({ text: 'hello' }))).cues).toBeUndefined(); }); // A subtitle document states the transcript only as its cues' texts, so the transcript on @@ -67,16 +94,20 @@ describe('the renderings of a transcription', () => { // claimed to be the string a `text` request would have returned. // https://github.com/openai/whisper/blob/v20250625/whisper/utils.py#L109-L116 it('rebuilds the transcript of a subtitle document from its cues', () => { - expect(parseAudioTranscription('srt', SRT).text).toBe(' The beach was a popular spot.\n People were swimming in the ocean.'); + expect(parseAudioTranscription('srt', bytes(SRT)).text).toBe(' The beach was a popular spot.\n People were swimming in the ocean.'); }); - it('cannot write a JSON rendering from a value that was not read from an object', () => { - expect(() => renderAudioTranscription('json', parseAudioTranscription('text', 'hello'))).toThrow('not read from an object'); + it('refuses a body the rendering cannot be read from', () => { + expect(() => parseAudioTranscription('json', bytes('{not-json'))).toThrow(SyntaxError); + expect(() => parseAudioTranscription('json', bytes(JSON.stringify({ transcript: 'hello' })))).toThrow('text must be a string'); + expect(() => parseAudioTranscription('vtt', bytes(SRT))).toThrow('must open with WEBVTT'); }); - it('refuses a body the rendering cannot be read from', () => { - expect(() => parseAudioTranscription('json', { transcript: 'hello' })).toThrow('text must be a string'); - expect(() => parseAudioTranscription('vtt', SRT)).toThrow('must open with WEBVTT'); + // The other half of that refusal: a caller that could not read the body still has the body, + // and what it writes back is what arrived. A 2xx nobody could parse is still the upstream's + // answer to the client's request. + it('writes back a body no reading could open', () => { + expect(carried(renderAudioTranscription('json', { document: bytes('{not-json') }))).toBe('{not-json'); }); }); @@ -92,11 +123,24 @@ describe('what a transcription says it will be billed for', () => { expect(parseAudioTranscriptionUsage(VERBOSE)).toEqual({ kind: 'duration', seconds: 8.47 }); }); - // A report that cannot be read is, from here, no report — which the caller states by - // naming the billed entity with no quantities rather than by naming zero. - it('is no report where the shape is not one the protocol names', () => { + // Nothing was stated, or what was stated is not something this reading names. A metric + // invented after this was written is not a malformed one, and the caller bills neither. + it('is no report where nothing this reading names was stated', () => { expect(parseAudioTranscriptionUsage({ text: 'hello' })).toBeUndefined(); - expect(parseAudioTranscriptionUsage({ usage: { type: 'tokens', input_tokens: -1, output_tokens: 2 } })).toBeUndefined(); expect(parseAudioTranscriptionUsage({ usage: { type: 'credits', spent: 3 } })).toBeUndefined(); }); + + // An upstream that reported under a name this reading does know, in a shape it cannot read, + // is a third situation and it says so — which is what lets the caller warn rather than + // record the request as though the upstream had metered nothing. + it('reports a block it names and cannot read, rather than reading it as nothing', () => { + expect(() => parseAudioTranscriptionUsage({ usage: { type: 'duration', seconds: 'invalid' } })) + .toThrow('usage.seconds must be a finite non-negative number'); + expect(() => parseAudioTranscriptionUsage({ usage: { type: 'tokens', input_tokens: -1, output_tokens: 2 } })) + .toThrow('usage.input_tokens must be a non-negative safe integer'); + expect(() => parseAudioTranscriptionUsage({ usage: { type: 'tokens', input_tokens: 4, input_token_details: { audio_tokens: 9 }, output_tokens: 2 } })) + .toThrow('audio_tokens must not exceed usage.input_tokens'); + expect(() => parseAudioTranscriptionUsage({ usage: 'billed' })).toThrow('usage must be an object'); + expect(() => parseAudioTranscriptionUsage({ duration: 'a while' })).toThrow('duration must be a finite non-negative number'); + }); }); diff --git a/packages/protocols/src/audio/index.ts b/packages/protocols/src/audio/index.ts index d93c17de68..ceaf14a7d7 100644 --- a/packages/protocols/src/audio/index.ts +++ b/packages/protocols/src/audio/index.ts @@ -4,7 +4,7 @@ // of events instead of any of them. export type { AudioTranscriptionCue, SubtitleDialect } from './subtitles.ts'; -export { parseSubtitleDocument, renderSubtitleDocument } from './subtitles.ts'; +export { parseSubtitleDocument } from './subtitles.ts'; export type { AudioTranscriptionObjectFormat, diff --git a/packages/protocols/src/audio/subtitles.ts b/packages/protocols/src/audio/subtitles.ts index 2dede717e9..100ff94419 100644 --- a/packages/protocols/src/audio/subtitles.ts +++ b/packages/protocols/src/audio/subtitles.ts @@ -1,12 +1,17 @@ // SubRip and WebVTT, the two renderings of a transcription that are subtitle documents -// rather than JSON. Both carry exactly one thing — timed cues — so both are a total -// reading of, and a total writing from, the canonical transcription's `cues`. +// rather than JSON. Both carry exactly one thing — timed cues — so reading one is a total +// reading of it. // -// The exact shapes are Whisper's own writers, which is what OpenAI's `whisper-1` runs: -// SubRip numbers its cues from 1, always writes the hour component and separates the -// milliseconds with a comma; WebVTT opens with a `WEBVTT` line, drops the hour component -// below one hour and separates the milliseconds with a period. Both terminate every cue -// with a blank line. +// There is no writer here, and that is the point: a subtitle rendering is carried to the +// client as the bytes the upstream sent, so nothing ever has to reproduce one. What the +// reading is for is the record and the transcript beside it. +// +// The shapes are Whisper's own writers, which is what OpenAI's `whisper-1` runs: SubRip +// numbers its cues from 1, always writes the hour component and separates the milliseconds +// with a comma; WebVTT opens with a `WEBVTT` line, drops the hour component below one hour +// and separates the milliseconds with a period. Both terminate every cue with a blank line. +// Every one of those is a convention rather than a rule, which is why reading is written to +// accept more than Whisper writes. // https://github.com/openai/whisper/blob/v20250625/whisper/utils.py#L238-L262 export interface AudioTranscriptionCue { @@ -20,27 +25,6 @@ export type SubtitleDialect = 'srt' | 'vtt'; const VTT_HEADER = 'WEBVTT'; -/** Whisper rounds to whole milliseconds before splitting the components, so a timestamp - * written from a parsed one is the same string it was parsed from. - * https://github.com/openai/whisper/blob/v20250625/whisper/utils.py#L50-L68 */ -const writeTimestamp = (seconds: number, dialect: SubtitleDialect): string => { - if (!Number.isFinite(seconds) || seconds < 0) { - throw new Error(`Subtitle timestamp must be a finite non-negative number: ${seconds}`); - } - let milliseconds = Math.round(seconds * 1000); - const hours = Math.floor(milliseconds / 3_600_000); - milliseconds -= hours * 3_600_000; - const minutes = Math.floor(milliseconds / 60_000); - milliseconds -= minutes * 60_000; - const whole = Math.floor(milliseconds / 1_000); - milliseconds -= whole * 1_000; - - const pad = (value: number, width: number) => String(value).padStart(width, '0'); - const hoursMarker = dialect === 'srt' || hours > 0 ? `${pad(hours, 2)}:` : ''; - const decimalMarker = dialect === 'srt' ? ',' : '.'; - return `${hoursMarker}${pad(minutes, 2)}:${pad(whole, 2)}${decimalMarker}${pad(milliseconds, 3)}`; -}; - const TIMESTAMP = /^(?:(\d+):)?(\d{1,2}):(\d{1,2})[,.](\d{1,3})$/; const readTimestamp = (value: string, dialect: SubtitleDialect): number => { @@ -94,14 +78,3 @@ export const parseSubtitleDocument = (dialect: SubtitleDialect, document: string } return blocks.map(block => readCueBlock(block, dialect)); }; - -export const renderSubtitleDocument = (dialect: SubtitleDialect, cues: readonly AudioTranscriptionCue[]): string => { - const body = cues - .map((cue, index) => { - const timing = `${writeTimestamp(cue.start, dialect)} --> ${writeTimestamp(cue.end, dialect)}`; - const ordinal = dialect === 'srt' ? `${index + 1}\n` : ''; - return `${ordinal}${timing}\n${cue.text}\n\n`; - }) - .join(''); - return dialect === 'vtt' ? `${VTT_HEADER}\n\n${body}` : body; -}; diff --git a/packages/protocols/src/audio/transcription.ts b/packages/protocols/src/audio/transcription.ts index 4050e561b1..274f7ab673 100644 --- a/packages/protocols/src/audio/transcription.ts +++ b/packages/protocols/src/audio/transcription.ts @@ -1,21 +1,26 @@ // The one transcription, and the several documents that are renderings of it. // -// `response_format` picks a rendering, and it travels to the upstream in the form the -// client sent, so the rendering the client asked for is the rendering the upstream answers -// in. That is what makes one canonical value enough: the gateway reads whichever document -// arrived and writes the same one back, and no rendering is ever derived from a value that -// was read out of a different one. +// `response_format` picks a rendering, and it travels to the upstream in the form the client +// sent, so the rendering the client asked for is the rendering the upstream answers in. That +// is what makes one canonical value enough: whichever rendering arrived is the one that goes +// back, and no rendering is ever derived from a value read out of a different one. +// +// A document rendering is **carried**, never written again. `text`, `srt` and `vtt` are +// documents the upstream wrote for the client to read, and re-writing one from what we read +// out of it changes bytes nobody asked to have changed: a cue writer of our own terminates +// the last cue with a blank line the upstream may not have written, renumbers a SubRip cue +// the upstream numbered from something else, and rounds a timestamp to its own precision. So +// the canonical value holds the bytes that arrived and hands them straight back, and what it +// reads out of them is what the record shows and what usage is measured from. // // Which is as well, because the renderings are not equally rich. `verbose_json` carries a // whole transcript and its cues; `json` carries the transcript with usage beside it; `srt` // and `vtt` carry the cues alone; `text` carries the transcript alone and nothing else at -// all. So a `srt` answer holds no `language`, no `duration` and no usage, and its transcript -// exists only as the cues' texts. Those are properties of the renderings, not gaps in the -// reading. +// all. Those are properties of the renderings, not gaps in the reading. // https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L714-L745 import type { AudioTranscriptionCue } from './subtitles.ts'; -import { parseSubtitleDocument, renderSubtitleDocument } from './subtitles.ts'; +import { parseSubtitleDocument } from './subtitles.ts'; // `diarized_json` joined the enum with `gpt-4o-transcribe-diarize`; it is a JSON rendering // and needs nothing of its own here. @@ -57,20 +62,27 @@ export type AudioTranscriptionUsage = | { readonly kind: 'duration'; readonly seconds: number }; export interface CanonicalAudioTranscription { - /** The whole transcript. A subtitle document does not carry one, so for `srt` and `vtt` - * this is the cues' texts joined by newlines — which is what Whisper's own text writer - * produces from the same result, and is not claimed to be byte-identical to what a - * `text` request would have returned. + /** What the upstream sent, byte for byte. Every rendering but the objects is written back + * from this, and so is a body the reading below could not open. + * + * Bytes rather than a string, because a decode is a reading like any other: a document + * under a charset nothing here asked about survives being carried and does not survive + * being decoded and encoded again. */ + readonly document: Uint8Array; + /** The whole transcript, as the rendering stated it. A subtitle document does not carry + * one, so for `srt` and `vtt` this is the cues' texts joined by newlines — which is what + * Whisper's own text writer produces from the same result, and is not the string a `text` + * request would have returned. Absent on a document the reading could not open. * https://github.com/openai/whisper/blob/v20250625/whisper/utils.py#L109-L116 */ - readonly text: string; + readonly text?: string; /** The timed cues. This is the whole of `srt` and `vtt`; `verbose_json` and * `diarized_json` carry the same thing as `segments`. */ readonly cues?: readonly AudioTranscriptionCue[]; /** The upstream's own object, kept whole for the renderings that are objects. A JSON * answer is re-serialized from this, so a field this type does not name — a segment's - * `avg_logprob`, a `logprobs` array, a `speaker` label — reaches the client unchanged. */ + * `avg_logprob`, a `logprobs` array, a `speaker` label — reaches the client unchanged, + * and it is also what usage is read out of. */ readonly raw?: Record; - readonly usage?: AudioTranscriptionUsage; } const isRecord = (value: unknown): value is Record => @@ -78,39 +90,58 @@ const isRecord = (value: unknown): value is Record => const isFiniteNumber = (value: unknown): value is number => typeof value === 'number' && Number.isFinite(value); -const isCount = (value: unknown): value is number => - typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; +const seconds = (value: unknown, where: string): number => { + if (!isFiniteNumber(value) || value < 0) throw new Error(`Audio transcription ${where} must be a finite non-negative number`); + return value; +}; + +const count = (value: unknown, where: string): number => { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) { + throw new Error(`Audio transcription ${where} must be a non-negative safe integer`); + } + return value; +}; /** - * What the upstream said it will bill for, or `undefined` when it said nothing this reading - * recognizes. There is no third answer: a report that cannot be read is, from here, no - * report, and the caller says so by naming the entity with no quantities. + * What the upstream said it will bill for. + * + * `undefined` is "nothing this reading recognizes was stated": no report at all, or one under + * a discriminator this protocol does not name — a metric invented after this was written is + * not a malformed one. A report that *is* one of the two and cannot be read is the other + * situation entirely, and it throws, so the caller can say so rather than bill an upstream + * that did meter as though it had not. * - * Whisper's `verbose_json` predates the `usage` block and states the same duration at the - * top level, so that is read where no block is present. + * Whisper's `verbose_json` predates the `usage` block and states the same duration at the top + * level, so that is read where no block is present. * https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L36513-L36545 * https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L61968-L62051 */ export const parseAudioTranscriptionUsage = (value: unknown): AudioTranscriptionUsage | undefined => { if (!isRecord(value)) return undefined; - const usage = value.usage; - if (!isRecord(usage)) { - return isFiniteNumber(value.duration) && value.duration >= 0 - ? { kind: 'duration', seconds: value.duration } - : undefined; - } - if (usage.type === 'duration') { - return isFiniteNumber(usage.seconds) && usage.seconds >= 0 ? { kind: 'duration', seconds: usage.seconds } : undefined; + if (value.usage === undefined) { + if (value.duration === undefined) return undefined; + return { kind: 'duration', seconds: seconds(value.duration, 'duration') }; } - if (usage.type !== 'tokens' || !isCount(usage.input_tokens) || !isCount(usage.output_tokens)) return undefined; + const usage = value.usage; + if (!isRecord(usage)) throw new Error('Audio transcription usage must be an object'); + if (usage.type === 'duration') return { kind: 'duration', seconds: seconds(usage.seconds, 'usage.seconds') }; + if (usage.type !== 'tokens') return undefined; + + const inputTokens = count(usage.input_tokens, 'usage.input_tokens'); + const outputTokens = count(usage.output_tokens, 'usage.output_tokens'); const details = usage.input_token_details; - const audioTokens = isRecord(details) && isCount(details.audio_tokens) ? details.audio_tokens : undefined; - if (audioTokens !== undefined && audioTokens > usage.input_tokens) return undefined; + if (details !== undefined && !isRecord(details)) throw new Error('Audio transcription usage.input_token_details must be an object'); + const audioTokens = details?.audio_tokens === undefined + ? undefined + : count(details.audio_tokens, 'usage.input_token_details.audio_tokens'); + if (audioTokens !== undefined && audioTokens > inputTokens) { + throw new Error('Audio transcription usage.input_token_details.audio_tokens must not exceed usage.input_tokens'); + } return { kind: 'tokens', - inputTokens: usage.input_tokens, + inputTokens, ...(audioTokens === undefined ? {} : { inputAudioTokens: audioTokens }), - outputTokens: usage.output_tokens, + outputTokens, }; }; @@ -132,48 +163,45 @@ const objectCues = (value: Record): readonly AudioTranscription }; /** - * The one reading. A JSON rendering hands in the parsed object; `text`, `srt` and `vtt` - * hand in the document itself. + * The one reading, over the bytes the upstream sent. + * + * It throws on a body it cannot open, and a caller that carries the document rather than + * serializing one from what it read answers anyway: what a failed reading costs is the + * transcript in the record and the usage measured beside it, never the answer. */ export const parseAudioTranscription = ( format: AudioTranscriptionResponseFormat, - body: unknown, + document: Uint8Array, ): CanonicalAudioTranscription => { + const decoded = new TextDecoder().decode(document); if (isAudioTranscriptionObjectFormat(format)) { + const body: unknown = JSON.parse(decoded); if (!isRecord(body)) throw new Error('Audio transcription response body must be an object'); const cues = objectCues(body); - const usage = parseAudioTranscriptionUsage(body); return { + document, text: requiredText(body.text, 'text'), ...(cues === undefined ? {} : { cues }), raw: body, - ...(usage === undefined ? {} : { usage }), }; } - const document = requiredText(body, 'response body'); - if (format === 'text') return { text: document }; - const cues = parseSubtitleDocument(format, document); - return { text: cues.map(cue => cue.text).join('\n'), cues }; + if (format === 'text') return { document, text: decoded }; + const cues = parseSubtitleDocument(format, decoded); + return { document, text: cues.map(cue => cue.text).join('\n'), cues }; }; /** - * The one writing, back into the rendering the client asked for. A JSON rendering is the - * upstream's own object re-serialized; a subtitle rendering is written from the cues; `text` - * is the transcript alone. + * The one writing, back into the rendering the client asked for. + * + * An object rendering is the upstream's own object re-serialized, which is what keeps a field + * this protocol does not name on its way to the client. Everything else is the document that + * arrived, handed back untouched — and so is an object rendering whose body the reading could + * not open, because bytes nobody could read are still the answer the upstream gave. */ export const renderAudioTranscription = ( format: AudioTranscriptionResponseFormat, transcription: CanonicalAudioTranscription, -): Record | string => { - if (isAudioTranscriptionObjectFormat(format)) { - if (transcription.raw === undefined) { - throw new Error(`Audio transcription cannot be rendered as ${format}: it was not read from an object`); - } - return transcription.raw; - } - if (format === 'text') return transcription.text; - if (transcription.cues === undefined) { - throw new Error(`Audio transcription cannot be rendered as ${format}: it carries no cues`); - } - return renderSubtitleDocument(format, transcription.cues); -}; +): Record | Uint8Array => + isAudioTranscriptionObjectFormat(format) && transcription.raw !== undefined + ? transcription.raw + : transcription.document; From 6355a3b311867c47f3594402c17e4060127777e1 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 04:40:26 +0800 Subject: [PATCH 32/79] feat(protocols): give the images protocol its stream contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both images endpoints accept `stream: true` and answer with SSE, and this protocol could read only the single JSON body — so a gateway carrying it had nothing to parse a streamed answer into. The stream is now the protocol's own: `parseImagesStream` reads the upstream's body into events, `imagesStreamEventToSSEFrame` writes them back, and `isImagesTerminalEvent` says when the image is done. The terminal is the completed event and there is no `[DONE]` — the two endpoints name their terminal apart (`image_generation.completed`, `image_edit.completed`), so the suffix is what the reading matches, which also serves an endpoint added later. An event's name is written twice on the wire, as the SSE label and as `type` in the payload, and the two are reconciled at the boundary so every reader takes it off the payload. `imagesRequestWantsStream` reads the flag the client sent. It admits both the boolean and the string, because a multipart edit carries every field as the text the client typed. --- packages/protocols/src/images/index.ts | 26 +++++++++++- packages/protocols/src/images/request.ts | 14 ++++++- packages/protocols/src/images/stream.ts | 51 ++++++++++++++++++++++++ packages/protocols/src/images/to-sse.ts | 9 +++++ 4 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 packages/protocols/src/images/stream.ts create mode 100644 packages/protocols/src/images/to-sse.ts diff --git a/packages/protocols/src/images/index.ts b/packages/protocols/src/images/index.ts index 6e050e7876..9f0ead18e1 100644 --- a/packages/protocols/src/images/index.ts +++ b/packages/protocols/src/images/index.ts @@ -104,5 +104,29 @@ export interface CanonicalImagesResponse { usage?: CanonicalImagesUsage; } -export { parseImagesEditsRequest, parseImagesGenerationsRequest } from './request.ts'; +/** One event on a streamed answer. A stream carries partial images and then the completed one, + * and the completed one carries the `usage` block the non-streamed body carries — so what a + * call is billed by is stated the same way whichever shape it came back in. + * https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L51337-L51424 + * https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L51158-L51246 + * + * `type` is an open string rather than the four names the specification lists: an event type + * is the upstream's own word, and a shape added later has to reach the client whether or not + * this gateway has heard of it. The index signature carries everything not named here. */ +export interface ImagesStreamEvent { + type: string; + b64_json?: string; + created_at?: number; + size?: string; + quality?: string; + background?: string; + output_format?: string; + partial_image_index?: number; + usage?: unknown; + [key: string]: unknown; +} + +export { imagesRequestWantsStream, parseImagesEditsRequest, parseImagesGenerationsRequest } from './request.ts'; export { parseImagesResponse, parseImagesUsage, renderImagesResponse } from './response.ts'; +export { IMAGES_MISSING_TERMINAL_MESSAGE, isImagesTerminalEvent, parseImagesStream, type ParseImagesStreamOptions } from './stream.ts'; +export { imagesStreamEventToSSEFrame } from './to-sse.ts'; diff --git a/packages/protocols/src/images/request.ts b/packages/protocols/src/images/request.ts index 287e17ab10..65234b74ba 100644 --- a/packages/protocols/src/images/request.ts +++ b/packages/protocols/src/images/request.ts @@ -2,12 +2,24 @@ // bodies — generations is JSON, edits is JSON or a multipart form — and each message here is // what the client is told, so they name the endpoint and the field they are about. -import type { ImageEditReference, ImagesEditImage, ImagesUploadedFile, ParsedImagesRequest } from './index.ts'; +import type { CanonicalImagesRequest, ImageEditReference, ImagesEditImage, ImagesUploadedFile, ParsedImagesRequest } from './index.ts'; import { isJsonMediaType, isMultipartFormDataMediaType } from '../common/media-type.ts'; const isRecord = (value: unknown): value is Record => typeof value === 'object' && value !== null && !Array.isArray(value); +/** + * Whether the client asked for the answer as a stream. + * + * The flag rides on to the upstream inside `parameters` exactly as it arrived, and this is the + * gateway's own reading of it. It is two values rather than one because an edit may be sent as + * a multipart form, where every field arrives as the text the client typed, while generations + * and a JSON edit carry the boolean the specification names. + * https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L47542-L47673 + */ +export const imagesRequestWantsStream = (request: CanonicalImagesRequest): boolean => + request.parameters.stream === true || request.parameters.stream === 'true'; + const jsonObject = (body: Uint8Array, endpoint: string): Record => { let parsed: unknown; try { diff --git a/packages/protocols/src/images/stream.ts b/packages/protocols/src/images/stream.ts new file mode 100644 index 0000000000..8d3b3b96f4 --- /dev/null +++ b/packages/protocols/src/images/stream.ts @@ -0,0 +1,51 @@ +// Reading a streamed answer. Both endpoints accept `stream: true` and answer with SSE that +// carries the same two shapes under their own names — `image_generation.partial_image` and +// `image_generation.completed` on generations, `image_edit.*` on edits — and the sequence ends +// at the completed event rather than at a sentinel: this protocol has no `[DONE]`. +// https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L13077-L13086 +// https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L12846-L12857 + +import type { ImagesStreamEvent } from './index.ts'; +import { parseTargetStreamFrames } from '../common/parse-events.ts'; +import { parseSSEStream } from '../common/parse-sse.ts'; + +export interface ParseImagesStreamOptions { + signal?: AbortSignal; +} + +export const IMAGES_MISSING_TERMINAL_MESSAGE = 'Images stream ended without a completed event.'; + +/** The stream is over when the image is. Each endpoint names its own terminal — + * `image_generation.completed` and `image_edit.completed` — so the suffix is the whole of + * what the two have in common, which is also what makes this reading serve an endpoint added + * later without being taught its prefix. */ +export const isImagesTerminalEvent = (event: ImagesStreamEvent): boolean => event.type.endsWith('.completed'); + +/** + * The upstream's SSE body as this protocol's events. Transport framing ends here: what comes + * out carries images events, and nothing that reads them knows how they arrived. + */ +export const parseImagesStream = ( + body: ReadableStream, + options: ParseImagesStreamOptions = {}, +): AsyncGenerator => (async function* () { + for await (const frame of parseTargetStreamFrames>(parseSSEStream(body, options), { protocol: 'Images' })) { + // `[DONE]` is not part of this protocol. An OpenAI-compatible upstream that appends the + // sentinel it writes elsewhere is saying the body is over, which is all it can mean here — + // whether the image finished is what the completed event says, and that has already passed. + if (frame.type === 'done') return; + yield named(frame.data, frame.frame.event); + } +})(); + +/** The event's own name, which the specification writes twice: as the SSE `event:` label and + * as `type` inside the payload. Upstreams that write only the label have been seen on the + * Responses protocol, and every reader here takes the name off the payload, so the two are + * reconciled once at the boundary rather than at each reader. An event that carries neither + * is not one this protocol can place — it could be the terminal or a partial — so it ends the + * read rather than being passed on as an unknown. */ +const named = (event: Record, label: string | undefined): ImagesStreamEvent => { + if (typeof event.type === 'string') return event as ImagesStreamEvent; + if (label === undefined) throw new Error('Images stream event carries no type, on the payload or as an SSE event name'); + return { ...event, type: label } as ImagesStreamEvent; +}; diff --git a/packages/protocols/src/images/to-sse.ts b/packages/protocols/src/images/to-sse.ts new file mode 100644 index 0000000000..0075f9eb50 --- /dev/null +++ b/packages/protocols/src/images/to-sse.ts @@ -0,0 +1,9 @@ +import type { ImagesStreamEvent } from './index.ts'; +import { type SseFrame, sseFrame } from '../common/index.ts'; + +// The specification writes an event's name twice — as the SSE `event:` label and as `type` +// inside the payload — and the stream parser reconciles the two on the way in, so writing the +// label back off the payload writes the name that arrived. +// https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L13077-L13086 +export const imagesStreamEventToSSEFrame = (event: ImagesStreamEvent): SseFrame => + sseFrame(JSON.stringify(event), event.type); From b36f9707da8265465ba71ec7f44db991e251e3b4 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 04:40:47 +0800 Subject: [PATCH 33/79] feat(images): carry a streamed answer as facts, and serve both endpoints through the pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A request carrying `stream: true` reached the upstream and was charged for, and the ending stage then had nothing to read it with: the images fact space held a parsed JSON body and nothing else, so the answer became a synthesized 502 after the image had already been paid for. That is why the route was withdrawn to its existing surface, which forwarded those frames. The stream is now carried the way completions and audio carry theirs. `response.images.canonical` holds a stream, a value or a failure at one key — telling them apart is reading a value — and `response.images.streamedUsage` holds what the upstream will have reported by the time the events run out. The ending reads the SSE when the client asked to stream and the upstream answered with one (an upstream that ignores the flag still answers the single body the other arm reads), meters usage off the events as they pass, and stops at the completed event, failing the answer if the body ends without one. The upstream's body is now owned and the fork declares it, so a losing attempt is released. The edge renders SSE frames or the JSON object, and `writeSettlement` is told when a run is still reading, so a streaming turn settles once — from the promise, in the epilogue — rather than twice. Both endpoints route through `serveThrough` again. What the endpoint decides is which contract reads the body; after that both hand the same canonical request to the same run. --- .../data-plane/images-pipeline_test.ts | 31 +- .../data-plane/images/stream_test.ts | 250 ++++++++++++++++ .../gateway/src/data-plane/images/http.ts | 217 ++++---------- .../gateway/src/data-plane/images/pipeline.ts | 275 ++++++++++++++---- 4 files changed, 540 insertions(+), 233 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/images/stream_test.ts diff --git a/packages/gateway/__tests__/data-plane/images-pipeline_test.ts b/packages/gateway/__tests__/data-plane/images-pipeline_test.ts index 650e506486..d10ec447ae 100644 --- a/packages/gateway/__tests__/data-plane/images-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/images-pipeline_test.ts @@ -1,14 +1,6 @@ // Images' pipeline, assembled. `compose` derives the entry contract and rejects an array that // cannot work, so most of what this file establishes is established by the assembly succeeding // at all — and what is worth writing down is the contract it derives and the keys it cannot see. -// -// One thing no assertion here can reach, and the reason the route table must keep serving this -// family the old way until it is settled: both endpoints accept `stream: true` and answer with -// an SSE sequence of partial images, while this pipeline carries a parsed JSON body and nothing -// else. Such a request reaches the upstream, spends it, and comes back as the 502 the ending -// stage synthesizes for a body the protocol cannot read. Carrying it needs a stream of protocol -// frames in the fact space, which is the work the first streaming family brings with it. -// https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L13077-L13086 import { describe, expect, it } from 'vitest'; @@ -32,30 +24,37 @@ describe('the images pipeline', () => { expect([...imagesServePipeline(edits).entryNeeds].sort()).toEqual(['serve.model']); }); - // `callImagesUpstream` reads `request.images.canonical` and `ingress.http.headers`, and the - // derived contract mentions neither. That is not this family's defect: a stage whose only - // trait is `return` declares no request side at all, by ruling — "when it short-circuits, - // only `provides`" — so assembly cannot see what an ending stage reads, and every family's - // ending stage reads something. + // `callImagesUpstream` reads `request.images.canonical`, `ingress.images.wantsStream` and + // `ingress.http.headers`, and the derived contract mentions none of them. That is not this + // family's defect: a stage whose only trait is `return` declares no request side at all, by + // ruling — "when it short-circuits, only `provides`" — so assembly cannot see what an ending + // stage reads, and every family's ending stage reads something. // // Written as a test rather than a comment because the hole has a consequence: a caller who - // omits either key gets a runtime failure at the deepest stage instead of the assembly error + // omits any of them gets a runtime failure at the deepest stage instead of the assembly error // the entry contract exists to produce. it('cannot see what its ending stage reads, because a return-only stage declares no needs', () => { const derived = imagesServePipeline(generations).entryNeeds; expect(derived).not.toContain('request.images.canonical'); + expect(derived).not.toContain('ingress.images.wantsStream'); expect(derived).not.toContain('ingress.http.headers'); }); // What covers that hole, and the reason it is a gap rather than a break: the entry type names - // all three, so the caller `entryNeeds` would have let through does not compile. + // all four, so the caller `entryNeeds` would have let through does not compile. it('names every key a caller must bring in its entry type', () => { const entry: ImagesServeEntry = { 'ingress.http.headers': [['content-type', 'application/json']], + 'ingress.images.wantsStream': false, 'request.images.canonical': generations, 'serve.model': 'gpt-image-1', }; - expect(Object.keys(entry).sort()).toEqual(['ingress.http.headers', 'request.images.canonical', 'serve.model']); + expect(Object.keys(entry).sort()).toEqual([ + 'ingress.http.headers', + 'ingress.images.wantsStream', + 'request.images.canonical', + 'serve.model', + ]); // @ts-expect-error — dropping one of them is a compile error, which is the statement this // makes; the assertion below only keeps the binding from being unused. diff --git a/packages/gateway/__tests__/data-plane/images/stream_test.ts b/packages/gateway/__tests__/data-plane/images/stream_test.ts new file mode 100644 index 0000000000..b8c69c4b91 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/images/stream_test.ts @@ -0,0 +1,250 @@ +import { test, vi } from 'vitest'; + +import type { InMemoryRepo } from '../../repo/memory.ts'; +import { buildCustomUpstreamRecord, flushAsyncWork, requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; +import { withMockedFetch, assertEquals, assertExists } from '@floway-dev/test-utils'; + +// The upstream is declared rather than discovered: both images endpoints are named on one +// manual model row, so nothing in these cases depends on the id heuristics that infer a +// catalog's capabilities. +const registerImagesModel = async (repo: InMemoryRepo): Promise => { + await repo.upstreams.deleteAll(); + clearInProcessCopilotTokenCache(); + await repo.upstreams.save(buildCustomUpstreamRecord({ + id: 'up_images', + name: 'Image Provider', + sortOrder: 100, + config: { + baseUrl: 'https://images.example.com', + authStyle: 'bearer', + ingressHeadersRules: [], + apiKey: 'sk-images', + endpoints: {}, + modelsFetch: { enabled: false }, + models: [{ + upstreamModelId: 'gpt-image-2-upstream', + publicModelId: 'gpt-image-2', + kind: 'image', + endpoints: { imagesGenerations: {}, imagesEdits: {} }, + }], + }, + })); +}; + +const PARTIAL = 'event: image_generation.partial_image\ndata: {"type":"image_generation.partial_image","b64_json":"UDA=","partial_image_index":0}\n\n'; +const COMPLETED = 'event: image_generation.completed\ndata: {"type":"image_generation.completed","b64_json":"RklO","usage":{"total_tokens":100,"input_tokens":50,"output_tokens":50,"input_tokens_details":{"text_tokens":10,"image_tokens":40}}}\n\n'; + +const sseResponse = (body: string, headers: Record = {}): Response => + new Response(body, { status: 200, headers: { 'content-type': 'text/event-stream', ...headers } }); + +test('/v1/images/generations answers a streaming request with the upstream events as SSE', async () => { + const { apiKey, repo } = await setupAppTest(); + await registerImagesModel(repo); + let forwarded: Record | undefined; + + await withMockedFetch( + async request => { + const url = new URL(request.url); + if (url.hostname === 'images.example.com' && url.pathname === '/v1/images/generations') { + forwarded = await request.json() as Record; + return sseResponse(PARTIAL + COMPLETED, { 'x-image-trace': 'trace-stream', 'set-cookie': 'upstream-session=secret' }); + } + throw new Error(`Unhandled fetch ${request.url}`); + }, + async () => { + const response = await requestApp('/v1/images/generations', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + body: JSON.stringify({ model: 'gpt-image-2', prompt: 'a shiba in space', stream: true, partial_images: 1 }), + }); + assertEquals(response.status, 200); + assertEquals(response.headers.get('content-type'), 'text/event-stream'); + // Vendor traces stay visible; an upstream session does not. + assertEquals(response.headers.get('x-image-trace'), 'trace-stream'); + assertEquals(response.headers.get('set-cookie'), null); + + const stream = await response.text(); + assertEquals(stream.includes('event: image_generation.partial_image'), true); + assertEquals(stream.includes('"partial_image_index":0'), true); + assertEquals(stream.includes('event: image_generation.completed'), true); + assertEquals(stream.includes('"b64_json":"RklO"'), true); + // This protocol ends at its completed event; the sentinel the chat dialects use is not + // part of it and is not invented here. + assertEquals(stream.includes('[DONE]'), false); + }, + ); + + // The flag rides to the upstream exactly as the client wrote it — the gateway asks for the + // stream the client asked for, having no usage chunk of its own to turn on. + assertExists(forwarded); + assertEquals(forwarded.stream, true); + assertEquals(forwarded.partial_images, 1); + assertEquals(forwarded.model, 'gpt-image-2-upstream'); +}); + +test('/v1/images/generations answers a non-streaming request with the JSON object', async () => { + const { apiKey, repo } = await setupAppTest(); + await registerImagesModel(repo); + + await withMockedFetch( + request => { + const url = new URL(request.url); + if (url.hostname === 'images.example.com' && url.pathname === '/v1/images/generations') { + return Promise.resolve(Response.json({ data: [{ b64_json: 'aGVsbG8=' }], usage: { input_tokens: 10, output_tokens: 50 } })); + } + throw new Error(`Unhandled fetch ${request.url}`); + }, + async () => { + const response = await requestApp('/v1/images/generations', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + body: JSON.stringify({ model: 'gpt-image-2', prompt: 'a shiba in space', stream: false }), + }); + assertEquals(response.status, 200); + assertEquals(response.headers.get('content-type'), 'application/json'); + assertEquals(await response.json(), { data: [{ b64_json: 'aGVsbG8=' }], usage: { input_tokens: 10, output_tokens: 50 } }); + }, + ); +}); + +test('/v1/images/generations bills a stream from the usage its completed event reported, once', async () => { + const { apiKey, repo } = await setupAppTest(); + await registerImagesModel(repo); + + await withMockedFetch( + () => Promise.resolve(sseResponse(PARTIAL + COMPLETED)), + async () => { + const response = await requestApp('/v1/images/generations', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + body: JSON.stringify({ model: 'gpt-image-2', prompt: 'a shiba in space', stream: true }), + }); + assertEquals(response.status, 200); + await response.text(); + }, + ); + + await flushAsyncWork(); + const rows = await repo.usage.listAll(); + assertEquals(rows.length, 1); + // The numbers arrive with the completed event, long after the run answered, so a row that + // carries them is a row settled from the promise the run handed up. Settling in the stage as + // well would have counted the request twice. + assertEquals(rows[0]?.requests, 1); + assertEquals(rows[0]?.metrics.map(row => ({ metric: row.metric, quantity: row.quantity })), [ + { metric: 'input_tokens', quantity: '10' }, + { metric: 'input_image_tokens', quantity: '40' }, + { metric: 'output_tokens', quantity: '50' }, + ]); + const [performance] = await repo.performance.listAll(); + assertEquals(performance?.requests, 1); +}); + +test('/v1/images/generations answers a refused stream in the upstream status and words', async () => { + const { apiKey, repo } = await setupAppTest(); + await registerImagesModel(repo); + + await withMockedFetch( + () => Promise.resolve(new Response(JSON.stringify({ error: { message: 'Rate limit reached for images.', code: 'rate_limit_exceeded' } }), { + status: 429, + headers: { 'content-type': 'application/json', 'retry-after': '7', 'x-error-trace': 'trace-refused' }, + })), + async () => { + const response = await requestApp('/v1/images/generations', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + body: JSON.stringify({ model: 'gpt-image-2', prompt: 'a shiba in space', stream: true }), + }); + // A request that asked to stream but was refused is answered as the refusal it is. + assertEquals(response.status, 429); + assertEquals(response.headers.get('content-type'), 'application/json'); + assertEquals(response.headers.get('retry-after'), '7'); + assertEquals(response.headers.get('x-error-trace'), 'trace-refused'); + assertEquals(await response.json(), { error: { message: 'Rate limit reached for images.', code: 'rate_limit_exceeded' } }); + }, + ); + + await flushAsyncWork(); + const [performance] = await repo.performance.listAll(); + assertEquals(performance?.errorsNoOutput, 1); +}); + +test('/v1/images/generations fails a stream that ended without a completed event', async () => { + const { apiKey, repo } = await setupAppTest(); + await registerImagesModel(repo); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + + try { + await withMockedFetch( + () => Promise.resolve(sseResponse(PARTIAL)), + async () => { + const response = await requestApp('/v1/images/generations', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + body: JSON.stringify({ model: 'gpt-image-2', prompt: 'a shiba in space', stream: true }), + }); + // The answer had already begun, so what the client keeps is what it was already sent. + assertEquals(response.status, 200); + const stream = await response.text(); + assertEquals(stream.includes('event: image_generation.partial_image'), true); + assertEquals(stream.includes('image_generation.completed'), false); + }, + ); + // An upstream that sent partial images and no image never answered the request, and the + // run says so rather than letting a truncated stream read as a complete one. + assertEquals( + errorSpy.mock.calls.some(call => call.some(arg => arg instanceof Error && arg.message === 'Images stream ended without a completed event.')), + true, + ); + } finally { + errorSpy.mockRestore(); + } +}); + +test('/v1/images/edits streams a multipart request, whose stream field arrives as text', async () => { + const { apiKey, repo } = await setupAppTest(); + await registerImagesModel(repo); + let upstreamForm: FormData | undefined; + + await withMockedFetch( + async request => { + const url = new URL(request.url); + if (url.hostname === 'images.example.com' && url.pathname === '/v1/images/edits') { + upstreamForm = await request.formData(); + return sseResponse( + 'event: image_edit.partial_image\ndata: {"type":"image_edit.partial_image","b64_json":"UDA=","partial_image_index":0}\n\n' + + 'event: image_edit.completed\ndata: {"type":"image_edit.completed","b64_json":"RklO","usage":{"total_tokens":20,"input_tokens":8,"output_tokens":12,"input_tokens_details":{"text_tokens":3,"image_tokens":5}}}\n\n', + ); + } + throw new Error(`Unhandled fetch ${request.url}`); + }, + async () => { + const form = new FormData(); + form.append('model', 'gpt-image-2'); + form.append('prompt', 'replace the sky'); + form.append('stream', 'true'); + form.append('image', new Blob([new Uint8Array([1, 2, 3])], { type: 'image/png' }), 'photo.png'); + const response = await requestApp('/v1/images/edits', { + method: 'POST', headers: { 'x-api-key': apiKey.key }, body: form, + }); + assertEquals(response.status, 200); + assertEquals(response.headers.get('content-type'), 'text/event-stream'); + const stream = await response.text(); + assertEquals(stream.includes('event: image_edit.partial_image'), true); + assertEquals(stream.includes('event: image_edit.completed'), true); + }, + ); + + assertExists(upstreamForm); + assertEquals(upstreamForm.get('stream'), 'true'); + + await flushAsyncWork(); + const rows = await repo.usage.listAll(); + assertEquals(rows.length, 1); + assertEquals(rows[0]?.metrics.map(row => ({ metric: row.metric, quantity: row.quantity })), [ + { metric: 'input_tokens', quantity: '3' }, + { metric: 'input_image_tokens', quantity: '5' }, + { metric: 'output_tokens', quantity: '12' }, + ]); +}); diff --git a/packages/gateway/src/data-plane/images/http.ts b/packages/gateway/src/data-plane/images/http.ts index 411127db9a..099259e718 100644 --- a/packages/gateway/src/data-plane/images/http.ts +++ b/packages/gateway/src/data-plane/images/http.ts @@ -1,174 +1,75 @@ -// POST /v1/images/generations and POST /v1/images/edits — route image -// requests to the provider that declares the requested model and the -// matching image endpoint capability. +// POST /v1/images/generations and POST /v1/images/edits, served through the pipeline. // -// The edits handler accepts multipart uploads and JSON `images` references. -// Both are buffered once for dump capture and normalized into a semantic -// request; each provider owns the final JSON or multipart serialization. -// https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L12558-L12620 +// Two endpoints and one family. What the endpoint decides is which contract reads the body — +// generations is always JSON, edits is JSON or a multipart form carrying the files — and +// after that both hand the same canonical request to the same run. import type { Context } from 'hono'; -import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; -import { createGatewayCtxFromHono, finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; -import { prepareJsonModelRequest } from '../shared/passthrough-request.ts'; -import { passthroughApiError, passthroughServe } from '../shared/passthrough-serve.ts'; -import { readRequestBody, takeRequestBody, type RequestBody } from '../shared/request-body.ts'; -import { tokenUsageFromImagesBody } from '../shared/telemetry/usage.ts'; -import { isJsonMediaType, isMultipartFormDataMediaType } from '@floway-dev/protocols/common'; -import type { ImageEditReference } from '@floway-dev/protocols/images'; -import { isBase64ImageDataUrl, type ImagesEditsRequest, type ImagesEditsSource } from '@floway-dev/provider'; +import { imagesServePipeline } from './pipeline.ts'; +import { isFrames, openPrologue, readIngress, serveThrough, type Ingress } from '../pipeline/serve.ts'; +import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; +import { move } from '@floway-dev/pipeline'; +import { imagesRequestWantsStream, parseImagesEditsRequest, parseImagesGenerationsRequest, type ParsedImagesRequest } from '@floway-dev/protocols/images'; -type PreparedImagesEdit = - | { type: 'ok'; request: ImagesEditsRequest } - | { type: 'invalid'; message: string }; - -const imageEditSource = (value: unknown, path: string): ImagesEditsSource | string => { - if (value === null || typeof value !== 'object' || Array.isArray(value)) { - return `${path} must be an object.`; - } - const reference = value as { image_url?: unknown; file_id?: unknown }; - const { image_url: imageUrl, file_id: fileId } = reference; - if (typeof imageUrl === 'string' && fileId === undefined) { - const imageReference = reference as ImageEditReference & { image_url: string }; - return isBase64ImageDataUrl(imageUrl) - ? { type: 'inline', reference: imageReference } - : { type: 'reference', reference: imageReference }; - } - if (typeof fileId === 'string' && imageUrl === undefined) { - return { type: 'reference', reference: reference as ImageEditReference }; - } - return `${path} must contain exactly one string field: image_url or file_id.`; -}; - -const prepareJsonImagesEdit = (body: Record): PreparedImagesEdit => { - if (!Array.isArray(body.images)) { - return { type: 'invalid', message: 'Image edits request body must include an images array.' }; - } - const images: ImagesEditsSource[] = []; - for (const [index, value] of body.images.entries()) { - const source = imageEditSource(value, `Image edits images[${index}]`); - if (typeof source === 'string') return { type: 'invalid', message: source }; - images.push(source); - } - let mask: ImagesEditsSource | undefined; - if (body.mask !== undefined) { - const source = imageEditSource(body.mask, 'Image edits mask'); - if (typeof source === 'string') return { type: 'invalid', message: source }; - mask = source; +/** The half both endpoints share: whatever the contract read, hand it over and turn what the + * run answered with into a response. The contract reports a malformed request by throwing; + * what the client is owed is a 400 carrying the reason. */ +const serveImages = async ( + c: Context, + ingress: Ingress, + read: () => ParsedImagesRequest | Promise, +): Promise => { + let parsed: ParsedImagesRequest; + try { + parsed = await read(); + } catch (error) { + // A request the gateway could not read never reaches a pipeline: there is no model to + // resolve and no attempt to make, so there is nothing for a run to record. + const refused = openPrologue(c, ingress, { wantsStream: false }); + refused.gateway.dump?.error('gateway'); + return finalizeGatewayResponse( + refused.gateway, + Response.json( + { error: { message: error instanceof Error ? error.message : String(error), type: 'api_error' } }, + { status: 400 }, + ), + ); } - const { model: _model, images: _images, mask: _mask, ...parameters } = body; - return { - type: 'ok', - request: { - images, - ...(mask === undefined ? {} : { mask }), - parameters, - }, - }; -}; -export const imagesGenerations = async (c: Context): Promise => { - const requestBody = await readRequestBody(c); - const request = prepareJsonModelRequest(requestBody.bytes, 'Images generations'); - const ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - if (request.type === 'invalid') { - ctx.dump?.error('gateway'); - return finalizeGatewayResponse(ctx, passthroughApiError(c, request.message, 400)); - } + const { model, request } = parsed; + // Whether the answer streams is written in the request the client sent, and the run has to be + // opened knowing it: the abort controller a streaming run cancels its read with is minted + // from this flag. + const wantsStream = imagesRequestWantsStream(request); + const prologue = openPrologue(c, ingress, { wantsStream, model }); - ctx.dump?.requestedModel(request.model); - const response = await passthroughServe({ + return await serveThrough( c, - ctx, - sourceApi: '/images/generations', - operation: 'image_generation', - model: request.model, - kind: 'image', - modelServesEndpoint: model => model.endpoints.imagesGenerations !== undefined, - call: (provider, model, opts) => { - const { model: _model, ...body } = request.body; - return provider.instance.callImagesGenerations(model, body, undefined, opts); + prologue, + imagesServePipeline(request), + move({ + 'ingress.http.headers': prologue.headers, + 'ingress.images.wantsStream': wantsStream, + 'request.images.canonical': request, + 'serve.model': model, + }) as never, + facts => { + const answer = facts['response.images.rendered']; + return isFrames(answer) ? { frames: answer } : { body: JSON.stringify(answer), contentType: 'application/json' }; }, - response: { format: 'json', extractBilling: tokenUsageFromImagesBody }, - }); - return finalizeGatewayResponse(ctx, response); + facts => facts['response.images.streamedUsage'], + ); }; -const serveImagesEditRequest = async ( - c: Context, - requestBody: RequestBody, - model: string, - request: ImagesEditsRequest, -): Promise => { - const ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - ctx.dump?.requestedModel(model); - const response = await passthroughServe({ - c, - ctx, - sourceApi: '/images/edits', - operation: 'image_edit', - model, - kind: 'image', - modelServesEndpoint: model => model.endpoints.imagesEdits !== undefined, - call: (provider, model, opts) => provider.instance.callImagesEdits(model, request, undefined, opts), - response: { format: 'json', extractBilling: tokenUsageFromImagesBody }, - }); - return finalizeGatewayResponse(ctx, response); +export const imagesGenerations = async (c: Context): Promise => { + const ingress = await readIngress(c); + return await serveImages(c, ingress, () => parseImagesGenerationsRequest(ingress.body.bytes)); }; export const imagesEdits = async (c: Context): Promise => { - const requestBody = await readRequestBody(c); - const invalid = (message: string): Response => { - const errorCtx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - errorCtx.dump?.error('gateway'); - return finalizeGatewayResponse(errorCtx, passthroughApiError(c, message, 400)); - }; - - const contentType = c.req.header('content-type'); - if (contentType === undefined) { - return invalid('Image edits request body must use application/json or multipart/form-data.'); - } - if (isJsonMediaType(contentType)) { - const body = prepareJsonModelRequest(requestBody.bytes, 'Image edits'); - if (body.type === 'invalid') return invalid(body.message); - const request = prepareJsonImagesEdit(body.body); - if (request.type === 'invalid') return invalid(request.message); - return await serveImagesEditRequest(c, requestBody, body.model, request.request); - } - - if (!isMultipartFormDataMediaType(contentType)) { - return invalid('Image edits request body must use application/json or multipart/form-data.'); - } - let form: FormData; - try { - form = await new Response(requestBody.bytes as BodyInit, { headers: { 'content-type': contentType } }).formData(); - } catch { - return invalid('Image edits request body must be valid multipart/form-data.'); - } - const model = form.get('model'); - if (typeof model !== 'string' || model.length === 0) { - return invalid('Image edits request body must include a model field.'); - } - const images: File[] = []; - let mask: File | undefined; - const parameters: Record = {}; - for (const [name, value] of form.entries()) { - if (name === 'model') continue; - if (name === 'image' || name === 'image[]') { - if (!(value instanceof File)) return invalid(`Image edits ${name} fields must be files.`); - images.push(value); - } else if (name === 'mask') { - if (!(value instanceof File)) return invalid('Image edits mask field must be a file.'); - mask = value; - } else { - if (typeof value !== 'string') return invalid(`Image edits ${name} field must be text.`); - parameters[name] = value; - } - } - return await serveImagesEditRequest(c, requestBody, model, { - images: images.map(file => ({ type: 'upload', file })), - ...(mask === undefined ? {} : { mask: { type: 'upload' as const, file: mask } }), - parameters, - }); + const ingress = await readIngress(c); + // Which of the two bodies arrived is a header's statement and not the payload's, so the + // contract is handed the media type alongside the bytes. + return await serveImages(c, ingress, () => parseImagesEditsRequest(c.req.header('content-type'), ingress.body.bytes)); }; diff --git a/packages/gateway/src/data-plane/images/pipeline.ts b/packages/gateway/src/data-plane/images/pipeline.ts index febb378f14..5ee7da45de 100644 --- a/packages/gateway/src/data-plane/images/pipeline.ts +++ b/packages/gateway/src/data-plane/images/pipeline.ts @@ -1,14 +1,18 @@ // Images as a pipeline. Two endpoints and one family: `generations` sends JSON and `edits` // sends either JSON or a multipart form, so what the operation decides is the request fact and -// the provider method the ending calls, and the four stages are the same either way. +// the provider method the ending calls, and the four stages are the same either way. Both +// endpoints accept `stream: true`, so the answer is a stream whenever the client asked for one +// — the same request field decides both what the upstream is asked for and what shape comes +// back. // -// emitImages the edge: serializes the answer into the images protocol +// emitImages the edge: serializes the answer into the images protocol, SSE +// framing included // resolveCandidates narrows to the upstreams that expose this operation's endpoint // failover runs what follows once per candidate // callImagesUpstream the ending: dials, and provides what came back import type { UsageQuantities } from '../../repo/types.ts'; -import type { Failure, GatewayFacts } from '../pipeline/facts.ts'; +import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; import { isFailure } from '../pipeline/facts.ts'; import type { GatewayServices } from '../pipeline/services.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; @@ -17,11 +21,24 @@ import { dialFailure, readUpstreamBody } from '../pipeline/upstream-body.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; -import type { Pipeline } from '@floway-dev/pipeline'; -import { compose, defineStage, move } from '@floway-dev/pipeline'; -import { mediaTypeEssence, parseDecimalString, renderErrorEnvelope, upstreamErrorMessage, type ModelEndpointKey } from '@floway-dev/protocols/common'; +import { compose, defineStage, move, own, type Owned, type Pipeline } from '@floway-dev/pipeline'; import { + eventFrame, + isEventStreamMediaType, + mediaTypeEssence, + parseDecimalString, + renderErrorEnvelope, + upstreamErrorMessage, + type ModelEndpointKey, + type SseFrame, +} from '@floway-dev/protocols/common'; +import { + IMAGES_MISSING_TERMINAL_MESSAGE, + imagesStreamEventToSSEFrame, + isImagesTerminalEvent, parseImagesResponse, + parseImagesStream, + parseImagesUsage, renderImagesResponse, type CanonicalImagesEditsRequest, type CanonicalImagesRequest, @@ -30,18 +47,47 @@ import { type ImageEditReference, type ImagesEditImage, type ImagesOperation, + type ImagesStreamEvent, } from '@floway-dev/protocols/images'; import { isBase64ImageDataUrl, providerModelOf } from '@floway-dev/provider'; -import type { ImagesEditsRequest, ImagesEditsSource, ModelCandidate, PerformanceOperation } from '@floway-dev/provider'; +import type { ImagesEditsRequest, ImagesEditsSource, ModelCandidate, PerformanceOperation, TelemetryModelIdentity } from '@floway-dev/provider'; + +/** The answer while it is still the upstream's, one event at a time. There is no transport + * sentinel below the events — this protocol ends at its completed event — so what travels is + * the events themselves rather than frames with a terminal arm the protocol has not got. */ +export type ImagesFrames = AsyncIterable; + +/** A stream as a value the record can hold. It has to be a wrapper around the generator rather + * than the generator itself: the language puts `Symbol.asyncDispose` on every async generator, + * so handing one over would make the runner adopt it as a resource, and releasing a resource + * calls `return()` — which cancels the iteration instead of draining it. The one resource in + * an images run is the upstream's body, at `response.http.body`, and this is what keeps it + * that way. */ +const view = (frames: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => frames }); /** Images' own keys. They extend the shared space and never merge into it, so a stage written * against the gateway alone cannot name one. */ export interface ImagesFacts extends GatewayFacts { + /** Whether the client asked for the answer as a stream. It stays put, as every `ingress.*` + * key does: the same flag travels to the upstream inside the parameters, and the answer is + * written back in the shape it asked for. */ + 'ingress.images.wantsStream': boolean; 'request.images.canonical': CanonicalImagesRequest; - 'response.images.canonical': CanonicalImagesResponse | Failure; - /** What the client is actually sent, in the images protocol. The edge provides it, so a dump - * shows the body the client received rather than the gateway's canonical form. */ - 'response.images.rendered': Record; + /** The answer, whichever kind it turned out to be. A stream, a value and a failure sit at + * one key: telling them apart is reading a value, and each stage does that where it needs + * to. */ + 'response.images.canonical': CanonicalImagesResponse | ImagesFrames | Failure; + /** What the upstream will have reported by the time the events run out, and `null` on every + * path that does not stream. A streamed image states its usage in the completed event, + * which is after this run has answered — so the numbers cannot be in + * `response.usage.billable`, which says what had been reported when the ending stage handed + * up: the entity, and no quantities. Settling from this is the epilogue's job, after the + * drain. */ + 'response.images.streamedUsage': Promise | null; + /** What the client is actually sent, in the images protocol — a JSON body, or the SSE frames + * of a stream. The edge provides it, so a dump shows the body the client received rather + * than the gateway's canonical form. */ + 'response.images.rendered': Record | AsyncIterable; } type I = { [P in K]: ImagesFacts[P] }; @@ -56,11 +102,18 @@ const PERFORMANCE_OPERATION = { edits: 'image_edit', } as const satisfies Record; +const isFrames = (answer: CanonicalImagesResponse | ImagesFrames): answer is ImagesFrames => + Symbol.asyncIterator in answer; + /** * The outermost edge. It names nothing on the way down — the family has one protocol, so there * is no request to read to know how to answer — and coming back it renders the answer, keeps * the upstream headers a client may see, and decides the status. The status is decided here * rather than carried up because a refusal that never reached an upstream has none to carry. + * + * SSE framing is produced here and nowhere else: below this stage a stream carries protocol + * events, so the same assembly would serve another transport by rendering differently at this + * one point. */ const emitImages = defineStage< Record, @@ -84,39 +137,46 @@ const emitImages = defineStage< // would misdescribe a body this gateway serialized itself, does not. A filter that removed // nothing hands the same array on, so the record shows no change where none happened. const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); - const forClient = forwardable.length === headers.length ? headers : move(forwardable); - if (isFailure(answer)) { - return { - ...rest, - 'response.http.headers': forClient, - 'response.images.rendered': move(renderErrorEnvelope(answer.message, answer.body)), - 'response.http.status': answer.status, - }; - } return { ...rest, - 'response.http.headers': forClient, - 'response.images.rendered': move(renderImagesResponse(answer)), - 'response.http.status': 200, + 'response.http.headers': forwardable.length === headers.length ? headers : move(forwardable), + 'response.http.status': isFailure(answer) ? answer.status : 200, + 'response.images.rendered': move(rendered(answer)), }; }, }); +const rendered = (answer: ImagesFacts['response.images.canonical']): ImagesFacts['response.images.rendered'] => + isFailure(answer) ? renderErrorEnvelope(answer.message, answer.body) + : isFrames(answer) ? renderSSE(answer) + : renderImagesResponse(answer); + +const renderSSE = (frames: ImagesFrames): AsyncIterable => view((async function* () { + for await (const event of frames) yield imagesStreamEventToSSEFrame(event); +})()); + /** - * The ending. It dials, reads the upstream's body, and provides the canonical answer, the - * headers that came with it, and what the call is billable for. A failure is a value: an - * upstream that refused, and one that answered with something this protocol cannot read, are - * both outcomes the fork above can take to the next candidate rather than faults that end the - * run. + * The ending. It dials, reads what came back on the shape the request asked for, and provides + * the canonical answer, the headers that came with it, the raw HTTP body beneath it, and what + * the call is billable for. A failure is a value: an upstream that refused, and one that + * answered with something this protocol cannot read, are both outcomes the fork above can take + * to the next candidate rather than faults that end the run. */ const callImagesUpstream = defineStage< - I<'request.images.canonical' | 'route.attempt' | 'ingress.http.headers'>, - I<'response.images.canonical' | 'response.http.headers' | 'response.usage.billable'>, + I<'ingress.images.wantsStream' | 'request.images.canonical' | 'route.attempt' | 'ingress.http.headers'>, + I<'response.images.canonical' | 'response.images.streamedUsage' | 'response.http.headers' + | 'response.http.body' | 'response.usage.billable'>, GatewayServices >({ name: 'callImagesUpstream', return: { - provides: ['response.images.canonical', 'response.http.headers', 'response.usage.billable'], + provides: [ + 'response.images.canonical', + 'response.images.streamedUsage', + 'response.http.headers', + 'response.http.body', + 'response.usage.billable', + ], }, execute: async (facts, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); @@ -138,7 +198,8 @@ const callImagesUpstream = defineStage< try { // No abort signal, as on this family's endpoints from the beginning: an image the upstream // has already begun is charged for whether or not the client waited for it, so dropping the - // call would lose the usage reading and save nothing. + // call would lose the usage reading and save nothing. Reading a stream is the other half of + // that and does take the signal — what it drops there is a connection, not an image. result = request.operation === 'generations' ? await candidate.provider.instance.callImagesGenerations(model, request.parameters, undefined, options) : await candidate.provider.instance.callImagesEdits(model, providerEditsRequest(request), undefined, options); @@ -149,7 +210,9 @@ const callImagesUpstream = defineStage< return move({ ...facts, 'response.images.canonical': dialFailure(error), + 'response.images.streamedUsage': null, 'response.http.headers': [], + 'response.http.body': spentBody(null), 'response.usage.billable': [], }); } @@ -157,36 +220,72 @@ const callImagesUpstream = defineStage< const identity = telemetryModelIdentity(candidate, result.modelKey); // What came back, unfiltered: the edge is where a client's view of it is decided. const headers = [...result.response.headers]; - const answered = (canonical: CanonicalImagesResponse | Failure, quantities: UsageQuantities) => move({ + // An entity with no quantities is how "the upstream was called and reported nothing" is + // said, which is a different situation from reporting zero. + const called: readonly BillableEntity[] = [{ identity, quantities: {} }]; + const read = (canonical: CanonicalImagesResponse | Failure, billable: readonly BillableEntity[]) => move({ ...facts, 'response.images.canonical': canonical, + 'response.images.streamedUsage': null, 'response.http.headers': headers, - 'response.usage.billable': [{ identity, quantities }], + 'response.http.body': spentBody(result.response.body), + 'response.usage.billable': billable, }); - // An entity with no quantities is how "the upstream was called and reported nothing" is - // said, which is a different situation from reporting zero. - const reportedNothing: UsageQuantities = {}; - const body = await readUpstreamBody(result.response); if (!result.response.ok) { use.log.warn('upstream refused', { status: result.response.status }); - return answered({ + // An upstream error body is JSON like any other body. Reading it here is also what + // leaves a losing attempt with nothing open behind it. + const body = await readUpstreamBody(result.response); + return read({ status: result.response.status, message: upstreamErrorMessage(body.json) ?? body.text, ...('json' in body ? { body: body.json } : {}), - }, reportedNothing); + }, called); + } + + const mediaType = result.response.headers.get('content-type'); + // Both halves have to hold. `stream` is what the client asked for, and an upstream that + // ignores it answers the single JSON body the arm below reads — so what settles which + // shape arrived is the media type, and what settles which shape the client is owed is the + // request. + if (facts['ingress.images.wantsStream'] && isEventStreamMediaType(mediaType)) { + if (result.response.body === null) { + return read({ status: 502, message: 'Upstream returned a streaming response with no body.' }, called); + } + // Usage is observed here, closest to the upstream and on the protocol it spoke, by + // folding the events as they pass — so the reading costs one pass and the client's own + // stream is what drives it. What it finds arrives with the completed event, long after + // this stage has handed up, which is why the entity above carries no quantities. + const metered = meterFrames( + result.response.body, + identity, + use.gateway.abortSignal, + event => { use.gateway.dump?.frame(eventFrame(event)); }, + ); + return move({ + ...facts, + 'response.images.canonical': metered.frames, + 'response.images.streamedUsage': metered.billable, + 'response.http.headers': headers, + // Releasing this body is reading those events to the end: they are one reader over one + // connection, and a second reader is not something a `ReadableStream` allows. + 'response.http.body': own(result.response.body, async (): Promise => { for await (const _event of metered.frames) { /* to end of stream */ } }), + 'response.usage.billable': called, + }); } + + const body = await readUpstreamBody(result.response); if (!('json' in body)) { // Every protocol the gateway carries is one it fully understands, so a body it cannot - // read is not handed on unread. A client that asked for `stream: true` lands here: the - // images stream is not carried as facts yet. - const mediaType = mediaTypeEssence(result.response.headers.get('content-type')) ?? 'no media type'; - use.log.warn('upstream answered with a body that is not JSON', { status: result.response.status, mediaType }); - return answered({ + // read is not handed on unread. + const essence = mediaTypeEssence(mediaType) ?? 'no media type'; + use.log.warn('upstream answered with a body that is not JSON', { status: result.response.status, mediaType: essence }); + return read({ status: 502, - message: `The upstream answered ${result.response.status} with ${mediaType}, and the images protocol is JSON.`, + message: `The upstream answered ${result.response.status} with ${essence}, and the images protocol is JSON.`, body: body.text, - }, reportedNothing); + }, called); } let canonical: CanonicalImagesResponse; @@ -195,12 +294,62 @@ const callImagesUpstream = defineStage< } catch (error) { const message = error instanceof Error ? error.message : String(error); use.log.warn('upstream answered with a body the images protocol cannot read', { message }); - return answered({ status: 502, message, body: body.json }, reportedNothing); + return read({ status: 502, message, body: body.json }, called); } - return answered(canonical, billed(canonical.usage)); + return read(canonical, [{ identity, quantities: billed(canonical.usage) }]); }, }); +/** A body this stage has already read to the end, or one the upstream never sent. The record + * holds a body as a stream and `failover` releases the losing attempts', so every path hands + * one up; what says an answer was unusable is the failure at the canonical key, not this. */ +const spentBody = (body: ReadableStream | null): ReadableStream & Owned => + own(body ?? new ReadableStream({ start: controller => controller.close() }), (): Promise => Promise.resolve()); + +interface MeteredFrames { + readonly frames: ImagesFrames; + readonly billable: Promise; +} + +const meterFrames = ( + body: ReadableStream, + identity: TelemetryModelIdentity, + signal: AbortSignal | undefined, + record: (event: ImagesStreamEvent) => void, +): MeteredFrames => { + let settle!: (billable: readonly BillableEntity[]) => void; + const billable = new Promise(resolve => { settle = resolve; }); + const frames = view((async function* () { + let usage: CanonicalImagesUsage | undefined; + try { + for await (const event of parseImagesStream(body, { signal })) { + // What the upstream sent, before the edge decides what the client sees — a dump reader + // is owed the events that arrived. + record(event); + if (isImagesTerminalEvent(event)) { + usage = parseImagesUsage(event); + yield event; + // The image is complete, so there is nothing further to read. An upstream that holds + // the connection open past this point would otherwise keep the client's own stream + // open with it; returning here closes the read, which cancels the upstream. + return; + } + yield event; + } + } finally { + // Reached however the events ended — the completed one, a client that stopped reading, + // or a broken upstream — because what the upstream already metered is billable whatever + // happened to the downstream half. + settle([{ identity, quantities: billed(usage) }]); + } + // Only an upstream that ended its body without ever completing the image reaches here: the + // arm above returns on the terminal event. A client has been sent partial images and no + // image, which is a failed answer however far it got. + throw new Error(IMAGES_MISSING_TERMINAL_MESSAGE); + })()); + return { frames, billable }; +}; + /** An image is billed by the tokens its upstream reports: `BILLING_METRICS` names no per-image * or per-size unit, and per-size pricing is a selector coordinate rather than a metric, so * there is nothing else here to record a count against. A reported zero is kept — it says the @@ -243,26 +392,34 @@ const narrowing = (request: CanonicalImagesRequest) => ({ ? `the upstream does not expose the images ${request.operation} endpoint` : null, unsupported: (model: string) => `Model ${model} does not support the /images/${request.operation} endpoint.`, - refuse: (status: number, message: string) => ({ 'response.images.canonical': { status, message } }), - refuses: ['response.images.canonical'] as const, + refuse: (status: number, message: string) => ({ + 'response.images.canonical': { status, message } as Failure, + 'response.images.streamedUsage': null, + }), + refuses: ['response.images.canonical', 'response.images.streamedUsage'] as const, }); -/** What a caller must bring. `ingress.http.headers` and `request.images.canonical` are in it - * although `compose` cannot derive them — the ending stage reads both and a return-only stage - * declares no request side — so this type is the whole statement and `entryNeeds` is part of - * it. */ -export type ImagesServeEntry = I<'ingress.http.headers' | 'request.images.canonical' | 'serve.model'>; +/** What a caller must bring. `ingress.http.headers`, `ingress.images.wantsStream` and + * `request.images.canonical` are in it although `compose` cannot derive them — the ending + * stage reads all three and a return-only stage declares no request side — so this type is the + * whole statement and `entryNeeds` is part of it. */ +export type ImagesServeEntry = I<'ingress.http.headers' | 'ingress.images.wantsStream' | 'request.images.canonical' | 'serve.model'>; -export type ImagesServeExit = I<'response.images.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'>; +export type ImagesServeExit = I< + 'response.images.rendered' | 'response.images.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable' +>; export const imagesServePipeline = (request: CanonicalImagesRequest): Pipeline => compose('imagesServe', [ emitImages, - writeSettlement(handedUp => isFailure((handedUp as { 'response.images.canonical'?: unknown })['response.images.canonical'])), + writeSettlement( + handedUp => isFailure((handedUp as { 'response.images.canonical'?: unknown })['response.images.canonical']), + handedUp => (handedUp as { 'response.images.streamedUsage'?: unknown })['response.images.streamedUsage'] !== null, + ), resolveCandidates(narrowing(request)), failover({ failed: handedUp => isFailure((handedUp as { 'response.images.canonical'?: unknown })['response.images.canonical']), - owns: [], + owns: ['response.http.body'], }), callImagesUpstream, ]); From f5ae077948fa9b07429dd0d875c1661eafa7c981 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 04:43:52 +0800 Subject: [PATCH 34/79] feat(audio): serve /v1/audio/transcriptions through the pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The route was the last one left on the replaced surface, held there by one question: that surface forwarded a transcription's document untouched, and the pipeline parsed it into cues and wrote it back. Carrying the document settles it, and three more of the replaced surface's statements had to follow it across. A 2xx body is the answer whether or not this endpoint could read it. Nothing is serialized from the parse any more, so a body no reading could open is still what goes back — with a warning and a request-only usage row, never a 502 and never a failover to a candidate that would answer no differently. The two readings are independent for the same reason: an upstream whose usage block this gateway cannot model still had its transcript carried, and one whose document it could not open is still recorded as having metered nothing. A stream states its own outcome in its last event. An upstream that stops before `transcript.text.done` answered 200 and then did not finish, which is a failed request however much of the transcript reached the client, so the promise the ending stage hands up now carries that alongside what was billed. The handler writes its own epilogue rather than going through `serveThrough`, on two counts that belong to the seam: `Rendered.contentType` has no value for an upstream that declared no media type, and `DeferredUsage` cannot carry a stream's outcome. Fold both into the seam and this handler is `serveThrough` again. --- .../__tests__/data-plane/audio/http_test.ts | 38 +++++ packages/gateway/src/data-plane/audio/http.ts | 141 ++++++++++++---- .../gateway/src/data-plane/audio/pipeline.ts | 156 +++++++++++------- 3 files changed, 247 insertions(+), 88 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/audio/http_test.ts b/packages/gateway/__tests__/data-plane/audio/http_test.ts index 7e01171779..7c88f0428a 100644 --- a/packages/gateway/__tests__/data-plane/audio/http_test.ts +++ b/packages/gateway/__tests__/data-plane/audio/http_test.ts @@ -150,6 +150,44 @@ test('/v1/audio/transcriptions forwards VTT verbatim and records request-only us assertEquals(usage.metrics, []); }); +// A subtitle document is carried, so every convention the upstream did not follow survives: +// the ordinal it started at, the line ending it chose, and the fraction digits it wrote. +// Reading the cues and writing them back would have normalized all three. +test('/v1/audio/transcriptions forwards SubRip with the numbering and line endings the upstream chose', async () => { + const { apiKey, repo } = await setupAppTest(); + await registerAudioModel(repo); + const document = '7\r\n00:00:00,0 --> 00:00:01,5\r\nhello\r\n'; + await withMockedFetch( + () => new Response(document, { headers: { 'content-type': 'application/x-subrip' } }), + async () => { + const response = await requestApp('/v1/audio/transcriptions', { + method: 'POST', headers: { 'x-api-key': apiKey.key }, body: transcriptionForm([['response_format', 'srt']]), + }); + assertEquals(response.headers.get('content-type'), 'application/x-subrip'); + assertEquals(await response.text(), document); + }, + ); +}); + +// The document never becomes a string on the way through, so an upstream that answered under +// a charset nobody here asked about is forwarded rather than mangled: decoding these bytes as +// UTF-8 and encoding them again would replace both of them with U+FFFD. +test('/v1/audio/transcriptions forwards a document the gateway never decoded', async () => { + const { apiKey, repo } = await setupAppTest(); + await registerAudioModel(repo); + const latin1 = new Uint8Array([0x63, 0x61, 0x66, 0xE9, 0x20, 0xE0, 0x20, 0x31]); + await withMockedFetch( + () => new Response(latin1, { headers: { 'content-type': 'text/plain; charset=iso-8859-1' } }), + async () => { + const response = await requestApp('/v1/audio/transcriptions', { + method: 'POST', headers: { 'x-api-key': apiKey.key }, body: transcriptionForm([['response_format', 'text']]), + }); + assertEquals(response.headers.get('content-type'), 'text/plain; charset=iso-8859-1'); + assertEquals(new Uint8Array(await response.arrayBuffer()), latin1); + }, + ); +}); + test('/v1/audio/transcriptions skips JSON parsing for text responses without warning', async () => { const { apiKey, repo } = await setupAppTest(); await registerAudioModel(repo); diff --git a/packages/gateway/src/data-plane/audio/http.ts b/packages/gateway/src/data-plane/audio/http.ts index 6e56447d06..1ebba12288 100644 --- a/packages/gateway/src/data-plane/audio/http.ts +++ b/packages/gateway/src/data-plane/audio/http.ts @@ -1,23 +1,31 @@ -// POST /v1/audio/transcriptions — buffered OpenAI-compatible multipart -// transcription. The full body is parsed before routing because multipart -// field order is unconstrained; providers receive ordered semantic entries -// and rebuild a fresh body per candidate. +// POST /v1/audio/transcriptions, served through the pipeline. +// +// The multipart body is read and parsed before routing because field order is unconstrained +// and every candidate builds a fresh body from the entries. What the handler decides for +// itself is written in that form: which of the six renderings the client asked for, and +// whether the request streams — the run has to be opened knowing the second. // https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L714-L1040 import type { Context } from 'hono'; +import { streamSSE } from 'hono/streaming'; +import type { ContentfulStatusCode } from 'hono/utils/http-status'; -import { respondAudioTranscription } from './respond.ts'; -import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; -import { createGatewayCtxFromHono, finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; -import { passthroughApiError, passthroughServe } from '../shared/passthrough-serve.ts'; -import { readRequestBody, takeRequestBody } from '../shared/request-body.ts'; -import { isMultipartFormDataMediaType } from '@floway-dev/protocols/common'; +import { audioTranscriptionServePipeline } from './pipeline.ts'; +import { consoleLogSink } from '../../runtime/log.ts'; +import { isFrames, openPrologue, readIngress } from '../pipeline/serve.ts'; +import { settleBillable } from '../pipeline/settlement.ts'; +import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; +import { writeSSEFrames } from '../shared/sse.ts'; +import { move, run } from '@floway-dev/pipeline'; +import { parseAudioTranscriptionResponseFormat, type AudioTranscriptionResponseFormat } from '@floway-dev/protocols/audio'; +import { isMultipartFormDataMediaType, sseCommentFrame } from '@floway-dev/protocols/common'; import type { AudioTranscriptionFormEntry } from '@floway-dev/provider'; type PreparedTranscription = | { readonly type: 'ok'; readonly model: string; + readonly responseFormat: AudioTranscriptionResponseFormat; readonly wantsStream: boolean; readonly entries: readonly AudioTranscriptionFormEntry[]; } @@ -43,6 +51,16 @@ const prepareTranscription = async (bytes: Uint8Array, contentType: string | und if (files.length === 0 || files.some(file => !(file instanceof File))) { return { type: 'invalid', message: 'Audio transcription request body must include a file upload.' }; } + const declaredFormat = form.get('response_format'); + if (declaredFormat !== null && typeof declaredFormat !== 'string') { + return { type: 'invalid', message: 'Audio transcription response_format must be a text field.' }; + } + let responseFormat: AudioTranscriptionResponseFormat; + try { + responseFormat = parseAudioTranscriptionResponseFormat(declaredFormat ?? undefined); + } catch (error) { + return { type: 'invalid', message: error instanceof Error ? error.message : String(error) }; + } const entries: AudioTranscriptionFormEntry[] = []; for (const [name, value] of form.entries()) { @@ -51,35 +69,94 @@ const prepareTranscription = async (bytes: Uint8Array, contentType: string | und return { type: 'ok', model, + responseFormat, wantsStream: form.get('stream') === 'true', entries, }; }; +/** The client is sent bytes on every path, never a string, so nothing downstream of here + * guesses a media type: a `Response` built over a string is labelled `text/plain` by the + * platform, and an upstream that declared no media type has not asked for that. */ +const bodyOf = (rendered: Record | Uint8Array): Uint8Array => + rendered instanceof Uint8Array ? rendered : new TextEncoder().encode(JSON.stringify(rendered)); + +/** + * The epilogue: what the run answered with, as a response. + * + * This family writes its own rather than going through `serveThrough`, on two counts that + * belong to the seam rather than to it. A carried document goes out under the upstream's own + * media type — including the upstream that declared none, which `Rendered.contentType` has no + * value for — and a transcription's stream states its own outcome in its last event, which is + * what settlement has to be told and what `DeferredUsage` cannot carry. Fold both into the + * seam and this handler is `serveThrough` again. + */ export const audioTranscriptions = async (c: Context): Promise => { - const requestBody = await readRequestBody(c); - const request = await prepareTranscription(requestBody.bytes, c.req.header('content-type')); - const ctx = createGatewayCtxFromHono(c, { - wantsStream: request.type === 'ok' ? request.wantsStream : false, - requestBody: takeRequestBody(requestBody), - backgroundScheduler: backgroundSchedulerFromContext(c), - }); + const ingress = await readIngress(c); + const request = await prepareTranscription(ingress.body.bytes, c.req.header('content-type')); if (request.type === 'invalid') { - ctx.dump?.error('gateway'); - return finalizeGatewayResponse(ctx, passthroughApiError(c, request.message, 400)); + // A request the gateway could not read never reaches a pipeline: there is no model to + // resolve and no attempt to make, so there is nothing for a run to record. + const refused = openPrologue(c, ingress, { wantsStream: false }); + refused.gateway.dump?.error('gateway'); + return finalizeGatewayResponse( + refused.gateway, + Response.json({ error: { message: request.message, type: 'api_error' } }, { status: 400 }), + ); + } + + const prologue = openPrologue(c, ingress, { wantsStream: request.wantsStream, model: request.model }); + const { facts, drain } = await run( + audioTranscriptionServePipeline, + move({ + 'ingress.http.headers': prologue.headers, + 'ingress.audioTranscription.responseFormat': request.responseFormat, + 'request.audioTranscription.form': request.entries, + 'serve.model': request.model, + }) as never, + prologue.services as never, + ); + + // A stream states what it billed, and whether the transcript ever finished, in its last + // event — which is after the run has answered. So the stage above the fork stood down and + // settlement is scheduled here, on the promise the ending stage handed up. + const outcome = facts['response.audioTranscription.streamedOutcome']; + if (outcome !== null) { + prologue.services.background(outcome.then(({ billable, failed }) => { + settleBillable({ ...prologue.services, log: consoleLogSink }, billable, failed); + })); + } + + const rendered = facts['response.audioTranscription.rendered']; + const status = facts['response.http.status'] as ContentfulStatusCode; + if (isFrames(rendered)) { + // Hono's streamSSE builds the response itself, so what the client is to see has to be + // staged on the context before it is called rather than passed to a constructor. + for (const [name, value] of facts['response.http.headers']) c.header(name, value); + c.status(status); + return finalizeGatewayResponse(prologue.gateway, streamSSE(c, async stream => { + try { + await writeSSEFrames(stream, rendered, { + keepAlive: { frame: sseCommentFrame('keepalive') }, + ...(prologue.gateway.downstreamAbortController === undefined + ? {} + : { downstreamAbortController: prologue.gateway.downstreamAbortController }), + }); + } finally { + // Reading the frames to the client *is* releasing the body they came from, so the + // drain waits for that to finish. Draining alongside it would take frames out of the + // client's own stream — one connection has one reader. A client that stopped reading + // still gets here, which is what leaves nothing open behind it. + await drain(); + } + })); } - ctx.dump?.requestedModel(request.model); - const response = await passthroughServe({ - c, - ctx, - sourceApi: '/audio/transcriptions', - operation: 'audio_transcription', - model: request.model, - kind: 'transcription', - modelServesEndpoint: model => model.endpoints.audioTranscriptions !== undefined, - call: (provider, model, opts) => provider.instance.callAudioTranscriptions(model, { entries: request.entries }, ctx.abortSignal, opts), - response: { format: 'strategy', respond: respondAudioTranscription }, - }); - return finalizeGatewayResponse(ctx, response); + // Nothing is left to read: what the client is sent is either the document the run already + // holds or an object serialized from it, so releasing can start at once. + prologue.services.background(drain()); + const headers = new Headers(facts['response.http.headers'].map(([name, value]): [string, string] => [name, value])); + const mediaType = facts['response.audioTranscription.mediaType']; + if (mediaType !== null) headers.set('content-type', mediaType); + return finalizeGatewayResponse(prologue.gateway, new Response(bodyOf(rendered) as BodyInit, { status, headers })); }; diff --git a/packages/gateway/src/data-plane/audio/pipeline.ts b/packages/gateway/src/data-plane/audio/pipeline.ts index 7a675e89c1..0388310f92 100644 --- a/packages/gateway/src/data-plane/audio/pipeline.ts +++ b/packages/gateway/src/data-plane/audio/pipeline.ts @@ -23,13 +23,13 @@ import { dialFailure } from '../pipeline/upstream-body.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; -import { compose, defineStage, move, own, type Owned, type Pipeline } from '@floway-dev/pipeline'; +import { compose, defineStage, move, own, type Logger, type Owned, type Pipeline } from '@floway-dev/pipeline'; import { isAudioTranscriptionDoneEvent, - isAudioTranscriptionObjectFormat, parseAudioTranscription, parseAudioTranscriptionStreamEvent, parseAudioTranscriptionStreamUsage, + parseAudioTranscriptionUsage, renderAudioTranscription, type AudioTranscriptionResponseFormat, type AudioTranscriptionStreamEvent, @@ -52,6 +52,16 @@ export type AudioTranscriptionEvents = AsyncIterable(events: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => events }); +/** What settling this run will be told once the events run out: what the upstream metered, + * and whether the transcript ever finished. */ +export interface AudioTranscriptionStreamOutcome { + readonly billable: readonly BillableEntity[]; + /** An upstream that stopped before `transcript.text.done` answered 200 and then did not + * finish what it started, which is a failed request however much of the transcript + * reached the client — the same reading the replaced surface made. */ + readonly failed: boolean; +} + /** Audio transcription's own keys, extending the shared space by intersection. */ export interface AudioTranscriptionFacts extends GatewayFacts { /** Which of the six renderings the client asked for. It belongs to the ingress and stays @@ -69,22 +79,23 @@ export interface AudioTranscriptionFacts extends GatewayFacts { * telling them apart is reading a value, and each stage does that where it needs to. */ 'response.audioTranscription.canonical': CanonicalAudioTranscription | AudioTranscriptionEvents | Failure; /** What the answer goes out under. A media type is upstream-owned — OpenAI answers - * `text`, `srt` and `vtt` under one media type and other upstreams label them apart — so - * the upstream's own travels, and the edge re-provides it only where the gateway wrote - * the body itself. `null` is an upstream that declared none, which is what a client of - * this endpoint sees today. */ + * `text`, `srt` and `vtt` under one media type and other upstreams label them apart, and + * the document beneath it is the upstream's own either way — so the upstream's travels, + * and the edge names one only where the gateway wrote the body out of nothing the + * upstream sent. `null` is an upstream that declared none, and it stays `null`: labelling + * a body nobody described is a statement this gateway has no grounds to make. */ 'response.audioTranscription.mediaType': string | null; - /** What the upstream will have reported by the time the events run out, and `null` on - * every path that does not stream. A streamed transcription states its usage in the - * terminal event, which is after this run has answered — so the numbers cannot be in + /** What the stream will have come to by the time the events run out, and `null` on every + * path that does not stream. A streamed transcription states its usage in the terminal + * event, which is after this run has answered — so the numbers cannot be in * `response.usage.billable`, which says what had been reported when the ending stage - * handed up: the entity, and no quantities. Settling from this is the prologue's job, + * handed up: the entity, and no quantities. Settling from this is the epilogue's job, * after the drain. */ - 'response.audioTranscription.streamedUsage': Promise | null; - /** What the client is actually sent — a JSON object, a text document, or the SSE frames - * of a stream. The edge provides it, so a dump shows what the client received rather - * than the gateway's own reading of it. */ - 'response.audioTranscription.rendered': Record | string | AsyncIterable; + 'response.audioTranscription.streamedOutcome': Promise | null; + /** What the client is actually sent — a JSON object, the upstream's own document, or the + * SSE frames of a stream. The edge provides it, so a dump shows what the client received + * rather than the gateway's own reading of it. */ + 'response.audioTranscription.rendered': Record | Uint8Array | AsyncIterable; } type A = { [P in K]: AudioTranscriptionFacts[P] }; @@ -93,8 +104,11 @@ const isEvents = (answer: CanonicalAudioTranscription | AudioTranscriptionEvents Symbol.asyncIterator in answer; /** - * The outermost edge. Writes the answer back in the rendering the client asked for, and - * owns the media type of anything the gateway wrote itself. + * The outermost edge. Writes the answer back in the rendering the client asked for, and names + * a media type for the one body the gateway wrote out of nothing the upstream sent — its + * error envelope. Every other answer goes out under the media type it arrived with, because + * for a document that is carried rather than rewritten the upstream's label is the only true + * description there is. * * SSE framing is produced here and nowhere else — below this stage the answer is parsed * events, so the same assembly would serve another transport by rendering differently at @@ -163,13 +177,14 @@ const renderSSE = (events: AudioTranscriptionEvents): AsyncIterable => /** * The ending. It dials, reads what came back in the rendering the request asked for, and * provides the answer, the raw HTTP response beneath it, and what the call is billable for. - * A failure is a value: a 429 here is what an earlier stage fails over, and so is a 200 whose - * body this rendering cannot be read from, because the next candidate may answer with one - * that can. + * A failure is a value: a 429 here is what an earlier stage fails over, and so is a dial that + * never reached anyone. A 200 is never one of them — this endpoint carries the document the + * upstream sent rather than serializing one from what it read, so a body no reading could + * open is still that upstream's answer and there is nothing to try the next candidate for. */ const callAudioTranscriptionUpstream = defineStage< A<'ingress.audioTranscription.responseFormat' | 'request.audioTranscription.form' | 'route.attempt' | 'ingress.http.headers'>, - A<'response.audioTranscription.canonical' | 'response.audioTranscription.mediaType' | 'response.audioTranscription.streamedUsage'> + A<'response.audioTranscription.canonical' | 'response.audioTranscription.mediaType' | 'response.audioTranscription.streamedOutcome'> & { 'response.usage.billable': readonly BillableEntity[]; 'response.http.status': number; 'response.http.headers': readonly (readonly [string, string])[]; 'response.http.body': ReadableStream & Owned; }, @@ -180,7 +195,7 @@ const callAudioTranscriptionUpstream = defineStage< provides: [ 'response.audioTranscription.canonical', 'response.audioTranscription.mediaType', - 'response.audioTranscription.streamedUsage', + 'response.audioTranscription.streamedOutcome', 'response.usage.billable', 'response.http.status', 'response.http.headers', @@ -212,7 +227,7 @@ const callAudioTranscriptionUpstream = defineStage< ...facts, 'response.audioTranscription.canonical': dialFailure(error), 'response.audioTranscription.mediaType': null, - 'response.audioTranscription.streamedUsage': null, + 'response.audioTranscription.streamedOutcome': null, 'response.usage.billable': [], 'response.http.status': 502, 'response.http.headers': [], @@ -236,7 +251,7 @@ const callAudioTranscriptionUpstream = defineStage< ...facts, 'response.audioTranscription.canonical': await refusal(status, result.response), 'response.audioTranscription.mediaType': mediaType, - 'response.audioTranscription.streamedUsage': null, + 'response.audioTranscription.streamedOutcome': null, 'response.usage.billable': called, 'response.http.status': status, 'response.http.headers': headers, @@ -254,22 +269,22 @@ const callAudioTranscriptionUpstream = defineStage< ...facts, 'response.audioTranscription.canonical': { status: 502, message: 'Upstream returned a streaming response with no body.' }, 'response.audioTranscription.mediaType': mediaType, - 'response.audioTranscription.streamedUsage': null, + 'response.audioTranscription.streamedOutcome': null, 'response.usage.billable': called, 'response.http.status': status, 'response.http.headers': headers, 'response.http.body': spentBody(null), }); } - // Usage is observed here, closest to the upstream and on the protocol it spoke, by - // folding the events as they pass — so the reading costs one pass and the client's own - // stream is what drives it. - const metered = meterEvents(result.response.body, identity, use.gateway.abortSignal); + // What the upstream metered, and whether the transcript finished, are both observed + // here — closest to the upstream and on the protocol it spoke — by folding the events + // as they pass, so the reading costs one pass and the client's own stream drives it. + const metered = meterEvents(result.response.body, identity, use.gateway.abortSignal, use.log); return move({ ...facts, 'response.audioTranscription.canonical': metered.events, 'response.audioTranscription.mediaType': mediaType, - 'response.audioTranscription.streamedUsage': metered.billable, + 'response.audioTranscription.streamedOutcome': metered.outcome, 'response.usage.billable': called, 'response.http.status': status, 'response.http.headers': headers, @@ -279,13 +294,17 @@ const callAudioTranscriptionUpstream = defineStage< }); } - const read = await readTranscription(format, result.response); + // A 2xx body is the answer whether or not this endpoint could read it, because what the + // client is sent is the document that arrived and not something serialized from a parse. + // So there is nothing here to fail over from: the reading feeds the record and the usage + // row, and the bytes travel either way. + const read = readTranscription(format, new Uint8Array(await result.response.arrayBuffer()), use.log); return move({ ...facts, - 'response.audioTranscription.canonical': isFailure(read) ? read : read.value, + 'response.audioTranscription.canonical': read.canonical, 'response.audioTranscription.mediaType': mediaType, - 'response.audioTranscription.streamedUsage': null, - 'response.usage.billable': isFailure(read) ? called : [{ identity, quantities: billed(read.value.usage) }], + 'response.audioTranscription.streamedOutcome': null, + 'response.usage.billable': [{ identity, quantities: billed(read.usage) }], 'response.http.status': status, 'response.http.headers': headers, 'response.http.body': spentBody(result.response.body), @@ -310,43 +329,68 @@ const refusal = async (status: number, response: Response): Promise => } }; -/** A body that answered 200 and cannot be read in the rendering it was asked for is an - * attempt that failed, because the gateway serializes what it sends from the value it - * parsed and has nothing to serialize. */ -const readTranscription = async ( +/** + * What a 2xx body was worth to this endpoint, in two readings that do not depend on each + * other. + * + * Neither can cost the client the answer: the document is carried, so a body no reading + * could open is still what goes back, and what a failed reading costs is the transcript in + * the record and whatever usage the body would have stated. Nor can either cost the other — + * an upstream whose usage block this gateway cannot model still had its transcript read, and + * one whose document it could not open is still billed for nothing rather than mis-billed. + */ +const readTranscription = ( format: AudioTranscriptionResponseFormat, - response: Response, -): Promise<{ readonly value: CanonicalAudioTranscription } | Failure> => { + document: Uint8Array, + log: Logger, +): { readonly canonical: CanonicalAudioTranscription; readonly usage: AudioTranscriptionUsage | undefined } => { + let canonical: CanonicalAudioTranscription; try { - const body: unknown = isAudioTranscriptionObjectFormat(format) ? await response.json() : await response.text(); - return { value: parseAudioTranscription(format, body) }; + canonical = parseAudioTranscription(format, document); } catch (error) { - return { - status: 502, - message: `Upstream answered ${response.status} with a body this endpoint cannot read as ${format}: ${error instanceof Error ? error.message : String(error)}`, - }; + log.warn('failed to parse 2xx upstream body for /audio/transcriptions; forwarding it as it arrived', { error: String(error) }); + return { canonical: { document }, usage: undefined }; + } + // Only an object rendering states usage: `text`, `srt` and `vtt` have nowhere to put it, + // and asking them for one is what would warn about every subtitle a client requests. + return { canonical, usage: canonical.raw === undefined ? undefined : readUsage(() => parseAudioTranscriptionUsage(canonical.raw), log) }; +}; + +/** A transcription states its usage in two places — the body of an object rendering and the + * terminal event of a stream — and either can state it in a shape this gateway cannot read. + * That upstream metered something, and the request is recorded saying exactly that: the + * entity, and no quantities. */ +const readUsage = (read: () => AudioTranscriptionUsage | undefined, log: Logger): AudioTranscriptionUsage | undefined => { + try { + return read(); + } catch (error) { + log.warn('invalid usage in 2xx upstream response; recording the request only', { error: String(error) }); + return undefined; } }; interface MeteredEvents { readonly events: AudioTranscriptionEvents; - readonly billable: Promise; + readonly outcome: Promise; } const meterEvents = ( body: ReadableStream, identity: TelemetryModelIdentity, signal: AbortSignal | undefined, + log: Logger, ): MeteredEvents => { - let settle!: (billable: readonly BillableEntity[]) => void; - const billable = new Promise(resolve => { settle = resolve; }); + let settle!: (outcome: AudioTranscriptionStreamOutcome) => void; + const outcome = new Promise(resolve => { settle = resolve; }); const events = viewOf((async function* () { let usage: AudioTranscriptionUsage | undefined; + let completed = false; try { for await (const frame of parseSSEStream(body, { signal })) { const event = parseAudioTranscriptionStreamEvent(JSON.parse(frame.data) as unknown); if (isAudioTranscriptionDoneEvent(event)) { - usage = parseAudioTranscriptionStreamUsage(event); + usage = readUsage(() => parseAudioTranscriptionStreamUsage(event), log); + completed = true; yield event; // The transcript is complete, so there is nothing further to read. An upstream that // holds the connection open past this point would otherwise keep the client's own @@ -359,10 +403,10 @@ const meterEvents = ( // Reached however the events ended — the terminal one, a client that stopped reading, // or a broken upstream — because what the upstream already metered is billable // whatever happened to the downstream half. - settle([{ identity, quantities: billed(usage) }]); + settle({ billable: [{ identity, quantities: billed(usage) }], failed: !completed }); } })()); - return { events, billable }; + return { events, outcome }; }; const billed = (usage: AudioTranscriptionUsage | undefined): UsageQuantities => { @@ -390,25 +434,25 @@ const narrowing = { refuse: (status: number, message: string) => ({ 'response.audioTranscription.canonical': { status, message } as Failure, 'response.audioTranscription.mediaType': null, - 'response.audioTranscription.streamedUsage': null, + 'response.audioTranscription.streamedOutcome': null, }), refuses: [ 'response.audioTranscription.canonical', 'response.audioTranscription.mediaType', - 'response.audioTranscription.streamedUsage', + 'response.audioTranscription.streamedOutcome', ] as const, }; export const audioTranscriptionServePipeline: Pipeline< A<'ingress.http.headers' | 'ingress.audioTranscription.responseFormat' | 'request.audioTranscription.form' | 'serve.model'>, - A<'response.audioTranscription.rendered' | 'response.audioTranscription.mediaType' | 'response.audioTranscription.streamedUsage'> + A<'response.audioTranscription.rendered' | 'response.audioTranscription.mediaType' | 'response.audioTranscription.streamedOutcome'> & { 'response.http.status': number; 'response.usage.billable': readonly BillableEntity[]; 'response.http.headers': readonly (readonly [string, string])[]; } > = compose('audioTranscriptionServe', [ emitAudioTranscription, writeSettlement( handedUp => isFailure((handedUp as { 'response.audioTranscription.canonical'?: unknown })['response.audioTranscription.canonical']), - handedUp => (handedUp as { 'response.audioTranscription.streamedUsage'?: unknown })['response.audioTranscription.streamedUsage'] !== null, + handedUp => (handedUp as { 'response.audioTranscription.streamedOutcome'?: unknown })['response.audioTranscription.streamedOutcome'] !== null, ), resolveCandidates(narrowing), failover({ From 038463a46c9a14933158bd63903a02abcb58247f Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 04:44:10 +0800 Subject: [PATCH 35/79] test(images): assert a completed stream closes the read behind it The ending returns at the completed event so an upstream that holds the connection open does not hold the client's stream open with it, and nothing observed that: every other case in the suite reads an upstream that closes its own body, where returning and running to EOF look the same. --- .../data-plane/images/stream_test.ts | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/packages/gateway/__tests__/data-plane/images/stream_test.ts b/packages/gateway/__tests__/data-plane/images/stream_test.ts index b8c69c4b91..0b1314858b 100644 --- a/packages/gateway/__tests__/data-plane/images/stream_test.ts +++ b/packages/gateway/__tests__/data-plane/images/stream_test.ts @@ -170,6 +170,37 @@ test('/v1/images/generations answers a refused stream in the upstream status and assertEquals(performance?.errorsNoOutput, 1); }); +test('/v1/images/generations completes and cancels an upstream kept open after the completed event', async () => { + const { apiKey, repo } = await setupAppTest(); + await registerImagesModel(repo); + let upstreamCancelled = false; + const encoder = new TextEncoder(); + + await withMockedFetch( + () => Promise.resolve(new Response(new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(PARTIAL)); + controller.enqueue(encoder.encode(COMPLETED)); + // Deliberately never closed: the image is done and the connection is not. + }, + cancel() { + upstreamCancelled = true; + }, + }), { headers: { 'content-type': 'text/event-stream' } })), + async () => { + const response = await requestApp('/v1/images/generations', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + body: JSON.stringify({ model: 'gpt-image-2', prompt: 'a shiba in space', stream: true }), + }); + const stream = await response.text(); + assertEquals(stream.includes('event: image_generation.completed'), true); + }, + ); + + assertEquals(upstreamCancelled, true); +}); + test('/v1/images/generations fails a stream that ended without a completed event', async () => { const { apiKey, repo } = await setupAppTest(); await registerImagesModel(repo); From c1bc7927a6b22098068799f58883d1f831a78ec9 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 04:45:28 +0800 Subject: [PATCH 36/79] feat(gateway): give a dump record a shape, and keep one metadata under both MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A dump record had exactly one shape: the turn's two edges, `{request, response}`. A pipelined endpoint has no such pair to record — what happened is the run, every stage and both directions, which `@floway-dev/pipeline` already encodes as an NDJSON event stream. There was nowhere for that stream to go. A stored record is now either shape and `shape` is what a reader dispatches on. The storage below is untouched: a run's stream is one more gzipped body file under `dumps/v1/`, carried by the response descriptor with `type: 'run'`, staged in the spilled-files registry before it is written and pointed at by a row written last — so retention, the sweep and the files-before-row ordering apply to it without knowing it exists. The row's request-headers column, which is NOT NULL and means nothing for a run, holds the empty list; nothing reads it back because `get` dispatches on the descriptor first. `DumpMetadata` stays common — the dashboard lists both kinds together — so the attribution hooks and the assembly that fills it move out of the edge accumulator into `DumpAttribution`, which both shapes will stamp. --- .../chat/responses/websocket_test.ts | 9 +- .../data-plane/completions/http_test.ts | 6 +- .../gateway/__tests__/dump/test-fixtures.ts | 35 ++++- packages/gateway/__tests__/dump/wire_test.ts | 53 ++++++-- .../gateway/__tests__/repo/dump-store_test.ts | 128 +++++++++++++++--- .../scheduled/expiration-sweeps_test.ts | 1 + packages/gateway/__tests__/scheduled_test.ts | 1 + packages/gateway/src/dump/accumulator.ts | 112 ++++----------- packages/gateway/src/dump/attribution.ts | 122 +++++++++++++++++ packages/gateway/src/dump/schemas.ts | 6 +- packages/gateway/src/dump/types.ts | 53 +++++++- packages/gateway/src/dump/wire.ts | 42 ++++-- packages/gateway/src/repo/dump-store.ts | 78 +++++++---- 13 files changed, 476 insertions(+), 170 deletions(-) create mode 100644 packages/gateway/src/dump/attribution.ts diff --git a/packages/gateway/__tests__/data-plane/chat/responses/websocket_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/websocket_test.ts index f08688c73f..77a131dc45 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses/websocket_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses/websocket_test.ts @@ -7,7 +7,7 @@ import { responsesServe } from '../../../../src/data-plane/chat/responses/serve. import { KEEP_ALIVE_EVENT_TYPE } from '../../../../src/data-plane/chat/responses/websocket.ts'; import { DOWNSTREAM_KEEP_ALIVE_INTERVAL_MS } from '../../../../src/data-plane/shared/sse.ts'; import { initDumpBroker, initDumpStore } from '../../../../src/dump/registry.ts'; -import { installDumpStubs } from '../../../dump/test-fixtures.ts'; +import { edgeRecordOf, installDumpStubs } from '../../../dump/test-fixtures.ts'; import { FakeTime } from '../../../test-time.ts'; import { buildCodexUpstreamRecord, codexModels, copilotModels, flushAsyncWork, setupAppTest, sseResponse, sseResponsesResponse } from '../../../test-utils/app.ts'; import { installWorkerWebSocketRuntime, type TestWorkerWebSocket } from '../../../test-utils/worker-websocket.ts'; @@ -226,9 +226,10 @@ test('Responses WebSocket starts capturing on the next turn when dump retention const stored = dumps.stored[0]; assertExists(stored); assertEquals(stored.keyId, apiKey.id); - assertEquals(stored.record.request.method, 'WS'); - assertEquals(stored.record.request.path, '/v1/responses'); - assertEquals(JSON.parse(new TextDecoder().decode(stored.record.request.body)), { + const request = edgeRecordOf(stored.record).request; + assertEquals(request.method, 'WS'); + assertEquals(request.path, '/v1/responses'); + assertEquals(JSON.parse(new TextDecoder().decode(request.body)), { type: 'response.create', event_id: 'capture-after-enable', response: { diff --git a/packages/gateway/__tests__/data-plane/completions/http_test.ts b/packages/gateway/__tests__/data-plane/completions/http_test.ts index eacb12aa49..8f3dd4185e 100644 --- a/packages/gateway/__tests__/data-plane/completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/completions/http_test.ts @@ -2,7 +2,7 @@ import { test } from 'vitest'; import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts'; import { tokenCountsFromUsage } from '../../../src/repo/usage-metrics.ts'; -import { installDumpStubs } from '../../dump/test-fixtures.ts'; +import { edgeRecordOf, installDumpStubs } from '../../dump/test-fixtures.ts'; import { buildCustomUpstreamRecord, flushAsyncWork, requestApp, setupAppTest } from '../../test-utils/app.ts'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; @@ -311,7 +311,7 @@ test('/v1/completions non-streaming records usage row, performance neutral row ( assertEquals(performance[0]?.errorsNoOutput, 0); assertEquals(dumpStubs.stored.length, 1); - const dump = dumpStubs.stored[0]!.record; + const dump = edgeRecordOf(dumpStubs.stored[0]?.record); assertEquals(dump.meta.path, '/v1/completions'); assertEquals(dump.meta.status, 200); assertEquals(dump.meta.model, 'davinci-002'); @@ -353,7 +353,7 @@ test('/v1/completions streaming records usage row, performance neutral row (text assertEquals(performance[0]?.errorsNoOutput, 0); assertEquals(dumpStubs.stored.length, 1); - const dump = dumpStubs.stored[0]!.record; + const dump = edgeRecordOf(dumpStubs.stored[0]?.record); assertEquals(dump.meta.path, '/v1/completions'); assertEquals(dump.meta.status, 200); assertEquals(dump.meta.model, 'davinci-002'); diff --git a/packages/gateway/__tests__/dump/test-fixtures.ts b/packages/gateway/__tests__/dump/test-fixtures.ts index cf9d384140..dd0093e586 100644 --- a/packages/gateway/__tests__/dump/test-fixtures.ts +++ b/packages/gateway/__tests__/dump/test-fixtures.ts @@ -1,6 +1,7 @@ import type { DumpBroker } from '../../src/dump/broker.ts'; import type { DumpStore } from '../../src/dump/store-contract.ts'; -import type { DumpMetadata, StoredDumpRecord } from '../../src/dump/types.ts'; +import type { DumpMetadata, StoredDumpEdgeRecord, StoredDumpRecord, StoredDumpRunRecord } from '../../src/dump/types.ts'; +import { toNdjson, type DumpEvent } from '@floway-dev/pipeline'; export const fakeMeta = (overrides: Partial = {}): DumpMetadata => ({ id: 'test-id', @@ -20,12 +21,29 @@ export const fakeMeta = (overrides: Partial = {}): DumpMetadata => ...overrides, }); -export const fakeRecord = (overrides: Partial = {}): StoredDumpRecord => ({ +export const fakeRecord = (overrides: Partial = {}): StoredDumpEdgeRecord => ({ + shape: 'edge', meta: fakeMeta(overrides), request: { method: 'POST', path: '/v1/x', headers: [], body: new Uint8Array() }, response: { status: 200, headers: [], body: { type: 'none' } }, }); +// A run record holds one NDJSON stream and no edge halves, so a fixture only +// needs the lines the reader is meant to get back. +export const fakeRunRecord = (events: readonly DumpEvent[], overrides: Partial = {}): StoredDumpRunRecord => ({ + shape: 'run', + meta: fakeMeta(overrides), + events: new TextEncoder().encode(toNdjson(events)), +}); + +// A reader hands back whichever shape was written, so a test asserting on the +// request or response halves says which one it wrote. +export const edgeRecordOf = (record: StoredDumpRecord | null | undefined): StoredDumpEdgeRecord => { + if (!record) throw new Error('expected a stored dump record'); + if (record.shape !== 'edge') throw new Error(`expected the edge shape, got ${record.shape}`); + return record; +}; + type DumpStubFailMethod = | 'put' | 'list' @@ -60,11 +78,14 @@ export const installDumpStubs = ( }, async put(keyId, record) { if (throws.put) throw throws.put; - if (record.request.body.encoding !== 'identity') throw new Error('dump test stub expected identity request body'); - const storedRecord: StoredDumpRecord = { - ...record, - request: { ...record.request, body: record.request.body.bytes }, - }; + // The run shape is already stored-shaped; only the edge shape carries a + // separately prepared request body to rehydrate. + if (record.shape === 'edge' && record.request.body.encoding !== 'identity') { + throw new Error('dump test stub expected identity request body'); + } + const storedRecord: StoredDumpRecord = record.shape === 'run' + ? record + : { ...record, request: { ...record.request, body: record.request.body.bytes } }; stored.push({ keyId, record: storedRecord }); const list = records.get(keyId) ?? []; list.unshift(storedRecord); diff --git a/packages/gateway/__tests__/dump/wire_test.ts b/packages/gateway/__tests__/dump/wire_test.ts index 98bfabf688..e4abae2283 100644 --- a/packages/gateway/__tests__/dump/wire_test.ts +++ b/packages/gateway/__tests__/dump/wire_test.ts @@ -1,10 +1,11 @@ -import { test } from 'vitest'; +import { expect, test } from 'vitest'; -import type { StoredDumpRecord } from '../../src/dump/types.ts'; +import type { DumpEdgeRecord, StoredDumpEdgeRecord } from '../../src/dump/types.ts'; import { dumpRecordToWire } from '../../src/dump/wire.ts'; import { assertEquals } from '@floway-dev/test-utils'; -const baseStored = (overrides: Partial = {}): StoredDumpRecord => ({ +const baseStored = (overrides: Partial = {}): StoredDumpEdgeRecord => ({ + shape: 'edge', meta: { id: 'rec', startedAt: 0, @@ -26,10 +27,16 @@ const baseStored = (overrides: Partial = {}): StoredDumpRecord ...overrides, }); +const edgeWire = (record: StoredDumpEdgeRecord): DumpEdgeRecord => { + const wire = dumpRecordToWire(record); + if (wire.shape !== 'edge') throw new Error('expected the edge shape'); + return wire; +}; + // A textual request content-type with valid UTF-8 bytes serializes as utf8 on // the wire — the dashboard reads `data` directly as a string. test('dumpRecordToWire encodes a textual request body as utf8', () => { - const wire = dumpRecordToWire(baseStored({ + const wire = edgeWire(baseStored({ request: { method: 'POST', path: '/v1/messages', @@ -42,7 +49,7 @@ test('dumpRecordToWire encodes a textual request body as utf8', () => { }); test('dumpRecordToWire recognizes structured textual suffixes without accepting near matches', () => { - const structured = dumpRecordToWire(baseStored({ + const structured = edgeWire(baseStored({ request: { method: 'POST', path: '/v1/x', @@ -52,7 +59,7 @@ test('dumpRecordToWire recognizes structured textual suffixes without accepting })); assertEquals(structured.request.body.encoding, 'utf8'); - const nearMatch = dumpRecordToWire(baseStored({ + const nearMatch = edgeWire(baseStored({ request: { method: 'POST', path: '/v1/x', @@ -67,7 +74,7 @@ test('dumpRecordToWire recognizes structured textual suffixes without accepting // preserves every byte; the dashboard decodes base64 client-side. test('dumpRecordToWire encodes a binary response body as base64', () => { const png = new Uint8Array([0x89, 0x50, 0x4E, 0x47]); // PNG magic - const wire = dumpRecordToWire(baseStored({ + const wire = edgeWire(baseStored({ response: { status: 200, headers: [['content-type', 'image/png']], @@ -85,7 +92,7 @@ test('dumpRecordToWire encodes a binary response body as base64', () => { // as UTF-8 falls through to base64 so the wire never silently corrupts. test('dumpRecordToWire falls back to base64 when textual content-type carries non-UTF-8 bytes', () => { const bytes = new Uint8Array([0xFF, 0xFE, 0xFD]); - const wire = dumpRecordToWire(baseStored({ + const wire = edgeWire(baseStored({ request: { method: 'POST', path: '/v1/x', @@ -101,13 +108,39 @@ test('dumpRecordToWire falls back to base64 when textual content-type carries no // `stream` and `none` response bodies pass through wire serialization // unchanged because they carry no raw bytes. test('dumpRecordToWire passes stream + none response bodies through', () => { - const streamWire = dumpRecordToWire(baseStored({ + const streamWire = edgeWire(baseStored({ response: { status: 200, headers: [], body: { type: 'stream', events: [] } }, })); assertEquals(streamWire.response.body.type, 'stream'); - const noneWire = dumpRecordToWire(baseStored({ + const noneWire = edgeWire(baseStored({ response: { status: null, headers: [], body: { type: 'none' } }, })); assertEquals(noneWire.response.body.type, 'none'); }); + +// A run record crosses to the wire as the stored bytes decoded, because a line +// of NDJSON is one event and one SSE `data:` payload — what the dashboard reads +// here is what a live observer will read frame by frame. +test('dumpRecordToWire hands a run record its NDJSON verbatim', () => { + const ndjson = '{"type":"stage.entered","stageId":1,"name":"serve","parentStageId":null}\n' + + '{"type":"stage.leaved","stageId":1,"facts":{"response.http.status":200}}\n'; + const wire = dumpRecordToWire({ + shape: 'run', + meta: baseStored().meta, + events: new TextEncoder().encode(ndjson), + }); + if (wire.shape !== 'run') throw new Error('expected the run shape'); + assertEquals(wire.events, ndjson); + assertEquals(wire.meta.id, 'rec'); +}); + +// The gateway wrote those bytes itself, so bytes that are not UTF-8 mean a +// corrupted record and the reader says so rather than serving mojibake. +test('dumpRecordToWire refuses a run stream that is not UTF-8', () => { + expect(() => dumpRecordToWire({ + shape: 'run', + meta: baseStored().meta, + events: new Uint8Array([0xFF, 0xFE, 0xFD]), + })).toThrow(); +}); diff --git a/packages/gateway/__tests__/repo/dump-store_test.ts b/packages/gateway/__tests__/repo/dump-store_test.ts index 2211f99b80..194d9ce036 100644 --- a/packages/gateway/__tests__/repo/dump-store_test.ts +++ b/packages/gateway/__tests__/repo/dump-store_test.ts @@ -6,11 +6,12 @@ import { expect, test } from 'vitest'; import { createSqliteTestDb, mapRunChangeCount } from './test-sqlite.ts'; import { decodeDumpBodyDescriptor } from '../../src/dump/storage-codec.ts'; -import type { DumpWriteRecord } from '../../src/dump/types.ts'; +import type { DumpWriteEdgeRecord, StoredDumpEdgeRecord, StoredDumpRecord, StoredDumpRunRecord } from '../../src/dump/types.ts'; import { FileDumpStore } from '../../src/repo/dump-store.ts'; import { initRepo } from '../../src/repo/index.ts'; import { SqlRepo } from '../../src/repo/sql.ts'; import { collectSpilledFiles } from '../../src/scheduled/spilled-files.ts'; +import { encodeRun, toNdjson, type Facts } from '@floway-dev/pipeline'; import { initFileStore, MemoryFileStore } from '@floway-dev/platform'; import type { FileStore, SqlDatabase } from '@floway-dev/platform'; import { assertEquals, assertExists } from '@floway-dev/test-utils'; @@ -36,7 +37,8 @@ const utf8 = (s: string): Uint8Array => new TextEncoder().encode(s); const requestBody = utf8('{"hello":"world"}'); -const baseRecord = (id: string, completedAt: number): DumpWriteRecord => ({ +const baseRecord = (id: string, completedAt: number): DumpWriteEdgeRecord => ({ + shape: 'edge', meta: { id, startedAt: completedAt - 1, completedAt, method: 'POST', path: '/v1/x', status: 200, upstream: null, model: 'm', inputTokens: 1, outputTokens: 2, @@ -54,6 +56,14 @@ const baseRecord = (id: string, completedAt: number): DumpWriteRecord => ({ }, }); +// Every fixture above writes the edge shape, and a reader hands back what was +// written — so a test that asserts on the request or response halves says so. +const edgeOf = (record: StoredDumpRecord | null): StoredDumpEdgeRecord => { + assertExists(record); + if (record.shape !== 'edge') throw new Error('expected the edge shape'); + return record; +}; + test('FileDumpStore prepares request gzip before terminal persistence', async () => { const db = await openDb(); const files = new MemoryFileStore(); @@ -61,7 +71,7 @@ test('FileDumpStore prepares request gzip before terminal persistence', async () const raw = utf8(`{"content":"${'repeatable '.repeat(4096)}"}`); const prepared = await store.prepareRequestBody(raw); const base = baseRecord('01HZZ0000000000000000000P1', Date.UTC(2026, 5, 1, 12, 0, 0)); - const record: DumpWriteRecord = { + const record: DumpWriteEdgeRecord = { ...base, meta: { ...base.meta, requestBytes: raw.byteLength }, request: { @@ -76,8 +86,7 @@ test('FileDumpStore prepares request gzip before terminal persistence', async () assertEquals(prepared.decodedByteLength, raw.byteLength); assertEquals(prepared.bytes.byteLength < raw.byteLength, true); await store.put('key_x', record); - const fetched = await store.get('key_x', record.meta.id); - assertExists(fetched); + const fetched = edgeOf(await store.get('key_x', record.meta.id)); assertEquals(Array.from(fetched.request.body), Array.from(raw)); }); @@ -88,8 +97,7 @@ test('FileDumpStore round-trips a JSON record through gzip', async () => { const record = baseRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 12, 0, 0)); await store.put('key_x', record); - const fetched = await store.get('key_x', '01HZZ0000000000000000000A1'); - assertExists(fetched); + const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000A1')); assertEquals(fetched.meta.id, record.meta.id); assertEquals(new TextDecoder().decode(fetched.request.body), '{"hello":"world"}'); if (fetched.response.body.type !== 'bytes') throw new Error('expected bytes'); @@ -101,7 +109,7 @@ test('FileDumpStore preserves the original content-type header on binary bodies' const files = new MemoryFileStore(); const store = new FileDumpStore(db, files); const pngMagic = new Uint8Array([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]); - const record: DumpWriteRecord = { + const record: DumpWriteEdgeRecord = { ...baseRecord('01HZZ0000000000000000000PNG', Date.UTC(2026, 5, 1, 12, 0, 0)), response: { status: 200, @@ -111,8 +119,7 @@ test('FileDumpStore preserves the original content-type header on binary bodies' }; await store.put('key_x', record); - const fetched = await store.get('key_x', '01HZZ0000000000000000000PNG'); - assertExists(fetched); + const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000PNG')); // The header pair must survive verbatim — no `;base64` suffix tacked on. assertEquals(fetched.response.headers.find(([k]) => k === 'content-type')?.[1], 'image/png'); if (fetched.response.body.type !== 'bytes') throw new Error('expected bytes'); @@ -126,7 +133,7 @@ test('FileDumpStore preserves the bytes discriminator on an empty-body response' // 204-style: real upstream response with status + headers but a zero-length // body. Persistence drops the body file (nothing to gzip), but headers are // still written — the read path must surface this as `bytes`, not `none`. - const record: DumpWriteRecord = { + const record: DumpWriteEdgeRecord = { ...baseRecord('01HZZ0000000000000000000E1', Date.UTC(2026, 5, 1, 12, 0, 0)), response: { status: 204, @@ -136,8 +143,7 @@ test('FileDumpStore preserves the bytes discriminator on an empty-body response' }; await store.put('key_x', record); - const fetched = await store.get('key_x', '01HZZ0000000000000000000E1'); - assertExists(fetched); + const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000E1')); if (fetched.response.body.type !== 'bytes') throw new Error('expected bytes'); assertEquals(fetched.response.body.body.byteLength, 0); assertEquals(fetched.response.headers.find(([k]) => k === 'content-type')?.[1], 'application/json'); @@ -147,7 +153,7 @@ test('FileDumpStore round-trips an SSE record as a stream events array', async ( const db = await openDb(); const files = new MemoryFileStore(); const store = new FileDumpStore(db, files); - const record: DumpWriteRecord = { + const record: DumpWriteEdgeRecord = { ...baseRecord('01HZZ0000000000000000000A2', Date.UTC(2026, 5, 1, 12, 0, 0)), response: { status: 200, @@ -162,13 +168,98 @@ test('FileDumpStore round-trips an SSE record as a stream events array', async ( }, }; await store.put('key_x', record); - const fetched = await store.get('key_x', '01HZZ0000000000000000000A2'); - assertExists(fetched); + const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000A2')); if (fetched.response.body.type !== 'stream') throw new Error('expected stream'); assertEquals(fetched.response.body.events.length, 2); assertEquals(fetched.response.body.events[0]!.frame.type, 'event'); }); +// A pipelined turn stores the run rather than its edges: one NDJSON body file +// under the same contract, and a row that carries the same metadata. +const runRecord = (id: string, completedAt: number): StoredDumpRunRecord => { + const entered: Facts = { 'request.http.path': '/v1/embeddings' }; + const left: Facts = { ...entered, 'response.http.status': 200 }; + return { + shape: 'run', + meta: baseRecord(id, completedAt).meta, + events: new TextEncoder().encode(toNdjson(encodeRun([ + { type: 'stage.entered', stageId: 1, name: 'serve', parentStageId: null, facts: entered }, + { type: 'stage.entered', stageId: 2, name: 'dial', parentStageId: 1, facts: entered }, + { type: 'stage.leaved', stageId: 2, facts: left }, + { type: 'stage.leaved', stageId: 1, facts: left }, + ]))), + }; +}; + +test('FileDumpStore round-trips a run record as its NDJSON event stream', async () => { + const db = await openDb(); + const files = new MemoryFileStore(); + const store = new FileDumpStore(db, files); + const record = runRecord('01HZZ00000000000000000RUN1', Date.UTC(2026, 5, 1, 12, 0, 0)); + + await store.put('key_x', record); + const fetched = await store.get('key_x', record.meta.id); + assertExists(fetched); + if (fetched.shape !== 'run') throw new Error(`expected the run shape, got ${fetched.shape}`); + assertEquals(new TextDecoder().decode(fetched.events), new TextDecoder().decode(record.events)); + + // The stream is a body file like the other two: gzipped, named for what it + // holds, and pointed at by the row's descriptor. + const row = await db.prepare('SELECT request_body_descriptor, response_body_descriptor FROM dump_records WHERE key_id = ? AND id = ?') + .bind('key_x', record.meta.id) + .first<{ request_body_descriptor: string | null; response_body_descriptor: string }>(); + assertExists(row); + assertEquals(row.request_body_descriptor, null); + const descriptor = decodeDumpBodyDescriptor(row.response_body_descriptor, 'test run descriptor'); + assertEquals(descriptor.type, 'run'); + assertEquals(descriptor.key.endsWith('.run.gz'), true); +}); + +test('FileDumpStore lists a run record alongside an edge-shaped one', async () => { + const db = await openDb(); + const store = new FileDumpStore(db, new MemoryFileStore()); + const base = Date.UTC(2026, 5, 1, 12, 0, 0); + await store.put('key_x', baseRecord('01HZZ00000000000000000EDG1', base)); + await store.put('key_x', runRecord('01HZZ00000000000000000RUN2', base + 1)); + + const listed = await store.list('key_x', { limit: 10 }); + assertEquals(listed.map(meta => meta.id), ['01HZZ00000000000000000RUN2', '01HZZ00000000000000000EDG1']); + // One metadata shape for both: the list says nothing about how a turn was + // served, and every column a row renders is filled either way. + assertEquals(listed.map(meta => meta.model), ['m', 'm']); + assertEquals(listed.map(meta => meta.status), [200, 200]); +}); + +test('FileDumpStore retires an expired run record and collects its stream file', async () => { + const db = await openDb(); + const repo = new SqlRepo(db); + initRepo(repo); + const files = new MemoryFileStore(); + initFileStore(files); + const store = new FileDumpStore(db, files); + const now = Date.UTC(2026, 5, 1, 12, 0, 0); + await store.put('key_x', runRecord('01HZZ00000000000000000RUN3', Date.UTC(2026, 5, 1, 9, 0, 0))); + await store.put('key_x', runRecord('01HZZ00000000000000000RUN4', now)); + await repo.apiKeys.update('key_x', { dumpRetentionSeconds: 2 * 3600 }); + + const originalNow = Date.now; + Date.now = () => now + 1; + try { + assertEquals((await store.list('key_x', { limit: 10 })).map(meta => meta.id), ['01HZZ00000000000000000RUN4']); + assertEquals(await store.get('key_x', '01HZZ00000000000000000RUN3'), null); + assertEquals(await store.deleteExpiredBatch('key_x', now + 1, 100), 1); + await collectSpilledFiles(now + 1); + } finally { + Date.now = originalNow; + } + + const { results: remaining } = await db + .prepare("SELECT file_key FROM spilled_files WHERE state = 'owned' ORDER BY file_key") + .all<{ file_key: string }>(); + assertEquals(remaining.map(row => row.file_key.endsWith('.run.gz')), [true]); + assertEquals(remaining.every(row => !row.file_key.includes('2026060109')), true); +}); + test('FileDumpStore rejects malformed metadata with its row identity', async () => { const db = await openDb(); const store = new FileDumpStore(db, new MemoryFileStore()); @@ -212,7 +303,7 @@ test('FileDumpStore rejects malformed stream events with record and file context const db = await openDb(); const files = new MemoryFileStore(); const store = new FileDumpStore(db, files); - const record: DumpWriteRecord = { + const record: DumpWriteEdgeRecord = { ...baseRecord('01HZZ000000000000000000BADE', Date.UTC(2026, 5, 1, 12)), response: { status: 200, @@ -415,8 +506,7 @@ test('FileDumpStore: put + get round-trips through real-filesystem IO', async () const record = baseRecord('01HZZ0000000000000000000A1', Date.UTC(2026, 5, 1, 12, 0, 0)); await store.put('key_x', record); - const fetched = await store.get('key_x', '01HZZ0000000000000000000A1'); - assertExists(fetched); + const fetched = edgeOf(await store.get('key_x', '01HZZ0000000000000000000A1')); assertEquals(new TextDecoder().decode(fetched.request.body), '{"hello":"world"}'); if (fetched.response.body.type !== 'bytes') throw new Error('expected bytes'); assertEquals(new TextDecoder().decode(fetched.response.body.body), '{"id":"abc"}'); diff --git a/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts b/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts index 04a894fd26..7d4e6d1836 100644 --- a/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts +++ b/packages/gateway/__tests__/scheduled/expiration-sweeps_test.ts @@ -38,6 +38,7 @@ const responseItem = (id: string, refreshedAt: number, apiKeyId = 'key-a'): Stor }); const dumpRecord = (id: string, completedAt: number): DumpWriteRecord => ({ + shape: 'edge', meta: { id, startedAt: completedAt - 1, diff --git a/packages/gateway/__tests__/scheduled_test.ts b/packages/gateway/__tests__/scheduled_test.ts index bd2a2e5903..66cf14bd13 100644 --- a/packages/gateway/__tests__/scheduled_test.ts +++ b/packages/gateway/__tests__/scheduled_test.ts @@ -30,6 +30,7 @@ const apiKey = (id: string, now: number, secretDigit: number): ApiKey => ({ }); const fileBackedDumpRecord = (id: string, completedAt: number): DumpWriteRecord => ({ + shape: 'edge', meta: { id, startedAt: completedAt - 1, diff --git a/packages/gateway/src/dump/accumulator.ts b/packages/gateway/src/dump/accumulator.ts index 65b869b7c1..de92a428b7 100644 --- a/packages/gateway/src/dump/accumulator.ts +++ b/packages/gateway/src/dump/accumulator.ts @@ -1,23 +1,26 @@ -// Per-request dump pipeline. Opens the dump session (request snapshot + -// opt-in decision) and exposes the mid-flight hooks the respond layer -// calls to record outcomes and frames. When the api key has no retention -// configured, opening returns null and the data plane pays no per-request -// cost. +// Per-request dump pipeline for an endpoint served by the onion: the record is +// the turn's two **edges** — what the client sent, what the client got back. +// Opens the dump session (request snapshot + opt-in decision) and exposes the +// mid-flight hooks the respond layer calls to record outcomes and frames. When +// the api key has no retention configured, opening returns null and the data +// plane pays no per-request cost. +// +// A pipelined endpoint records the whole run instead; `run-sink.ts` is that +// half, and both fill the same `DumpMetadata` through `DumpAttribution`. import type { Context } from 'hono'; +import { DumpAttribution, oneLineError } from './attribution.ts'; import { getDumpBroker, getDumpStore } from './registry.ts'; import type { DumpErrorMeta, DumpMetadata, DumpStreamEvent, - DumpUpstreamRef, DumpWriteRecord, PreparedDumpRequestBody, StoredDumpResponseBody, } from './types.ts'; import type { RequestBody } from '../data-plane/shared/request-body.ts'; -import { getRepo } from '../repo/index.ts'; import type { ApiKey, TokenUsage } from '../repo/types.ts'; import { ulid } from '../shared/ulid.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; @@ -43,68 +46,16 @@ interface ResponseSnapshot { readonly streamError: string | null; } -// Four independent attribution slots the mid-flight hooks fill: `model` and -// `upstreamId` identify what the turn was about, `inputTokens` / -// `outputTokens` quantify what the upstream reported. They're independent -// because different outcomes set different subsets: -// -// • Every protocol handler calls `requestedModel(model)` immediately after -// parsing the payload, so `model` is set regardless of outcome. -// • `success(identity, usage)` fills all four; the upstream-resolved model -// id may overwrite what `requestedModel` had. -// • `error(kind, upstream?)` records a categorized api-error envelope -// (`kind` matches `ApiErrorResult.source`). Real upstream non-2xx pass -// `upstream` so a 4xx/5xx row in the dashboard names the upstream that -// rejected the call; the gateway arm may also pass it when a candidate -// was already chosen (item-not-found rewrite, server-tool input -// rejection). -// • `failed(reason)` records an uncategorized terminal failure: a thrown -// exception (caught by the respond layer or passthrough-serve), a -// source-emitted error frame, a downstream cancel, or a writer error. -// Caller passes a string or Error; the accumulator one-line-formats -// it (`.message` only — never the stack, which lives in the response -// body's debug envelope). -// -// `requestedModel`-set model survives across both error variants so even an -// outright-failed turn carries model attribution. - -// Anthropic-style disjoint per-category counts: input excludes cache reads -// and cache writes; sum the present ones onto the dump's single inputTokens -// column. Missing categories stay null (not measured) instead of zero so a -// recorded zero genuinely means "upstream said zero". -const tokenUsageInput = (usage: TokenUsage | null): number | null => { - if (!usage) return null; - const { input, input_cache_read, input_cache_write } = usage; - if (input === undefined && input_cache_read === undefined && input_cache_write === undefined) return null; - return (input ?? 0) + (input_cache_read ?? 0) + (input_cache_write ?? 0); -}; - -const oneLineError = (err: unknown): string => { - const msg = (err instanceof Error ? err.message : String(err)).replace(/\s+/g, ' ').trim(); - return msg.length > 500 ? `${msg.slice(0, 497)}…` : msg; -}; - const headerPairs = (headers: Headers): Array<[string, string]> => { const pairs: Array<[string, string]> = []; headers.forEach((value, name) => { pairs.push([name, value]); }); return pairs; }; -const resolveUpstreamRef = async (id: string | null): Promise => { - if (!id) return null; - const upstream = await getRepo().upstreams.getById(id); - if (!upstream) return null; - return { id: upstream.id, name: upstream.name, kind: upstream.kind, hue: upstream.hue }; -}; - export class DumpAccumulator { + private readonly attribution = new DumpAttribution(); private readonly events: DumpStreamEvent[] = []; private sentPayloadBytes = 0; - private model: string | null = null; - private upstreamId: string | null = null; - private inputTokens: number | null = null; - private outputTokens: number | null = null; - private errorMeta: DumpErrorMeta | null = null; private readonly preparedRequestBody: Promise; constructor( @@ -124,16 +75,15 @@ export class DumpAccumulator { // --- mid-flight hooks (called from per-protocol respond layer) --- requestedModel(model: string): void { - this.model = model; + this.attribution.requestedModel(model); } error(kind: 'upstream' | 'gateway', upstream?: string): void { - this.errorMeta = { kind }; - if (upstream !== undefined) this.upstreamId = upstream; + this.attribution.error(kind, upstream); } failed(reason: unknown): void { - this.errorMeta = { kind: 'failed', reason: typeof reason === 'string' ? reason : oneLineError(reason) }; + this.attribution.failed(reason); } // Records one protocol frame. Stored as the canonical ProtocolFrame so @@ -149,10 +99,7 @@ export class DumpAccumulator { } success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void { - this.model = identity.model; - this.upstreamId = identity.upstream; - this.inputTokens = tokenUsageInput(usage); - this.outputTokens = usage?.output ?? null; + this.attribution.success(identity, usage); } // --- response-side: handler exit --- @@ -254,32 +201,27 @@ export class DumpAccumulator { : { type: 'bytes', body: response.bytes } : { type: 'none' }; - const meta: DumpMetadata = { + const meta: DumpMetadata = await this.attribution.metadata({ id: recordId, startedAt: this.startedAt, completedAt, method: this.requestSnapshot.method, path: this.requestSnapshot.path, status: response.status, - upstream: await resolveUpstreamRef(this.upstreamId), - model: this.model, - inputTokens: this.inputTokens, - outputTokens: this.outputTokens, requestBytes: this.requestSnapshot.bodyByteLength, responseBytes: response.payloadBytes, - durationMs: completedAt - this.startedAt, - // Precedence: an explicit error stamp from the respond path wins; - // otherwise a request-body read failure (operator-side payload didn't - // arrive intact) outranks a response-body read failure. Both stream- - // read failures surface as `kind: 'failed'`. - error: this.errorMeta - ?? (this.requestSnapshot.streamError !== null ? { kind: 'failed', reason: this.requestSnapshot.streamError } : null) - ?? (response.streamError !== null ? { kind: 'failed', reason: response.streamError } : null), - }; + // Precedence: an explicit error stamp from the respond path wins (the + // assembler applies this only when there is none); otherwise a + // request-body read failure (operator-side payload didn't arrive intact) + // outranks a response-body read failure. Both stream-read failures + // surface as `kind: 'failed'`. + fallbackError: this.streamReadError(response), + }); // Commit the row before publishing so subscribers fetching detail off the meta frame find it. try { const record: DumpWriteRecord = { + shape: 'edge', meta, request: { method: this.requestSnapshot.method, @@ -299,6 +241,12 @@ export class DumpAccumulator { console.error(`[dump] write failed for key=${this.apiKey.id} record=${recordId}`, oneLineError(err)); } } + + private streamReadError(response: ResponseSnapshot): DumpErrorMeta | null { + if (this.requestSnapshot.streamError !== null) return { kind: 'failed', reason: this.requestSnapshot.streamError }; + if (response.streamError !== null) return { kind: 'failed', reason: response.streamError }; + return null; + } } // Returns null when the api key opts out of dumps; callers then skip all diff --git a/packages/gateway/src/dump/attribution.ts b/packages/gateway/src/dump/attribution.ts new file mode 100644 index 0000000000..e6b98d5425 --- /dev/null +++ b/packages/gateway/src/dump/attribution.ts @@ -0,0 +1,122 @@ +// What a turn's record says about the turn, and what fills it. +// +// `DumpMetadata` is common to both record shapes — the dashboard lists a run +// and a pair of edges in the same list, and a turn's model, upstream and token +// counts do not depend on which mechanism served it — so the hooks that stamp +// attribution and the assembly that turns it into metadata are one thing rather +// than one per shape. +// +// Four independent slots the mid-flight hooks fill: `model` and `upstreamId` +// identify what the turn was about, `inputTokens` / `outputTokens` quantify +// what the upstream reported. They're independent because different outcomes +// set different subsets: +// +// • Every protocol handler calls `requestedModel(model)` immediately after +// parsing the payload, so `model` is set regardless of outcome. +// • `success(identity, usage)` fills all four; the upstream-resolved model +// id may overwrite what `requestedModel` had. +// • `error(kind, upstream?)` records a categorized api-error envelope +// (`kind` matches `ApiErrorResult.source`). Real upstream non-2xx pass +// `upstream` so a 4xx/5xx row in the dashboard names the upstream that +// rejected the call; the gateway arm may also pass it when a candidate +// was already chosen (item-not-found rewrite, server-tool input +// rejection). +// • `failed(reason)` records an uncategorized terminal failure: a thrown +// exception, a source-emitted error frame, a downstream cancel, or a +// writer error. Caller passes a string or Error; this one-line-formats it +// (`.message` only — never the stack, which lives in the response body's +// debug envelope). +// +// `requestedModel`-set model survives across both error variants so even an +// outright-failed turn carries model attribution. + +import type { DumpErrorMeta, DumpMetadata, DumpUpstreamRef } from './types.ts'; +import { getRepo } from '../repo/index.ts'; +import type { TokenUsage } from '../repo/types.ts'; +import type { TelemetryModelIdentity } from '@floway-dev/provider'; + +export const oneLineError = (err: unknown): string => { + const msg = (err instanceof Error ? err.message : String(err)).replace(/\s+/g, ' ').trim(); + return msg.length > 500 ? `${msg.slice(0, 497)}…` : msg; +}; + +// Anthropic-style disjoint per-category counts: input excludes cache reads +// and cache writes; sum the present ones onto the dump's single inputTokens +// column. Missing categories stay null (not measured) instead of zero so a +// recorded zero genuinely means "upstream said zero". +const tokenUsageInput = (usage: TokenUsage | null): number | null => { + if (!usage) return null; + const { input, input_cache_read, input_cache_write } = usage; + if (input === undefined && input_cache_read === undefined && input_cache_write === undefined) return null; + return (input ?? 0) + (input_cache_read ?? 0) + (input_cache_write ?? 0); +}; + +const resolveUpstreamRef = async (id: string | null): Promise => { + if (!id) return null; + const upstream = await getRepo().upstreams.getById(id); + if (!upstream) return null; + return { id: upstream.id, name: upstream.name, kind: upstream.kind, hue: upstream.hue }; +}; + +// What only the recording side knows: identity, timing and the measured sizes +// of the two edges. Everything else on the metadata comes from the hooks. +export interface DumpTurnOutcome { + readonly id: string; + readonly startedAt: number; + readonly completedAt: number; + readonly method: string; + readonly path: string; + readonly status: number | null; + readonly requestBytes: number; + readonly responseBytes: number; + // Applied only when no hook stamped an error, so an explicit stamp from the + // respond path always outranks a transport-level read failure. + readonly fallbackError: DumpErrorMeta | null; +} + +export class DumpAttribution { + private model: string | null = null; + private upstreamId: string | null = null; + private inputTokens: number | null = null; + private outputTokens: number | null = null; + private errorMeta: DumpErrorMeta | null = null; + + requestedModel(model: string): void { + this.model = model; + } + + error(kind: 'upstream' | 'gateway', upstream?: string): void { + this.errorMeta = { kind }; + if (upstream !== undefined) this.upstreamId = upstream; + } + + failed(reason: unknown): void { + this.errorMeta = { kind: 'failed', reason: typeof reason === 'string' ? reason : oneLineError(reason) }; + } + + success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void { + this.model = identity.model; + this.upstreamId = identity.upstream; + this.inputTokens = tokenUsageInput(usage); + this.outputTokens = usage?.output ?? null; + } + + async metadata(outcome: DumpTurnOutcome): Promise { + return { + id: outcome.id, + startedAt: outcome.startedAt, + completedAt: outcome.completedAt, + method: outcome.method, + path: outcome.path, + status: outcome.status, + upstream: await resolveUpstreamRef(this.upstreamId), + model: this.model, + inputTokens: this.inputTokens, + outputTokens: this.outputTokens, + requestBytes: outcome.requestBytes, + responseBytes: outcome.responseBytes, + durationMs: outcome.completedAt - outcome.startedAt, + error: this.errorMeta ?? outcome.fallbackError, + }; + } +} diff --git a/packages/gateway/src/dump/schemas.ts b/packages/gateway/src/dump/schemas.ts index d5c820ada8..ac4d3ee3e7 100644 --- a/packages/gateway/src/dump/schemas.ts +++ b/packages/gateway/src/dump/schemas.ts @@ -35,9 +35,13 @@ export const persistedDumpMetadataSchema = dumpMetadataSchema.omit({ upstream: t export const dumpHeadersSchema = z.array(z.tuple([z.string(), z.string()])); +// `type` says how to read the file the descriptor points at: raw bytes, the +// JSON array of captured protocol frames, or a run's NDJSON event stream. It is +// also what tells the reader which shape the record is — a run's stream is one +// more body file under the same contract, so the row needs nothing else. export const dumpBodyDescriptorSchema = z.object({ key: z.string(), - type: z.enum(['bytes', 'events']), + type: z.enum(['bytes', 'events', 'run']), }).strict(); const dumpProtocolFrameSchema = z.discriminatedUnion('type', [ diff --git a/packages/gateway/src/dump/types.ts b/packages/gateway/src/dump/types.ts index a6a9af955b..723bd60fb2 100644 --- a/packages/gateway/src/dump/types.ts +++ b/packages/gateway/src/dump/types.ts @@ -8,6 +8,17 @@ // view served to the dashboard by `dumpRecordToWire`. // // `DumpMetadata` and `DumpStreamEvent` are body-free and shared verbatim. +// +// Across all three, a record is one of two shapes, and `shape` is what a reader +// dispatches on. An endpoint served by the onion records its **edges** — what +// the client sent and what the client got back. An endpoint served by a +// pipeline records the **whole run**: every stage, both directions, as the +// NDJSON event stream `@floway-dev/pipeline` encodes. The edges are still in +// that stream; they are the first and last things it holds. The shape follows +// the endpoint, so both are alive for as long as the two mechanisms are. +// +// What stays common is `DumpMetadata`: the dashboard lists both kinds together, +// and one turn's attribution does not depend on which mechanism served it. import type { z } from 'zod'; @@ -17,6 +28,11 @@ import type { dumpStreamEventSchema, dumpUpstreamRefSchema, } from './schemas.ts'; +import type { DumpEvent } from '@floway-dev/pipeline'; + +// Re-exported because the dashboard reads a run record's stream through this +// module and has no other reach into the pipeline package. +export type { DumpEvent }; export type DumpRecordId = string; @@ -79,18 +95,37 @@ export interface StoredDumpResponse { body: StoredDumpResponseBody; } -export type StoredDumpRecord = { +// The run's NDJSON, still as bytes: it is a body file under the same contract +// as the other two, gzipped into the file store and pointed at by the row's +// descriptor. One `put` carries it whole — measured on production, P99 of a +// turn's request and response together is 2.86 MB, well under the 5 MiB below +// which multipart has nothing to divide. +export interface StoredDumpRunRecord { + shape: 'run'; + meta: DumpMetadata; + events: Uint8Array; +} + +export type StoredDumpEdgeRecord = { + shape: 'edge'; meta: DumpMetadata; request: StoredDumpRequest; response: StoredDumpResponse; }; -export type DumpWriteRecord = { +export type StoredDumpRecord = StoredDumpEdgeRecord | StoredDumpRunRecord; + +export type DumpWriteEdgeRecord = { + shape: 'edge'; meta: DumpMetadata; request: DumpWriteRequest; response: StoredDumpResponse; }; +// No prepared half: a run's stream is encoded once the run is over, so there is +// nothing to compress ahead of the terminal write the way a request body is. +export type DumpWriteRecord = DumpWriteEdgeRecord | StoredDumpRunRecord; + // --- Wire shape (serialized JSON over the dashboard's control plane) --- // `utf8` is chosen from the upstream content-type, with a UTF-8-fatal @@ -119,8 +154,20 @@ interface DumpResponse { body: DumpResponseBody; } -export type DumpRecord = { +export type DumpEdgeRecord = { + shape: 'edge'; meta: DumpMetadata; request: DumpRequest; response: DumpResponse; }; + +// The stored NDJSON verbatim, decoded as UTF-8. One line is one event and one +// SSE `data:` payload, so what a reader parses here is what a live observer +// will parse frame by frame once the fan-out exists. +export type DumpRunRecord = { + shape: 'run'; + meta: DumpMetadata; + events: string; +}; + +export type DumpRecord = DumpEdgeRecord | DumpRunRecord; diff --git a/packages/gateway/src/dump/wire.ts b/packages/gateway/src/dump/wire.ts index 7659ae2914..a7d3b84518 100644 --- a/packages/gateway/src/dump/wire.ts +++ b/packages/gateway/src/dump/wire.ts @@ -31,18 +31,30 @@ const responseBodyToWire = (body: StoredDumpResponseBody, contentType: string): }; // Sole place the storage shape crosses into the wire shape. Called once, -// at the control-plane HTTP boundary, just before `c.json(...)`. -export const dumpRecordToWire = (record: StoredDumpRecord): DumpRecord => ({ - meta: record.meta, - request: { - method: record.request.method, - path: record.request.path, - headers: record.request.headers, - body: encodeBodyForWire(record.request.body, contentTypeOf(record.request.headers)), - }, - response: { - status: record.response.status, - headers: record.response.headers, - body: responseBodyToWire(record.response.body, contentTypeOf(record.response.headers)), - }, -}); +// at the control-plane HTTP boundary, just before `c.json(...)`. A run's stream +// is decoded UTF-8-fatal rather than sniffed: the gateway wrote those bytes +// itself, so anything that fails to decode is a corrupted record and says so. +export const dumpRecordToWire = (record: StoredDumpRecord): DumpRecord => { + if (record.shape === 'run') { + return { + shape: 'run', + meta: record.meta, + events: new TextDecoder('utf-8', { fatal: true }).decode(record.events), + }; + } + return { + shape: 'edge', + meta: record.meta, + request: { + method: record.request.method, + path: record.request.path, + headers: record.request.headers, + body: encodeBodyForWire(record.request.body, contentTypeOf(record.request.headers)), + }, + response: { + status: record.response.status, + headers: record.response.headers, + body: responseBodyToWire(record.response.body, contentTypeOf(record.response.headers)), + }, + }; +}; diff --git a/packages/gateway/src/repo/dump-store.ts b/packages/gateway/src/repo/dump-store.ts index ab29082fa6..4437589425 100644 --- a/packages/gateway/src/repo/dump-store.ts +++ b/packages/gateway/src/repo/dump-store.ts @@ -26,7 +26,7 @@ import type { import { gunzipBytes, gzipBytes } from '../shared/gzip.ts'; import type { FileStore, SqlDatabase } from '@floway-dev/platform'; -// Bodies live at `dumps/v1/{keyId}/{YYYYMMDDHH}/{recordId}-{uniqueSuffix}.{req|resp}.gz`. +// Bodies live at `dumps/v1/{keyId}/{YYYYMMDDHH}/{recordId}-{uniqueSuffix}.{req|resp|run}.gz`. // The hour segment remains useful for operator inspection; lifecycle and // collection are driven by the shared spilled_files registry. @@ -72,14 +72,14 @@ const hourBucket = (ms: number): string => { return `${y}${m}${d}${h}`; }; -const bodyPath = (keyId: string, bucket: string, recordId: string, side: 'req' | 'resp'): string => +const bodyPath = (keyId: string, bucket: string, recordId: string, side: 'req' | 'resp' | 'run'): string => `${DUMP_FILE_PREFIX}${keyId}/${bucket}/${recordId}-${crypto.randomUUID()}.${side}.gz`; const putRawBody = async ( files: FileStore, key: string, rawBytes: Uint8Array, - type: 'bytes' | 'events', + type: DumpBodyDescriptor['type'], ): Promise => { const gz = await gzipBytes(rawBytes); await files.put(key, gz); @@ -102,6 +102,12 @@ const fetchBody = async (files: FileStore, descriptor: DumpBodyDescriptor): Prom return await gunzipBytes(gz); }; +// A run record has no edge halves at row level — its request and response are +// events inside the stream — and `request_headers_json` is NOT NULL. The empty +// list is how the row spells "this shape has none"; nothing reads it back, +// because `get` dispatches on the body descriptor first. +const NO_EDGE_HEADERS = '[]'; + export class FileDumpStore implements DumpStore { constructor(private readonly db: SqlDatabase, private readonly files: FileStore) {} @@ -113,14 +119,20 @@ export class FileDumpStore implements DumpStore { }; } + // Both shapes take the same route: files are staged in the registry, written, + // and only then pointed at by a row. A run's NDJSON is one more body file + // under that contract, carried by the response descriptor — which is what + // leaves retention, the sweep and the files-before-row ordering untouched by + // its arrival. async put(keyId: string, record: DumpWriteRecord): Promise { const bucket = hourBucket(record.meta.completedAt); - const requestFileKey = record.request.body.decodedByteLength === 0 + const requestFileKey = record.shape === 'run' || record.request.body.decodedByteLength === 0 ? null : bodyPath(keyId, bucket, record.meta.id, 'req'); - const responseFileKey = record.response.body.type === 'bytes' && record.response.body.body.byteLength === 0 - ? null + const responseFileKey = record.shape === 'run' + ? bodyPath(keyId, bucket, record.meta.id, 'run') : record.response.body.type === 'none' + || (record.response.body.type === 'bytes' && record.response.body.body.byteLength === 0) ? null : bodyPath(keyId, bucket, record.meta.id, 'resp'); const staged = [ @@ -142,22 +154,27 @@ export class FileDumpStore implements DumpStore { .bind(keyId, record.meta.id, Date.now() + SPILLED_FILE_STAGE_GRACE_MS, JSON.stringify(staged)) .run(); } - const requestDescriptor = record.request.body.decodedByteLength === 0 - ? null - : await putPreparedBody(this.files, requestFileKey!, record.request.body); + let requestDescriptor: DumpBodyDescriptor | null = null; let responseDescriptor: DumpBodyDescriptor | null = null; - if (record.response.body.type === 'bytes') { - if (record.response.body.body.byteLength > 0) { - responseDescriptor = await putRawBody(this.files, responseFileKey!, record.response.body.body, 'bytes'); + if (record.shape === 'run') { + responseDescriptor = await putRawBody(this.files, responseFileKey!, record.events, 'run'); + } else { + if (requestFileKey !== null) { + requestDescriptor = await putPreparedBody(this.files, requestFileKey, record.request.body); + } + if (record.response.body.type === 'bytes') { + if (record.response.body.body.byteLength > 0) { + responseDescriptor = await putRawBody(this.files, responseFileKey!, record.response.body.body, 'bytes'); + } + } else if (record.response.body.type === 'stream') { + responseDescriptor = await putRawBody( + this.files, + responseFileKey!, + new TextEncoder().encode(encodeDumpStreamEvents(record.response.body.events, `dump record ${record.meta.id} response events`)), + 'events', + ); } - } else if (record.response.body.type === 'stream') { - responseDescriptor = await putRawBody( - this.files, - responseFileKey!, - new TextEncoder().encode(encodeDumpStreamEvents(record.response.body.events, `dump record ${record.meta.id} response events`)), - 'events', - ); } // Files before row — a partial failure leaves orphan files the sweep @@ -172,8 +189,10 @@ export class FileDumpStore implements DumpStore { record.meta.completedAt, record.meta.upstream?.id ?? null, encodePersistedDumpMetadata(record.meta, `dump record ${record.meta.id} metadata`), - encodeDumpHeaders(record.request.headers, `dump record ${record.meta.id} request headers`), - record.response.body.type === 'none' + record.shape === 'run' + ? NO_EDGE_HEADERS + : encodeDumpHeaders(record.request.headers, `dump record ${record.meta.id} request headers`), + record.shape === 'run' || record.response.body.type === 'none' ? null : encodeDumpHeaders(record.response.headers, `dump record ${record.meta.id} response headers`), requestDescriptor === null @@ -231,17 +250,24 @@ export class FileDumpStore implements DumpStore { ...decodePersistedDumpMetadata(row.meta_json, `dump record ${recordId} metadata`), upstream: hydrateUpstream(row), }; - const requestHeaders = decodeDumpHeaders(row.request_headers_json, `dump record ${recordId} request headers`); const requestDescriptor = row.request_body_descriptor === null ? null : decodeDumpBodyDescriptor(row.request_body_descriptor, `dump record ${recordId} request body descriptor`); - const responseHeaders = row.response_headers_json === null - ? null - : decodeDumpHeaders(row.response_headers_json, `dump record ${recordId} response headers`); const responseDescriptor = row.response_body_descriptor === null ? null : decodeDumpBodyDescriptor(row.response_body_descriptor, `dump record ${recordId} response body descriptor`); + // The body kind is the shape: a run was written as one NDJSON stream and + // has no edge halves to rebuild. + if (responseDescriptor?.type === 'run') { + return { shape: 'run', meta, events: await fetchBody(this.files, responseDescriptor) }; + } + + const requestHeaders = decodeDumpHeaders(row.request_headers_json, `dump record ${recordId} request headers`); + const responseHeaders = row.response_headers_json === null + ? null + : decodeDumpHeaders(row.response_headers_json, `dump record ${recordId} response headers`); + const request: StoredDumpRequest = { method: meta.method, path: meta.path, @@ -272,7 +298,7 @@ export class FileDumpStore implements DumpStore { headers: responseHeaders ?? [], body: responseBody, }; - return { meta, request, response }; + return { shape: 'edge', meta, request, response }; } async deleteExpiredBatch(keyId: string, now: number, limit: number): Promise { From a8966ded825d36c1ea9f030618ca9893853f2dfd Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 04:50:06 +0800 Subject: [PATCH 37/79] docs(gateway): record why a truncated stream still settles as a success MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A stream that stops short is recorded as a turn that produced what it said it would: the deferred settlement passes failed=false unconditionally, so a neutral performance row and a dump success are written either way. It affects every streaming family. Three shapes were tried. The write has to be registered while the request is still live for the platform to bind it, and the outcome is only known when the stream ends; each attempt to satisfy both lost the usage row entirely. The fix that would work is for a family's meter to report how its stream ended, since that is the one place that knows — a change across all four of them. Left as it was, with the constraint written where the next reader meets it. --- packages/gateway/src/data-plane/pipeline/serve.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/gateway/src/data-plane/pipeline/serve.ts b/packages/gateway/src/data-plane/pipeline/serve.ts index 7834ae1080..92bb05ddac 100644 --- a/packages/gateway/src/data-plane/pipeline/serve.ts +++ b/packages/gateway/src/data-plane/pipeline/serve.ts @@ -145,6 +145,13 @@ export const serveThrough = async < const answer = render(facts); const pending = deferredUsage?.(facts) ?? null; + // OPEN DECISION, deliberately taken the simple way for now. `failed` is false here even + // for a stream that stopped short, so a truncated turn still writes a neutral performance + // row and a dump success. Settling it correctly needs the write registered while the + // request is still live *and* the outcome known only when the stream ends, and the two + // pull against each other — three shapes were tried and each lost the row entirely. + // The honest fix is for the family's meter to report how its stream ended, since that is + // the one place that knows; that is a change across all four streaming families. if (pending !== null) { prologue.services.background(pending.then(billable => { settleBillable({ ...prologue.services, log: consoleLogSink }, billable, false); @@ -177,7 +184,7 @@ export const serveThrough = async < })); } - // Nothing is left to read: what the client is sent was serialized from facts the run + // Nothing is left to read either: what the client is sent was serialized from facts the run // already held, so releasing can start at once. prologue.services.background(drain()); const headers = new Headers(facts['response.http.headers'].map(([name, value]): [string, string] => [name, value])); From 8ceee12e1a06f9bf270f2785331aa0d0d6bde7d5 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 04:51:25 +0800 Subject: [PATCH 38/79] feat(gateway): give a run's event stream somewhere to be written MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pipeline emitted a run's events and had no sink to emit them to: `run()` recorded into `RunResult.events` and nothing carried them to storage, so a pipelined turn's dump was the edges of its HTTP request and nothing about the run that produced them. `openRunDump` opens one recording per turn. `sink` is what the prologue hands to `run` as `services.dump`; every event it receives is folded into the encoded stream as it happens rather than re-walked at the end, so a stage's exit that hands up what its child handed up costs nothing and a payload the run never rewrote is one object id. `finalize` turns the stream into NDJSON and writes one record through `DumpStore.put` — the same route the edge record takes. Recording is conditional structurally: a key with no retention gets no sink, so `services.dump` is absent and the runner does none of the recording. There is no mode flag and nothing that accumulates into a value it discards. --- .../gateway/__tests__/dump/run-sink_test.ts | 168 ++++++++++++++++ packages/gateway/src/dump/run-sink.ts | 189 ++++++++++++++++++ 2 files changed, 357 insertions(+) create mode 100644 packages/gateway/__tests__/dump/run-sink_test.ts create mode 100644 packages/gateway/src/dump/run-sink.ts diff --git a/packages/gateway/__tests__/dump/run-sink_test.ts b/packages/gateway/__tests__/dump/run-sink_test.ts new file mode 100644 index 0000000000..190f916b7e --- /dev/null +++ b/packages/gateway/__tests__/dump/run-sink_test.ts @@ -0,0 +1,168 @@ +import { test } from 'vitest'; + +import { installDumpStubs } from './test-fixtures.ts'; +import { initDumpBroker, initDumpStore } from '../../src/dump/registry.ts'; +import { openRunDump } from '../../src/dump/run-sink.ts'; +import type { StoredDumpRecord, StoredDumpRunRecord } from '../../src/dump/types.ts'; +import type { ApiKey } from '../../src/repo/types.ts'; +import { flushBackground, trackBackground } from '../test-utils/background-tracker.ts'; +import { compose, defineStage, move, run, type DumpEvent, type Event } from '@floway-dev/pipeline'; +import { assertEquals } from '@floway-dev/test-utils'; + +const apiKey = (dumpRetentionSeconds: number | null): ApiKey => ({ + id: 'key_run', + userId: 1, + name: 'Run key', + key: 'raw-run-key', + serverSecret: '11'.repeat(32), + createdAt: '2026-01-01T00:00:00.000Z', + upstreamIds: null, + deletedAt: null, + dumpRetentionSeconds, + responsesRetentionSeconds: 0, +}); + +const requestBody = { bytes: new TextEncoder().encode('{"input":"hi"}'), streamError: null }; + +const turn = { method: 'POST', path: '/v1/embeddings', body: requestBody }; + +// Two stages, so the record has a shape to hold: one that hands down and one +// that answers. +interface Facts { + 'in.text': string; + 'out.result': string; +} + +const answer = defineStage, Pick>({ + name: 'answer', + return: { provides: ['out.result'] }, + execute: async facts => move({ ...facts, 'out.result': facts['in.text'].toUpperCase() }), +}); + +const shout = defineStage, Pick, Pick, Pick>({ + name: 'shout', + through: { + request: { needs: ['in.text'], consumes: [], provides: [] }, + response: { needs: ['out.result'], consumes: [], provides: [] }, + }, + execute: async (facts, next, use) => { + use.log.info('shouting', { length: facts['in.text'].length }); + return await next(move({ ...facts, 'in.text': `${facts['in.text']}!` })); + }, +}); + +const pipeline = compose, Pick>('shout-it', [shout, answer]); + +const runRecordOf = (stored: { record: StoredDumpRecord } | undefined): StoredDumpRunRecord => { + if (!stored) throw new Error('expected a stored dump record'); + if (stored.record.shape !== 'run') throw new Error(`expected the run shape, got ${stored.record.shape}`); + return stored.record; +}; + +const ndjson = (record: StoredDumpRunRecord): string => new TextDecoder().decode(record.events); + +const lines = (record: StoredDumpRunRecord): DumpEvent[] => + ndjson(record).split('\n').filter(Boolean).map(line => JSON.parse(line) as DumpEvent); + +test('a run under a key with retention stores its whole event stream as NDJSON', async () => { + const stubs = installDumpStubs(initDumpStore, initDumpBroker); + const dump = openRunDump(apiKey(3600), turn, trackBackground); + if (dump === null) throw new Error('a key with retention must open a run dump'); + + const { facts } = await run(pipeline, move({ 'in.text': 'hey' }), { dump: dump.sink }); + assertEquals(facts['out.result'], 'HEY!'); + dump.finalize(200, 12); + await flushBackground(); + + const record = runRecordOf(stubs.stored[0]); + const events = lines(record); + // Every stage is in the tree and the shape of the run is in the parent ids. + assertEquals( + events.filter(event => event.type === 'stage.entered').map(event => [event.name, event.parentStageId]), + [['shout', null], ['answer', 1]], + ); + // A stage's own log line is content about that stage, like the other five kinds. + assertEquals(events.filter(event => event.type === 'stage.log').map(event => event.message), ['shouting']); + // A `stage.leaved` that hands up exactly what its last child handed up carries + // nothing and is not emitted, so `shout`'s exit goes and `answer`'s stays. + assertEquals(events.filter(event => event.type === 'stage.leaved').map(event => event.stageId), [2]); + // NDJSON: one event per line, appended in order, so the stored file and what a + // live observer would be handed are the same bytes. + assertEquals(ndjson(record).endsWith('\n'), true); + assertEquals(ndjson(record).trimEnd().split('\n').length, events.length); + + assertEquals(record.meta.method, 'POST'); + assertEquals(record.meta.path, '/v1/embeddings'); + assertEquals(record.meta.status, 200); + assertEquals(record.meta.requestBytes, requestBody.bytes.byteLength); + assertEquals(record.meta.responseBytes, 12); + assertEquals(stubs.published.map(entry => entry.meta.id), [record.meta.id]); +}); + +test('a key without retention opens no sink, so a run records and stores nothing', async () => { + const stubs = installDumpStubs(initDumpStore, initDumpBroker); + const dump = openRunDump(apiKey(null), turn, trackBackground); + assertEquals(dump, null); + + // The absence is the mechanism: with nothing to put in `services.dump` the + // runner does none of the recording, rather than feeding a sink that throws + // the result away. + const emitted: Event[] = []; + const services = dump === null ? {} : { dump: (event: Event) => { emitted.push(event); } }; + const { facts } = await run(pipeline, move({ 'in.text': 'hey' }), services); + assertEquals(facts['out.result'], 'HEY!'); + await flushBackground(); + + assertEquals(emitted, []); + assertEquals(stubs.stored, []); + assertEquals(stubs.published, []); +}); + +test('a run record carries the attribution the turn stamped on it', async () => { + const stubs = installDumpStubs(initDumpStore, initDumpBroker); + const dump = openRunDump(apiKey(3600), turn, trackBackground); + if (dump === null) throw new Error('a key with retention must open a run dump'); + + dump.requestedModel('text-embedding-3-small'); + dump.failed(new Error('upstream went\naway')); + dump.finalize(null, 0); + await flushBackground(); + + const record = runRecordOf(stubs.stored[0]); + assertEquals(record.meta.model, 'text-embedding-3-small'); + assertEquals(record.meta.status, null); + assertEquals(record.meta.error, { kind: 'failed', reason: 'upstream went away' }); +}); + +test('a run whose request never arrived intact records that as the turn\'s failure', async () => { + const stubs = installDumpStubs(initDumpStore, initDumpBroker); + const dump = openRunDump( + apiKey(3600), + { ...turn, body: { bytes: new Uint8Array(), streamError: 'client aborted the upload' } }, + trackBackground, + ); + if (dump === null) throw new Error('a key with retention must open a run dump'); + + dump.finalize(400, 0); + await flushBackground(); + + assertEquals(runRecordOf(stubs.stored[0]).meta.error, { kind: 'failed', reason: 'client aborted the upload' }); +}); + +test('finalizing on a response measures what the client reads and leaves it intact', async () => { + const stubs = installDumpStubs(initDumpStore, initDumpBroker); + const dump = openRunDump(apiKey(3600), turn, trackBackground); + if (dump === null) throw new Error('a key with retention must open a run dump'); + + await run(pipeline, move({ 'in.text': 'hey' }), { dump: dump.sink }); + const answered = dump.finalize(new Response('data: one\n\ndata: two\n\n', { + status: 200, + headers: { 'content-type': 'text/event-stream' }, + })); + + assertEquals(await answered.text(), 'data: one\n\ndata: two\n\n'); + await flushBackground(); + + const record = runRecordOf(stubs.stored[0]); + assertEquals(record.meta.responseBytes, 22); +}); diff --git a/packages/gateway/src/dump/run-sink.ts b/packages/gateway/src/dump/run-sink.ts new file mode 100644 index 0000000000..0ca5893c31 --- /dev/null +++ b/packages/gateway/src/dump/run-sink.ts @@ -0,0 +1,189 @@ +// The pipeline's half of the dump: the record of a **whole run**. +// +// A run emits events — every stage, both directions — and this is where they +// go. What the runner hands `services.dump` is `sink`; what it accumulates is +// the encoded stream, folded event by event by `createRunEncoder` so a run is +// written down as it happens rather than re-walked at the end. At the terminal +// point the stream becomes NDJSON and one record goes through `DumpStore.put`, +// which is the same contract the edge record is written under: the stream is +// one more gzipped body file, retained and swept by the same row. +// +// Recording is conditional and that is structural: with no retention configured +// there is no sink to hand to `run`, so `services.dump` is absent and the +// runner does none of the recording — not a no-op that accumulates and throws +// the result away. + +import { DumpAttribution, oneLineError } from './attribution.ts'; +import { getDumpBroker, getDumpStore } from './registry.ts'; +import type { DumpErrorMeta, DumpMetadata } from './types.ts'; +import type { RequestBody } from '../data-plane/shared/request-body.ts'; +import type { ApiKey, TokenUsage } from '../repo/types.ts'; +import { ulid } from '../shared/ulid.ts'; +import { createRunEncoder, toNdjson, type DumpEvent, type Event } from '@floway-dev/pipeline'; +import type { BackgroundScheduler } from '@floway-dev/platform'; +import type { TelemetryModelIdentity } from '@floway-dev/provider'; + +// What the client sent, as the metadata needs it. The headers and the body are +// facts the run itself records, so nothing is snapshotted here beyond what a +// list row shows without opening the record. +interface RequestSnapshot { + readonly method: string; + readonly path: string; + readonly bodyByteLength: number; + readonly streamError: string | null; +} + +export class RunDump { + private readonly attribution = new DumpAttribution(); + private readonly encode = createRunEncoder(); + private readonly events: DumpEvent[] = []; + + constructor( + private readonly apiKey: ApiKey, + private readonly requestSnapshot: RequestSnapshot, + private readonly startedAt: number, + private readonly backgroundScheduler: BackgroundScheduler, + ) {} + + /** What the prologue hands to `run` as `services.dump`. Bound to this + * recording, so it travels as a value. */ + readonly sink = (event: Event): void => { + for (const encoded of this.encode(event)) this.events.push(encoded); + }; + + // --- mid-flight hooks, the same four the edge record is stamped with --- + + requestedModel(model: string): void { + this.attribution.requestedModel(model); + } + + error(kind: 'upstream' | 'gateway', upstream?: string): void { + this.attribution.error(kind, upstream); + } + + failed(reason: unknown): void { + this.attribution.failed(reason); + } + + success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void { + this.attribution.success(identity, usage); + } + + // --- terminal point --- + + // Two input shapes, matching the edge accumulator's seam: + // + // • `(status, responseBytes)` — the caller already knows what it wrote. + // • `(response)` — tees the answer so the client gets bytes flowing while a + // background reader measures the other half. Only the byte count is kept: + // what the client was sent is already in the run's own record, so + // retaining a second copy of a streamed answer would buy nothing. + // + // The drain → encode → store put → broker publish runs on the runtime's + // BackgroundScheduler so a dump write failure cannot turn a served answer + // into a 502. + finalize(status: number | null, responseBytes: number): void; + finalize(response: Response): Response; + finalize(...args: [number | null, number] | [Response]): void | Response { + if (args.length === 2) { + const [status, responseBytes] = args; + this.backgroundScheduler(this.write(status, responseBytes, null)); + return; + } + + const [response] = args; + if (response.body === null) { + this.finalize(response.status, 0); + return response; + } + + const [forClient, forMeasure] = response.body.tee(); + this.backgroundScheduler((async () => { + const reader = forMeasure.getReader(); + let payloadBytes = 0; + let streamError: string | null = null; + try { + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + payloadBytes += value.byteLength; + } + } catch (err) { + streamError = oneLineError(err); + } + await this.write(response.status, payloadBytes, streamError); + })()); + + return new Response(forClient, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }); + } + + private async write(status: number | null, responseBytes: number, responseStreamError: string | null): Promise { + // ULID-from-completedAt keeps ids increasing with row creation time; the + // random tail provides the deterministic tie-breaker for one millisecond. + const completedAt = Date.now(); + const recordId = ulid(completedAt); + const meta: DumpMetadata = await this.attribution.metadata({ + id: recordId, + startedAt: this.startedAt, + completedAt, + method: this.requestSnapshot.method, + path: this.requestSnapshot.path, + status, + requestBytes: this.requestSnapshot.bodyByteLength, + responseBytes, + // A request-body read failure — the operator-side payload did not arrive + // intact — outranks a failure reading back what was answered. + fallbackError: streamReadError(this.requestSnapshot.streamError, responseStreamError), + }); + + // Commit the row before publishing so subscribers fetching detail off the meta frame find it. + try { + await getDumpStore().put(this.apiKey.id, { + shape: 'run', + meta, + events: new TextEncoder().encode(toNdjson(this.events)), + }); + await getDumpBroker().publish(this.apiKey.id, meta); + } catch (err) { + console.error(`[dump] run write failed for key=${this.apiKey.id} record=${recordId}`, oneLineError(err)); + } + } +} + +const streamReadError = (request: string | null, response: string | null): DumpErrorMeta | null => { + if (request !== null) return { kind: 'failed', reason: request }; + if (response !== null) return { kind: 'failed', reason: response }; + return null; +}; + +/** + * Returns null when the api key opts out of dumps, and the absence is the + * mechanism: the prologue has nothing to put in `services.dump`, so the run + * emits nothing and accumulates nothing. + * + * `method` and `path` are passed rather than read off a request so a transport + * that carries several turns over one connection can name each one as what it + * is — the same reason the edge accumulator takes its method explicitly. + */ +export const openRunDump = ( + apiKey: ApiKey, + turn: { readonly method: string; readonly path: string; readonly body: RequestBody }, + backgroundScheduler: BackgroundScheduler, +): RunDump | null => { + if (apiKey.dumpRetentionSeconds === null) return null; + return new RunDump( + apiKey, + { + method: turn.method, + path: turn.path, + bodyByteLength: turn.body.bytes.byteLength, + streamError: turn.body.streamError, + }, + Date.now(), + backgroundScheduler, + ); +}; From edddcf3b539894e3fe67151ee867b8500701f230 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 04:53:13 +0800 Subject: [PATCH 39/79] docs(audio): tie the family's own epilogue to the seam's open decision The seam now records why a truncated stream settles as a success, and names the fix as the family's meter reporting how its stream ended. This family already does that, so the epilogue says so where the next reader meets it, along with what registering the write while the request is still live costs it. --- packages/gateway/src/data-plane/audio/http.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/gateway/src/data-plane/audio/http.ts b/packages/gateway/src/data-plane/audio/http.ts index 1ebba12288..9316186902 100644 --- a/packages/gateway/src/data-plane/audio/http.ts +++ b/packages/gateway/src/data-plane/audio/http.ts @@ -85,11 +85,17 @@ const bodyOf = (rendered: Record | Uint8Array): Uint8Array => * The epilogue: what the run answered with, as a response. * * This family writes its own rather than going through `serveThrough`, on two counts that - * belong to the seam rather than to it. A carried document goes out under the upstream's own - * media type — including the upstream that declared none, which `Rendered.contentType` has no - * value for — and a transcription's stream states its own outcome in its last event, which is - * what settlement has to be told and what `DeferredUsage` cannot carry. Fold both into the - * seam and this handler is `serveThrough` again. + * belong to the seam rather than to it. + * + * A carried document goes out under the upstream's own media type, and that includes the + * upstream which declared none — a value `Rendered.contentType` cannot hold. + * + * And a transcription's stream states its own outcome in its last event, which is exactly the + * shape `serveThrough`'s open decision on truncated streams names as the fix: the family's + * meter reports how the stream ended, because it is the one place that knows. `DeferredUsage` + * carries only what was billed, so this family settles its own — the write still registered + * while the request is live, on a promise that resolves with both. Give the seam these two + * and this handler is `serveThrough` again. */ export const audioTranscriptions = async (c: Context): Promise => { const ingress = await readIngress(c); From 4ff5ed279724844aa84d4ed9712b0fce01e9640f Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 05:02:03 +0800 Subject: [PATCH 40/79] feat(web): read a dump record of either shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The request detail panel read every record as a pair of edges — request headers, request body, response headers, response body. A pipelined turn is recorded as its whole run instead, so the panel would have read `record.request` off a record that has none. It now dispatches on the record's shape. An edge record draws the four sections it always did; a run record draws its event stream, one block per NDJSON line, labelled with the event's own kind and with whichever stage, object or stream it names. A line that does not parse is shown as it was stored with the failure named, because a record that renders as nothing would read as an empty run. The severity badge reads `meta.status` rather than the response half's copy of it — the metadata is what both shapes have, and the two were always the same number. --- .../components/requests/detail_test.tsx | 80 +++++++++++++++++++ .../components/requests/run-render_test.ts | 35 ++++++++ apps/web/src/components/requests/detail.tsx | 49 +++++++++++- .../web/src/components/requests/run-render.ts | 46 +++++++++++ apps/web/src/i18n/locales/en.ts | 2 + apps/web/src/i18n/locales/zh-Hans.ts | 2 + .../src/routes/dashboard-monitor-requests.tsx | 4 +- 7 files changed, 213 insertions(+), 5 deletions(-) create mode 100644 apps/web/__tests__/components/requests/detail_test.tsx create mode 100644 apps/web/__tests__/components/requests/run-render_test.ts create mode 100644 apps/web/src/components/requests/run-render.ts diff --git a/apps/web/__tests__/components/requests/detail_test.tsx b/apps/web/__tests__/components/requests/detail_test.tsx new file mode 100644 index 0000000000..dd158c1457 --- /dev/null +++ b/apps/web/__tests__/components/requests/detail_test.tsx @@ -0,0 +1,80 @@ +import { describe, expect, it } from 'vitest'; + +import { RequestDetailPanel } from '../../../src/components/requests/detail'; +import { renderInApp } from '../../render'; +import type { DumpMetadata, DumpRecord } from '@floway-dev/gateway/dump-types'; + +const meta: DumpMetadata = { + id: 'rec-1', + startedAt: 0, + completedAt: 10, + method: 'POST', + path: '/v1/embeddings', + status: 200, + upstream: null, + model: 'text-embedding-3-small', + inputTokens: 7, + outputTokens: 0, + requestBytes: 14, + responseBytes: 22, + durationMs: 10, + error: null, +}; + +const edgeRecord: DumpRecord = { + shape: 'edge', + meta, + request: { + method: 'POST', + path: '/v1/embeddings', + headers: [['content-type', 'application/json']], + body: { encoding: 'utf8', data: '{"input":"hi"}' }, + }, + response: { + status: 200, + headers: [['content-type', 'application/json']], + body: { type: 'bytes', body: { encoding: 'utf8', data: '{"object":"list"}' } }, + }, +}; + +const runRecord: DumpRecord = { + shape: 'run', + meta, + events: '{"type":"stage.entered","stageId":1,"name":"serve","parentStageId":null}\n' + + '{"type":"object","fromObjectId":1,"nodes":[{"model":"text-embedding-3-small"}]}\n' + + '{"type":"stage.leaved","stageId":1,"facts":{"response.http.status":200}}\n', +}; + +const panel = (record: DumpRecord) => + renderInApp(); + +// The shape follows the endpoint, so the panel is handed both and has to tell +// them apart: an endpoint on the onion is recorded as its two edges, a pipelined +// one as the whole run. +describe('request detail panel', () => { + it('draws the two edges of an edge-shaped record', () => { + const { container } = panel(edgeRecord); + const headings = [...container.querySelectorAll('h3')].map(node => node.textContent); + expect(headings).toEqual(['Request', 'Request body', 'Response', 'Response body']); + expect(container.textContent).toContain('"input": "hi"'); + expect(container.textContent).toContain('"object": "list"'); + }); + + it('draws the event stream of a run-shaped record', () => { + const { container } = panel(runRecord); + const headings = [...container.querySelectorAll('h3')].map(node => node.textContent); + expect(headings).toEqual(['Run']); + // One block per NDJSON line, each labelled with the event's own kind. + expect([...container.querySelectorAll('pre')]).toHaveLength(3); + expect(container.textContent).toContain('stage.entered'); + expect(container.textContent).toContain('response.http.status'); + // Nothing from the edge shape leaks into it: a run has no header tables and + // no separate request body. + expect(container.querySelector('table')).toBe(null); + }); + + it('says so when a run recorded no events at all', () => { + const { container } = panel({ shape: 'run', meta, events: '' }); + expect(container.textContent).toContain('This run recorded no events.'); + }); +}); diff --git a/apps/web/__tests__/components/requests/run-render_test.ts b/apps/web/__tests__/components/requests/run-render_test.ts new file mode 100644 index 0000000000..74e5e676e3 --- /dev/null +++ b/apps/web/__tests__/components/requests/run-render_test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'vitest'; + +import { renderRunEvents } from '../../../src/components/requests/run-render'; + +// The stored record is NDJSON, one event per line, and each line is also one SSE +// `data:` payload — so this is the same reading a live observer will do. +describe('run event reader', () => { + it('names what each of the six event kinds is about', () => { + const events = renderRunEvents([ + '{"type":"stage.entered","stageId":1,"name":"serve","parentStageId":null}', + '{"type":"stage.leaved","stageId":1,"facts":{}}', + '{"type":"stage.log","stageId":1,"level":"warn","message":"retrying"}', + '{"type":"object","fromObjectId":97,"nodes":[]}', + '{"type":"stream.frame","streamId":3,"frames":[]}', + '{"type":"stream.end","streamId":3}', + '', + ].join('\n')); + + expect(events.map(event => [event.type, event.subject])).toEqual([ + ['stage.entered', 'serve'], + ['stage.leaved', '#1'], + ['stage.log', 'warn'], + ['object', '#97'], + ['stream.frame', '#3'], + ['stream.end', '#3'], + ]); + expect(events.every(event => event.parseError === null)).toBe(true); + }); + + it('shows a line it cannot parse as it was stored, and says why', () => { + const [event] = renderRunEvents('{"type":"stage.entered",\n'); + expect(event?.text).toBe('{"type":"stage.entered",'); + expect(event?.parseError).toBeTruthy(); + }); +}); diff --git a/apps/web/src/components/requests/detail.tsx b/apps/web/src/components/requests/detail.tsx index d42ac06086..4ce5725aea 100644 --- a/apps/web/src/components/requests/detail.tsx +++ b/apps/web/src/components/requests/detail.tsx @@ -8,6 +8,7 @@ import type { PropsWithChildren, ReactNode } from 'react'; import { contentTypeOf, EMPTY_BODY, renderBody, type RenderedBody } from './body-render'; import { errorLabel, requestSeverity } from './format'; import { isSensitiveHeader, redactHeaderValue } from './header-redact'; +import { renderRunEvents } from './run-render'; import { detectCollectKind, renderStreamEvents, @@ -194,14 +195,22 @@ export function RequestDetailPanel({ collected: loadedCollected, error: loadedEr } const { collected, error, record, recordId } = shown; - const requestBody = record ? renderBody(record.request.body, contentTypeOf(record.request.headers)) : EMPTY_BODY; - const responseBody = record?.response.body.type === 'bytes' ? renderBody(record.response.body.body, contentTypeOf(record.response.headers)) : EMPTY_BODY; + // The two shapes a record comes in. An endpoint served by the onion is + // recorded as its edges and draws the four sections below; a pipelined one is + // recorded as its whole run, and the run's event stream is the whole of it. + const edge = record?.shape === 'edge' ? record : null; + const requestBody = edge ? renderBody(edge.request.body, contentTypeOf(edge.request.headers)) : EMPTY_BODY; + const responseBody = edge?.response.body.type === 'bytes' ? renderBody(edge.response.body.body, contentTypeOf(edge.response.headers)) : EMPTY_BODY; const streamEvents = useMemo( - () => record?.response.body.type === 'stream' ? record.response.body.events : [], + () => record?.shape === 'edge' && record.response.body.type === 'stream' ? record.response.body.events : [], [record], ); const collectKind = record ? detectCollectKind(record.meta.path) : null; const renderedEvents = useMemo(() => renderStreamEvents(collectKind, streamEvents), [collectKind, streamEvents]); + const runEvents = useMemo( + () => record?.shape === 'run' ? renderRunEvents(record.events) : [], + [record], + ); if (!recordId) return
{t('dashboard.requests.selectPrompt')}
; // This replaces every section rather than sitting in one, so it takes the @@ -210,8 +219,40 @@ export function RequestDetailPanel({ collected: loadedCollected, error: loadedEr if (error) return {error}; if (!record) return null; - const severity = requestSeverity(record.response.status, record.meta.error); + const severity = requestSeverity(record.meta.status, record.meta.error); const responseError = errorLabel(record.meta.error); + + if (record.shape === 'run') { + return ( + +
+ + + {record.meta.path} + {record.meta.status ?? t('dashboard.requests.noStatus')} + {responseError && {responseError}} + } + copyText={record.events || undefined} + /> + + {runEvents.length === 0 ? {t('dashboard.requests.noRunEvents')} : runEvents.map((event, index) => ( +
+
+ {event.type || t('dashboard.requests.unlabeled')} + {event.subject && {event.subject}} + {event.parseError && {t('dashboard.requests.jsonParseFailed')}} +
+ +
+ ))} + +
+ + ); + } + const requestHeadersCopy = record.request.headers.map(([name, value]) => `${name}: ${value}`).join('\n'); const responseHeadersCopy = record.response.headers.map(([name, value]) => `${name}: ${value}`).join('\n'); const collectedCopyText = collected?.result === null || collected?.result === undefined diff --git a/apps/web/src/components/requests/run-render.ts b/apps/web/src/components/requests/run-render.ts new file mode 100644 index 0000000000..9679c0f54c --- /dev/null +++ b/apps/web/src/components/requests/run-render.ts @@ -0,0 +1,46 @@ +import { errorMessage } from '../../lib/error-message'; +import type { DumpEvent } from '@floway-dev/gateway/dump-types'; + +// A pipelined turn is recorded as its whole run: one NDJSON line per event, in +// the order the run emitted them. A line is also one SSE `data:` payload, so +// what this reads out of a stored record is what a live observer would be handed +// frame by frame. + +export interface RenderedRunEvent { + /** The event's own kind — `stage.entered`, `object`, `stream.frame`, … */ + type: string; + /** Which stage, object or stream it is about. Ids, not prose. */ + subject: string | null; + text: string; + parseError: string | null; +} + +// Only three of the six name a stage, and each of the other three names its own +// namespace, so the subject is read off the event rather than looked up. +const subjectOf = (event: DumpEvent): string | null => { + switch (event.type) { + case 'stage.entered': return event.name; + case 'stage.leaved': return `#${event.stageId}`; + case 'stage.log': return event.level; + case 'object': return `#${event.fromObjectId}`; + case 'stream.frame': + case 'stream.end': return `#${event.streamId}`; + } +}; + +// A line the gateway wrote and this cannot parse is shown as it was stored, with +// the failure named: a record that renders as nothing would read as an empty run. +export const renderRunEvents = (ndjson: string): RenderedRunEvent[] => + ndjson.split('\n').filter(line => line.length > 0).map(line => { + try { + const event = JSON.parse(line) as DumpEvent; + return { + type: event.type, + subject: subjectOf(event), + text: JSON.stringify(event, null, 2), + parseError: null, + }; + } catch (error) { + return { type: '', subject: null, text: line, parseError: errorMessage(error) }; + } + }); diff --git a/apps/web/src/i18n/locales/en.ts b/apps/web/src/i18n/locales/en.ts index 600237cf15..c3b8b35ffe 100644 --- a/apps/web/src/i18n/locales/en.ts +++ b/apps/web/src/i18n/locales/en.ts @@ -957,6 +957,8 @@ const en = { apiKeysLink: 'API Keys', request: 'Request', requestBody: 'Request body', + run: 'Run', + noRunEvents: 'This run recorded no events.', response: 'Response', responseBody: 'Response body', noRequestBody: 'No request body', diff --git a/apps/web/src/i18n/locales/zh-Hans.ts b/apps/web/src/i18n/locales/zh-Hans.ts index a2383fba15..3df0337c0e 100644 --- a/apps/web/src/i18n/locales/zh-Hans.ts +++ b/apps/web/src/i18n/locales/zh-Hans.ts @@ -910,6 +910,8 @@ const zhHansCN = { apiKeysLink: 'API 密钥', request: '请求', requestBody: '请求体', + run: '运行过程', + noRunEvents: '这次运行没有记录到任何事件。', response: '响应', responseBody: '响应体', noRequestBody: '没有请求体', diff --git a/apps/web/src/routes/dashboard-monitor-requests.tsx b/apps/web/src/routes/dashboard-monitor-requests.tsx index aaf1054b3e..f7caa264f6 100644 --- a/apps/web/src/routes/dashboard-monitor-requests.tsx +++ b/apps/web/src/routes/dashboard-monitor-requests.tsx @@ -66,7 +66,9 @@ export async function clientLoader({ request }: Route.ClientLoaderArgs): Promise ]); const record = recordResult?.data ?? null; const collectKind = record ? detectCollectKind(record.meta.path) : null; - const streamEvents = record?.response.body.type === 'stream' ? record.response.body.events : []; + // Only an edge-shaped record carries a captured frame log to collect; a run + // records its stream inside its own events and the detail panel reads it there. + const streamEvents = record?.shape === 'edge' && record.response.body.type === 'stream' ? record.response.body.events : []; const collected = collectKind && streamEvents.length ? await collectStream(collectKind, streamEvents) : null; return { collected, From 1d0cb8ba29d0bb8e4decabe34b8146da5be49645 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 05:05:15 +0800 Subject: [PATCH 41/79] fix(gateway): record a stream by how it ended, not by the fact that it ended MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A streaming turn settled as a success whatever happened to it, so a stream that stopped before its terminator wrote a neutral performance row and a dump success — a turn that never finished, reported as one that produced what it said it would. What blocked this was that the write has to be registered while the request is still live for the platform to bind it, and the outcome is only known when the stream ends. The two stop pulling against each other once the family's meter reports both together: it knows the usage and the terminator at the same moment, so it resolves one value carrying each. The audio family arrived at this shape independently and had to write its own epilogue around the seam to use it. Widening the seam is what lets that handler collapse back into it. --- .../data-plane/completions-pipeline_test.ts | 23 ++++++----- .../data-plane/completions/http_test.ts | 29 ++++++++++++++ .../src/data-plane/completions/pipeline.ts | 20 ++++++---- .../gateway/src/data-plane/images/pipeline.ts | 19 ++++++---- .../gateway/src/data-plane/pipeline/serve.ts | 38 ++++++++++++------- 5 files changed, 92 insertions(+), 37 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts index 8f9638d78c..2af5df2db1 100644 --- a/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts @@ -163,12 +163,15 @@ describe('the completions pipeline', () => { { identity: { model: 'text-model', upstream: 'up_a', modelKey: 'text-model-key', pricing: null }, quantities: {} }, ]); await collect(facts['response.completions.rendered']); - expect(await facts['response.completions.streamedUsage']).toEqual([ - { - identity: { model: 'text-model', upstream: 'up_a', modelKey: 'text-model-key', pricing: null }, - quantities: { input_tokens: '5', output_tokens: '7' }, - }, - ]); + expect(await facts['response.completions.streamedUsage']).toMatchObject({ + failed: false, + billable: [ + { + identity: { model: 'text-model', upstream: 'up_a', modelKey: 'text-model-key', pricing: null }, + quantities: { input_tokens: '5', output_tokens: '7' }, + }, + ], + }); await drain(); }); @@ -271,9 +274,11 @@ describe('the completions pipeline', () => { expect(recorded.usage).toHaveLength(0); const streamed = facts['response.completions.streamedUsage']; expect(streamed).not.toBeNull(); - const billable = await streamed!; - expect(billable).toHaveLength(1); - expect(billable[0]!.quantities).toMatchObject({ input_tokens: '5', output_tokens: '7' }); + const outcome = await streamed!; + expect(outcome.billable).toHaveLength(1); + expect(outcome.billable[0]!.quantities).toMatchObject({ input_tokens: '5', output_tokens: '7' }); + // The stream reached its terminator, so the turn produced what it said it would. + expect(outcome.failed).toBe(false); }); // A run that reached no upstream bills nothing and samples nothing, which is what the diff --git a/packages/gateway/__tests__/data-plane/completions/http_test.ts b/packages/gateway/__tests__/data-plane/completions/http_test.ts index 8f3dd4185e..0c35f10e65 100644 --- a/packages/gateway/__tests__/data-plane/completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/completions/http_test.ts @@ -322,6 +322,35 @@ test('/v1/completions non-streaming records usage row, performance neutral row ( assertEquals(dump.response.body.type, 'bytes'); }); +// A stream that stops before its terminator did not produce what it said it would, and the +// performance row has to say so — a neutral row would report a turn that never finished as +// one that did. +test('/v1/completions a stream that never terminated is recorded as a failed request', async () => { + const { apiKey, repo } = await setupAppTest(); + await registerCompletionsUpstream(repo); + + await withMockedFetch( + () => Promise.resolve(new Response('data: {"id":"c","object":"text_completion","created":1,"model":"davinci-002","choices":[{"index":0,"text":"hi"}]}\n\n', { + status: 200, headers: { 'content-type': 'text/event-stream' }, + })), + async () => { + const response = await requestApp('/v1/completions', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + body: JSON.stringify({ model: 'davinci-002', prompt: 'hello', stream: true }), + }); + assertEquals(response.status, 200); + await response.text(); + }, + ); + + await flushAsyncWork(); + + const performance = await repo.performance.listAll(); + assertEquals(performance.length, 1); + assertEquals(performance[0]?.errorsNoOutput, 1); +}); + test('/v1/completions streaming records usage row, performance neutral row (text_completion operation, no TTFT/TPOT), and a frame-log dump record', async () => { const { apiKey, repo } = await setupAppTest(); await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); diff --git a/packages/gateway/src/data-plane/completions/pipeline.ts b/packages/gateway/src/data-plane/completions/pipeline.ts index 962db039c1..e59f92ddd8 100644 --- a/packages/gateway/src/data-plane/completions/pipeline.ts +++ b/packages/gateway/src/data-plane/completions/pipeline.ts @@ -17,6 +17,7 @@ import type { UsageQuantities } from '../../repo/types.ts'; import { tokenUsageQuantities } from '../../repo/usage-metrics.ts'; import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; import { isFailure } from '../pipeline/facts.ts'; +import type { StreamOutcome } from '../pipeline/serve.ts'; import type { GatewayServices } from '../pipeline/services.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; import { failover, resolveCandidates } from '../pipeline/stages.ts'; @@ -73,7 +74,7 @@ export interface CompletionsFacts extends GatewayFacts { * after this run has answered — so the numbers cannot be in `response.usage.billable`, * which says what had been reported when the ending stage handed up: the entity, and no * quantities. Settling billing from this is the prologue's job, after the drain. */ - 'response.completions.streamedUsage': Promise | null; + 'response.completions.streamedUsage': Promise | null; /** What the client is actually sent, in its own protocol — a JSON body, or the SSE frames * of a stream. The edge provides it, so a dump shows what the client received rather than * the gateway's own reading of it. */ @@ -288,7 +289,7 @@ const readUpstream = async ( const metered = meterFrames(parseCompletionsStream(response.body, { signal }), identity, candidate, record); return { payload: metered.frames, - streamedUsage: metered.billable, + streamedUsage: metered.outcome, billable: called, // Releasing this body is draining those frames: they are one reader over one connection, // and a second reader is not something a `ReadableStream` allows. @@ -326,7 +327,7 @@ const readResult = async (response: Response): Promise<{ readonly value: Complet interface MeteredFrames { readonly frames: CompletionsFrames; - readonly billable: Promise; + readonly outcome: Promise; } const meterFrames = ( @@ -335,8 +336,11 @@ const meterFrames = ( candidate: ModelCandidate, record: (frame: ProtocolFrame) => void, ): MeteredFrames => { - let settle!: (billable: readonly BillableEntity[]) => void; - const billable = new Promise(resolve => { settle = resolve; }); + let settle!: (outcome: StreamOutcome) => void; + const outcome = new Promise(resolve => { settle = resolve; }); + // Running out without the terminal frame is what "it did not finish" means, and it is known + // at the same moment the usage is. + let sawTerminal = false; const frames = view((async function* () { let usage: unknown; let tier: string | null | undefined; @@ -351,6 +355,8 @@ const meterFrames = ( } // What the upstream sent, before the edge decides what the client sees — a dump // reader is owed the frames that arrived, including the one metering asked for. + // The transport's own terminator is this protocol's end-of-turn. + if (frame.type === 'done') sawTerminal = true; record(frame); yield frame; } @@ -358,10 +364,10 @@ const meterFrames = ( // Reached however the frames ended — the terminal chunk, a client that stopped // reading, or a broken upstream — because tokens the upstream already metered are // billable whatever happened to the downstream half. - settle([{ identity, quantities: billed(usage, tier, candidate) }]); + settle({ billable: [{ identity, quantities: billed(usage, tier, candidate) }], failed: !sawTerminal }); } })()); - return { frames, billable }; + return { frames, outcome }; }; const billed = (usage: unknown, tier: string | null | undefined, candidate: ModelCandidate): UsageQuantities => { diff --git a/packages/gateway/src/data-plane/images/pipeline.ts b/packages/gateway/src/data-plane/images/pipeline.ts index 5ee7da45de..0ab79d9633 100644 --- a/packages/gateway/src/data-plane/images/pipeline.ts +++ b/packages/gateway/src/data-plane/images/pipeline.ts @@ -14,6 +14,7 @@ import type { UsageQuantities } from '../../repo/types.ts'; import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; import { isFailure } from '../pipeline/facts.ts'; +import type { StreamOutcome } from '../pipeline/serve.ts'; import type { GatewayServices } from '../pipeline/services.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; import { failover, resolveCandidates } from '../pipeline/stages.ts'; @@ -83,7 +84,7 @@ export interface ImagesFacts extends GatewayFacts { * `response.usage.billable`, which says what had been reported when the ending stage handed * up: the entity, and no quantities. Settling from this is the epilogue's job, after the * drain. */ - 'response.images.streamedUsage': Promise | null; + 'response.images.streamedUsage': Promise | null; /** What the client is actually sent, in the images protocol — a JSON body, or the SSE frames * of a stream. The edge provides it, so a dump shows the body the client received rather * than the gateway's canonical form. */ @@ -266,7 +267,7 @@ const callImagesUpstream = defineStage< return move({ ...facts, 'response.images.canonical': metered.frames, - 'response.images.streamedUsage': metered.billable, + 'response.images.streamedUsage': metered.outcome, 'response.http.headers': headers, // Releasing this body is reading those events to the end: they are one reader over one // connection, and a second reader is not something a `ReadableStream` allows. @@ -308,7 +309,7 @@ const spentBody = (body: ReadableStream | null): ReadableStream; + readonly outcome: Promise; } const meterFrames = ( @@ -317,8 +318,11 @@ const meterFrames = ( signal: AbortSignal | undefined, record: (event: ImagesStreamEvent) => void, ): MeteredFrames => { - let settle!: (billable: readonly BillableEntity[]) => void; - const billable = new Promise(resolve => { settle = resolve; }); + let settle!: (outcome: StreamOutcome) => void; + const outcome = new Promise(resolve => { settle = resolve; }); + // Running out without the completed event is what "it did not finish" means, and it is known + // at the same moment the usage is. + let sawTerminal = false; const frames = view((async function* () { let usage: CanonicalImagesUsage | undefined; try { @@ -328,6 +332,7 @@ const meterFrames = ( record(event); if (isImagesTerminalEvent(event)) { usage = parseImagesUsage(event); + sawTerminal = true; yield event; // The image is complete, so there is nothing further to read. An upstream that holds // the connection open past this point would otherwise keep the client's own stream @@ -340,14 +345,14 @@ const meterFrames = ( // Reached however the events ended — the completed one, a client that stopped reading, // or a broken upstream — because what the upstream already metered is billable whatever // happened to the downstream half. - settle([{ identity, quantities: billed(usage) }]); + settle({ billable: [{ identity, quantities: billed(usage) }], failed: !sawTerminal }); } // Only an upstream that ended its body without ever completing the image reaches here: the // arm above returns on the terminal event. A client has been sent partial images and no // image, which is a failed answer however far it got. throw new Error(IMAGES_MISSING_TERMINAL_MESSAGE); })()); - return { frames, billable }; + return { frames, outcome }; }; /** An image is billed by the tokens its upstream reports: `BILLING_METRICS` names no per-image diff --git a/packages/gateway/src/data-plane/pipeline/serve.ts b/packages/gateway/src/data-plane/pipeline/serve.ts index 92bb05ddac..8a30487e96 100644 --- a/packages/gateway/src/data-plane/pipeline/serve.ts +++ b/packages/gateway/src/data-plane/pipeline/serve.ts @@ -103,8 +103,10 @@ export type Rendered = | { readonly body: BodyInit; /** Owned by whichever stage serialized the body, never by the upstream — a media type - * describing bytes the gateway wrote itself is the one thing an upstream cannot say. */ - readonly contentType: string; + * describing bytes the gateway wrote itself is the one thing an upstream cannot say. + * `null` where a family carries an upstream's own body and the upstream declared none: + * inventing one would describe bytes nobody described. */ + readonly contentType: string | null; } | { /** An answer that *is* a stream. The frames go out as they arrive, which is why nothing @@ -117,10 +119,18 @@ export type Rendered = export const isFrames = (rendered: unknown): rendered is AsyncIterable => typeof rendered === 'object' && rendered !== null && Symbol.asyncIterator in rendered; -/** What a streaming family will have been billed once its frames run out. A stream's usage +/** How a streaming turn ended, once its frames ran out. Both halves arrive together because + * both are known at the same moment — the last chunk states the usage, and running out + * without a terminal event is what "it did not finish" means. */ +export interface StreamOutcome { + readonly billable: readonly BillableEntity[]; + readonly failed: boolean; +} + +/** What a streaming family will have been billed, and whether it got there. A stream's usage * arrives with its last chunk, which is after the run has answered — so the run hands up a * promise and settlement of it belongs here, after the answer is on its way. */ -export type DeferredUsage = (facts: Exit) => Promise | null; +export type DeferredUsage = (facts: Exit) => Promise | null; /** * Runs a family's pipeline and turns what it answered with into a response. @@ -145,16 +155,13 @@ export const serveThrough = async < const answer = render(facts); const pending = deferredUsage?.(facts) ?? null; - // OPEN DECISION, deliberately taken the simple way for now. `failed` is false here even - // for a stream that stopped short, so a truncated turn still writes a neutral performance - // row and a dump success. Settling it correctly needs the write registered while the - // request is still live *and* the outcome known only when the stream ends, and the two - // pull against each other — three shapes were tried and each lost the row entirely. - // The honest fix is for the family's meter to report how its stream ended, since that is - // the one place that knows; that is a change across all four streaming families. + // Registered while the request is still live, so the platform binds the write to it — and + // resolved only when the stream ends, which is the one moment both what it billed and + // whether it finished are known. A turn that stopped short is not recorded as one that + // produced what it said it would. if (pending !== null) { - prologue.services.background(pending.then(billable => { - settleBillable({ ...prologue.services, log: consoleLogSink }, billable, false); + prologue.services.background(pending.then(outcome => { + settleBillable({ ...prologue.services, log: consoleLogSink }, outcome.billable, outcome.failed); })); } @@ -188,6 +195,9 @@ export const serveThrough = async < // already held, so releasing can start at once. prologue.services.background(drain()); const headers = new Headers(facts['response.http.headers'].map(([name, value]): [string, string] => [name, value])); - headers.set('content-type', answer.contentType); + // An upstream that declared no media type is answered without one, rather than having one + // invented for bytes nobody described. + if (answer.contentType !== null) headers.set('content-type', answer.contentType); + else headers.delete('content-type'); return finalizeGatewayResponse(prologue.gateway, new Response(answer.body, { status, headers })); }; From 7e0e113d39b678e626be70cedf14601050f8fb28 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 05:07:28 +0800 Subject: [PATCH 42/79] test(gateway): serve both record shapes through the control plane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reader dispatch was covered per layer and not end to end, so nothing said that a run record survives the list query, the detail fetch and the wire codec together — which is the path the dashboard actually takes. One list carries a run and a pair of edges, and a detail fetch of each hands back what it is: the run's NDJSON, and the edge's request half. The run fixture encodes the events the way the sink does, object space and folding included, rather than spelling the stored lines out by hand. --- .../__tests__/control-plane/dump_test.ts | 37 ++++++++++++++++++- .../gateway/__tests__/dump/test-fixtures.ts | 11 +++--- packages/gateway/src/dump/types.ts | 21 ++++++----- 3 files changed, 53 insertions(+), 16 deletions(-) diff --git a/packages/gateway/__tests__/control-plane/dump_test.ts b/packages/gateway/__tests__/control-plane/dump_test.ts index 6bc9dd9f0d..2bebb7dd7c 100644 --- a/packages/gateway/__tests__/control-plane/dump_test.ts +++ b/packages/gateway/__tests__/control-plane/dump_test.ts @@ -3,7 +3,7 @@ import { test } from 'vitest'; import { initDumpBroker, initDumpStore } from '../../src/dump/registry.ts'; import type { DumpStore } from '../../src/dump/store-contract.ts'; import type { DumpMetadata, DumpRecord, StoredDumpRecord } from '../../src/dump/types.ts'; -import { fakeMeta as baseFakeMeta, fakeRecord as baseFakeRecord, installDumpStubs } from '../dump/test-fixtures.ts'; +import { fakeMeta as baseFakeMeta, fakeRecord as baseFakeRecord, fakeRunRecord, installDumpStubs } from '../dump/test-fixtures.ts'; import { requestApp, setupAppTest } from '../test-utils/app.ts'; import { assertEquals, assertExists } from '@floway-dev/test-utils'; @@ -80,6 +80,41 @@ test('GET /api/dump/keys/:keyId/records/:recordId returns the rehydrated record' assertEquals(body.meta.id, '01HZZ0000000000000000000XX'); }); +// One list, both shapes — the metadata is common, so the dashboard's list says +// nothing about which mechanism served a turn — and a detail fetch hands each +// one back as what it is. +test('GET /api/dump/keys/:keyId/records serves a run record beside an edge one', async () => { + const { repo, apiKey } = await setupAppTest(); + await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); + const stubs = installDumpStubs(initDumpStore, initDumpBroker); + stubs.seed(apiKey.id, fakeRecord('01HZZ0000000000000000EDGE', 1000)); + stubs.seed(apiKey.id, fakeRunRecord( + [ + { type: 'stage.entered', stageId: 1, name: 'serve', parentStageId: null, facts: { 'request.payload': { model: 'm' } } }, + { type: 'stage.leaved', stageId: 1, facts: { 'response.http.status': 200 } }, + ], + { id: '01HZZ00000000000000000RUN', completedAt: 2000, startedAt: 1999 }, + )); + + const listed = await requestApp(`/api/dump/keys/${apiKey.id}/records`, { headers: { 'x-api-key': apiKey.key } }); + assertEquals(listed.status, 200); + const { records } = await listed.json() as { records: DumpMetadata[] }; + assertEquals(records.map(meta => meta.id), ['01HZZ00000000000000000RUN', '01HZZ0000000000000000EDGE']); + + const run = await requestApp(`/api/dump/keys/${apiKey.id}/records/01HZZ00000000000000000RUN`, { headers: { 'x-api-key': apiKey.key } }); + const runBody = await run.json() as DumpRecord; + if (runBody.shape !== 'run') throw new Error(`expected the run shape, got ${runBody.shape}`); + // The object space rides with the events that reference it, so the `object` + // event carrying the payload arrives before the entry that points at it. + assertEquals(runBody.events.trimEnd().split('\n').map(line => (JSON.parse(line) as { type: string }).type), + ['object', 'stage.entered', 'stage.leaved']); + + const edge = await requestApp(`/api/dump/keys/${apiKey.id}/records/01HZZ0000000000000000EDGE`, { headers: { 'x-api-key': apiKey.key } }); + const edgeBody = await edge.json() as DumpRecord; + if (edgeBody.shape !== 'edge') throw new Error(`expected the edge shape, got ${edgeBody.shape}`); + assertEquals(edgeBody.request.method, 'POST'); +}); + test('GET /api/dump/keys/:keyId/records/:recordId 404s on unknown id', async () => { const { repo, apiKey } = await setupAppTest(); await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); diff --git a/packages/gateway/__tests__/dump/test-fixtures.ts b/packages/gateway/__tests__/dump/test-fixtures.ts index dd0093e586..4ba95752ec 100644 --- a/packages/gateway/__tests__/dump/test-fixtures.ts +++ b/packages/gateway/__tests__/dump/test-fixtures.ts @@ -1,7 +1,7 @@ import type { DumpBroker } from '../../src/dump/broker.ts'; import type { DumpStore } from '../../src/dump/store-contract.ts'; import type { DumpMetadata, StoredDumpEdgeRecord, StoredDumpRecord, StoredDumpRunRecord } from '../../src/dump/types.ts'; -import { toNdjson, type DumpEvent } from '@floway-dev/pipeline'; +import { encodeRun, toNdjson, type Event } from '@floway-dev/pipeline'; export const fakeMeta = (overrides: Partial = {}): DumpMetadata => ({ id: 'test-id', @@ -28,12 +28,13 @@ export const fakeRecord = (overrides: Partial = {}): StoredDumpEdg response: { status: 200, headers: [], body: { type: 'none' } }, }); -// A run record holds one NDJSON stream and no edge halves, so a fixture only -// needs the lines the reader is meant to get back. -export const fakeRunRecord = (events: readonly DumpEvent[], overrides: Partial = {}): StoredDumpRunRecord => ({ +// A run record holds one NDJSON stream and no edge halves, so a fixture takes +// the events a run emitted and encodes them the way the sink does — object +// space, folding and all. +export const fakeRunRecord = (events: readonly Event[], overrides: Partial = {}): StoredDumpRunRecord => ({ shape: 'run', meta: fakeMeta(overrides), - events: new TextEncoder().encode(toNdjson(events)), + events: new TextEncoder().encode(toNdjson(encodeRun(events))), }); // A reader hands back whichever shape was written, so a test asserting on the diff --git a/packages/gateway/src/dump/types.ts b/packages/gateway/src/dump/types.ts index 723bd60fb2..6d8faae5ab 100644 --- a/packages/gateway/src/dump/types.ts +++ b/packages/gateway/src/dump/types.ts @@ -95,22 +95,22 @@ export interface StoredDumpResponse { body: StoredDumpResponseBody; } +export type StoredDumpEdgeRecord = { + shape: 'edge'; + meta: DumpMetadata; + request: StoredDumpRequest; + response: StoredDumpResponse; +}; + // The run's NDJSON, still as bytes: it is a body file under the same contract // as the other two, gzipped into the file store and pointed at by the row's // descriptor. One `put` carries it whole — measured on production, P99 of a // turn's request and response together is 2.86 MB, well under the 5 MiB below // which multipart has nothing to divide. -export interface StoredDumpRunRecord { +export type StoredDumpRunRecord = { shape: 'run'; meta: DumpMetadata; events: Uint8Array; -} - -export type StoredDumpEdgeRecord = { - shape: 'edge'; - meta: DumpMetadata; - request: StoredDumpRequest; - response: StoredDumpResponse; }; export type StoredDumpRecord = StoredDumpEdgeRecord | StoredDumpRunRecord; @@ -122,8 +122,9 @@ export type DumpWriteEdgeRecord = { response: StoredDumpResponse; }; -// No prepared half: a run's stream is encoded once the run is over, so there is -// nothing to compress ahead of the terminal write the way a request body is. +// The run half is the stored shape unchanged: a run's stream is encoded once the +// run is over, so there is nothing to compress ahead of the terminal write the +// way a request body is. export type DumpWriteRecord = DumpWriteEdgeRecord | StoredDumpRunRecord; // --- Wire shape (serialized JSON over the dashboard's control plane) --- From 940673f0c1cad67acc47e9e8047874ebdd4f47c4 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 05:09:47 +0800 Subject: [PATCH 43/79] refactor(gateway): state the dump's error precedence once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both record shapes fell back to a transport-level read failure when no hook had stamped an error, and each spelled the precedence out for itself — a request-body read failure outranking a response-body one, twice. It belongs beside the assembly it feeds, so it moves there and both call it. --- packages/gateway/src/dump/accumulator.ts | 18 ++++-------------- packages/gateway/src/dump/attribution.ts | 9 +++++++++ packages/gateway/src/dump/run-sink.ts | 12 ++---------- 3 files changed, 15 insertions(+), 24 deletions(-) diff --git a/packages/gateway/src/dump/accumulator.ts b/packages/gateway/src/dump/accumulator.ts index de92a428b7..9a832b7cbd 100644 --- a/packages/gateway/src/dump/accumulator.ts +++ b/packages/gateway/src/dump/accumulator.ts @@ -10,10 +10,9 @@ import type { Context } from 'hono'; -import { DumpAttribution, oneLineError } from './attribution.ts'; +import { DumpAttribution, oneLineError, streamReadError } from './attribution.ts'; import { getDumpBroker, getDumpStore } from './registry.ts'; import type { - DumpErrorMeta, DumpMetadata, DumpStreamEvent, DumpWriteRecord, @@ -210,12 +209,9 @@ export class DumpAccumulator { status: response.status, requestBytes: this.requestSnapshot.bodyByteLength, responseBytes: response.payloadBytes, - // Precedence: an explicit error stamp from the respond path wins (the - // assembler applies this only when there is none); otherwise a - // request-body read failure (operator-side payload didn't arrive intact) - // outranks a response-body read failure. Both stream-read failures - // surface as `kind: 'failed'`. - fallbackError: this.streamReadError(response), + // Precedence: an explicit error stamp from the respond path wins — the + // assembler applies this only when there is none. + fallbackError: streamReadError(this.requestSnapshot.streamError, response.streamError), }); // Commit the row before publishing so subscribers fetching detail off the meta frame find it. @@ -241,12 +237,6 @@ export class DumpAccumulator { console.error(`[dump] write failed for key=${this.apiKey.id} record=${recordId}`, oneLineError(err)); } } - - private streamReadError(response: ResponseSnapshot): DumpErrorMeta | null { - if (this.requestSnapshot.streamError !== null) return { kind: 'failed', reason: this.requestSnapshot.streamError }; - if (response.streamError !== null) return { kind: 'failed', reason: response.streamError }; - return null; - } } // Returns null when the api key opts out of dumps; callers then skip all diff --git a/packages/gateway/src/dump/attribution.ts b/packages/gateway/src/dump/attribution.ts index e6b98d5425..9118a21efe 100644 --- a/packages/gateway/src/dump/attribution.ts +++ b/packages/gateway/src/dump/attribution.ts @@ -74,6 +74,15 @@ export interface DumpTurnOutcome { readonly fallbackError: DumpErrorMeta | null; } +// A request-body read failure — the operator-side payload did not arrive intact +// — outranks a failure reading back what was answered. Both surface as +// `kind: 'failed'`. +export const streamReadError = (request: string | null, response: string | null): DumpErrorMeta | null => { + if (request !== null) return { kind: 'failed', reason: request }; + if (response !== null) return { kind: 'failed', reason: response }; + return null; +}; + export class DumpAttribution { private model: string | null = null; private upstreamId: string | null = null; diff --git a/packages/gateway/src/dump/run-sink.ts b/packages/gateway/src/dump/run-sink.ts index 0ca5893c31..146833c29e 100644 --- a/packages/gateway/src/dump/run-sink.ts +++ b/packages/gateway/src/dump/run-sink.ts @@ -13,9 +13,9 @@ // runner does none of the recording — not a no-op that accumulates and throws // the result away. -import { DumpAttribution, oneLineError } from './attribution.ts'; +import { DumpAttribution, oneLineError, streamReadError } from './attribution.ts'; import { getDumpBroker, getDumpStore } from './registry.ts'; -import type { DumpErrorMeta, DumpMetadata } from './types.ts'; +import type { DumpMetadata } from './types.ts'; import type { RequestBody } from '../data-plane/shared/request-body.ts'; import type { ApiKey, TokenUsage } from '../repo/types.ts'; import { ulid } from '../shared/ulid.ts'; @@ -135,8 +135,6 @@ export class RunDump { status, requestBytes: this.requestSnapshot.bodyByteLength, responseBytes, - // A request-body read failure — the operator-side payload did not arrive - // intact — outranks a failure reading back what was answered. fallbackError: streamReadError(this.requestSnapshot.streamError, responseStreamError), }); @@ -154,12 +152,6 @@ export class RunDump { } } -const streamReadError = (request: string | null, response: string | null): DumpErrorMeta | null => { - if (request !== null) return { kind: 'failed', reason: request }; - if (response !== null) return { kind: 'failed', reason: response }; - return null; -}; - /** * Returns null when the api key opts out of dumps, and the absence is the * mechanism: the prologue has nothing to put in `services.dump`, so the run From e91863aa917390abf0fe979c7e85920eaa9d23d8 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 05:27:10 +0800 Subject: [PATCH 44/79] feat(gateway): record a pipelined turn as the run it was MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The run's event stream had a writer and no producer: nothing opened one, and a pipelined endpoint still wrote the edge record its handler no longer builds. The shape follows the endpoint. A pipelined turn opens the run recording in its prologue, so one turn is written once and the two shapes never both appear for it. What a turn stamps on either — the requested model, the upstream that answered, what it billed, how it failed — is now a named surface both implement, which is what keeps the metadata common and the stages ignorant of which they are stamping. A frame becomes the stream.frame event the format already specified, which is that event's first producer. --- .../data-plane/completions/http_test.ts | 49 ++++++++----------- .../gateway/__tests__/dump/test-fixtures.ts | 11 +++++ .../data-plane/chat/responses/websocket.ts | 8 +-- .../gateway/src/data-plane/pipeline/serve.ts | 19 ++++++- .../src/data-plane/shared/gateway-ctx.ts | 11 +++-- packages/gateway/src/dump/run-sink.ts | 26 +++++++++- packages/gateway/src/dump/turn-dump.ts | 25 ++++++++++ 7 files changed, 110 insertions(+), 39 deletions(-) create mode 100644 packages/gateway/src/dump/turn-dump.ts diff --git a/packages/gateway/__tests__/data-plane/completions/http_test.ts b/packages/gateway/__tests__/data-plane/completions/http_test.ts index 0c35f10e65..78ad261317 100644 --- a/packages/gateway/__tests__/data-plane/completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/completions/http_test.ts @@ -2,7 +2,7 @@ import { test } from 'vitest'; import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts'; import { tokenCountsFromUsage } from '../../../src/repo/usage-metrics.ts'; -import { edgeRecordOf, installDumpStubs } from '../../dump/test-fixtures.ts'; +import { eventsOf, installDumpStubs, runRecordOf } from '../../dump/test-fixtures.ts'; import { buildCustomUpstreamRecord, flushAsyncWork, requestApp, setupAppTest } from '../../test-utils/app.ts'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; @@ -311,15 +311,16 @@ test('/v1/completions non-streaming records usage row, performance neutral row ( assertEquals(performance[0]?.errorsNoOutput, 0); assertEquals(dumpStubs.stored.length, 1); - const dump = edgeRecordOf(dumpStubs.stored[0]?.record); + const dump = runRecordOf(dumpStubs.stored[0]?.record); assertEquals(dump.meta.path, '/v1/completions'); assertEquals(dump.meta.status, 200); assertEquals(dump.meta.model, 'davinci-002'); assertEquals(dump.meta.inputTokens, 7); assertEquals(dump.meta.outputTokens, 2); - // Non-streaming: the upstream sent a one-shot JSON, so the dump - // captures the bytes (not a frame log). - assertEquals(dump.response.body.type, 'bytes'); + // The shape follows the endpoint: a pipelined turn is recorded as its whole run, so what + // is stored is the event stream rather than the two edges. The metadata is common to both, + // which is what lets the dashboard list them together. + assertEquals(runRecordOf(dumpStubs.stored[0]!.record).shape, 'run'); }); // A stream that stops before its terminator did not produce what it said it would, and the @@ -382,32 +383,24 @@ test('/v1/completions streaming records usage row, performance neutral row (text assertEquals(performance[0]?.errorsNoOutput, 0); assertEquals(dumpStubs.stored.length, 1); - const dump = edgeRecordOf(dumpStubs.stored[0]?.record); + const dump = runRecordOf(dumpStubs.stored[0]?.record); assertEquals(dump.meta.path, '/v1/completions'); assertEquals(dump.meta.status, 200); assertEquals(dump.meta.model, 'davinci-002'); assertEquals(dump.meta.inputTokens, 4); assertEquals(dump.meta.outputTokens, 2); - // Streaming: dump stores the protocol frames the gateway saw from - // upstream BEFORE transformFrame ran. The fixture stream emits two - // content events, one usage-only event (which the client did not opt - // into and so it was stripped from the forwarded stream), and a done - // terminator. - assertEquals(dump.response.body.type, 'stream'); - if (dump.response.body.type === 'stream') { - const frames = dump.response.body.events.map(e => e.frame); - assertEquals(frames.length, 4); - assertEquals(frames[0]?.type, 'event'); - assertEquals(frames[1]?.type, 'event'); - assertEquals(frames[2]?.type, 'event'); - // Upstream's usage chunk is preserved in the dump even though it was - // stripped from the client-facing stream. - const usageFrame = frames[2]; - if (usageFrame?.type === 'event') { - const event = usageFrame.event as { choices: unknown[]; usage: { prompt_tokens: number } }; - assertEquals(event.choices.length, 0); - assertEquals(event.usage.prompt_tokens, 4); - } - assertEquals(frames[3]?.type, 'done'); - } + // The frames the gateway saw from upstream, before the edge decided which of them the + // client is shown — recorded as `stream.frame` events in the run's own stream. The fixture + // emits two content events, one usage-only event (which this client did not opt into and + // so was stripped from what it received), and a done terminator. + const events = eventsOf(runRecordOf(dumpStubs.stored[0]!.record)); + const frames = events + .filter(event => event.type === 'stream.frame') + .flatMap(event => (event as unknown as { frames: readonly unknown[] }).frames); + assertEquals(frames.length, 4); + // The usage chunk the client did not opt into is still in the run's own record: what the + // gateway saw is not narrowed to what it forwarded. The encoder shares repeated values, so + // what is asserted is that the numbers are in the stream, not the shape they took in it. + const stored = new TextDecoder().decode(runRecordOf(dumpStubs.stored[0]!.record).events); + assertEquals(stored.includes('"prompt_tokens":4'), true); }); diff --git a/packages/gateway/__tests__/dump/test-fixtures.ts b/packages/gateway/__tests__/dump/test-fixtures.ts index 4ba95752ec..30e5068dd2 100644 --- a/packages/gateway/__tests__/dump/test-fixtures.ts +++ b/packages/gateway/__tests__/dump/test-fixtures.ts @@ -45,6 +45,17 @@ export const edgeRecordOf = (record: StoredDumpRecord | null | undefined): Store return record; }; +export const runRecordOf = (record: StoredDumpRecord | null | undefined): StoredDumpRunRecord => { + if (!record) throw new Error('expected a stored dump record'); + if (record.shape !== 'run') throw new Error(`expected the run shape, got ${record.shape}`); + return record; +}; + +/** The events a run recorded, decoded from the NDJSON one line at a time. */ +export const eventsOf = (record: StoredDumpRunRecord): readonly Record[] => + new TextDecoder().decode(record.events).split('\n').filter(line => line.length > 0) + .map(line => JSON.parse(line) as Record); + type DumpStubFailMethod = | 'put' | 'list' diff --git a/packages/gateway/src/data-plane/chat/responses/websocket.ts b/packages/gateway/src/data-plane/chat/responses/websocket.ts index 4edf42684d..cfbc5fa47b 100644 --- a/packages/gateway/src/data-plane/chat/responses/websocket.ts +++ b/packages/gateway/src/data-plane/chat/responses/websocket.ts @@ -4,7 +4,7 @@ import { wrapResponsesClientEgress } from './client-output.ts'; import { createResponsesWsSession } from './items/store.ts'; import { PreviousResponseNotFoundError } from './serve-prep.ts'; import { responsesServe } from './serve.ts'; -import type { DumpAccumulator } from '../../../dump/accumulator.ts'; +import type { TurnDump } from '../../../dump/turn-dump.ts'; import { apiKeyFromContext, authenticateApiKey, type AuthedContext } from '../../../middleware/auth.ts'; import { backgroundSchedulerFromContext } from '../../../runtime/background.ts'; import { inboundHeaders } from '../../shared/inbound-headers.ts'; @@ -706,7 +706,7 @@ const sendError = ( status: number, error: Record, eventId?: string, - dump?: DumpAccumulator | null, + dump?: TurnDump | null, ): void => { sendJson(socket, { type: 'error', status, error }, eventId, dump); }; @@ -723,14 +723,14 @@ const sendResponsesEvent = ( socket: ResponsesWebSocketSocket, event: ClientResponsesStreamEvent, eventId?: string, - dump?: DumpAccumulator | null, + dump?: TurnDump | null, ): boolean => sendJson(socket, event, eventId, dump); const sendJson = ( socket: ResponsesWebSocketSocket, value: unknown, eventId?: string, - dump?: DumpAccumulator | null, + dump?: TurnDump | null, ): boolean => { if (socket.readyState !== 1) return false; const payload = eventId === undefined || !value || typeof value !== 'object' diff --git a/packages/gateway/src/data-plane/pipeline/serve.ts b/packages/gateway/src/data-plane/pipeline/serve.ts index 8a30487e96..fc3fcbe2a9 100644 --- a/packages/gateway/src/data-plane/pipeline/serve.ts +++ b/packages/gateway/src/data-plane/pipeline/serve.ts @@ -13,6 +13,8 @@ import type { ContentfulStatusCode } from 'hono/utils/http-status'; import type { AttemptSelector, BillableEntity, GatewayFacts } from './facts.ts'; import type { GatewayServices } from './services.ts'; import { settleBillable } from './settlement.ts'; +import { openRunDump } from '../../dump/run-sink.ts'; +import { apiKeyFromContext, type AuthedContext } from '../../middleware/auth.ts'; import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; import { consoleLogSink } from '../../runtime/log.ts'; import { createGatewayCtxFromHono, finalizeGatewayResponse, type GatewayCtx } from '../shared/gateway-ctx.ts'; @@ -63,15 +65,25 @@ export interface Prologue { * live ones here, and the stage that dials asks for one back by selector. */ export const openPrologue = ( - c: Context, + c: AuthedContext, ingress: Ingress, options: { readonly wantsStream: boolean; readonly model?: string }, ): Prologue => { + const backgroundScheduler = backgroundSchedulerFromContext(c); + // The shape follows the endpoint. A pipelined turn is recorded as its whole run — every + // stage, both directions — so it opens that recording here instead of the edge one, and + // no turn is ever written twice. + const runDump = openRunDump( + apiKeyFromContext(c), + { method: c.req.method, path: new URL(c.req.raw.url).pathname, body: ingress.body }, + backgroundScheduler, + ); const gateway = createGatewayCtxFromHono(c, { wantsStream: options.wantsStream, ...(options.model === undefined ? {} : { model: options.model }), requestBody: takeRequestBody(ingress.body), - backgroundScheduler: backgroundSchedulerFromContext(c), + backgroundScheduler, + dump: runDump, }); const live = new Map(); @@ -86,6 +98,9 @@ export const openPrologue = ( rememberCandidates: candidates => { for (const candidate of candidates) live.set(candidate.provider.upstreamId, candidate); }, + // Absent when this key has no retention configured, which is what keeps recording + // conditional: the runner does none of it rather than doing it and discarding. + ...(runDump === null ? {} : { dump: runDump.sink }), resolveAttempt: (selector: AttemptSelector) => { const candidate = live.get(selector.upstreamId); if (candidate === undefined) { diff --git a/packages/gateway/src/data-plane/shared/gateway-ctx.ts b/packages/gateway/src/data-plane/shared/gateway-ctx.ts index 5889261c59..f53b480dab 100644 --- a/packages/gateway/src/data-plane/shared/gateway-ctx.ts +++ b/packages/gateway/src/data-plane/shared/gateway-ctx.ts @@ -1,5 +1,6 @@ import type { RequestBody } from './request-body.ts'; -import { type DumpAccumulator, openDumpAccumulator } from '../../dump/accumulator.ts'; +import { openDumpAccumulator } from '../../dump/accumulator.ts'; +import type { TurnDump } from '../../dump/turn-dump.ts'; import { apiKeyFromContext, type AuthedContext, effectiveUpstreamIdsFromContext } from '../../middleware/auth.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; @@ -40,11 +41,15 @@ export interface GatewayCtx { // Null when the api key has no dump retention configured, in which case // `finalizeGatewayResponse` short-circuits the dump tee and returns the // response untouched. - readonly dump: DumpAccumulator | null; + readonly dump: TurnDump | null; } export interface CreateGatewayCtxOptions { wantsStream: boolean; + // What this turn is recorded as. The shape follows the endpoint: a pipelined one hands in + // its run recording, and everything else lets the factory open the edge accumulator. + // Absent and null differ — absent means "open the usual one", null means "record nothing". + dump?: TurnDump | null; // WebSocket-style call sites own the AbortController (so the upgrade // handler can cancel mid-stream); HTTP call sites let the factory mint one // when wantsStream is true. @@ -80,7 +85,7 @@ export const createGatewayCtxFromHono = (c: AuthedContext, opts: CreateGatewayCt const controller = opts.downstreamAbortController ?? (opts.wantsStream ? new AbortController() : undefined); const apiKey = apiKeyFromContext(c); const upstreamIds = effectiveUpstreamIdsFromContext(c); - const dump = openDumpAccumulator(c, opts.method ?? c.req.method, apiKey, opts.requestBody, opts.backgroundScheduler); + const dump = 'dump' in opts ? opts.dump ?? null : openDumpAccumulator(c, opts.method ?? c.req.method, apiKey, opts.requestBody, opts.backgroundScheduler); if (opts.model !== undefined) dump?.requestedModel(opts.model); return { apiKeyId: apiKey.id, diff --git a/packages/gateway/src/dump/run-sink.ts b/packages/gateway/src/dump/run-sink.ts index 146833c29e..0b2d4bd905 100644 --- a/packages/gateway/src/dump/run-sink.ts +++ b/packages/gateway/src/dump/run-sink.ts @@ -21,6 +21,7 @@ import type { ApiKey, TokenUsage } from '../repo/types.ts'; import { ulid } from '../shared/ulid.ts'; import { createRunEncoder, toNdjson, type DumpEvent, type Event } from '@floway-dev/pipeline'; import type { BackgroundScheduler } from '@floway-dev/platform'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { TelemetryModelIdentity } from '@floway-dev/provider'; // What the client sent, as the metadata needs it. The headers and the body are @@ -37,6 +38,7 @@ export class RunDump { private readonly attribution = new DumpAttribution(); private readonly encode = createRunEncoder(); private readonly events: DumpEvent[] = []; + private sentPayloadBytes = 0; constructor( private readonly apiKey: ApiKey, @@ -51,7 +53,7 @@ export class RunDump { for (const encoded of this.encode(event)) this.events.push(encoded); }; - // --- mid-flight hooks, the same four the edge record is stamped with --- + // --- mid-flight hooks, the same ones the edge record is stamped with --- requestedModel(model: string): void { this.attribution.requestedModel(model); @@ -65,6 +67,18 @@ export class RunDump { this.attribution.failed(reason); } + /** + * A frame the client was sent, as the event the format names for one. + * + * The edge dump kept a frame log of its own; here a frame is content about a stream, so it + * is `stream.frame` and it folds through the same encoder as everything else. One stream + * per turn is what the endpoints this carries produce, so the id is fixed — a family that + * opens two would need to say which, and none does. + */ + frame(frame: ProtocolFrame): void { + this.sink({ type: 'stream.frame', streamId: 1, frames: [frame] }); + } + success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void { this.attribution.success(identity, usage); } @@ -82,12 +96,20 @@ export class RunDump { // The drain → encode → store put → broker publish runs on the runtime's // BackgroundScheduler so a dump write failure cannot turn a served answer // into a 502. + /** A transport that writes its own frames counts what it sent, because nothing downstream + * of it can. The run's own bytes are its events; this is the answer's. */ + recordSentPayloadBytes(byteLength: number): void { + this.sentPayloadBytes += byteLength; + } + finalize(status: number | null, responseBytes: number): void; finalize(response: Response): Response; finalize(...args: [number | null, number] | [Response]): void | Response { if (args.length === 2) { const [status, responseBytes] = args; - this.backgroundScheduler(this.write(status, responseBytes, null)); + // A transport that wrote its own frames counted them as it went; what it passes here + // is whatever else it sent alongside them. + this.backgroundScheduler(this.write(status, responseBytes + this.sentPayloadBytes, null)); return; } diff --git a/packages/gateway/src/dump/turn-dump.ts b/packages/gateway/src/dump/turn-dump.ts new file mode 100644 index 0000000000..5e8483a4e6 --- /dev/null +++ b/packages/gateway/src/dump/turn-dump.ts @@ -0,0 +1,25 @@ +// What a turn stamps on its recording, whichever shape that recording is. +// +// Two shapes are alive while the migration runs: the edge record, which holds what the client +// sent and what it got back, and the run record, which holds every stage of the pipeline as +// an event stream. The shape follows the endpoint — a pipelined one produces the run shape — +// and the stages in between neither know nor care which they are stamping. + +import type { TokenUsage } from '../repo/types.ts'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; +import type { TelemetryModelIdentity } from '@floway-dev/provider'; + +export interface TurnDump { + requestedModel(model: string): void; + success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void; + error(kind: 'upstream' | 'gateway', upstream?: string): void; + failed(reason: unknown): void; + frame(frame: ProtocolFrame): void; + /** How much of the answer actually went out. A transport that writes its own frames counts + * them itself, because nothing downstream of it can. */ + recordSentPayloadBytes(byteLength: number): void; + /** Closes the recording. A transport that owns its own response — the WebSocket turn — + * states what it sent; an HTTP one hands over the response so the bytes can be teed. */ + finalize(status: number | null, responseBytes: number | readonly unknown[]): void; + finalize(response: Response): Response; +} From 0729e48712550b7b38cf043c3b37d3f68089107c Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 05:36:45 +0800 Subject: [PATCH 45/79] refactor(gateway): delete the passthrough surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every non-chat family serves through its pipeline now, so the scaffold whose whole job was to forward a body it had not parsed and hand back one it had not read has no callers. passthrough-serve.ts, passthrough-attempt.ts and their tests go, and PassthroughServeApiName with them — the set of endpoints it named is not a set anything groups by any more. What survives is the one piece that was never about forwarding: reading a JSON body with a model field, which is a request parser and now says so in its name. Audio's respond.ts and usage.ts went the same way — the strategy responder and the usage reader the pipeline replaced, both without a referrer since the family was routed. Audio's handler collapses back into the seam. It wrote its own epilogue because the seam could not carry an upstream's absent media type or a stream's outcome; both now travel, so the two agree again. --- .../__tests__/data-plane/audio/usage_test.ts | 91 ----- .../data-plane/chat/messages/serve_test.ts | 2 +- .../shared/passthrough-attempt_test.ts | 118 ------- .../shared/passthrough-serve_test.ts | 333 ------------------ packages/gateway/src/data-plane/audio/http.ts | 68 +--- .../gateway/src/data-plane/audio/respond.ts | 92 ----- .../gateway/src/data-plane/audio/usage.ts | 100 ------ .../src/data-plane/completions/http.ts | 2 +- .../src/data-plane/shared/api-names.ts | 11 - ...rough-request.ts => json-model-request.ts} | 0 .../data-plane/shared/passthrough-attempt.ts | 82 ----- .../data-plane/shared/passthrough-serve.ts | 257 -------------- packages/gateway/src/dump/accumulator.ts | 4 +- packages/gateway/src/dump/types.ts | 2 +- 14 files changed, 20 insertions(+), 1142 deletions(-) delete mode 100644 packages/gateway/__tests__/data-plane/audio/usage_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/shared/passthrough-attempt_test.ts delete mode 100644 packages/gateway/__tests__/data-plane/shared/passthrough-serve_test.ts delete mode 100644 packages/gateway/src/data-plane/audio/respond.ts delete mode 100644 packages/gateway/src/data-plane/audio/usage.ts delete mode 100644 packages/gateway/src/data-plane/shared/api-names.ts rename packages/gateway/src/data-plane/shared/{passthrough-request.ts => json-model-request.ts} (100%) delete mode 100644 packages/gateway/src/data-plane/shared/passthrough-attempt.ts delete mode 100644 packages/gateway/src/data-plane/shared/passthrough-serve.ts diff --git a/packages/gateway/__tests__/data-plane/audio/usage_test.ts b/packages/gateway/__tests__/data-plane/audio/usage_test.ts deleted file mode 100644 index ea9240dcdf..0000000000 --- a/packages/gateway/__tests__/data-plane/audio/usage_test.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { test } from 'vitest'; - -import { audioTranscriptionUsageMeasurement } from '../../../src/data-plane/audio/usage.ts'; -import { assertEquals, assertThrows } from '@floway-dev/test-utils'; - -test('audio transcription usage preserves duration seconds as a base-unit metric', () => { - assertEquals(audioTranscriptionUsageMeasurement({ - usage: { type: 'duration', seconds: 91 }, - duration: 91.8, - }), { - quantities: { input_audio_seconds: '91' }, - pricingFacts: {}, - dumpTokenUsage: null, - }); -}); - -test('audio transcription usage reads Whisper verbose JSON duration', () => { - assertEquals(audioTranscriptionUsageMeasurement({ - task: 'transcribe', - duration: 91.8, - text: 'hello', - }), { - quantities: { input_audio_seconds: '91.8' }, - pricingFacts: {}, - dumpTokenUsage: null, - }); -}); - -test('audio transcription usage maps text and audio input token details to disjoint metrics', () => { - assertEquals(audioTranscriptionUsageMeasurement({ - usage: { - type: 'tokens', - input_tokens: 14, - input_token_details: { text_tokens: 4, audio_tokens: 10 }, - output_tokens: 45, - total_tokens: 59, - }, - }), { - quantities: { input_tokens: '4', input_audio_tokens: '10', output_tokens: '45' }, - pricingFacts: { inputTokens: 14 }, - dumpTokenUsage: { input: 14, output: 45 }, - }); -}); - -test('audio transcription usage keeps aggregate input tokens general when details are absent', () => { - assertEquals(audioTranscriptionUsageMeasurement({ - usage: { type: 'tokens', input_tokens: 14, output_tokens: 45, total_tokens: 59 }, - }).quantities, { input_tokens: '14', output_tokens: '45' }); -}); - -test('audio transcription usage accepts partial details and leaves unclassified input general', () => { - for (const [input_token_details, quantities] of [ - [{}, { input_tokens: '14', output_tokens: '45' }], - [{ text_tokens: 4 }, { input_tokens: '14', output_tokens: '45' }], - [{ audio_tokens: 10 }, { input_tokens: '4', input_audio_tokens: '10', output_tokens: '45' }], - ] as const) { - assertEquals(audioTranscriptionUsageMeasurement({ - usage: { type: 'tokens', input_tokens: 14, input_token_details, output_tokens: 45, total_tokens: 59 }, - }).quantities, quantities); - } -}); - -test('audio transcription usage without a recognized metric is request-only', () => { - for (const body of [ - { usage: { seconds: 10 } }, - { usage: { type: 'future_metric', samples: 10 } }, - ]) { - assertEquals(audioTranscriptionUsageMeasurement(body), { - quantities: {}, pricingFacts: {}, dumpTokenUsage: null, - }); - } -}); - -test('audio transcription usage rejects malformed declared metrics', () => { - for (const [body, message] of [ - [{ duration: '10' }, 'duration must be'], - [{ usage: null }, 'usage must be an object'], - [{ usage: 'tokens' }, 'usage must be an object'], - [{ usage: { type: 'duration' } }, 'duration usage.seconds'], - [{ usage: { type: 'duration', seconds: '10' } }, 'duration usage.seconds'], - [{ usage: { type: 'tokens', input_tokens: -1, output_tokens: 45, total_tokens: 44 } }, 'token usage.input_tokens'], - [{ usage: { type: 'tokens', input_tokens: 14, output_tokens: Number.NaN, total_tokens: 59 } }, 'token usage.output_tokens'], - [{ usage: { type: 'tokens', input_tokens: 14, output_tokens: 45, total_tokens: '59' } }, 'token usage.total_tokens'], - [{ usage: { type: 'tokens', input_tokens: 14, output_tokens: 45, total_tokens: 58 } }, 'total_tokens must equal'], - [{ usage: { type: 'tokens', input_tokens: 14, input_token_details: null, output_tokens: 45, total_tokens: 59 } }, 'input_token_details must be an object'], - [{ usage: { type: 'tokens', input_tokens: 14, input_token_details: { text_tokens: 4, audio_tokens: '10' }, output_tokens: 45, total_tokens: 59 } }, 'audio_tokens must be'], - [{ usage: { type: 'tokens', input_tokens: 14, input_token_details: { text_tokens: 6, audio_tokens: 9 }, output_tokens: 45, total_tokens: 59 } }, 'input_token_details must not exceed'], - ] as const) { - assertThrows(() => audioTranscriptionUsageMeasurement(body), Error, message); - } -}); diff --git a/packages/gateway/__tests__/data-plane/chat/messages/serve_test.ts b/packages/gateway/__tests__/data-plane/chat/messages/serve_test.ts index f1435fd3d0..122d4bb1aa 100644 --- a/packages/gateway/__tests__/data-plane/chat/messages/serve_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/messages/serve_test.ts @@ -705,7 +705,7 @@ test('alias whose targets have no kind-matching binding surfaces as the regular // callback so the http.ts catch can build an internal-error result carrying // the correct upstream, and `recordFailedRequest` lands a row rather than // short-circuiting on missing telemetry. Passthrough's equivalent regression -// lives in passthrough-serve_test.ts (R3 fix 303c4e89). +// lived in passthrough-serve_test.ts, which went with that surface (R3 fix 303c4e89). test('mid-attempt throw stamps telemetry with the throwing candidate, not the previous one', async () => { installRepo(); const firstError = new Response(JSON.stringify({ error: { message: 'nope' } }), { diff --git a/packages/gateway/__tests__/data-plane/shared/passthrough-attempt_test.ts b/packages/gateway/__tests__/data-plane/shared/passthrough-attempt_test.ts deleted file mode 100644 index c9116c112c..0000000000 --- a/packages/gateway/__tests__/data-plane/shared/passthrough-attempt_test.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { Hono } from 'hono'; -import { test } from 'vitest'; - -import { passthroughAttempt, type PassthroughAttemptResult } from '../../../src/data-plane/shared/passthrough-attempt.ts'; -import type { AuthVars } from '../../../src/middleware/auth.ts'; -import { mockGatewayCtx } from '../../test-utils/gateway-ctx.ts'; -import { assertEquals, assertExists, stubModelCandidate, stubProvider } from '@floway-dev/test-utils'; - -test('passthroughAttempt applies the selected provider ingress policy', async () => { - let observed: Headers | undefined; - const base = stubModelCandidate(); - const candidate = stubModelCandidate({ - provider: { - ...base.provider, - kind: 'custom', - instance: stubProvider({ - callEmbeddings: async (_model, _body, _signal, opts) => { - observed = opts.headers; - return { response: new Response('{}'), modelKey: 'test-model' }; - }, - }), - }, - }); - const app = new Hono<{ Variables: AuthVars }>(); - app.post('/test', async c => { - await passthroughAttempt({ - c, - ctx: mockGatewayCtx(), - candidate, - operation: 'embeddings', - call: (provider, model, opts) => provider.instance.callEmbeddings(model, { input: 'hi' }, undefined, opts), - }); - return c.text('ok'); - }); - await app.request('/test', { - method: 'POST', - headers: { - authorization: 'Bearer secret', - 'x-client-request-id': 'request-1', - 'x-debug': 'discard', - }, - }); - - assertExists(observed); - assertEquals([...observed], []); -}); - -// Drives one attempt against a canned upstream response, through the same Hono -// context the serve layer builds. -const runPassthroughAttempt = async (upstream: Response): Promise => { - const base = stubModelCandidate(); - const candidate = stubModelCandidate({ - provider: { - ...base.provider, - kind: 'custom', - instance: stubProvider({ - callEmbeddings: async () => ({ response: upstream, modelKey: 'test-model' }), - }), - }, - }); - let result: PassthroughAttemptResult | undefined; - const app = new Hono<{ Variables: AuthVars }>(); - app.post('/test', async c => { - result = await passthroughAttempt({ - c, - ctx: mockGatewayCtx(), - candidate, - operation: 'embeddings', - call: (provider, model, opts) => provider.instance.callEmbeddings(model, { input: 'hi' }, undefined, opts), - }); - return c.text('ok'); - }); - await app.request('/test', { method: 'POST' }); - assertExists(result); - return result; -}; - -// The fallback loop keeps only the most recent failure, so every superseded -// attempt is dropped. On the direct-connect egress a dropped response strands -// its socket, which is why a failed attempt must not carry a live body. -test('passthroughAttempt materializes a failed upstream response so a discarded attempt holds no transport', async () => { - let sourceCancelled = false; - let sourceRead = false; - const upstream = new Response( - new ReadableStream({ - start(controller) { - sourceRead = true; - controller.enqueue(new TextEncoder().encode('{"error":"upstream busy"}')); - controller.close(); - }, - cancel() { sourceCancelled = true; }, - }), - { status: 503, statusText: 'Service Unavailable', headers: { 'content-type': 'application/json', 'retry-after': '12' } }, - ); - - const result = await runPassthroughAttempt(upstream); - - assertEquals(result.status, 503); - // The upstream body was consumed rather than left for someone else to close. - assertEquals(sourceRead, true); - assertEquals(sourceCancelled, false); - assertEquals(upstream.bodyUsed, true); - // Status, headers and bytes still forward verbatim. - assertEquals(result.response.status, 503); - assertEquals(result.response.statusText, 'Service Unavailable'); - assertEquals(result.response.headers.get('retry-after'), '12'); - assertEquals(await result.response.text(), '{"error":"upstream busy"}'); -}); - -test('passthroughAttempt forwards a successful upstream response without reading it', async () => { - const upstream = new Response('ok', { status: 200 }); - const result = await runPassthroughAttempt(upstream); - - // A 2xx is returned to the caller immediately and is never discarded, so it - // must keep streaming rather than be buffered here. - assertEquals(result.response, upstream); - assertEquals(upstream.bodyUsed, false); -}); diff --git a/packages/gateway/__tests__/data-plane/shared/passthrough-serve_test.ts b/packages/gateway/__tests__/data-plane/shared/passthrough-serve_test.ts deleted file mode 100644 index bc7e342f91..0000000000 --- a/packages/gateway/__tests__/data-plane/shared/passthrough-serve_test.ts +++ /dev/null @@ -1,333 +0,0 @@ -// Behavioral coverage for the shared passthrough serve scaffold used by -// /v1/embeddings, /v1/images/{generations,edits}, and audio transcription. Each test exercises a -// full client request through the in-memory app rather than constructing a -// synthetic hono Context so the integration with model resolution, -// upstream HTTP, and background scheduling stays honest. -// -// We pick /v1/embeddings as the source under test because: -// - its acceptBinding gate is `kind === 'embedding'`, satisfied by any -// embedding-only custom upstream, with no per-endpoint Copilot setup -// required; -// - its extractBilling reads the OpenAI-style `usage.prompt_tokens` off -// a 2xx JSON body, so a body with that shape triggers a real usage -// write; -// - it shares the exact same forwardUpstreamResponse + settle -// path as the images endpoints — the behaviors under test are owned by -// passthroughServe, not the endpoint shape. - -import { test, vi } from 'vitest'; - -import { buildCustomUpstreamRecord, flushAsyncWork, requestApp, setupAppTest } from '../../test-utils/app.ts'; -import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; -import { jsonResponse, withMockedFetch, assertEquals, assertExists } from '@floway-dev/test-utils'; - -const registerEmbeddingsUpstream = async ( - repo: Awaited>['repo'], - ingressHeadersRules: { key: string; value: string | null }[] = [], -): Promise => { - await repo.upstreams.deleteAll(); - clearInProcessCopilotTokenCache(); - await repo.upstreams.save(buildCustomUpstreamRecord({ - id: 'up_passthrough', - name: 'Passthrough Embedding Provider', - sortOrder: 100, - config: { - baseUrl: 'https://passthrough.example.com', - authStyle: 'bearer', - ingressHeadersRules, - apiKey: 'sk-passthrough', - endpoints: {}, - }, - })); -}; - -test('passthrough-serve: Custom resolves configured ingress header rules before provider dispatch', async () => { - const { apiKey, repo } = await setupAppTest(); - await registerEmbeddingsUpstream(repo, [ - { key: 'x-passthrough', value: null }, - { key: 'x-overwrite', value: 'configured' }, - { key: 'x-empty', value: '' }, - ]); - - await withMockedFetch( - request => { - const url = new URL(request.url); - if (url.pathname === '/v1/models') { - return jsonResponse({ object: 'list', data: [{ id: 'custom-embed-model' }] }); - } - if (url.pathname === '/v1/embeddings') { - assertEquals(request.headers.get('x-passthrough'), 'client'); - assertEquals(request.headers.get('x-overwrite'), 'configured'); - assertEquals(request.headers.get('x-empty'), ''); - assertEquals(request.headers.get('authorization'), 'Bearer sk-passthrough'); - assertEquals(request.headers.get('x-unlisted'), null); - return jsonResponse({ object: 'list', data: [], model: 'custom-embed-model' }); - } - throw new Error(`Unhandled fetch ${request.url}`); - }, - async () => { - const response = await requestApp('/v1/embeddings', { - method: 'POST', - headers: { - 'content-type': 'application/json', - 'x-api-key': apiKey.key, - 'x-empty': 'client', - 'x-overwrite': 'client', - 'x-passthrough': 'client', - 'x-unlisted': 'discard', - }, - body: JSON.stringify({ model: 'custom-embed-model', input: 'hi' }), - }); - assertEquals(response.status, 200); - }, - ); -}); - -test('passthrough-serve: response header blocklist preserves vendor metadata and drops unsafe headers', async () => { - const { apiKey, repo } = await setupAppTest(); - await registerEmbeddingsUpstream(repo); - - await withMockedFetch( - request => { - const url = new URL(request.url); - if (url.hostname === 'passthrough.example.com' && url.pathname === '/v1/models') { - return jsonResponse({ object: 'list', data: [{ id: 'custom-embed-model' }] }); - } - if (url.hostname === 'passthrough.example.com' && url.pathname === '/v1/embeddings') { - return new Response(JSON.stringify({ - object: 'list', - model: 'custom-embed-model', - data: [{ object: 'embedding', index: 0, embedding: [0.1] }], - usage: { prompt_tokens: 1, total_tokens: 1 }, - }), { - status: 200, - headers: { - 'content-type': 'application/json', - 'x-request-id': 'req-123', - 'openai-organization': 'org-1', - 'x-ratelimit-remaining': '100', - 'retry-after': '30', - 'cf-ray': 'abc', - 'x-internal-secret': 'leak', - 'set-cookie': 'nope=1', - }, - }); - } - throw new Error(`Unhandled fetch ${request.url}`); - }, - async () => { - const response = await requestApp('/v1/embeddings', { - method: 'POST', - headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, - body: JSON.stringify({ model: 'custom-embed-model', input: 'hi' }), - }); - - assertEquals(response.status, 200); - assertExists(response.headers.get('x-request-id')); - assertEquals(response.headers.get('x-request-id'), 'req-123'); - assertEquals(response.headers.get('openai-organization'), 'org-1'); - assertEquals(response.headers.get('x-ratelimit-remaining'), '100'); - assertEquals(response.headers.get('retry-after'), '30'); - assertEquals(response.headers.get('cf-ray'), 'abc'); - assertEquals(response.headers.get('x-internal-secret'), 'leak'); - assertEquals(response.headers.get('set-cookie'), null); - await response.json(); - }, - ); -}); - -test('passthrough-serve: alias whose targets have no kind-matching binding surfaces as the regular model-missing 404', async () => { - // The inlined alias resolver walks alias targets in `selection` order and - // stops at the first target with kind-matching candidates. When every - // target is unroutable (as here, where the single target id doesn't - // exist in any upstream catalog), the resolver returns empty candidates - // + sawModel=false, and the passthrough seam surfaces the regular - // model-missing 404. No upstream call should fire. - const { apiKey, repo } = await setupAppTest(); - await registerEmbeddingsUpstream(repo); - await repo.modelAliases.insert({ - id: 'alias_embed-fast', - name: 'embed-fast', - kind: 'embedding', - selection: 'first-available', - displayName: null, - visibleInModelsList: true, - targets: [{ target_model_id: 'unknown-embed', rules: {} }], - announcedMetadata: null, - sortOrder: 0, - createdAt: '2026-01-01T00:00:00.000Z', - updatedAt: '2026-01-01T00:00:00.000Z', - }); - - await withMockedFetch( - request => { - const url = new URL(request.url); - if (url.hostname === 'passthrough.example.com' && url.pathname === '/v1/models') { - return jsonResponse({ object: 'list', data: [{ id: 'custom-embed-model' }] }); - } - if (url.hostname === 'passthrough.example.com' && url.pathname === '/v1/embeddings') { - throw new Error('passthrough-serve: upstream must not be called when alias has no routable target'); - } - throw new Error(`Unhandled fetch ${request.url}`); - }, - async () => { - const response = await requestApp('/v1/embeddings', { - method: 'POST', - headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, - body: JSON.stringify({ model: 'embed-fast', input: 'hi' }), - }); - - assertEquals(response.status, 404); - const body = await response.json() as { error: { message: string; type: string } }; - assertEquals(body.error.type, 'api_error'); - // The alias name (still on `payload.model` because no candidate was - // rewritten in) reaches the wording verbatim. - assertEquals(body.error.message, 'Model embed-fast is not available on any configured upstream.'); - }, - ); -}); - -// Register two custom upstreams both exposing the same embedding model, so -// the shared narrow phase produces a two-element candidate list ordered by -// `sortOrder`. The passthrough loop must try `up_a` first (sortOrder 100) -// and `up_b` second (sortOrder 200). -const registerTwoEmbeddingsUpstreams = async (repo: Awaited>['repo']): Promise => { - await repo.upstreams.deleteAll(); - clearInProcessCopilotTokenCache(); - await repo.upstreams.save(buildCustomUpstreamRecord({ - id: 'up_a', name: 'Upstream A', sortOrder: 100, - config: { baseUrl: 'https://up-a.example.com', authStyle: 'bearer', apiKey: 'sk-a', endpoints: {}, ingressHeadersRules: [] }, - })); - await repo.upstreams.save(buildCustomUpstreamRecord({ - id: 'up_b', name: 'Upstream B', sortOrder: 200, - config: { baseUrl: 'https://up-b.example.com', authStyle: 'bearer', apiKey: 'sk-b', endpoints: {}, ingressHeadersRules: [] }, - })); -}; - -test('passthrough-serve: 5xx from the first candidate falls through to the next successful upstream', async () => { - const { apiKey, repo } = await setupAppTest(); - await registerTwoEmbeddingsUpstreams(repo); - - let firstEmbeddingsCalls = 0; - let secondEmbeddingsCalls = 0; - await withMockedFetch( - request => { - const url = new URL(request.url); - if (url.pathname === '/v1/models') { - return jsonResponse({ object: 'list', data: [{ id: 'custom-embed-model' }] }); - } - if (url.hostname === 'up-a.example.com' && url.pathname === '/v1/embeddings') { - firstEmbeddingsCalls += 1; - return new Response('upstream boom', { status: 503 }); - } - if (url.hostname === 'up-b.example.com' && url.pathname === '/v1/embeddings') { - secondEmbeddingsCalls += 1; - return jsonResponse({ - object: 'list', - model: 'custom-embed-model', - data: [{ object: 'embedding', index: 0, embedding: [0.25] }], - usage: { prompt_tokens: 2, total_tokens: 2 }, - }); - } - throw new Error(`Unhandled fetch ${request.url}`); - }, - async () => { - const response = await requestApp('/v1/embeddings', { - method: 'POST', - headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, - body: JSON.stringify({ model: 'custom-embed-model', input: 'hi' }), - }); - - assertEquals(response.status, 200); - const body = await response.json() as { data: Array<{ embedding: number[] }> }; - assertEquals(body.data[0].embedding, [0.25]); - await flushAsyncWork(); - }, - ); - - assertEquals(firstEmbeddingsCalls, 1); - assertEquals(secondEmbeddingsCalls, 1); -}); - -test('passthrough-serve: when every candidate returns non-2xx the most recent upstream response is forwarded verbatim', async () => { - const { apiKey, repo } = await setupAppTest(); - await registerTwoEmbeddingsUpstreams(repo); - - await withMockedFetch( - request => { - const url = new URL(request.url); - if (url.pathname === '/v1/models') { - return jsonResponse({ object: 'list', data: [{ id: 'custom-embed-model' }] }); - } - if (url.hostname === 'up-a.example.com' && url.pathname === '/v1/embeddings') { - return new Response('first upstream unavailable', { status: 503 }); - } - if (url.hostname === 'up-b.example.com' && url.pathname === '/v1/embeddings') { - return new Response(JSON.stringify({ error: { message: 'rate limited' } }), { - status: 429, headers: { 'content-type': 'application/json', 'retry-after': '17' }, - }); - } - throw new Error(`Unhandled fetch ${request.url}`); - }, - async () => { - const response = await requestApp('/v1/embeddings', { - method: 'POST', - headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, - body: JSON.stringify({ model: 'custom-embed-model', input: 'hi' }), - }); - - // The last-attempted upstream's status, body, and allow-listed - // headers pass through unchanged so clients see real upstream - // telemetry — no synthetic gateway envelope. - assertEquals(response.status, 429); - assertEquals(response.headers.get('retry-after'), '17'); - const body = await response.json() as { error: { message: string } }; - assertEquals(body.error.message, 'rate limited'); - await flushAsyncWork(); - }, - ); -}); - -// A throw during candidate rollover attributes the error row to the -// throwing candidate, not the previously-succeeded one. -test('passthrough-serve: throw during rollover attributes the error perf row to the throwing candidate, not the previous one', async () => { - const { apiKey, repo } = await setupAppTest(); - await registerTwoEmbeddingsUpstreams(repo); - - const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - - try { - await withMockedFetch( - request => { - const url = new URL(request.url); - if (url.pathname === '/v1/models') { - return jsonResponse({ object: 'list', data: [{ id: 'custom-embed-model' }] }); - } - if (url.hostname === 'up-a.example.com' && url.pathname === '/v1/embeddings') { - return new Response('first upstream unavailable', { status: 503 }); - } - if (url.hostname === 'up-b.example.com' && url.pathname === '/v1/embeddings') { - throw new Error('simulated network error to up_b'); - } - throw new Error(`Unhandled fetch ${request.url}`); - }, - async () => { - const response = await requestApp('/v1/embeddings', { - method: 'POST', - headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, - body: JSON.stringify({ model: 'custom-embed-model', input: 'hi' }), - }); - - assertEquals(response.status, 502); - await flushAsyncWork(); - }, - ); - - const perfRows = await repo.performance.listAll(); - const errorRows = perfRows.filter(row => row.errorsNoOutput + row.errorsWithOutput > 0); - assertEquals(errorRows.length, 1); - assertEquals(errorRows[0].upstream, 'up_b'); - } finally { - errorSpy.mockRestore(); - } -}); diff --git a/packages/gateway/src/data-plane/audio/http.ts b/packages/gateway/src/data-plane/audio/http.ts index 9316186902..63a4e89615 100644 --- a/packages/gateway/src/data-plane/audio/http.ts +++ b/packages/gateway/src/data-plane/audio/http.ts @@ -7,18 +7,13 @@ // https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L714-L1040 import type { Context } from 'hono'; -import { streamSSE } from 'hono/streaming'; -import type { ContentfulStatusCode } from 'hono/utils/http-status'; import { audioTranscriptionServePipeline } from './pipeline.ts'; -import { consoleLogSink } from '../../runtime/log.ts'; -import { isFrames, openPrologue, readIngress } from '../pipeline/serve.ts'; -import { settleBillable } from '../pipeline/settlement.ts'; +import { isFrames, openPrologue, readIngress, serveThrough } from '../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; -import { writeSSEFrames } from '../shared/sse.ts'; -import { move, run } from '@floway-dev/pipeline'; +import { move } from '@floway-dev/pipeline'; import { parseAudioTranscriptionResponseFormat, type AudioTranscriptionResponseFormat } from '@floway-dev/protocols/audio'; -import { isMultipartFormDataMediaType, sseCommentFrame } from '@floway-dev/protocols/common'; +import { isMultipartFormDataMediaType } from '@floway-dev/protocols/common'; import type { AudioTranscriptionFormEntry } from '@floway-dev/provider'; type PreparedTranscription = @@ -112,7 +107,10 @@ export const audioTranscriptions = async (c: Context): Promise => { } const prologue = openPrologue(c, ingress, { wantsStream: request.wantsStream, model: request.model }); - const { facts, drain } = await run( + + return await serveThrough( + c, + prologue, audioTranscriptionServePipeline, move({ 'ingress.http.headers': prologue.headers, @@ -120,49 +118,13 @@ export const audioTranscriptions = async (c: Context): Promise => { 'request.audioTranscription.form': request.entries, 'serve.model': request.model, }) as never, - prologue.services as never, + facts => { + const rendered = facts['response.audioTranscription.rendered']; + if (isFrames(rendered)) return { frames: rendered }; + // The upstream's own media type, or none where it declared none: a document this + // gateway carried rather than wrote is not one it can describe. + return { body: bodyOf(rendered) as BodyInit, contentType: facts['response.audioTranscription.mediaType'] }; + }, + facts => facts['response.audioTranscription.streamedOutcome'], ); - - // A stream states what it billed, and whether the transcript ever finished, in its last - // event — which is after the run has answered. So the stage above the fork stood down and - // settlement is scheduled here, on the promise the ending stage handed up. - const outcome = facts['response.audioTranscription.streamedOutcome']; - if (outcome !== null) { - prologue.services.background(outcome.then(({ billable, failed }) => { - settleBillable({ ...prologue.services, log: consoleLogSink }, billable, failed); - })); - } - - const rendered = facts['response.audioTranscription.rendered']; - const status = facts['response.http.status'] as ContentfulStatusCode; - if (isFrames(rendered)) { - // Hono's streamSSE builds the response itself, so what the client is to see has to be - // staged on the context before it is called rather than passed to a constructor. - for (const [name, value] of facts['response.http.headers']) c.header(name, value); - c.status(status); - return finalizeGatewayResponse(prologue.gateway, streamSSE(c, async stream => { - try { - await writeSSEFrames(stream, rendered, { - keepAlive: { frame: sseCommentFrame('keepalive') }, - ...(prologue.gateway.downstreamAbortController === undefined - ? {} - : { downstreamAbortController: prologue.gateway.downstreamAbortController }), - }); - } finally { - // Reading the frames to the client *is* releasing the body they came from, so the - // drain waits for that to finish. Draining alongside it would take frames out of the - // client's own stream — one connection has one reader. A client that stopped reading - // still gets here, which is what leaves nothing open behind it. - await drain(); - } - })); - } - - // Nothing is left to read: what the client is sent is either the document the run already - // holds or an object serialized from it, so releasing can start at once. - prologue.services.background(drain()); - const headers = new Headers(facts['response.http.headers'].map(([name, value]): [string, string] => [name, value])); - const mediaType = facts['response.audioTranscription.mediaType']; - if (mediaType !== null) headers.set('content-type', mediaType); - return finalizeGatewayResponse(prologue.gateway, new Response(bodyOf(rendered) as BodyInit, { status, headers })); }; diff --git a/packages/gateway/src/data-plane/audio/respond.ts b/packages/gateway/src/data-plane/audio/respond.ts deleted file mode 100644 index d56575b945..0000000000 --- a/packages/gateway/src/data-plane/audio/respond.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { streamSSE } from 'hono/streaming'; - -import { measureAudioTranscriptionUsage } from './usage.ts'; -import { passthroughApiError } from '../shared/passthrough-serve.ts'; -import type { PassthroughResponseStrategyContext } from '../shared/passthrough-serve.ts'; -import { type StreamCompletion, writeSSEFrames } from '../shared/sse.ts'; -import { settleUsageMeasurement } from '../shared/telemetry/settle.ts'; -import { requestOnlyUsageMeasurement } from '../shared/telemetry/usage.ts'; -import { forwardUpstreamHeaders, forwardUpstreamResponse } from '../shared/upstream-response.ts'; -import { isAudioTranscriptionDoneEvent } from '@floway-dev/protocols/audio'; -import { eventFrame, isEventStreamMediaType, isJsonMediaType, parseSSEStream, sseCommentFrame } from '@floway-dev/protocols/common'; - -const respondNonStreaming = async ({ ctx, sourceApi, response, performance, identity }: PassthroughResponseStrategyContext): Promise => { - let measurement = requestOnlyUsageMeasurement(); - if (isJsonMediaType(response.headers.get('content-type'))) { - let parsed: unknown; - try { - parsed = await response.clone().json(); - } catch (error) { - console.warn( - `audio-transcription: failed to parse 2xx upstream body for ${sourceApi}; usage row will be request-only`, - error instanceof Error ? error.message : String(error), - ); - } - if (parsed !== undefined) { - measurement = measureAudioTranscriptionUsage(parsed, sourceApi); - } - } - ctx.dump?.success(identity, measurement.dumpTokenUsage); - settleUsageMeasurement(ctx, performance, identity, measurement, false); - return forwardUpstreamResponse(response, { defaultContentType: null }); -}; - -const respondStreaming = ({ c, ctx, sourceApi, response, performance, identity }: PassthroughResponseStrategyContext): Response => { - const upstreamBody = response.body; - if (!upstreamBody) { - ctx.dump?.failed(`${sourceApi} streaming upstream returned no body`); - settleUsageMeasurement(ctx, performance, identity, requestOnlyUsageMeasurement(), true); - forwardUpstreamHeaders(c, response.headers); - return passthroughApiError(c, 'Upstream returned a streaming response with no body.', 502); - } - forwardUpstreamHeaders(c, response.headers); - return streamSSE(c, async stream => { - let completion: StreamCompletion = 'error'; - let streamError: unknown; - let terminalEventSeen = false; - let measurement = requestOnlyUsageMeasurement(); - try { - const frames = (async function* () { - for await (const frame of parseSSEStream(upstreamBody, { signal: ctx.abortSignal })) { - let event: unknown; - try { - event = JSON.parse(frame.data) as unknown; - } catch (error) { - throw new Error(`Malformed upstream ${sourceApi} SSE JSON: ${frame.data}`, { cause: error }); - } - ctx.dump?.frame(eventFrame(event)); - if (isAudioTranscriptionDoneEvent(event)) { - terminalEventSeen = true; - measurement = measureAudioTranscriptionUsage(event, sourceApi); - yield frame; - return; - } - yield frame; - } - })(); - completion = await writeSSEFrames(stream, frames, { - keepAlive: { frame: sseCommentFrame('keepalive') }, - downstreamAbortController: ctx.downstreamAbortController, - }); - } catch (error) { - streamError = error; - } finally { - const failed = streamError !== undefined || completion === 'error' || !terminalEventSeen; - if (failed) ctx.dump?.failed(streamError ?? `${sourceApi} stream ended with completion=${completion}`); - else ctx.dump?.success(identity, measurement.dumpTokenUsage); - settleUsageMeasurement(ctx, performance, identity, measurement, failed); - } - }); -}; - -export const respondAudioTranscription = async (context: PassthroughResponseStrategyContext): Promise => { - const { ctx, response, performance, identity } = context; - if (!response.ok) { - settleUsageMeasurement(ctx, performance, identity, requestOnlyUsageMeasurement(), true); - ctx.dump?.error('upstream', identity.upstream); - return forwardUpstreamResponse(response, { defaultContentType: null }); - } - return isEventStreamMediaType(response.headers.get('content-type')) - ? respondStreaming(context) - : await respondNonStreaming(context); -}; diff --git a/packages/gateway/src/data-plane/audio/usage.ts b/packages/gateway/src/data-plane/audio/usage.ts deleted file mode 100644 index 57baa7835f..0000000000 --- a/packages/gateway/src/data-plane/audio/usage.ts +++ /dev/null @@ -1,100 +0,0 @@ -import type { UsageQuantities } from '../../repo/types.ts'; -import { requestOnlyUsageMeasurement, tokenUsage, type UsageMeasurement } from '../shared/telemetry/usage.ts'; -import { parseDecimalString } from '@floway-dev/protocols/common'; - -// OpenAI transcription responses discriminate usage by `type`. Token-based -// models split input_token_details into text and audio metrics; without that -// optional split, the aggregate stays on the general input metric. Duration- -// based usage exposes seconds, while Whisper verbose JSON reports the same -// quantity as a top-level `duration`. Unknown breakdowns record the request -// only, while malformed fields under a known discriminator remain observable. -// https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L36378-L36562 -const audioDurationMeasurement = (seconds: unknown, label: string): UsageMeasurement => { - if (typeof seconds !== 'number' || !Number.isFinite(seconds) || seconds < 0) { - throw new Error(`Audio transcription ${label} must be a finite non-negative number`); - } - return { - quantities: { input_audio_seconds: parseDecimalString(String(seconds)) }, - pricingFacts: {}, - dumpTokenUsage: null, - }; -}; - -export const audioTranscriptionUsageMeasurement = (body: unknown): UsageMeasurement => { - if (!body || typeof body !== 'object') return requestOnlyUsageMeasurement(); - if (!Object.hasOwn(body, 'usage')) { - if (!Object.hasOwn(body, 'duration')) return requestOnlyUsageMeasurement(); - return audioDurationMeasurement((body as { duration: unknown }).duration, 'duration'); - } - const usage = (body as { usage: unknown }).usage; - if (!usage || typeof usage !== 'object' || Array.isArray(usage)) { - throw new Error('Audio transcription usage must be an object'); - } - const metric = usage as { type?: unknown; seconds?: unknown; input_tokens?: unknown; input_token_details?: unknown; output_tokens?: unknown; total_tokens?: unknown }; - - if (metric.type === 'duration') { - return audioDurationMeasurement(metric.seconds, 'duration usage.seconds'); - } - - if (metric.type !== 'tokens') return requestOnlyUsageMeasurement(); - for (const [field, value] of [ - ['input_tokens', metric.input_tokens], - ['output_tokens', metric.output_tokens], - ['total_tokens', metric.total_tokens], - ] as const) { - if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) { - throw new Error(`Audio transcription token usage.${field} must be a non-negative safe integer`); - } - } - const inputTokens = metric.input_tokens as number; - const outputTokens = metric.output_tokens as number; - const totalTokens = metric.total_tokens as number; - if (totalTokens !== inputTokens + outputTokens) { - throw new Error('Audio transcription token usage.total_tokens must equal input_tokens plus output_tokens'); - } - - let inputQuantities: UsageQuantities = { input_tokens: parseDecimalString(String(inputTokens)) }; - if (metric.input_token_details !== undefined) { - if (!metric.input_token_details || typeof metric.input_token_details !== 'object' || Array.isArray(metric.input_token_details)) { - throw new Error('Audio transcription token usage.input_token_details must be an object'); - } - const details = metric.input_token_details as { text_tokens?: unknown; audio_tokens?: unknown }; - for (const [field, value] of [ - ['text_tokens', details.text_tokens], - ['audio_tokens', details.audio_tokens], - ] as const) { - if (value !== undefined && (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0)) { - throw new Error(`Audio transcription token usage.input_token_details.${field} must be a non-negative safe integer`); - } - } - const textTokens = details.text_tokens as number | undefined; - const audioTokens = details.audio_tokens as number | undefined; - if ((textTokens ?? 0) + (audioTokens ?? 0) > inputTokens) { - throw new Error('Audio transcription token usage.input_token_details must not exceed input_tokens'); - } - inputQuantities = { - input_tokens: parseDecimalString(String(inputTokens - (audioTokens ?? 0))), - ...(audioTokens === undefined ? {} : { input_audio_tokens: parseDecimalString(String(audioTokens)) }), - }; - } - return { - quantities: { - ...inputQuantities, - output_tokens: parseDecimalString(String(outputTokens)), - }, - pricingFacts: { inputTokens }, - dumpTokenUsage: tokenUsage({ input: inputTokens, output: outputTokens }), - }; -}; - -export const measureAudioTranscriptionUsage = (value: unknown, sourceApi: string): UsageMeasurement => { - try { - return audioTranscriptionUsageMeasurement(value); - } catch (error) { - console.warn( - `audio-transcription: invalid usage in 2xx upstream response for ${sourceApi}; usage row will be request-only`, - error instanceof Error ? error.message : String(error), - ); - return requestOnlyUsageMeasurement(); - } -}; diff --git a/packages/gateway/src/data-plane/completions/http.ts b/packages/gateway/src/data-plane/completions/http.ts index e3cc9d016c..5cf9420944 100644 --- a/packages/gateway/src/data-plane/completions/http.ts +++ b/packages/gateway/src/data-plane/completions/http.ts @@ -10,7 +10,7 @@ import type { Context } from 'hono'; import { completionsServePipeline } from './pipeline.ts'; import { isFrames, openPrologue, readIngress, serveThrough } from '../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; -import { prepareJsonModelRequest } from '../shared/passthrough-request.ts'; +import { prepareJsonModelRequest } from '../shared/json-model-request.ts'; import { move } from '@floway-dev/pipeline'; export const completions = async (c: Context): Promise => { diff --git a/packages/gateway/src/data-plane/shared/api-names.ts b/packages/gateway/src/data-plane/shared/api-names.ts deleted file mode 100644 index 174855734f..0000000000 --- a/packages/gateway/src/data-plane/shared/api-names.ts +++ /dev/null @@ -1,11 +0,0 @@ -// Routing-side API name primitives shared across the data plane. -// -// `PassthroughServeApiName` is the set of API names served by the -// passthroughServe helper (see ./passthrough-serve.ts) rather than the -// chat source/target executor. It groups by transport shape (the body / -// frames are forwarded verbatim, possibly with a usage-extraction step), -// not by whether the endpoint is chat-shaped — `/completions` is a -// non-chat endpoint that lives here because there is nothing to translate -// to or from. The value is the public URL fragment, so it can be used -// directly in error messages and route comparisons without a lookup table. -export type PassthroughServeApiName = '/completions' | '/embeddings' | '/images/generations' | '/images/edits' | '/audio/transcriptions'; diff --git a/packages/gateway/src/data-plane/shared/passthrough-request.ts b/packages/gateway/src/data-plane/shared/json-model-request.ts similarity index 100% rename from packages/gateway/src/data-plane/shared/passthrough-request.ts rename to packages/gateway/src/data-plane/shared/json-model-request.ts diff --git a/packages/gateway/src/data-plane/shared/passthrough-attempt.ts b/packages/gateway/src/data-plane/shared/passthrough-attempt.ts deleted file mode 100644 index db997affe0..0000000000 --- a/packages/gateway/src/data-plane/shared/passthrough-attempt.ts +++ /dev/null @@ -1,82 +0,0 @@ -// Per-candidate passthrough attempt: does the upstream HTTP call for one -// resolved candidate and hands the serve loop back a `plain`-shaped result -// the shared `iterateCandidates` iterator can drive. -// -// Chat protocols run each attempt through a translation + interceptor -// stack that yields an `ExecuteResult` discriminated union. Passthrough -// endpoints have no translation — the request body is forwarded to the -// upstream's matching endpoint and the raw upstream Response is returned -// verbatim. The `plain` discriminant is enlarged here to carry that -// Response plus the per-call telemetry the serve site needs when it -// forwards the winning attempt (2xx) or the last failure (exhausted). - -import type { GatewayCtx } from './gateway-ctx.ts'; -import { inboundHeaders } from './inbound-headers.ts'; -import { telemetryModelIdentity, upstreamPerformanceContext } from './telemetry/attribution.ts'; -import type { PerformanceTelemetryContext } from './telemetry/performance.ts'; -import { buildUpstreamCallOptions } from './upstream-call-options.ts'; -import type { AuthedContext } from '../../middleware/auth.ts'; -import { providerModelOf } from '@floway-dev/provider'; -import type { ModelCandidate, PerformanceOperation, Provider, ProviderCallResult, ProviderModel, TelemetryModelIdentity, UpstreamCallOptions } from '@floway-dev/provider'; - -// Enlarged `plain` shape: `iterateCandidates` reads `type` + `status`; -// the passthrough serve reads the rest to forward the response and -// attribute dumps. `identity` carries the upstream id alongside the -// model/pricing metadata the dump and usage-record paths already consume -// together. -export interface PassthroughAttemptResult { - readonly type: 'plain'; - readonly status: number; - readonly response: Response; - readonly performance: PerformanceTelemetryContext; - readonly identity: TelemetryModelIdentity; -} - -export interface PassthroughAttemptArgs { - readonly c: AuthedContext; - readonly ctx: GatewayCtx; - readonly candidate: ModelCandidate; - readonly operation: PerformanceOperation; - // Delegated to the passthrough caller so each endpoint keeps its - // request-body shaping (`{ model: _, ...body }`) local. Any throw here - // is preserved and the serve layer turns it into a 502 with the - // internal-debug envelope. - readonly call: (provider: Provider, model: ProviderModel, opts: UpstreamCallOptions) => Promise; -} - -// Statuses whose response is defined to carry no body; constructing a Response -// with one throws. -// https://fetch.spec.whatwg.org/#null-body-status -const NULL_BODY_STATUSES = new Set([101, 103, 204, 205, 304]); - -// The fallback loop keeps only the most recent failure and drops the rest, and -// on the direct-connect egress a dropped response strands its socket — the -// transport is released only when the body is read to its end or cancelled. -// Reading a failed attempt's body now makes the result inert, so no later owner -// has to remember to release it. The bytes, status and headers still forward -// verbatim when this turns out to be the last candidate. This is what the chat -// path already does through `readUpstreamApiError`. -const materializeFailure = async (response: Response): Promise => { - const bytes = await response.arrayBuffer(); - return new Response(NULL_BODY_STATUSES.has(response.status) ? null : bytes, { - status: response.status, - statusText: response.statusText, - headers: response.headers, - }); -}; - -export const passthroughAttempt = async (args: PassthroughAttemptArgs): Promise => { - const { c, ctx, candidate, operation, call } = args; - const { response, modelKey } = await call( - candidate.provider, - providerModelOf(candidate), - buildUpstreamCallOptions(candidate, ctx, inboundHeaders(c)), - ); - return { - type: 'plain', - status: response.status, - response: response.ok ? response : await materializeFailure(response), - performance: upstreamPerformanceContext(ctx, candidate, operation), - identity: telemetryModelIdentity(candidate, modelKey), - }; -}; diff --git a/packages/gateway/src/data-plane/shared/passthrough-serve.ts b/packages/gateway/src/data-plane/shared/passthrough-serve.ts deleted file mode 100644 index 9ab44d1e82..0000000000 --- a/packages/gateway/src/data-plane/shared/passthrough-serve.ts +++ /dev/null @@ -1,257 +0,0 @@ -// Shared serve scaffold for passthrough data-plane endpoints. These -// bypass the chat source/target executor because they have no protocol -// translation — the request body is forwarded to the chosen provider's -// matching endpoint and the upstream response is passed through back to -// the client. Embeddings and images run the `json` branch (single-shot -// body, OpenAI-shape `usage` block); /v1/completions runs the `sse` branch -// (frame-level transformFrame closure + settleUsage). Endpoint-owned response -// strategies handle specialized media-type state machines. Usage and -// request-performance writes are scheduled through the runtime's -// background scheduler so transient repo failures cannot turn a -// successful 200 from upstream into a 502. - -import type { Context } from 'hono'; -import { streamSSE } from 'hono/streaming'; -import type { ContentfulStatusCode } from 'hono/utils/http-status'; - -import type { PassthroughServeApiName } from './api-names.ts'; -import { appendFailedUpstreams } from './failed-upstreams.ts'; -import type { GatewayCtx } from './gateway-ctx.ts'; -import { iterateCandidates } from './iterate-candidates.ts'; -import { passthroughAttempt } from './passthrough-attempt.ts'; -import { type StreamCompletion, writeSSEFrames } from './sse.ts'; -import { recordFailedRequest } from './telemetry/performance.ts'; -import { settle } from './telemetry/settle.ts'; -import { forwardUpstreamHeaders, forwardUpstreamResponse } from './upstream-response.ts'; -import type { AuthedContext } from '../../middleware/auth.ts'; -import type { TokenUsage } from '../../repo/types.ts'; -import { enumerateModelCandidates } from '../providers/resolution.ts'; -import { doneFrame, eventFrame, type ModelKind, parseSSEStream, parseTargetStreamFrames, type ProtocolFrame, sseCommentFrame, sseFrame } from '@floway-dev/protocols/common'; -import { httpResponseToResponse, ProviderModelsUnavailableError, toInternalDebugError } from '@floway-dev/provider'; -import type { PerformanceOperation, PerformanceTelemetryContext, InternalModel, Provider, ProviderCallResult, ProviderModel, TelemetryModelIdentity, UpstreamCallOptions } from '@floway-dev/provider'; - -// `json` (embeddings, images): single-shot body, `extractBilling` reads -// usage / metadata off the parsed root. `sse` (/v1/completions): frame -// stream, `transformFrame` mutates or drops frames (return null), then -// `settleUsage` reports billing once the stream ends. `strategy` delegates -// response handling after candidate selection to the owning endpoint. -type PassthroughResponseHandling = - | { - readonly format: 'json'; - readonly extractBilling: (body: unknown) => TokenUsage | null; - } - | { - readonly format: 'sse'; - readonly transformFrame: (frame: ProtocolFrame) => ProtocolFrame | null; - readonly settleUsage: () => TokenUsage | null; - } - | { - readonly format: 'strategy'; - readonly respond: (context: PassthroughResponseStrategyContext) => Promise; - }; - -export interface PassthroughResponseStrategyContext { - readonly c: AuthedContext; - readonly ctx: GatewayCtx; - readonly sourceApi: PassthroughServeApiName; - readonly response: Response; - readonly performance: PerformanceTelemetryContext; - readonly identity: TelemetryModelIdentity; -} - -interface PassthroughServeContext { - readonly c: AuthedContext; - readonly ctx: GatewayCtx; - readonly sourceApi: PassthroughServeApiName; - readonly operation: PerformanceOperation; - // Already-validated public model id the client requested. The helper - // resolves it against the provider registry; if no upstream serves the - // id with the requested kind, the client sees a 404 with the standard - // wording. - readonly model: string; - // The model kind this endpoint serves. The resolver filters candidates - // to `model.kind === kind`; `sawModel=true && candidates=[]` becomes - // the "model exists but doesn't support this endpoint" 400. - readonly kind: ModelKind; - // Endpoint-availability gate against a resolved candidate's `InternalModel`. - // Reads `.endpoints` on the candidate — the row narrows to exactly one - // contributing upstream, so those endpoints come verbatim from the emitting - // upstream's `ProviderModel`. - readonly modelServesEndpoint: (model: InternalModel) => boolean; - // Any throw here is preserved and becomes a 502 with the internal-debug - // envelope. `model` is the emitting upstream's `ProviderModel`. - readonly call: (provider: Provider, model: ProviderModel, opts: UpstreamCallOptions) => Promise; - readonly response: PassthroughResponseHandling; -} - -// Uniform error envelope for this endpoint family. -export const passthroughApiError = (c: Context, message: string, status: ContentfulStatusCode): Response => - c.json({ error: { message, type: 'api_error' } }, status); - -export const passthroughServe = async (input: PassthroughServeContext): Promise => { - const { c, ctx, sourceApi, operation, model, kind, modelServesEndpoint, call, response: responseHandling } = input; - - try { - // The shared resolver returns every candidate of the requested kind: - // unprefixed + prefixed addressable surfaces fan out across upstreams, - // a dated-suffix retry catches `-YYYYMMDD` ids the catalog only lists - // in base form, and the kind filter rejects models of the wrong - // family before they reach the endpoint check below. Iteration order - // follows configured sort_order across upstreams, with the unprefixed - // branch pushed before the prefixed one within a single upstream. - // The first candidate whose endpoint-key check passes wins. - // - // Alias resolution is a top-of-chain step inside the resolver: an alias - // id walks every target in `selection` order, tags each returned - // candidate with that target's rule overlay, and dedups across the - // flattened list. Passthrough endpoints never consult that overlay, so - // whatever rules a target carries are inert here — the alias flow only - // changes which id the gateway addresses upstream. - const { candidates, sawModel, failedUpstreams } = await enumerateModelCandidates({ - upstreamIds: ctx.upstreamIds, - model, - kind, - scheduler: ctx.backgroundScheduler, - runtimeLocation: ctx.runtimeLocation, - }); - if (candidates.length === 0) { - ctx.dump?.error('gateway'); - // `sawModel === false` means no upstream catalog knew the inbound id - // at all (404); `sawModel === true` with zero candidates means the - // id is known but every match was the wrong kind for this endpoint - // (400), which mirrors the empty-viable case below. - return sawModel - ? passthroughApiError(c, appendFailedUpstreams(`Model ${model} does not support the ${sourceApi} endpoint.`, failedUpstreams), 400) - : passthroughApiError(c, appendFailedUpstreams(`Model ${model} is not available on any configured upstream.`, failedUpstreams), 404); - } - - // Endpoint-level pre-filter: drop candidates whose upstream model - // exists for the requested kind but doesn't expose this endpoint's - // specific capability (e.g. an embeddings-kind model on an upstream - // that only exposes chat). An empty viable set is the same "model - // exists but no upstream serves this endpoint" 400 the empty-candidate - // branch above surfaces. - const viable = candidates.filter(c => modelServesEndpoint(c.model)); - if (viable.length === 0) { - ctx.dump?.error('gateway'); - return passthroughApiError(c, appendFailedUpstreams(`Model ${model} does not support the ${sourceApi} endpoint.`, failedUpstreams), 400); - } - - // Iterate the viable list. Each candidate's attempt runs the upstream - // HTTP call and records performance telemetry; the shared - // iterator returns the first 2xx or, on exhaustion, the last non-2xx - // result. Request-perf and dump attribution wait until this point so - // they land against the terminal candidate. - const result = await iterateCandidates( - viable, - 'passthroughServe', - ctx, - operation, - candidate => passthroughAttempt({ - c, ctx, candidate, operation, - call, - }), - ); - const { response, performance: performanceContext, identity } = result; - - if (responseHandling.format === 'strategy') { - return await responseHandling.respond({ c, ctx, sourceApi, response, performance: performanceContext, identity }); - } - - if (!response.ok) { - // Exhausted — forward the last upstream response verbatim so clients - // still see real upstream telemetry (status, retry-after, request-id, - // ...) rather than a synthetic gateway envelope. - recordFailedRequest(ctx, performanceContext); - ctx.dump?.error('upstream', identity.upstream); - return forwardUpstreamResponse(response); - } - - if (responseHandling.format === 'json') { - // A 2xx body that fails to parse must not 502 a client whose - // upstream call already succeeded; we skip usage extraction and - // log so missing rows stay traceable. - let parsed: unknown; - try { - parsed = await response.clone().json(); - } catch (e) { - console.warn(`passthrough-serve: failed to parse 2xx upstream body for ${sourceApi}; usage row will be skipped`, e instanceof Error ? e.message : String(e)); - parsed = undefined; - } - const usage = parsed !== undefined ? responseHandling.extractBilling(parsed) : null; - ctx.dump?.success(identity, usage); - settle(ctx, performanceContext, identity, usage, false); - return forwardUpstreamResponse(response); - } - - // Hono's streamSSE owns the response — forwardable upstream - // headers must be staged on `c` *before* the streamSSE call so - // they survive its internal newResponse. - const upstreamBody = response.body; - if (!upstreamBody) { - ctx.dump?.failed(`${sourceApi} streaming upstream returned no body`); - recordFailedRequest(ctx, performanceContext); - // Preserve upstream correlation headers (x-request-id, cf-ray, ...) - // on the synthesized 502 so this rare edge case is still traceable. - forwardUpstreamHeaders(c, response.headers); - return passthroughApiError(c, 'Upstream returned a streaming response with no body.', 502); - } - forwardUpstreamHeaders(c, response.headers); - return streamSSE(c, async stream => { - let completion: StreamCompletion = 'error'; - let streamError: unknown; - // Tracks whether the upstream's terminal (`done`) frame arrived - // before the writer settled. A client cancel after the terminal - // frame is graceful (upstream already finished its work); a - // mid-stream cancel or EOF without terminal is a real failure. - // Mirrors SourceStreamState.failedAfter on the chat endpoints. - let terminalFrameSeen = false; - try { - const frames = (async function* () { - const sseFramesIn = parseSSEStream(upstreamBody, { signal: ctx.abortSignal }); - for await (const parsed of parseTargetStreamFrames(sseFramesIn, { protocol: sourceApi })) { - const inputFrame: ProtocolFrame = parsed.type === 'done' ? doneFrame() : eventFrame(parsed.data); - // Dump pre-transform, so forensics see upstream truth even - // when the caller drops a frame from the client-facing stream. - ctx.dump?.frame(inputFrame); - if (inputFrame.type === 'done') terminalFrameSeen = true; - const outputFrame = responseHandling.transformFrame(inputFrame); - if (outputFrame === null) continue; - yield outputFrame.type === 'done' ? sseFrame('[DONE]') : sseFrame(JSON.stringify(outputFrame.event)); - } - })(); - completion = await writeSSEFrames(stream, frames, { - keepAlive: { frame: sseCommentFrame('keepalive') }, - downstreamAbortController: ctx.downstreamAbortController, - }); - } catch (e) { - streamError = e; - } finally { - const usage = responseHandling.settleUsage(); - const failed = streamError !== undefined || completion === 'error' || !terminalFrameSeen; - if (failed) { - ctx.dump?.failed(streamError ?? `${sourceApi} stream ended with completion=${completion}`); - } else { - ctx.dump?.success(identity, usage); - } - // Record any accumulated usage regardless of the failed flag — - // tokens already metered upstream should bill even when the - // downstream half of the round-trip turned out badly. The chat - // streaming endpoints follow the same rule. - settle(ctx, performanceContext, identity, usage, failed); - } - }); - } catch (e) { - if (e instanceof ProviderModelsUnavailableError) { - const forwarded = httpResponseToResponse(e.httpResponse); - if (forwarded) { - ctx.dump?.error('upstream'); - return forwarded; - } - } - // Attributes to whichever candidate iterateCandidates was on (or short-circuits if none started). - recordFailedRequest(ctx, ctx.attempt.telemetry); - ctx.dump?.failed(e); - return c.json({ error: toInternalDebugError(e) }, 502); - } -}; diff --git a/packages/gateway/src/dump/accumulator.ts b/packages/gateway/src/dump/accumulator.ts index 9a832b7cbd..c3ba1feb98 100644 --- a/packages/gateway/src/dump/accumulator.ts +++ b/packages/gateway/src/dump/accumulator.ts @@ -190,8 +190,8 @@ export class DumpAccumulator { const recordId = ulid(completedAt); // Prefer the accumulator's frame log so dumps reflect the gateway's - // frame sequence regardless of negotiated wire shape; passthrough - // endpoints with no frames fall back to captured bytes. + // frame sequence regardless of negotiated wire shape; a turn with no + // frames falls back to captured bytes. const responseBody: StoredDumpResponseBody = this.events.length > 0 ? { type: 'stream', events: this.events } : response.bytes.byteLength > 0 || response.streamError !== null diff --git a/packages/gateway/src/dump/types.ts b/packages/gateway/src/dump/types.ts index 6d8faae5ab..55ab15fddf 100644 --- a/packages/gateway/src/dump/types.ts +++ b/packages/gateway/src/dump/types.ts @@ -41,7 +41,7 @@ export type DumpUpstreamRef = z.infer; // What went wrong on a failed turn. Either a categorized api-error envelope // (real upstream non-2xx or a gateway-synthesized envelope — `kind` matches // `ApiErrorResult.source`) or an uncategorized failure (anything the -// respond layer / passthrough-serve caught or observed mid-flight: thrown +// respond layer caught or observed mid-flight: thrown // exceptions, source-emitted error events, downstream cancels, write // errors) carrying its one-line reason text. The categorized form stores // no status — `DumpMetadata.status` already does. From 326e2dd6f0bc98a4074404d407f11d767a15f45d Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 17 Aug 2026 13:07:39 +0800 Subject: [PATCH 46/79] fix(gateway): say where a resource is, not what the host marks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The audio test required `Symbol.asyncDispose` to be present on an async generator. That is true on Node 24 and false on Node 22, which CI runs and `apps/platform-node` supports, so the suite passed locally and failed there. What it means to assert is the run's answer, which is the same on every host. Four comments explained the stream wrapper by a mechanism that is gone. They said a bare generator in the record would be adopted as a resource and `return()`ed by the sweep — true when the runner detected ownership structurally, and not since it started claiming it through `own()`. The wrapper still earns its place, for the reason the audio test already stated: it says which value in the record is the resource. That is the upstream's body, and nothing else. --- .../__tests__/data-plane/audio-pipeline_test.ts | 6 ++++-- .../gateway/src/data-plane/audio/pipeline.ts | 8 +++----- .../src/data-plane/completions/pipeline.ts | 16 ++++++---------- .../gateway/src/data-plane/images/pipeline.ts | 10 ++++------ .../gateway/src/data-plane/pipeline/facts.ts | 6 +++--- 5 files changed, 20 insertions(+), 26 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/audio-pipeline_test.ts b/packages/gateway/__tests__/data-plane/audio-pipeline_test.ts index 9d6a3bc271..a32f5b7567 100644 --- a/packages/gateway/__tests__/data-plane/audio-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/audio-pipeline_test.ts @@ -160,9 +160,11 @@ describe('the audio transcription pipeline', () => { }); it('keeps the events view clear of what answers to release', () => { + // Neither the generator nor the view around it is a resource. Whether the host marks a + // generator disposable varies — Node 24 does, Node 22 does not — and the run's answer + // does not, because ownership is claimed rather than read off the value. const generator = (async function* () { yield 1; })(); - expect(Symbol.asyncDispose in generator).toBe(true); // the language marks it - expect(isOwned(generator)).toBe(false); // the run does not + expect(isOwned(generator)).toBe(false); expect(isOwned({ [Symbol.asyncIterator]: () => generator })).toBe(false); }); }); diff --git a/packages/gateway/src/data-plane/audio/pipeline.ts b/packages/gateway/src/data-plane/audio/pipeline.ts index 0388310f92..5a10514022 100644 --- a/packages/gateway/src/data-plane/audio/pipeline.ts +++ b/packages/gateway/src/data-plane/audio/pipeline.ts @@ -43,11 +43,9 @@ import { providerModelOf, type AudioTranscriptionFormEntry, type ModelCandidate, * resource: what owns the connection is `response.http.body`, which is where release and * failover's ownership both read. * - * A view has to be built as a wrapper around the generator rather than handed over as the - * generator itself. Measured on Node 24: `Symbol.asyncDispose in (async function*(){})()` - * is `true`, so a bare generator in the record is a releasable — the runner would adopt it, - * the top-level sweep would call its `return()`, and that cancels the iteration instead of - * draining it, which is the one thing release must never mean. */ + * A view is a wrapper around the generator rather than the generator itself, which is what + * says where the resource is: the upstream's body at `response.http.body`, claimed with + * `own()`, and nothing else here. */ export type AudioTranscriptionEvents = AsyncIterable; const viewOf = (events: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => events }); diff --git a/packages/gateway/src/data-plane/completions/pipeline.ts b/packages/gateway/src/data-plane/completions/pipeline.ts index e59f92ddd8..6305885abc 100644 --- a/packages/gateway/src/data-plane/completions/pipeline.ts +++ b/packages/gateway/src/data-plane/completions/pipeline.ts @@ -42,17 +42,13 @@ import type { ModelCandidate, TelemetryModelIdentity } from '@floway-dev/provide export type CompletionsFrames = AsyncIterable>; /** - * A stream as a value the record can hold, and the one thing here that does not follow from - * the rules as written: an async generator is itself an `AsyncDisposable` — V8 gives every - * generator a `Symbol.asyncDispose` that returns it — so handing one over makes a frame view - * a *resource* as far as the run is concerned. Two rules then bite, both correctly and both - * fatally: a stage that consumes a view while providing a derived one has its source closed - * the moment it returns, and `failover` refuses a fork that received a releasable at a key it - * did not declare, which is every key but `response.http.body`. + * A stream as a value the record can hold: a wrapper around the generator rather than the + * generator itself, which is what says where the resource is. There is exactly one resource in + * a completions run — the upstream's body, claimed with `own()` — and the wrapper keeps a frame + * view from reading as another. * - * There is exactly one resource in a completions run — the upstream's body — and this is what - * keeps it that way. What comes back is single-shot on purpose: a stream that has been read is - * a stream that is over, and a second reader learns that rather than being lied to. + * What comes back is single-shot on purpose: a stream that has been read is a stream that is + * over, and a second reader learns that rather than being lied to. */ const view = (frames: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => frames }); diff --git a/packages/gateway/src/data-plane/images/pipeline.ts b/packages/gateway/src/data-plane/images/pipeline.ts index 0ab79d9633..25dda9588b 100644 --- a/packages/gateway/src/data-plane/images/pipeline.ts +++ b/packages/gateway/src/data-plane/images/pipeline.ts @@ -58,12 +58,10 @@ import type { ImagesEditsRequest, ImagesEditsSource, ModelCandidate, Performance * the events themselves rather than frames with a terminal arm the protocol has not got. */ export type ImagesFrames = AsyncIterable; -/** A stream as a value the record can hold. It has to be a wrapper around the generator rather - * than the generator itself: the language puts `Symbol.asyncDispose` on every async generator, - * so handing one over would make the runner adopt it as a resource, and releasing a resource - * calls `return()` — which cancels the iteration instead of draining it. The one resource in - * an images run is the upstream's body, at `response.http.body`, and this is what keeps it - * that way. */ +/** A stream as a value the record can hold, as a wrapper around the generator rather than the + * generator itself. What the wrapper says is where the resource is: the one resource in an + * images run is the upstream's body at `response.http.body`, claimed with `own()`, and this + * keeps a frame view from reading as another. */ const view = (frames: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => frames }); /** Images' own keys. They extend the shared space and never merge into it, so a stage written diff --git a/packages/gateway/src/data-plane/pipeline/facts.ts b/packages/gateway/src/data-plane/pipeline/facts.ts index ad65f85777..e2a86227e7 100644 --- a/packages/gateway/src/data-plane/pipeline/facts.ts +++ b/packages/gateway/src/data-plane/pipeline/facts.ts @@ -81,9 +81,9 @@ export interface GatewayFacts { 'response.http.status': number; 'response.http.headers': readonly (readonly [string, string])[]; /** The upstream's body, still open, and marked as something the run answers for. `Owned` - * rather than `AsyncDisposable`: the language puts `Symbol.asyncDispose` on every async - * generator and on no `ReadableStream`, so a structural type would admit an iterator that - * is not a resource and reject the body that is. */ + * rather than `AsyncDisposable`, because a structural type would say what the host happens + * to mark rather than what this run answers for — and what a host marks differs between the + * Node versions this ships on. Ownership is claimed, so the type says so too. */ 'response.http.body': ReadableStream & Owned; /** The authoritative reading, provided closest to the upstream on the dialect it From 8f84ab7e9d692019457c351077636fd2629f763e Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 05:00:33 +0800 Subject: [PATCH 47/79] refactor: spell the five non-chat protocol names in full in the pipelines The rename that vendored `completions`, `embeddings`, `imagesGenerations`, `imagesEdits` and `audioTranscriptions` landed against `main`, where the non-chat pipelines did not exist yet. This branch's own protocol contracts, pipelines and handlers kept the bare names, so the merge left a tree where `ModelEndpoints` and the provider interface spoke the canonical vocabulary and everything this branch added spoke the old one. Every identifier, module path, fact key and internal string that names one of the five now carries its vendor and its endpoint: completions -> openaiCompletions / OpenAICompletions embeddings -> openaiEmbeddings / OpenAIEmbeddings imagesGenerations -> openaiImagesGenerations / OpenAIImagesGenerations imagesEdits -> openaiImagesEdits / OpenAIImagesEdits audioTranscriptions -> openaiAudioTranscriptions / OpenAIAudioTranscriptions The fact keys keep their dotted layout and change only the protocol segment: `response.completions.rendered` becomes `response.openaiCompletions.rendered`, `ingress.audioTranscription.responseFormat` becomes `ingress.openaiAudioTranscription.responseFormat`, and so on across all five families. The composed pipeline names travel with them, so a `run()` failure now names `openaiEmbeddingsServe` rather than `embeddingsServe`. Comments, test names and boundary messages take the prose spelling the chat rename established: "OpenAI Completions", "OpenAI Embeddings", "OpenAI Images Generations", "OpenAI Images Edits", "OpenAI Audio Transcriptions", and "OpenAI Images" for the family that spans two endpoints. Test files and directories follow the module they cover: `__tests__/data-plane/{audio,completions,embeddings,images}-pipeline_test.ts` take the `openai-` prefix, and `protocols/__tests__/{audio,embeddings,images}` move under `openai-audio`, `openai-embeddings` and `openai-images`. Deliberately left alone: - Public URL paths and the fragments built from them: `/v1/completions`, `/v1/embeddings`, `/v1/images/{generations,edits}`, `/v1/audio/transcriptions`, and the `Model X does not support the /embeddings endpoint.` refusals that quote them. - Persisted, CHECK-constrained `operation` values passed to `upstreamPerformanceContext`: `text_completion`, `embeddings`, `image_generation`, `image_edit`, `audio_transcription`. - `rerank`, which is a model kind fanning out to six vendor wire protocols, and its pipeline, facts and tests. - Stale keys seeded on purpose: the endpoint-key migration test's pre-migration rows, and the import test that pins a pre-rename backup being refused by name. - Ordinary English: "text completions" and "a text-completion model", the `embeddings` field of a canonical response and the local that builds it, the `images` field of a canonical edits request, `parseEmbedding` / `renderEmbedding` over a bare vector, and "this model is not an embeddings model" about a kind. - Foreign paths inside reference URLs, including `vscode-copilot-chat/.../remoteEmbeddingsComputer.ts`, which a mechanical pass had rewritten. The one behavioural fix the sweep turned up: `openai-images/stream_test.ts` seeded an upstream whose stored model declared `imagesGenerations` / `imagesEdits`. Those keys no longer exist, and the endpoint validator throws on the first key it does not recognize, so the row would have been refused at load. --- .../data-plane/images-pipeline_test.ts | 64 ----- ..._test.ts => openai-audio-pipeline_test.ts} | 44 ++-- ...ts => openai-completions-pipeline_test.ts} | 74 +++--- .../openai-completions/http_test.ts | 2 +- ....ts => openai-embeddings-pipeline_test.ts} | 24 +- .../openai-embeddings/failures_test.ts | 18 +- .../data-plane/openai-images-pipeline_test.ts | 66 ++++++ .../data-plane/openai-images/stream_test.ts | 24 +- .../data-plane/pipeline/assembly_test.ts | 18 +- .../src/data-plane/openai-audio/http.ts | 46 ++-- .../src/data-plane/openai-audio/pipeline.ts | 212 ++++++++--------- .../src/data-plane/openai-completions/http.ts | 26 +-- .../data-plane/openai-completions/pipeline.ts | 168 +++++++------- .../src/data-plane/openai-embeddings/http.ts | 20 +- .../data-plane/openai-embeddings/pipeline.ts | 131 +++++------ .../src/data-plane/openai-images/http.ts | 30 +-- .../src/data-plane/openai-images/pipeline.ts | 218 +++++++++--------- .../transcription_test.ts | 56 ++--- .../translate_test.ts | 72 +++--- .../{images => openai-images}/request_test.ts | 52 ++--- .../response_test.ts | 38 +-- packages/protocols/src/openai-audio/index.ts | 30 +-- packages/protocols/src/openai-audio/stream.ts | 22 +- .../protocols/src/openai-audio/subtitles.ts | 6 +- .../src/openai-audio/transcription.ts | 64 ++--- .../protocols/src/openai-completions/index.ts | 38 +-- .../protocols/src/openai-completions/parse.ts | 16 +- .../src/openai-completions/stream.ts | 14 +- .../protocols/src/openai-embeddings/index.ts | 42 ++-- .../src/openai-embeddings/translate.ts | 46 ++-- packages/protocols/src/openai-images/index.ts | 51 ++-- .../protocols/src/openai-images/request.ts | 50 ++-- .../protocols/src/openai-images/response.ts | 26 +-- .../protocols/src/openai-images/stream.ts | 26 +-- .../protocols/src/openai-images/to-sse.ts | 4 +- 35 files changed, 921 insertions(+), 917 deletions(-) delete mode 100644 packages/gateway/__tests__/data-plane/images-pipeline_test.ts rename packages/gateway/__tests__/data-plane/{audio-pipeline_test.ts => openai-audio-pipeline_test.ts} (77%) rename packages/gateway/__tests__/data-plane/{completions-pipeline_test.ts => openai-completions-pipeline_test.ts} (82%) rename packages/gateway/__tests__/data-plane/{embeddings-pipeline_test.ts => openai-embeddings-pipeline_test.ts} (63%) create mode 100644 packages/gateway/__tests__/data-plane/openai-images-pipeline_test.ts rename packages/protocols/__tests__/{audio => openai-audio}/transcription_test.ts (69%) rename packages/protocols/__tests__/{embeddings => openai-embeddings}/translate_test.ts (61%) rename packages/protocols/__tests__/{images => openai-images}/request_test.ts (64%) rename packages/protocols/__tests__/{images => openai-images}/response_test.ts (60%) diff --git a/packages/gateway/__tests__/data-plane/images-pipeline_test.ts b/packages/gateway/__tests__/data-plane/images-pipeline_test.ts deleted file mode 100644 index d10ec447ae..0000000000 --- a/packages/gateway/__tests__/data-plane/images-pipeline_test.ts +++ /dev/null @@ -1,64 +0,0 @@ -// Images' pipeline, assembled. `compose` derives the entry contract and rejects an array that -// cannot work, so most of what this file establishes is established by the assembly succeeding -// at all — and what is worth writing down is the contract it derives and the keys it cannot see. - -import { describe, expect, it } from 'vitest'; - -import { imagesServePipeline, type ImagesServeEntry } from '../../src/data-plane/images/pipeline.ts'; -import type { CanonicalImagesRequest } from '@floway-dev/protocols/images'; - -const generations: CanonicalImagesRequest = { - operation: 'generations', - parameters: { prompt: 'a shiba in space' }, -}; - -const edits: CanonicalImagesRequest = { - operation: 'edits', - images: [{ kind: 'reference', reference: { file_id: 'file-source' } }], - parameters: { prompt: 'replace the sky' }, -}; - -describe('the images pipeline', () => { - it('assembles both endpoints as one array, and asks its caller for what the descending stages need', () => { - expect([...imagesServePipeline(generations).entryNeeds].sort()).toEqual(['serve.model']); - expect([...imagesServePipeline(edits).entryNeeds].sort()).toEqual(['serve.model']); - }); - - // `callImagesUpstream` reads `request.images.canonical`, `ingress.images.wantsStream` and - // `ingress.http.headers`, and the derived contract mentions none of them. That is not this - // family's defect: a stage whose only trait is `return` declares no request side at all, by - // ruling — "when it short-circuits, only `provides`" — so assembly cannot see what an ending - // stage reads, and every family's ending stage reads something. - // - // Written as a test rather than a comment because the hole has a consequence: a caller who - // omits any of them gets a runtime failure at the deepest stage instead of the assembly error - // the entry contract exists to produce. - it('cannot see what its ending stage reads, because a return-only stage declares no needs', () => { - const derived = imagesServePipeline(generations).entryNeeds; - expect(derived).not.toContain('request.images.canonical'); - expect(derived).not.toContain('ingress.images.wantsStream'); - expect(derived).not.toContain('ingress.http.headers'); - }); - - // What covers that hole, and the reason it is a gap rather than a break: the entry type names - // all four, so the caller `entryNeeds` would have let through does not compile. - it('names every key a caller must bring in its entry type', () => { - const entry: ImagesServeEntry = { - 'ingress.http.headers': [['content-type', 'application/json']], - 'ingress.images.wantsStream': false, - 'request.images.canonical': generations, - 'serve.model': 'gpt-image-1', - }; - expect(Object.keys(entry).sort()).toEqual([ - 'ingress.http.headers', - 'ingress.images.wantsStream', - 'request.images.canonical', - 'serve.model', - ]); - - // @ts-expect-error — dropping one of them is a compile error, which is the statement this - // makes; the assertion below only keeps the binding from being unused. - const incomplete: ImagesServeEntry = { 'serve.model': 'gpt-image-1' }; - expect(Object.keys(incomplete)).toEqual(['serve.model']); - }); -}); diff --git a/packages/gateway/__tests__/data-plane/audio-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts similarity index 77% rename from packages/gateway/__tests__/data-plane/audio-pipeline_test.ts rename to packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts index a32f5b7567..3bf5712372 100644 --- a/packages/gateway/__tests__/data-plane/audio-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts @@ -1,4 +1,4 @@ -// Audio transcription's pipeline, assembled. `compose` derives the entry contract and +// OpenAI Audio Transcriptions' pipeline, assembled. `compose` derives the entry contract and // rejects an array that cannot work, so most of what this file establishes is established // by the module importing at all — the assembly runs at load. What is worth writing down is // the entry contract it derives, the keys it cannot see, and the one runtime property this @@ -6,7 +6,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { audioTranscriptionServePipeline } from '../../src/data-plane/audio/pipeline.ts'; +import { openaiAudioTranscriptionServePipeline } from '../../src/data-plane/openai-audio/pipeline.ts'; import { enumerateModelCandidates } from '../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../src/repo/index.ts'; import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts'; @@ -22,13 +22,13 @@ vi.mock('../../src/data-plane/providers/resolution.ts', async importOriginal => let live: readonly ModelCandidate[] = []; -const candidate = (callAudioTranscriptions: () => Promise): ModelCandidate => { - const endpoints = { audioTranscriptions: {} }; +const candidate = (callOpenAIAudioTranscriptions: () => Promise): ModelCandidate => { + const endpoints = { openaiAudioTranscriptions: {} }; return { provider: { upstreamId: 'up_a', kind: 'custom', name: 'up_a', inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, - instance: stubProvider({ callAudioTranscriptions }), + instance: stubProvider({ callOpenAIAudioTranscriptions }), }, model: stubInternalModel( { id: 'whisper-1', kind: 'transcription', endpoints, providerModels: { up_a: stubProviderModel({ id: 'whisper-1', endpoints }) } }, @@ -47,11 +47,11 @@ const sse = (...events: readonly string[]): Response => new Response(events.map(event => `data: ${event}\n\n`).join(''), { status: 200, headers: { 'content-type': 'text/event-stream' } }); const serve = async (responseFormat = 'json') => await run( - audioTranscriptionServePipeline, + openaiAudioTranscriptionServePipeline, move({ - 'ingress.audioTranscription.responseFormat': responseFormat, + 'ingress.openaiAudioTranscription.responseFormat': responseFormat, 'ingress.http.headers': [] as readonly (readonly [string, string])[], - 'request.audioTranscription.form': [{ name: 'model', value: 'whisper-1' }], + 'request.openaiAudioTranscription.form': [{ name: 'model', value: 'whisper-1' }], 'serve.model': 'whisper-1', }) as never, { @@ -74,18 +74,18 @@ beforeEach(() => { } as never); }); -describe('the audio transcription pipeline', () => { +describe('the OpenAI Audio Transcriptions pipeline', () => { it('assembles, and asks its caller for what the descending stages need', () => { - expect([...audioTranscriptionServePipeline.entryNeeds].sort()).toEqual([ - 'ingress.audioTranscription.responseFormat', + expect([...openaiAudioTranscriptionServePipeline.entryNeeds].sort()).toEqual([ + 'ingress.openaiAudioTranscription.responseFormat', 'serve.model', ]); }); - // The ending stage reads `request.audioTranscription.form` and `ingress.http.headers`, and - // the entry contract mentions neither. That is not this family's defect: a stage whose only - // trait is `return` declares no request side at all, by ruling — "when it short-circuits, - // only `provides`" — so assembly cannot see what an ending stage reads. + // The ending stage reads `request.openaiAudioTranscription.form` and `ingress.http.headers`, + // and the entry contract mentions neither. That is not this family's defect: a stage whose + // only trait is `return` declares no request side at all, by ruling — "when it + // short-circuits, only `provides`" — so assembly cannot see what an ending stage reads. // // It bites harder here than it does for a family whose edge needs the request payload to // render: this family's edge does not, so the payload the whole endpoint exists to send is @@ -93,13 +93,13 @@ describe('the audio transcription pipeline', () => { // before failing. The type layer still catches it at the definition site, which is why this // is a gap and not a break. it('cannot see the request payload, because only a return-only stage reads it', () => { - expect(audioTranscriptionServePipeline.entryNeeds).not.toContain('request.audioTranscription.form'); - expect(audioTranscriptionServePipeline.entryNeeds).not.toContain('ingress.http.headers'); + expect(openaiAudioTranscriptionServePipeline.entryNeeds).not.toContain('request.openaiAudioTranscription.form'); + expect(openaiAudioTranscriptionServePipeline.entryNeeds).not.toContain('ingress.http.headers'); }); it('names the entry key a caller did not bring, before any stage runs', async () => { - await expect(run(audioTranscriptionServePipeline, move({ 'serve.model': 'whisper-1' }) as never, {})) - .rejects.toThrow('run(audioTranscriptionServe): audioTranscriptionServe needs'); + await expect(run(openaiAudioTranscriptionServePipeline, move({ 'serve.model': 'whisper-1' }) as never, {})) + .rejects.toThrow('run(openaiAudioTranscriptionServe): openaiAudioTranscriptionServe needs'); }); // The streamed answer is a wrapper around its generator rather than the generator itself. @@ -119,7 +119,7 @@ describe('the audio transcription pipeline', () => { const { facts } = await serve(); expect(facts['response.http.status']).toBe(200); - expect(facts['response.audioTranscription.rendered']).toMatchObject({ text: 'hello there' }); + expect(facts['response.openaiAudioTranscription.rendered']).toMatchObject({ text: 'hello there' }); }); // A refusal reaches the client with the status the upstream gave it, and in the words the @@ -135,7 +135,7 @@ describe('the audio transcription pipeline', () => { const { facts } = await serve(); expect(facts['response.http.status']).toBe(413); - expect(facts['response.audioTranscription.rendered']).toEqual({ error: { message: 'too large' } }); + expect(facts['response.openaiAudioTranscription.rendered']).toEqual({ error: { message: 'too large' } }); }); // The reader stops at the terminal event. An upstream that holds the connection open past @@ -151,7 +151,7 @@ describe('the audio transcription pipeline', () => { const { facts, drain } = await serve('json'); const frames: SseFrame[] = []; - for await (const frame of facts['response.audioTranscription.rendered'] as AsyncIterable) frames.push(frame); + for await (const frame of facts['response.openaiAudioTranscription.rendered'] as AsyncIterable) frames.push(frame); await drain(); expect(frames).toHaveLength(2); diff --git a/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts similarity index 82% rename from packages/gateway/__tests__/data-plane/completions-pipeline_test.ts rename to packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts index 2af5df2db1..cc0a9af162 100644 --- a/packages/gateway/__tests__/data-plane/completions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts @@ -1,12 +1,12 @@ -// Completions' pipeline, assembled and run. `compose` derives the entry contract and rejects -// an array that cannot work, so the assembly succeeding is itself most of what a test of the -// wiring would say — what is written down here is the entry contract, the two things the +// OpenAI Completions' pipeline, assembled and run. `compose` derives the entry contract and +// rejects an array that cannot work, so the assembly succeeding is itself most of what a test +// of the wiring would say — what is written down here is the entry contract, the two things the // assembly cannot see, and one run of each shape the answer can take, because this is the // first family whose answer can be a stream and none of that is visible in a declaration. import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { completionsServePipeline } from '../../src/data-plane/completions/pipeline.ts'; +import { openaiCompletionsServePipeline } from '../../src/data-plane/openai-completions/pipeline.ts'; import { enumerateModelCandidates } from '../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../src/repo/index.ts'; import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts'; @@ -38,14 +38,14 @@ const resolveAttempt = (selector: { readonly upstreamId: string }): ModelCandida const candidate = ( upstream: string, - callCompletions: (model: unknown, body: unknown, signal: AbortSignal | undefined, opts: UpstreamCallOptions) => Promise, + callOpenAICompletions: (model: unknown, body: unknown, signal: AbortSignal | undefined, opts: UpstreamCallOptions) => Promise, ): ModelCandidate => { - const endpoints = { chatCompletions: {}, completions: {} }; + const endpoints = { openaiChatCompletions: {}, openaiCompletions: {} }; return { provider: { upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, - instance: stubProvider({ callCompletions }), + instance: stubProvider({ callOpenAICompletions }), }, model: stubInternalModel({ id: 'text-model', endpoints, providerModels: { [upstream]: stubProviderModel({ id: 'text-model', endpoints }) } }, upstream), fetcher: directFetcher, @@ -82,10 +82,10 @@ beforeEach(() => { }); const serve = async (facts: Record) => await run( - completionsServePipeline, + openaiCompletionsServePipeline, move(facts) as never, { - gateway: mockGatewayCtx({ wantsStream: facts['ingress.completions.wantsStream'] === true }), + gateway: mockGatewayCtx({ wantsStream: facts['ingress.openaiCompletions.wantsStream'] === true }), background: () => {}, rememberCandidates: () => {}, resolveAttempt, @@ -93,10 +93,10 @@ const serve = async (facts: Record) => await run( ); const entryFacts = (overrides: Record = {}) => ({ - 'ingress.completions.wantsStream': true, - 'ingress.completions.wantsUsageChunk': false, + 'ingress.openaiCompletions.wantsStream': true, + 'ingress.openaiCompletions.wantsUsageChunk': false, 'ingress.http.headers': [], - 'request.completions.payload': { model: 'text-model', prompt: 'hello', stream: true }, + 'request.openaiCompletions.payload': { model: 'text-model', prompt: 'hello', stream: true }, 'serve.model': 'text-model', ...overrides, }); @@ -109,24 +109,24 @@ const collect = async (rendered: unknown): Promise => { beforeEach(() => { vi.mocked(enumerateModelCandidates).mockReset(); }); -describe('the completions pipeline', () => { +describe('the OpenAI Completions pipeline', () => { it('assembles, and asks its caller for what the descending stages need', () => { - expect([...completionsServePipeline.entryNeeds].sort()).toEqual([ - 'ingress.completions.wantsStream', - 'ingress.completions.wantsUsageChunk', - 'request.completions.payload', + expect([...openaiCompletionsServePipeline.entryNeeds].sort()).toEqual([ + 'ingress.openaiCompletions.wantsStream', + 'ingress.openaiCompletions.wantsUsageChunk', + 'request.openaiCompletions.payload', 'serve.model', ]); }); - // `callCompletionsUpstream` reads `ingress.http.headers`, and the entry contract does not - // mention it. That is not this family's defect: a stage whose only trait is `return` + // `callOpenAICompletionsUpstream` reads `ingress.http.headers`, and the entry contract does + // not mention it. That is not this family's defect: a stage whose only trait is `return` // declares no request side at all, by ruling — "when it short-circuits, only `provides`" — // so assembly cannot see what an ending stage reads, and every family's ending stage reads // something. A caller who omits that key gets a runtime failure at the deepest stage // instead of the assembly error the entry contract exists to give it. it('cannot see what an ending stage reads, because a return-only stage declares no needs', () => { - expect(completionsServePipeline.entryNeeds).not.toContain('ingress.http.headers'); + expect(openaiCompletionsServePipeline.entryNeeds).not.toContain('ingress.http.headers'); }); it('renders the upstream frames as SSE, hiding the usage chunk the client did not ask for', async () => { @@ -140,7 +140,7 @@ describe('the completions pipeline', () => { // The answer comes back before the drain runs, which is what lets a streaming family // hand its stream on: the frames are still there to read. - expect(await collect(facts['response.completions.rendered'])).toEqual([ + expect(await collect(facts['response.openaiCompletions.rendered'])).toEqual([ { type: 'sse', event: undefined, data: chunk('he') }, { type: 'sse', event: undefined, data: chunk('llo') }, { type: 'sse', event: undefined, data: '[DONE]' }, @@ -162,8 +162,8 @@ describe('the completions pipeline', () => { expect(facts['response.usage.billable']).toEqual([ { identity: { model: 'text-model', upstream: 'up_a', modelKey: 'text-model-key', pricing: null }, quantities: {} }, ]); - await collect(facts['response.completions.rendered']); - expect(await facts['response.completions.streamedUsage']).toMatchObject({ + await collect(facts['response.openaiCompletions.rendered']); + expect(await facts['response.openaiCompletions.streamedUsage']).toMatchObject({ failed: false, billable: [ { @@ -178,9 +178,9 @@ describe('the completions pipeline', () => { it('shows the usage chunk to a client that asked for it', async () => { resolves([candidate('up_a', async () => ({ response: sse(chunk('hi'), usageChunk, '[DONE]'), modelKey: 'text-model-key' }))]); - const { facts, drain } = await serve(entryFacts({ 'ingress.completions.wantsUsageChunk': true })); + const { facts, drain } = await serve(entryFacts({ 'ingress.openaiCompletions.wantsUsageChunk': true })); - expect((await collect(facts['response.completions.rendered'])).map(frame => frame.data)).toEqual([chunk('hi'), usageChunk, '[DONE]']); + expect((await collect(facts['response.openaiCompletions.rendered'])).map(frame => frame.data)).toEqual([chunk('hi'), usageChunk, '[DONE]']); await drain(); }); @@ -189,12 +189,12 @@ describe('the completions pipeline', () => { resolves([candidate('up_a', async () => ({ response: Response.json(body), modelKey: 'text-model-key' }))]); const { facts, drain } = await serve(entryFacts({ - 'ingress.completions.wantsStream': false, - 'request.completions.payload': { model: 'text-model', prompt: 'hello' }, + 'ingress.openaiCompletions.wantsStream': false, + 'request.openaiCompletions.payload': { model: 'text-model', prompt: 'hello' }, })); - expect(facts['response.completions.rendered']).toEqual(body); - expect(facts['response.completions.streamedUsage']).toBeNull(); + expect(facts['response.openaiCompletions.rendered']).toEqual(body); + expect(facts['response.openaiCompletions.streamedUsage']).toBeNull(); expect(facts['response.usage.billable']).toEqual([ { identity: { model: 'text-model', upstream: 'up_a', modelKey: 'text-model-key', pricing: null }, @@ -235,7 +235,7 @@ describe('the completions pipeline', () => { const { facts, drain } = await serve(entryFacts()); // The client's stream is still live: the run answered before anything was drained. - expect(facts['response.completions.rendered']).toBeDefined(); + expect(facts['response.openaiCompletions.rendered']).toBeDefined(); await drain(); expect(drained).toContain('winner'); }); @@ -253,8 +253,8 @@ describe('the completions pipeline', () => { })), ]); const { drain } = await serve(entryFacts({ - 'ingress.completions.wantsStream': false, - 'request.completions.payload': { model: 'text-model', prompt: 'hello' }, + 'ingress.openaiCompletions.wantsStream': false, + 'request.openaiCompletions.payload': { model: 'text-model', prompt: 'hello' }, })); await drain(); expect(recorded.usage).toHaveLength(1); @@ -272,7 +272,7 @@ describe('the completions pipeline', () => { await drain(); expect(recorded.usage).toHaveLength(0); - const streamed = facts['response.completions.streamedUsage']; + const streamed = facts['response.openaiCompletions.streamedUsage']; expect(streamed).not.toBeNull(); const outcome = await streamed!; expect(outcome.billable).toHaveLength(1); @@ -306,8 +306,8 @@ describe('the completions pipeline', () => { ]); const { facts, drain } = await serve(entryFacts({ - 'ingress.completions.wantsStream': false, - 'request.completions.payload': { model: 'text-model', prompt: 'hello' }, + 'ingress.openaiCompletions.wantsStream': false, + 'request.openaiCompletions.payload': { model: 'text-model', prompt: 'hello' }, })); expect(tried).toEqual(['up_a', 'up_b']); @@ -315,7 +315,7 @@ describe('the completions pipeline', () => { // an upstream actually returned, and the words that upstream used, rather than a // synthesized envelope quoting its serialized body back as a message. expect(facts['response.http.status']).toBe(400); - expect(facts['response.completions.rendered']).toEqual({ error: { message: 'no' } }); + expect(facts['response.openaiCompletions.rendered']).toEqual({ error: { message: 'no' } }); await drain(); }); @@ -333,7 +333,7 @@ describe('the completions pipeline', () => { const { facts, drain } = await serve(entryFacts()); expect((facts as Record)['response.http.body']).toBeInstanceOf(ReadableStream); - await collect(facts['response.completions.rendered']); + await collect(facts['response.openaiCompletions.rendered']); await drain(); }); }); diff --git a/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts b/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts index 4661b9f3a2..f8df8fcd6e 100644 --- a/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts @@ -328,7 +328,7 @@ test('/v1/completions non-streaming records usage row, performance neutral row ( // one that did. test('/v1/completions a stream that never terminated is recorded as a failed request', async () => { const { apiKey, repo } = await setupAppTest(); - await registerCompletionsUpstream(repo); + await registerOpenAICompletionsUpstream(repo); await withMockedFetch( () => Promise.resolve(new Response('data: {"id":"c","object":"text_completion","created":1,"model":"davinci-002","choices":[{"index":0,"text":"hi"}]}\n\n', { diff --git a/packages/gateway/__tests__/data-plane/embeddings-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-embeddings-pipeline_test.ts similarity index 63% rename from packages/gateway/__tests__/data-plane/embeddings-pipeline_test.ts rename to packages/gateway/__tests__/data-plane/openai-embeddings-pipeline_test.ts index c42d762300..d9e19c3a28 100644 --- a/packages/gateway/__tests__/data-plane/embeddings-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-embeddings-pipeline_test.ts @@ -1,25 +1,25 @@ -// Embeddings' pipeline, assembled. `compose` derives the entry contract and rejects an +// OpenAI Embeddings' pipeline, assembled. `compose` derives the entry contract and rejects an // array that cannot work, so most of what this file establishes is established by the // assembly succeeding at all — and what is worth writing down is the entry contract it // derives, including the two keys it cannot see. import { describe, expect, it } from 'vitest'; -import { embeddingsServePipeline } from '../../src/data-plane/embeddings/pipeline.ts'; +import { openaiEmbeddingsServePipeline } from '../../src/data-plane/openai-embeddings/pipeline.ts'; import { move, run } from '@floway-dev/pipeline'; -describe('the embeddings pipeline', () => { +describe('the OpenAI Embeddings pipeline', () => { it('assembles, and asks its caller for what the descending stages need', () => { - expect([...embeddingsServePipeline.entryNeeds].sort()).toEqual([ - 'ingress.embeddings.encodingFormat', + expect([...openaiEmbeddingsServePipeline.entryNeeds].sort()).toEqual([ + 'ingress.openaiEmbeddings.encodingFormat', 'serve.model', ]); }); - // Rerank records the same hole against `ingress.http.headers`; embeddings shows it twice - // over, and the second one is the sharper case. A stage whose only trait is `return` + // Rerank records the same hole against `ingress.http.headers`; OpenAI Embeddings shows it + // twice over, and the second one is the sharper case. A stage whose only trait is `return` // declares no request side at all, by ruling — "when it short-circuits, only `provides`" - // — so assembly cannot see what an ending stage reads. `callEmbeddingsUpstream` reads + // — so assembly cannot see what an ending stage reads. `callOpenAIEmbeddingsUpstream` reads // both the headers and the request payload, and neither reaches the entry contract. // // Rerank's edge happened to need `request.rerank.canonical` for rendering, which put it @@ -31,12 +31,12 @@ describe('the embeddings pipeline', () => { // exactly that. The type layer still catches it at the definition site, which is why // this is a gap and not a break. it('cannot see what an ending stage reads, because a return-only stage declares no needs', () => { - expect(embeddingsServePipeline.entryNeeds).not.toContain('request.embeddings.canonical'); - expect(embeddingsServePipeline.entryNeeds).not.toContain('ingress.http.headers'); + expect(openaiEmbeddingsServePipeline.entryNeeds).not.toContain('request.openaiEmbeddings.canonical'); + expect(openaiEmbeddingsServePipeline.entryNeeds).not.toContain('ingress.http.headers'); }); it('names the entry key a caller did not bring, before any stage runs', async () => { - await expect(run(embeddingsServePipeline, move({ 'serve.model': 'text-embedding-3-small' }) as never, {})) - .rejects.toThrow('run(embeddingsServe): embeddingsServe needs'); + await expect(run(openaiEmbeddingsServePipeline, move({ 'serve.model': 'text-embedding-3-small' }) as never, {})) + .rejects.toThrow('run(openaiEmbeddingsServe): openaiEmbeddingsServe needs'); }); }); diff --git a/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts b/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts index c1142c6ece..3d812f671f 100644 --- a/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts @@ -8,7 +8,7 @@ import { buildCustomUpstreamRecord, flushAsyncWork, requestApp, setupAppTest } f import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { jsonResponse, withMockedFetch, assertEquals } from '@floway-dev/test-utils'; -const registerEmbeddingsUpstream = async ( +const registerOpenAIEmbeddingsUpstream = async ( repo: Awaited>['repo'], ): Promise => { await repo.upstreams.deleteAll(); @@ -29,7 +29,7 @@ const registerEmbeddingsUpstream = async ( const upstreamModels = () => jsonResponse({ object: 'list', data: [{ id: 'custom-embed-model' }] }); -const askForEmbeddings = async (key: string): Promise => await requestApp('/v1/embeddings', { +const askForOpenAIEmbeddings = async (key: string): Promise => await requestApp('/v1/embeddings', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': key }, body: JSON.stringify({ model: 'custom-embed-model', input: 'hi' }), @@ -40,7 +40,7 @@ const askForEmbeddings = async (key: string): Promise => await request // and the failure is still reported, because a write nobody hears about is one nobody fixes. test('a usage write that fails leaves the answer alone and still reports', async () => { const { apiKey, repo } = await setupAppTest(); - await registerEmbeddingsUpstream(repo); + await registerOpenAIEmbeddingsUpstream(repo); repo.usage.record = () => Promise.reject(new Error('simulated SQL write failure')); const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); @@ -61,7 +61,7 @@ test('a usage write that fails leaves the answer alone and still reports', async throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await askForEmbeddings(apiKey.key); + const response = await askForOpenAIEmbeddings(apiKey.key); assertEquals(response.status, 200); const body = await response.json() as { data: { embedding: number[] }[] }; @@ -79,9 +79,9 @@ test('a usage write that fails leaves the answer alone and still reports', async // The replaced surface forwarded such a body verbatim with the upstream's 200. A protocol // that requires JSON and did not get JSON has no answer to serve, so the gateway says so // itself — it cannot claim to have served a request whose answer it never read. -test('a 2xx body the embeddings protocol cannot read is refused rather than forwarded', async () => { +test('a 2xx body the OpenAI Embeddings protocol cannot read is refused rather than forwarded', async () => { const { apiKey, repo } = await setupAppTest(); - await registerEmbeddingsUpstream(repo); + await registerOpenAIEmbeddingsUpstream(repo); await withMockedFetch( request => { @@ -96,12 +96,12 @@ test('a 2xx body the embeddings protocol cannot read is refused rather than forw throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await askForEmbeddings(apiKey.key); + const response = await askForOpenAIEmbeddings(apiKey.key); assertEquals(response.status, 502); assertEquals(response.headers.get('content-type'), 'application/json'); const body = await response.json() as { error: { message: string } }; - assertEquals(body.error.message.includes('the embeddings protocol cannot read'), true); + assertEquals(body.error.message.includes('the OpenAI Embeddings protocol cannot read'), true); await flushAsyncWork(); }, ); @@ -141,7 +141,7 @@ test('when every candidate refuses the client gets the last upstream-s own refus throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await askForEmbeddings(apiKey.key); + const response = await askForOpenAIEmbeddings(apiKey.key); assertEquals(response.status, 429); assertEquals(response.headers.get('retry-after'), '17'); diff --git a/packages/gateway/__tests__/data-plane/openai-images-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-images-pipeline_test.ts new file mode 100644 index 0000000000..08f1d8aff5 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/openai-images-pipeline_test.ts @@ -0,0 +1,66 @@ +// OpenAI Images' pipeline, assembled. `compose` derives the entry contract and rejects an array +// that cannot work, so most of what this file establishes is established by the assembly +// succeeding at all — and what is worth writing down is the contract it derives and the keys it +// cannot see. + +import { describe, expect, it } from 'vitest'; + +import { openaiImagesServePipeline, type OpenAIImagesServeEntry } from '../../src/data-plane/openai-images/pipeline.ts'; +import type { CanonicalOpenAIImagesRequest } from '@floway-dev/protocols/openai-images'; + +const generations: CanonicalOpenAIImagesRequest = { + operation: 'generations', + parameters: { prompt: 'a shiba in space' }, +}; + +const edits: CanonicalOpenAIImagesRequest = { + operation: 'edits', + images: [{ kind: 'reference', reference: { file_id: 'file-source' } }], + parameters: { prompt: 'replace the sky' }, +}; + +describe('the OpenAI Images pipeline', () => { + it('assembles both endpoints as one array, and asks its caller for what the descending stages need', () => { + expect([...openaiImagesServePipeline(generations).entryNeeds].sort()).toEqual(['serve.model']); + expect([...openaiImagesServePipeline(edits).entryNeeds].sort()).toEqual(['serve.model']); + }); + + // `callOpenAIImagesUpstream` reads `request.openaiImages.canonical`, + // `ingress.openaiImages.wantsStream` and `ingress.http.headers`, and the derived contract + // mentions none of them. That is not this family's defect: a stage whose only trait is + // `return` declares no request side at all, by ruling — "when it short-circuits, only + // `provides`" — so assembly cannot see what an ending stage reads, and every family's ending + // stage reads something. + // + // Written as a test rather than a comment because the hole has a consequence: a caller who + // omits any of them gets a runtime failure at the deepest stage instead of the assembly error + // the entry contract exists to produce. + it('cannot see what its ending stage reads, because a return-only stage declares no needs', () => { + const derived = openaiImagesServePipeline(generations).entryNeeds; + expect(derived).not.toContain('request.openaiImages.canonical'); + expect(derived).not.toContain('ingress.openaiImages.wantsStream'); + expect(derived).not.toContain('ingress.http.headers'); + }); + + // What covers that hole, and the reason it is a gap rather than a break: the entry type names + // all four, so the caller `entryNeeds` would have let through does not compile. + it('names every key a caller must bring in its entry type', () => { + const entry: OpenAIImagesServeEntry = { + 'ingress.http.headers': [['content-type', 'application/json']], + 'ingress.openaiImages.wantsStream': false, + 'request.openaiImages.canonical': generations, + 'serve.model': 'gpt-image-1', + }; + expect(Object.keys(entry).sort()).toEqual([ + 'ingress.http.headers', + 'ingress.openaiImages.wantsStream', + 'request.openaiImages.canonical', + 'serve.model', + ]); + + // @ts-expect-error — dropping one of them is a compile error, which is the statement this + // makes; the assertion below only keeps the binding from being unused. + const incomplete: OpenAIImagesServeEntry = { 'serve.model': 'gpt-image-1' }; + expect(Object.keys(incomplete)).toEqual(['serve.model']); + }); +}); diff --git a/packages/gateway/__tests__/data-plane/openai-images/stream_test.ts b/packages/gateway/__tests__/data-plane/openai-images/stream_test.ts index 0b1314858b..d9f1273728 100644 --- a/packages/gateway/__tests__/data-plane/openai-images/stream_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-images/stream_test.ts @@ -5,10 +5,10 @@ import { buildCustomUpstreamRecord, flushAsyncWork, requestApp, setupAppTest } f import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { withMockedFetch, assertEquals, assertExists } from '@floway-dev/test-utils'; -// The upstream is declared rather than discovered: both images endpoints are named on one -// manual model row, so nothing in these cases depends on the id heuristics that infer a +// The upstream is declared rather than discovered: both OpenAI Images endpoints are named on +// one manual model row, so nothing in these cases depends on the id heuristics that infer a // catalog's capabilities. -const registerImagesModel = async (repo: InMemoryRepo): Promise => { +const registerOpenAIImagesModel = async (repo: InMemoryRepo): Promise => { await repo.upstreams.deleteAll(); clearInProcessCopilotTokenCache(); await repo.upstreams.save(buildCustomUpstreamRecord({ @@ -26,7 +26,7 @@ const registerImagesModel = async (repo: InMemoryRepo): Promise => { upstreamModelId: 'gpt-image-2-upstream', publicModelId: 'gpt-image-2', kind: 'image', - endpoints: { imagesGenerations: {}, imagesEdits: {} }, + endpoints: { openaiImagesGenerations: {}, openaiImagesEdits: {} }, }], }, })); @@ -40,7 +40,7 @@ const sseResponse = (body: string, headers: Record = {}): Respon test('/v1/images/generations answers a streaming request with the upstream events as SSE', async () => { const { apiKey, repo } = await setupAppTest(); - await registerImagesModel(repo); + await registerOpenAIImagesModel(repo); let forwarded: Record | undefined; await withMockedFetch( @@ -85,7 +85,7 @@ test('/v1/images/generations answers a streaming request with the upstream event test('/v1/images/generations answers a non-streaming request with the JSON object', async () => { const { apiKey, repo } = await setupAppTest(); - await registerImagesModel(repo); + await registerOpenAIImagesModel(repo); await withMockedFetch( request => { @@ -110,7 +110,7 @@ test('/v1/images/generations answers a non-streaming request with the JSON objec test('/v1/images/generations bills a stream from the usage its completed event reported, once', async () => { const { apiKey, repo } = await setupAppTest(); - await registerImagesModel(repo); + await registerOpenAIImagesModel(repo); await withMockedFetch( () => Promise.resolve(sseResponse(PARTIAL + COMPLETED)), @@ -143,7 +143,7 @@ test('/v1/images/generations bills a stream from the usage its completed event r test('/v1/images/generations answers a refused stream in the upstream status and words', async () => { const { apiKey, repo } = await setupAppTest(); - await registerImagesModel(repo); + await registerOpenAIImagesModel(repo); await withMockedFetch( () => Promise.resolve(new Response(JSON.stringify({ error: { message: 'Rate limit reached for images.', code: 'rate_limit_exceeded' } }), { @@ -172,7 +172,7 @@ test('/v1/images/generations answers a refused stream in the upstream status and test('/v1/images/generations completes and cancels an upstream kept open after the completed event', async () => { const { apiKey, repo } = await setupAppTest(); - await registerImagesModel(repo); + await registerOpenAIImagesModel(repo); let upstreamCancelled = false; const encoder = new TextEncoder(); @@ -203,7 +203,7 @@ test('/v1/images/generations completes and cancels an upstream kept open after t test('/v1/images/generations fails a stream that ended without a completed event', async () => { const { apiKey, repo } = await setupAppTest(); - await registerImagesModel(repo); + await registerOpenAIImagesModel(repo); const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); try { @@ -225,7 +225,7 @@ test('/v1/images/generations fails a stream that ended without a completed event // An upstream that sent partial images and no image never answered the request, and the // run says so rather than letting a truncated stream read as a complete one. assertEquals( - errorSpy.mock.calls.some(call => call.some(arg => arg instanceof Error && arg.message === 'Images stream ended without a completed event.')), + errorSpy.mock.calls.some(call => call.some(arg => arg instanceof Error && arg.message === 'OpenAI Images stream ended without a completed event.')), true, ); } finally { @@ -235,7 +235,7 @@ test('/v1/images/generations fails a stream that ended without a completed event test('/v1/images/edits streams a multipart request, whose stream field arrives as text', async () => { const { apiKey, repo } = await setupAppTest(); - await registerImagesModel(repo); + await registerOpenAIImagesModel(repo); let upstreamForm: FormData | undefined; await withMockedFetch( diff --git a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts index 88ff36a6d6..7d1fce498b 100644 --- a/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts +++ b/packages/gateway/__tests__/data-plane/pipeline/assembly_test.ts @@ -7,19 +7,19 @@ import { describe, expect, it } from 'vitest'; import { searchServePipeline } from '../../../src/data-plane/alpha-search/pipeline.ts'; -import { audioTranscriptionServePipeline } from '../../../src/data-plane/audio/pipeline.ts'; -import { completionsServePipeline } from '../../../src/data-plane/completions/pipeline.ts'; -import { embeddingsServePipeline } from '../../../src/data-plane/embeddings/pipeline.ts'; -import { imagesServePipeline } from '../../../src/data-plane/images/pipeline.ts'; +import { openaiAudioTranscriptionServePipeline } from '../../../src/data-plane/openai-audio/pipeline.ts'; +import { openaiCompletionsServePipeline } from '../../../src/data-plane/openai-completions/pipeline.ts'; +import { openaiEmbeddingsServePipeline } from '../../../src/data-plane/openai-embeddings/pipeline.ts'; +import { openaiImagesServePipeline } from '../../../src/data-plane/openai-images/pipeline.ts'; import { rerankServePipeline } from '../../../src/data-plane/rerank/pipeline.ts'; const FAMILIES: readonly (readonly [string, () => { readonly name: string }])[] = [ - ['embeddings', () => embeddingsServePipeline], + ['OpenAI Embeddings', () => openaiEmbeddingsServePipeline], ['rerank', () => rerankServePipeline({ sourceProtocol: 'cohere-v2', raw: {}, query: 'q', documents: ['a'] } as never)], - ['images generations', () => imagesServePipeline({ operation: 'generations', parameters: {} } as never)], - ['images edits', () => imagesServePipeline({ operation: 'edits', images: [], parameters: {} } as never)], - ['completions', () => completionsServePipeline], - ['audio transcription', () => audioTranscriptionServePipeline], + ['OpenAI Images Generations', () => openaiImagesServePipeline({ operation: 'generations', parameters: {} } as never)], + ['OpenAI Images Edits', () => openaiImagesServePipeline({ operation: 'edits', images: [], parameters: {} } as never)], + ['OpenAI Completions', () => openaiCompletionsServePipeline], + ['OpenAI Audio Transcriptions', () => openaiAudioTranscriptionServePipeline], ['alpha search', () => searchServePipeline({ kind: 'search' } as never)], ]; diff --git a/packages/gateway/src/data-plane/openai-audio/http.ts b/packages/gateway/src/data-plane/openai-audio/http.ts index 63a4e89615..036f15fb99 100644 --- a/packages/gateway/src/data-plane/openai-audio/http.ts +++ b/packages/gateway/src/data-plane/openai-audio/http.ts @@ -8,56 +8,56 @@ import type { Context } from 'hono'; -import { audioTranscriptionServePipeline } from './pipeline.ts'; +import { openaiAudioTranscriptionServePipeline } from './pipeline.ts'; import { isFrames, openPrologue, readIngress, serveThrough } from '../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; import { move } from '@floway-dev/pipeline'; -import { parseAudioTranscriptionResponseFormat, type AudioTranscriptionResponseFormat } from '@floway-dev/protocols/audio'; import { isMultipartFormDataMediaType } from '@floway-dev/protocols/common'; -import type { AudioTranscriptionFormEntry } from '@floway-dev/provider'; +import { parseOpenAIAudioTranscriptionResponseFormat, type OpenAIAudioTranscriptionResponseFormat } from '@floway-dev/protocols/openai-audio'; +import type { OpenAIAudioTranscriptionFormEntry } from '@floway-dev/provider'; -type PreparedTranscription = +type PreparedOpenAIAudioTranscription = | { readonly type: 'ok'; readonly model: string; - readonly responseFormat: AudioTranscriptionResponseFormat; + readonly responseFormat: OpenAIAudioTranscriptionResponseFormat; readonly wantsStream: boolean; - readonly entries: readonly AudioTranscriptionFormEntry[]; + readonly entries: readonly OpenAIAudioTranscriptionFormEntry[]; } | { readonly type: 'invalid'; readonly message: string }; -const prepareTranscription = async (bytes: Uint8Array, contentType: string | undefined): Promise => { +const prepareOpenAIAudioTranscription = async (bytes: Uint8Array, contentType: string | undefined): Promise => { if (!isMultipartFormDataMediaType(contentType)) { - return { type: 'invalid', message: 'Audio transcription request body must use multipart/form-data.' }; + return { type: 'invalid', message: 'OpenAI Audio Transcriptions request body must use multipart/form-data.' }; } let form: FormData; try { form = await new Response(bytes as BodyInit, { headers: { 'content-type': contentType } }).formData(); } catch { - return { type: 'invalid', message: 'Audio transcription request body must be valid multipart/form-data.' }; + return { type: 'invalid', message: 'OpenAI Audio Transcriptions request body must be valid multipart/form-data.' }; } const model = form.get('model'); if (typeof model !== 'string' || model.length === 0) { - return { type: 'invalid', message: 'Audio transcription request body must include a model field.' }; + return { type: 'invalid', message: 'OpenAI Audio Transcriptions request body must include a model field.' }; } const files = form.getAll('file'); if (files.length === 0 || files.some(file => !(file instanceof File))) { - return { type: 'invalid', message: 'Audio transcription request body must include a file upload.' }; + return { type: 'invalid', message: 'OpenAI Audio Transcriptions request body must include a file upload.' }; } const declaredFormat = form.get('response_format'); if (declaredFormat !== null && typeof declaredFormat !== 'string') { - return { type: 'invalid', message: 'Audio transcription response_format must be a text field.' }; + return { type: 'invalid', message: 'OpenAI Audio Transcriptions response_format must be a text field.' }; } - let responseFormat: AudioTranscriptionResponseFormat; + let responseFormat: OpenAIAudioTranscriptionResponseFormat; try { - responseFormat = parseAudioTranscriptionResponseFormat(declaredFormat ?? undefined); + responseFormat = parseOpenAIAudioTranscriptionResponseFormat(declaredFormat ?? undefined); } catch (error) { return { type: 'invalid', message: error instanceof Error ? error.message : String(error) }; } - const entries: AudioTranscriptionFormEntry[] = []; + const entries: OpenAIAudioTranscriptionFormEntry[] = []; for (const [name, value] of form.entries()) { entries.push({ name, value }); } @@ -92,9 +92,9 @@ const bodyOf = (rendered: Record | Uint8Array): Uint8Array => * while the request is live, on a promise that resolves with both. Give the seam these two * and this handler is `serveThrough` again. */ -export const audioTranscriptions = async (c: Context): Promise => { +export const openaiAudioTranscriptions = async (c: Context): Promise => { const ingress = await readIngress(c); - const request = await prepareTranscription(ingress.body.bytes, c.req.header('content-type')); + const request = await prepareOpenAIAudioTranscription(ingress.body.bytes, c.req.header('content-type')); if (request.type === 'invalid') { // A request the gateway could not read never reaches a pipeline: there is no model to // resolve and no attempt to make, so there is nothing for a run to record. @@ -111,20 +111,20 @@ export const audioTranscriptions = async (c: Context): Promise => { return await serveThrough( c, prologue, - audioTranscriptionServePipeline, + openaiAudioTranscriptionServePipeline, move({ 'ingress.http.headers': prologue.headers, - 'ingress.audioTranscription.responseFormat': request.responseFormat, - 'request.audioTranscription.form': request.entries, + 'ingress.openaiAudioTranscription.responseFormat': request.responseFormat, + 'request.openaiAudioTranscription.form': request.entries, 'serve.model': request.model, }) as never, facts => { - const rendered = facts['response.audioTranscription.rendered']; + const rendered = facts['response.openaiAudioTranscription.rendered']; if (isFrames(rendered)) return { frames: rendered }; // The upstream's own media type, or none where it declared none: a document this // gateway carried rather than wrote is not one it can describe. - return { body: bodyOf(rendered) as BodyInit, contentType: facts['response.audioTranscription.mediaType'] }; + return { body: bodyOf(rendered) as BodyInit, contentType: facts['response.openaiAudioTranscription.mediaType'] }; }, - facts => facts['response.audioTranscription.streamedOutcome'], + facts => facts['response.openaiAudioTranscription.streamedOutcome'], ); }; diff --git a/packages/gateway/src/data-plane/openai-audio/pipeline.ts b/packages/gateway/src/data-plane/openai-audio/pipeline.ts index 5a10514022..41420398c8 100644 --- a/packages/gateway/src/data-plane/openai-audio/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-audio/pipeline.ts @@ -1,17 +1,17 @@ -// Audio transcription as a pipeline. The family whose answer is not one document but six: -// `response_format` picks between three JSON objects and three text documents, `stream` +// OpenAI Audio Transcriptions as a pipeline. The family whose answer is not one document but +// six: `response_format` picks between three JSON objects and three text documents, `stream` // picks a sequence of events instead of any of them, and each is its own media type. That // is the whole reason this endpoint needed a response strategy of its own, and it is what // one canonical fact plus one media-type fact dissolve. // // The shape: // -// emitAudioTranscription the edge: writes the answer back in the rendering the -// client asked for, SSE framing included -// resolveCandidates narrows to the upstreams that expose the endpoint -// failover runs what follows once per candidate -// callAudioTranscriptionUpstream the ending: dials, reads what came back, and provides -// the answer plus what is billable +// emitOpenAIAudioTranscription the edge: writes the answer back in the rendering +// the client asked for, SSE framing included +// resolveCandidates narrows to the upstreams that expose the endpoint +// failover runs what follows once per candidate +// callOpenAIAudioTranscriptionUpstream the ending: dials, reads what came back, and +// provides the answer plus what is billable import type { UsageQuantities } from '../../repo/types.ts'; import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; @@ -24,20 +24,20 @@ import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/te import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import { compose, defineStage, move, own, type Logger, type Owned, type Pipeline } from '@floway-dev/pipeline'; -import { - isAudioTranscriptionDoneEvent, - parseAudioTranscription, - parseAudioTranscriptionStreamEvent, - parseAudioTranscriptionStreamUsage, - parseAudioTranscriptionUsage, - renderAudioTranscription, - type AudioTranscriptionResponseFormat, - type AudioTranscriptionStreamEvent, - type AudioTranscriptionUsage, - type CanonicalAudioTranscription, -} from '@floway-dev/protocols/audio'; import { isEventStreamMediaType, parseDecimalString, parseSSEStream, renderErrorEnvelope, sseFrame, type SseFrame } from '@floway-dev/protocols/common'; -import { providerModelOf, type AudioTranscriptionFormEntry, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; +import { + isOpenAIAudioTranscriptionDoneEvent, + parseOpenAIAudioTranscription, + parseOpenAIAudioTranscriptionStreamEvent, + parseOpenAIAudioTranscriptionStreamUsage, + parseOpenAIAudioTranscriptionUsage, + renderOpenAIAudioTranscription, + type OpenAIAudioTranscriptionResponseFormat, + type OpenAIAudioTranscriptionStreamEvent, + type OpenAIAudioTranscriptionUsage, + type CanonicalOpenAIAudioTranscription, +} from '@floway-dev/protocols/openai-audio'; +import { providerModelOf, type OpenAIAudioTranscriptionFormEntry, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; /** The answer while it is still the upstream's, one event at a time. It is a view and not a * resource: what owns the connection is `response.http.body`, which is where release and @@ -46,13 +46,13 @@ import { providerModelOf, type AudioTranscriptionFormEntry, type ModelCandidate, * A view is a wrapper around the generator rather than the generator itself, which is what * says where the resource is: the upstream's body at `response.http.body`, claimed with * `own()`, and nothing else here. */ -export type AudioTranscriptionEvents = AsyncIterable; +export type OpenAIAudioTranscriptionEvents = AsyncIterable; const viewOf = (events: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => events }); /** What settling this run will be told once the events run out: what the upstream metered, * and whether the transcript ever finished. */ -export interface AudioTranscriptionStreamOutcome { +export interface OpenAIAudioTranscriptionStreamOutcome { readonly billable: readonly BillableEntity[]; /** An upstream that stopped before `transcript.text.done` answered 200 and then did not * finish what it started, which is a failed request however much of the transcript @@ -60,45 +60,45 @@ export interface AudioTranscriptionStreamOutcome { readonly failed: boolean; } -/** Audio transcription's own keys, extending the shared space by intersection. */ -export interface AudioTranscriptionFacts extends GatewayFacts { +/** OpenAI Audio Transcriptions' own keys, extending the shared space by intersection. */ +export interface OpenAIAudioTranscriptionFacts extends GatewayFacts { /** Which of the six renderings the client asked for. It belongs to the ingress and stays * put: the same value travels to the upstream inside the form, so the rendering the * answer arrives in is the rendering the answer is written back in — and a text document * does not say which of the three it is, so nothing else could decide. */ - 'ingress.audioTranscription.responseFormat': AudioTranscriptionResponseFormat; + 'ingress.openaiAudioTranscription.responseFormat': OpenAIAudioTranscriptionResponseFormat; /** The multipart form as ordered semantic entries. The body is parsed before routing * because field order is unconstrained, and every candidate builds a fresh body from * these, so a retry never reuses a consumed one. The bytes the client sent are recorded * at `ingress.http.body`, which is where a dump reads the upload itself. */ - 'request.audioTranscription.form': readonly AudioTranscriptionFormEntry[]; + 'request.openaiAudioTranscription.form': readonly OpenAIAudioTranscriptionFormEntry[]; /** The one transcription, whichever rendering carried it — or the events it is arriving * as, or the failure that came instead. A stream, a value and a failure sit at one key: * telling them apart is reading a value, and each stage does that where it needs to. */ - 'response.audioTranscription.canonical': CanonicalAudioTranscription | AudioTranscriptionEvents | Failure; + 'response.openaiAudioTranscription.canonical': CanonicalOpenAIAudioTranscription | OpenAIAudioTranscriptionEvents | Failure; /** What the answer goes out under. A media type is upstream-owned — OpenAI answers * `text`, `srt` and `vtt` under one media type and other upstreams label them apart, and * the document beneath it is the upstream's own either way — so the upstream's travels, * and the edge names one only where the gateway wrote the body out of nothing the * upstream sent. `null` is an upstream that declared none, and it stays `null`: labelling * a body nobody described is a statement this gateway has no grounds to make. */ - 'response.audioTranscription.mediaType': string | null; + 'response.openaiAudioTranscription.mediaType': string | null; /** What the stream will have come to by the time the events run out, and `null` on every * path that does not stream. A streamed transcription states its usage in the terminal * event, which is after this run has answered — so the numbers cannot be in * `response.usage.billable`, which says what had been reported when the ending stage * handed up: the entity, and no quantities. Settling from this is the epilogue's job, * after the drain. */ - 'response.audioTranscription.streamedOutcome': Promise | null; + 'response.openaiAudioTranscription.streamedOutcome': Promise | null; /** What the client is actually sent — a JSON object, the upstream's own document, or the * SSE frames of a stream. The edge provides it, so a dump shows what the client received * rather than the gateway's own reading of it. */ - 'response.audioTranscription.rendered': Record | Uint8Array | AsyncIterable; + 'response.openaiAudioTranscription.rendered': Record | Uint8Array | AsyncIterable; } -type A = { [P in K]: AudioTranscriptionFacts[P] }; +type A = { [P in K]: OpenAIAudioTranscriptionFacts[P] }; -const isEvents = (answer: CanonicalAudioTranscription | AudioTranscriptionEvents): answer is AudioTranscriptionEvents => +const isEvents = (answer: CanonicalOpenAIAudioTranscription | OpenAIAudioTranscriptionEvents): answer is OpenAIAudioTranscriptionEvents => Symbol.asyncIterator in answer; /** @@ -114,30 +114,30 @@ const isEvents = (answer: CanonicalAudioTranscription | AudioTranscriptionEvents * OpenAI's clients read this endpoint's frames by their payload's `type` rather than by the * label. https://github.com/openai/openai-python/blob/10ee3f0da2ac6f93345c1204bd7bb1a2faa79ff2/src/openai/_streaming.py#L61-L107 */ -const emitAudioTranscription = defineStage< - A<'ingress.audioTranscription.responseFormat'>, - A<'ingress.audioTranscription.responseFormat'>, - A<'ingress.audioTranscription.responseFormat' | 'response.audioTranscription.canonical' | 'response.audioTranscription.mediaType'> +const emitOpenAIAudioTranscription = defineStage< + A<'ingress.openaiAudioTranscription.responseFormat'>, + A<'ingress.openaiAudioTranscription.responseFormat'>, + A<'ingress.openaiAudioTranscription.responseFormat' | 'response.openaiAudioTranscription.canonical' | 'response.openaiAudioTranscription.mediaType'> & { 'response.http.headers': readonly (readonly [string, string])[] }, - A<'response.audioTranscription.rendered' | 'response.audioTranscription.mediaType'> + A<'response.openaiAudioTranscription.rendered' | 'response.openaiAudioTranscription.mediaType'> & { 'response.http.status': number; 'response.http.headers': readonly (readonly [string, string])[] } >({ - name: 'emitAudioTranscription', + name: 'emitOpenAIAudioTranscription', through: { request: { - needs: ['ingress.audioTranscription.responseFormat'], + needs: ['ingress.openaiAudioTranscription.responseFormat'], consumes: [], provides: [], }, response: { - needs: ['response.audioTranscription.canonical', 'response.audioTranscription.mediaType', 'response.http.headers'], - consumes: ['response.audioTranscription.canonical', 'response.http.headers'], - provides: ['response.audioTranscription.rendered', 'response.audioTranscription.mediaType', 'response.http.status', 'response.http.headers'], + needs: ['response.openaiAudioTranscription.canonical', 'response.openaiAudioTranscription.mediaType', 'response.http.headers'], + consumes: ['response.openaiAudioTranscription.canonical', 'response.http.headers'], + provides: ['response.openaiAudioTranscription.rendered', 'response.openaiAudioTranscription.mediaType', 'response.http.status', 'response.http.headers'], }, }, execute: async (facts, next) => { const back = await next(facts); - const { 'response.audioTranscription.canonical': answer, 'response.http.headers': headers, ...rest } = back; + const { 'response.openaiAudioTranscription.canonical': answer, 'response.http.headers': headers, ...rest } = back; // Vendor traces and quota state stay visible; what an intermediary must strip, and what // would misdescribe a body this gateway serialized itself, does not. A filter that removed // nothing hands the same array on, so the record shows no change where none happened. @@ -149,8 +149,8 @@ const emitAudioTranscription = defineStage< ...rest, 'response.http.headers': forClient, 'response.http.status': answer.status, - 'response.audioTranscription.mediaType': 'application/json', - 'response.audioTranscription.rendered': move(renderErrorEnvelope(answer.message, answer.body)), + 'response.openaiAudioTranscription.mediaType': 'application/json', + 'response.openaiAudioTranscription.rendered': move(renderErrorEnvelope(answer.message, answer.body)), }; } // Everything that reaches here answered. The same key carried the upstream's own status @@ -160,14 +160,14 @@ const emitAudioTranscription = defineStage< ...rest, 'response.http.headers': forClient, 'response.http.status': 200, - 'response.audioTranscription.rendered': move(isEvents(answer) + 'response.openaiAudioTranscription.rendered': move(isEvents(answer) ? renderSSE(answer) - : renderAudioTranscription(back['ingress.audioTranscription.responseFormat'], answer)), + : renderOpenAIAudioTranscription(back['ingress.openaiAudioTranscription.responseFormat'], answer)), }; }, }); -const renderSSE = (events: AudioTranscriptionEvents): AsyncIterable => +const renderSSE = (events: OpenAIAudioTranscriptionEvents): AsyncIterable => viewOf((async function* () { for await (const event of events) yield sseFrame(JSON.stringify(event)); })()); @@ -180,20 +180,20 @@ const renderSSE = (events: AudioTranscriptionEvents): AsyncIterable => * upstream sent rather than serializing one from what it read, so a body no reading could * open is still that upstream's answer and there is nothing to try the next candidate for. */ -const callAudioTranscriptionUpstream = defineStage< - A<'ingress.audioTranscription.responseFormat' | 'request.audioTranscription.form' | 'route.attempt' | 'ingress.http.headers'>, - A<'response.audioTranscription.canonical' | 'response.audioTranscription.mediaType' | 'response.audioTranscription.streamedOutcome'> +const callOpenAIAudioTranscriptionUpstream = defineStage< + A<'ingress.openaiAudioTranscription.responseFormat' | 'request.openaiAudioTranscription.form' | 'route.attempt' | 'ingress.http.headers'>, + A<'response.openaiAudioTranscription.canonical' | 'response.openaiAudioTranscription.mediaType' | 'response.openaiAudioTranscription.streamedOutcome'> & { 'response.usage.billable': readonly BillableEntity[]; 'response.http.status': number; 'response.http.headers': readonly (readonly [string, string])[]; 'response.http.body': ReadableStream & Owned; }, GatewayServices >({ - name: 'callAudioTranscriptionUpstream', + name: 'callOpenAIAudioTranscriptionUpstream', return: { provides: [ - 'response.audioTranscription.canonical', - 'response.audioTranscription.mediaType', - 'response.audioTranscription.streamedOutcome', + 'response.openaiAudioTranscription.canonical', + 'response.openaiAudioTranscription.mediaType', + 'response.openaiAudioTranscription.streamedOutcome', 'response.usage.billable', 'response.http.status', 'response.http.headers', @@ -208,9 +208,9 @@ const callAudioTranscriptionUpstream = defineStage< let result; try { - result = await candidate.provider.instance.callAudioTranscriptions( + result = await candidate.provider.instance.callOpenAIAudioTranscriptions( providerModelOf(candidate), - { entries: facts['request.audioTranscription.form'] }, + { entries: facts['request.openaiAudioTranscription.form'] }, use.gateway.abortSignal, // The client's own headers reach the upstream from the record, not from a live request // object: what a provider is allowed to forward is filtered per provider, and the dump @@ -223,9 +223,9 @@ const callAudioTranscriptionUpstream = defineStage< // no headers to carry. What it leaves behind is the performance row settlement writes. return move({ ...facts, - 'response.audioTranscription.canonical': dialFailure(error), - 'response.audioTranscription.mediaType': null, - 'response.audioTranscription.streamedOutcome': null, + 'response.openaiAudioTranscription.canonical': dialFailure(error), + 'response.openaiAudioTranscription.mediaType': null, + 'response.openaiAudioTranscription.streamedOutcome': null, 'response.usage.billable': [], 'response.http.status': 502, 'response.http.headers': [], @@ -233,7 +233,7 @@ const callAudioTranscriptionUpstream = defineStage< }); } const identity = telemetryModelIdentity(candidate, result.modelKey); - const format = facts['ingress.audioTranscription.responseFormat']; + const format = facts['ingress.openaiAudioTranscription.responseFormat']; const status = result.response.status; const mediaType = result.response.headers.get('content-type'); const headers = move([...result.response.headers] as readonly (readonly [string, string])[]); @@ -247,9 +247,9 @@ const callAudioTranscriptionUpstream = defineStage< // leaves a losing attempt with nothing open behind it. return move({ ...facts, - 'response.audioTranscription.canonical': await refusal(status, result.response), - 'response.audioTranscription.mediaType': mediaType, - 'response.audioTranscription.streamedOutcome': null, + 'response.openaiAudioTranscription.canonical': await refusal(status, result.response), + 'response.openaiAudioTranscription.mediaType': mediaType, + 'response.openaiAudioTranscription.streamedOutcome': null, 'response.usage.billable': called, 'response.http.status': status, 'response.http.headers': headers, @@ -265,9 +265,9 @@ const callAudioTranscriptionUpstream = defineStage< if (result.response.body === null) { return move({ ...facts, - 'response.audioTranscription.canonical': { status: 502, message: 'Upstream returned a streaming response with no body.' }, - 'response.audioTranscription.mediaType': mediaType, - 'response.audioTranscription.streamedOutcome': null, + 'response.openaiAudioTranscription.canonical': { status: 502, message: 'Upstream returned a streaming response with no body.' }, + 'response.openaiAudioTranscription.mediaType': mediaType, + 'response.openaiAudioTranscription.streamedOutcome': null, 'response.usage.billable': called, 'response.http.status': status, 'response.http.headers': headers, @@ -280,9 +280,9 @@ const callAudioTranscriptionUpstream = defineStage< const metered = meterEvents(result.response.body, identity, use.gateway.abortSignal, use.log); return move({ ...facts, - 'response.audioTranscription.canonical': metered.events, - 'response.audioTranscription.mediaType': mediaType, - 'response.audioTranscription.streamedOutcome': metered.outcome, + 'response.openaiAudioTranscription.canonical': metered.events, + 'response.openaiAudioTranscription.mediaType': mediaType, + 'response.openaiAudioTranscription.streamedOutcome': metered.outcome, 'response.usage.billable': called, 'response.http.status': status, 'response.http.headers': headers, @@ -299,9 +299,9 @@ const callAudioTranscriptionUpstream = defineStage< const read = readTranscription(format, new Uint8Array(await result.response.arrayBuffer()), use.log); return move({ ...facts, - 'response.audioTranscription.canonical': read.canonical, - 'response.audioTranscription.mediaType': mediaType, - 'response.audioTranscription.streamedOutcome': null, + 'response.openaiAudioTranscription.canonical': read.canonical, + 'response.openaiAudioTranscription.mediaType': mediaType, + 'response.openaiAudioTranscription.streamedOutcome': null, 'response.usage.billable': [{ identity, quantities: billed(read.usage) }], 'response.http.status': status, 'response.http.headers': headers, @@ -338,27 +338,27 @@ const refusal = async (status: number, response: Response): Promise => * one whose document it could not open is still billed for nothing rather than mis-billed. */ const readTranscription = ( - format: AudioTranscriptionResponseFormat, + format: OpenAIAudioTranscriptionResponseFormat, document: Uint8Array, log: Logger, -): { readonly canonical: CanonicalAudioTranscription; readonly usage: AudioTranscriptionUsage | undefined } => { - let canonical: CanonicalAudioTranscription; +): { readonly canonical: CanonicalOpenAIAudioTranscription; readonly usage: OpenAIAudioTranscriptionUsage | undefined } => { + let canonical: CanonicalOpenAIAudioTranscription; try { - canonical = parseAudioTranscription(format, document); + canonical = parseOpenAIAudioTranscription(format, document); } catch (error) { log.warn('failed to parse 2xx upstream body for /audio/transcriptions; forwarding it as it arrived', { error: String(error) }); return { canonical: { document }, usage: undefined }; } // Only an object rendering states usage: `text`, `srt` and `vtt` have nowhere to put it, // and asking them for one is what would warn about every subtitle a client requests. - return { canonical, usage: canonical.raw === undefined ? undefined : readUsage(() => parseAudioTranscriptionUsage(canonical.raw), log) }; + return { canonical, usage: canonical.raw === undefined ? undefined : readUsage(() => parseOpenAIAudioTranscriptionUsage(canonical.raw), log) }; }; /** A transcription states its usage in two places — the body of an object rendering and the * terminal event of a stream — and either can state it in a shape this gateway cannot read. * That upstream metered something, and the request is recorded saying exactly that: the * entity, and no quantities. */ -const readUsage = (read: () => AudioTranscriptionUsage | undefined, log: Logger): AudioTranscriptionUsage | undefined => { +const readUsage = (read: () => OpenAIAudioTranscriptionUsage | undefined, log: Logger): OpenAIAudioTranscriptionUsage | undefined => { try { return read(); } catch (error) { @@ -368,8 +368,8 @@ const readUsage = (read: () => AudioTranscriptionUsage | undefined, log: Logger) }; interface MeteredEvents { - readonly events: AudioTranscriptionEvents; - readonly outcome: Promise; + readonly events: OpenAIAudioTranscriptionEvents; + readonly outcome: Promise; } const meterEvents = ( @@ -378,16 +378,16 @@ const meterEvents = ( signal: AbortSignal | undefined, log: Logger, ): MeteredEvents => { - let settle!: (outcome: AudioTranscriptionStreamOutcome) => void; - const outcome = new Promise(resolve => { settle = resolve; }); + let settle!: (outcome: OpenAIAudioTranscriptionStreamOutcome) => void; + const outcome = new Promise(resolve => { settle = resolve; }); const events = viewOf((async function* () { - let usage: AudioTranscriptionUsage | undefined; + let usage: OpenAIAudioTranscriptionUsage | undefined; let completed = false; try { for await (const frame of parseSSEStream(body, { signal })) { - const event = parseAudioTranscriptionStreamEvent(JSON.parse(frame.data) as unknown); - if (isAudioTranscriptionDoneEvent(event)) { - usage = readUsage(() => parseAudioTranscriptionStreamUsage(event), log); + const event = parseOpenAIAudioTranscriptionStreamEvent(JSON.parse(frame.data) as unknown); + if (isOpenAIAudioTranscriptionDoneEvent(event)) { + usage = readUsage(() => parseOpenAIAudioTranscriptionStreamUsage(event), log); completed = true; yield event; // The transcript is complete, so there is nothing further to read. An upstream that @@ -407,7 +407,7 @@ const meterEvents = ( return { events, outcome }; }; -const billed = (usage: AudioTranscriptionUsage | undefined): UsageQuantities => { +const billed = (usage: OpenAIAudioTranscriptionUsage | undefined): UsageQuantities => { if (usage === undefined) return {}; if (usage.kind === 'duration') return { input_audio_seconds: parseDecimalString(String(usage.seconds)) }; // Audio input is priced apart from text input, so what stays on the general input metric is @@ -425,37 +425,37 @@ const billed = (usage: AudioTranscriptionUsage | undefined): UsageQuantities => const narrowing = { kind: 'transcription' as const, reject: (candidate: ModelCandidate): string | null => - candidate.model.endpoints.audioTranscriptions === undefined - ? 'the upstream does not expose an audio transcription endpoint' + candidate.model.endpoints.openaiAudioTranscriptions === undefined + ? 'the upstream does not expose an OpenAI Audio Transcriptions endpoint' : null, unsupported: (model: string) => `Model ${model} does not support the /audio/transcriptions endpoint.`, refuse: (status: number, message: string) => ({ - 'response.audioTranscription.canonical': { status, message } as Failure, - 'response.audioTranscription.mediaType': null, - 'response.audioTranscription.streamedOutcome': null, + 'response.openaiAudioTranscription.canonical': { status, message } as Failure, + 'response.openaiAudioTranscription.mediaType': null, + 'response.openaiAudioTranscription.streamedOutcome': null, }), refuses: [ - 'response.audioTranscription.canonical', - 'response.audioTranscription.mediaType', - 'response.audioTranscription.streamedOutcome', + 'response.openaiAudioTranscription.canonical', + 'response.openaiAudioTranscription.mediaType', + 'response.openaiAudioTranscription.streamedOutcome', ] as const, }; -export const audioTranscriptionServePipeline: Pipeline< - A<'ingress.http.headers' | 'ingress.audioTranscription.responseFormat' | 'request.audioTranscription.form' | 'serve.model'>, - A<'response.audioTranscription.rendered' | 'response.audioTranscription.mediaType' | 'response.audioTranscription.streamedOutcome'> +export const openaiAudioTranscriptionServePipeline: Pipeline< + A<'ingress.http.headers' | 'ingress.openaiAudioTranscription.responseFormat' | 'request.openaiAudioTranscription.form' | 'serve.model'>, + A<'response.openaiAudioTranscription.rendered' | 'response.openaiAudioTranscription.mediaType' | 'response.openaiAudioTranscription.streamedOutcome'> & { 'response.http.status': number; 'response.usage.billable': readonly BillableEntity[]; 'response.http.headers': readonly (readonly [string, string])[]; } -> = compose('audioTranscriptionServe', [ - emitAudioTranscription, +> = compose('openaiAudioTranscriptionServe', [ + emitOpenAIAudioTranscription, writeSettlement( - handedUp => isFailure((handedUp as { 'response.audioTranscription.canonical'?: unknown })['response.audioTranscription.canonical']), - handedUp => (handedUp as { 'response.audioTranscription.streamedOutcome'?: unknown })['response.audioTranscription.streamedOutcome'] !== null, + handedUp => isFailure((handedUp as { 'response.openaiAudioTranscription.canonical'?: unknown })['response.openaiAudioTranscription.canonical']), + handedUp => (handedUp as { 'response.openaiAudioTranscription.streamedOutcome'?: unknown })['response.openaiAudioTranscription.streamedOutcome'] !== null, ), resolveCandidates(narrowing), failover({ - failed: handedUp => isFailure((handedUp as { 'response.audioTranscription.canonical'?: unknown })['response.audioTranscription.canonical']), + failed: handedUp => isFailure((handedUp as { 'response.openaiAudioTranscription.canonical'?: unknown })['response.openaiAudioTranscription.canonical']), owns: ['response.http.body'], }), - callAudioTranscriptionUpstream, + callOpenAIAudioTranscriptionUpstream, ]); diff --git a/packages/gateway/src/data-plane/openai-completions/http.ts b/packages/gateway/src/data-plane/openai-completions/http.ts index 5cf9420944..596c0931ea 100644 --- a/packages/gateway/src/data-plane/openai-completions/http.ts +++ b/packages/gateway/src/data-plane/openai-completions/http.ts @@ -1,21 +1,21 @@ // POST /v1/completions and /completions, served through the pipeline. // -// The handler is a prologue and an epilogue around `completionsServePipeline`: read what the -// client sent, hand it over, and turn what the run answered with into a response. The one -// thing it decides for itself is whether the request streams, because that is written in the -// body and the run has to be opened knowing it. +// The handler is a prologue and an epilogue around `openaiCompletionsServePipeline`: read +// what the client sent, hand it over, and turn what the run answered with into a response. The +// one thing it decides for itself is whether the request streams, because that is written in +// the body and the run has to be opened knowing it. import type { Context } from 'hono'; -import { completionsServePipeline } from './pipeline.ts'; +import { openaiCompletionsServePipeline } from './pipeline.ts'; import { isFrames, openPrologue, readIngress, serveThrough } from '../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; import { prepareJsonModelRequest } from '../shared/json-model-request.ts'; import { move } from '@floway-dev/pipeline'; -export const completions = async (c: Context): Promise => { +export const openaiCompletions = async (c: Context): Promise => { const ingress = await readIngress(c); - const request = prepareJsonModelRequest(ingress.body.bytes, 'Completions'); + const request = prepareJsonModelRequest(ingress.body.bytes, 'OpenAI Completions'); if (request.type === 'invalid') { // A request the gateway could not read never reaches a pipeline: there is no model to // resolve and no attempt to make, so there is nothing for a run to record. @@ -34,22 +34,22 @@ export const completions = async (c: Context): Promise => { return await serveThrough( c, prologue, - completionsServePipeline, + openaiCompletionsServePipeline, move({ 'ingress.http.headers': prologue.headers, - 'ingress.completions.wantsStream': wantsStream, + 'ingress.openaiCompletions.wantsStream': wantsStream, // Whether the client asked to *see* the usage chunk. The edge turns it on upstream // either way so billing always gets one, and drops it here when it was not asked for. - 'ingress.completions.wantsUsageChunk': streamOptions?.include_usage === true, - 'request.completions.payload': request.body, + 'ingress.openaiCompletions.wantsUsageChunk': streamOptions?.include_usage === true, + 'request.openaiCompletions.payload': request.body, 'serve.model': request.model, }) as never, facts => { - const rendered = facts['response.completions.rendered']; + const rendered = facts['response.openaiCompletions.rendered']; return isFrames(rendered) ? { frames: rendered } : { body: JSON.stringify(rendered), contentType: 'application/json' }; }, - facts => facts['response.completions.streamedUsage'], + facts => facts['response.openaiCompletions.streamedUsage'], ); }; diff --git a/packages/gateway/src/data-plane/openai-completions/pipeline.ts b/packages/gateway/src/data-plane/openai-completions/pipeline.ts index 6305885abc..aa9818432f 100644 --- a/packages/gateway/src/data-plane/openai-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-completions/pipeline.ts @@ -4,15 +4,15 @@ // // The shape: // -// emitCompletions the edge: asks the upstream for the usage chunk on the way -// down, and renders the answer into the client's protocol on the -// way back, SSE framing included -// resolveCandidates narrows to the upstreams that expose the endpoint -// failover runs what follows once per candidate -// callCompletionsUpstream the ending: dials, parses what came back, and provides the -// answer plus what is billable +// emitOpenAICompletions the edge: asks the upstream for the usage chunk on the way +// down, and renders the answer into the client's protocol on +// the way back, SSE framing included +// resolveCandidates narrows to the upstreams that expose the endpoint +// failover runs what follows once per candidate +// callOpenAICompletionsUpstream the ending: dials, parses what came back, and provides the +// answer plus what is billable -import { tokenUsageFromCompletionsUsage } from './usage.ts'; +import { tokenUsageFromOpenAICompletionsUsage } from './usage.ts'; import type { UsageQuantities } from '../../repo/types.ts'; import { tokenUsageQuantities } from '../../repo/usage-metrics.ts'; import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; @@ -28,58 +28,58 @@ import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import { compose, defineStage, move, own, type Owned, type Pipeline } from '@floway-dev/pipeline'; import { isOpenAIUsageOnlyEventShape, renderErrorEnvelope, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { - completionsProtocolFrameToSSEFrame, - parseCompletionsResult, - parseCompletionsStream, - type CompletionsPayload, - type CompletionsResult, - type CompletionsStreamEvent, -} from '@floway-dev/protocols/completions'; + openaiCompletionsProtocolFrameToSSEFrame, + parseOpenAICompletionsResult, + parseOpenAICompletionsStream, + type OpenAICompletionsPayload, + type OpenAICompletionsResult, + type OpenAICompletionsStreamEvent, +} from '@floway-dev/protocols/openai-completions'; import { providerModelOf } from '@floway-dev/provider'; import type { ModelCandidate, TelemetryModelIdentity } from '@floway-dev/provider'; /** The answer while it is still the upstream's, one frame at a time. */ -export type CompletionsFrames = AsyncIterable>; +export type OpenAICompletionsFrames = AsyncIterable>; /** * A stream as a value the record can hold: a wrapper around the generator rather than the * generator itself, which is what says where the resource is. There is exactly one resource in - * a completions run — the upstream's body, claimed with `own()` — and the wrapper keeps a frame - * view from reading as another. + * an OpenAI Completions run — the upstream's body, claimed with `own()` — and the wrapper keeps + * a frame view from reading as another. * * What comes back is single-shot on purpose: a stream that has been read is a stream that is * over, and a second reader learns that rather than being lied to. */ const view = (frames: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => frames }); -/** Completions' own keys, extending the shared space by intersection. */ -export interface CompletionsFacts extends GatewayFacts { +/** OpenAI Completions' own keys, extending the shared space by intersection. */ +export interface OpenAICompletionsFacts extends GatewayFacts { /** What the client asked for, which is not what the upstream is asked for: the gateway * meters every stream and so always turns the usage chunk on. These stay put for the same * reason every `ingress.*` key does — they describe the request that arrived, and the * answer is rendered back into it. */ - 'ingress.completions.wantsStream': boolean; - 'ingress.completions.wantsUsageChunk': boolean; - 'request.completions.payload': CompletionsPayload; + 'ingress.openaiCompletions.wantsStream': boolean; + 'ingress.openaiCompletions.wantsUsageChunk': boolean; + 'request.openaiCompletions.payload': OpenAICompletionsPayload; /** The answer, whichever kind it turned out to be. A stream, a value and a failure sit at * one key: telling them apart is reading a value, and each stage does that where it needs * to. */ - 'response.completions.payload': CompletionsResult | CompletionsFrames | Failure; + 'response.openaiCompletions.payload': OpenAICompletionsResult | OpenAICompletionsFrames | Failure; /** What the upstream will have reported by the time the frames run out, and `null` on * every path that does not stream. A stream's usage arrives with its last chunk, which is * after this run has answered — so the numbers cannot be in `response.usage.billable`, * which says what had been reported when the ending stage handed up: the entity, and no * quantities. Settling billing from this is the prologue's job, after the drain. */ - 'response.completions.streamedUsage': Promise | null; + 'response.openaiCompletions.streamedUsage': Promise | null; /** What the client is actually sent, in its own protocol — a JSON body, or the SSE frames * of a stream. The edge provides it, so a dump shows what the client received rather than * the gateway's own reading of it. */ - 'response.completions.rendered': Record | AsyncIterable; + 'response.openaiCompletions.rendered': Record | AsyncIterable; } -type C = { [P in K]: CompletionsFacts[P] }; +type C = { [P in K]: OpenAICompletionsFacts[P] }; -const isFrames = (answer: CompletionsFacts['response.completions.payload']): answer is CompletionsFrames => +const isFrames = (answer: OpenAICompletionsFacts['response.openaiCompletions.payload']): answer is OpenAICompletionsFrames => Symbol.asyncIterator in answer; /** @@ -91,35 +91,35 @@ const isFrames = (answer: CompletionsFacts['response.completions.payload']): ans * below this stage a stream carries protocol frames, so the same assembly would serve * another transport by rendering differently at this one point. */ -const emitCompletions = defineStage< - C<'ingress.completions.wantsStream' | 'ingress.completions.wantsUsageChunk' | 'request.completions.payload'>, - C<'ingress.completions.wantsStream' | 'ingress.completions.wantsUsageChunk' | 'request.completions.payload'>, - C<'ingress.completions.wantsUsageChunk' | 'response.completions.payload' | 'response.http.headers'>, - C<'response.completions.rendered' | 'response.http.status' | 'response.http.headers'> +const emitOpenAICompletions = defineStage< + C<'ingress.openaiCompletions.wantsStream' | 'ingress.openaiCompletions.wantsUsageChunk' | 'request.openaiCompletions.payload'>, + C<'ingress.openaiCompletions.wantsStream' | 'ingress.openaiCompletions.wantsUsageChunk' | 'request.openaiCompletions.payload'>, + C<'ingress.openaiCompletions.wantsUsageChunk' | 'response.openaiCompletions.payload' | 'response.http.headers'>, + C<'response.openaiCompletions.rendered' | 'response.http.status' | 'response.http.headers'> >({ - name: 'emitCompletions', + name: 'emitOpenAICompletions', through: { request: { - needs: ['ingress.completions.wantsStream', 'ingress.completions.wantsUsageChunk', 'request.completions.payload'], + needs: ['ingress.openaiCompletions.wantsStream', 'ingress.openaiCompletions.wantsUsageChunk', 'request.openaiCompletions.payload'], consumes: [], - provides: ['request.completions.payload'], + provides: ['request.openaiCompletions.payload'], }, response: { - needs: ['response.completions.payload', 'response.http.headers'], - consumes: ['response.completions.payload', 'response.http.headers'], - provides: ['response.completions.rendered', 'response.http.status', 'response.http.headers'], + needs: ['response.openaiCompletions.payload', 'response.http.headers'], + consumes: ['response.openaiCompletions.payload', 'response.http.headers'], + provides: ['response.openaiCompletions.rendered', 'response.http.status', 'response.http.headers'], }, }, execute: async (facts, next) => { - const asked = facts['request.completions.payload']; + const asked = facts['request.openaiCompletions.payload']; const back = await next({ ...facts, - 'request.completions.payload': move(facts['ingress.completions.wantsStream'] + 'request.openaiCompletions.payload': move(facts['ingress.openaiCompletions.wantsStream'] ? { ...asked, stream_options: { ...asked.stream_options, include_usage: true } } : asked), }); - const { 'response.completions.payload': answer, 'response.http.headers': headers, ...rest } = back; + const { 'response.openaiCompletions.payload': answer, 'response.http.headers': headers, ...rest } = back; // Vendor traces and quota state stay visible; what an intermediary must strip, and what // would misdescribe a body this gateway serialized itself, does not. A filter that removed // nothing hands the same array on, so the record shows no change where none happened. @@ -131,23 +131,23 @@ const emitCompletions = defineStage< ...rest, 'response.http.headers': forwardable.length === headers.length ? headers : move(forwardable), 'response.http.status': isFailure(answer) ? answer.status : 200, - 'response.completions.rendered': move(rendered(answer, back['ingress.completions.wantsUsageChunk'])), + 'response.openaiCompletions.rendered': move(rendered(answer, back['ingress.openaiCompletions.wantsUsageChunk'])), }; }, }); const rendered = ( - answer: CompletionsFacts['response.completions.payload'], + answer: OpenAICompletionsFacts['response.openaiCompletions.payload'], wantsUsageChunk: boolean, -): CompletionsFacts['response.completions.rendered'] => +): OpenAICompletionsFacts['response.openaiCompletions.rendered'] => isFailure(answer) ? renderErrorEnvelope(answer.message, answer.body) : isFrames(answer) ? renderSSE(answer, wantsUsageChunk) : answer; -const renderSSE = (frames: CompletionsFrames, wantsUsageChunk: boolean): AsyncIterable => view((async function* () { +const renderSSE = (frames: OpenAICompletionsFrames, wantsUsageChunk: boolean): AsyncIterable => view((async function* () { for await (const frame of frames) { if (!wantsUsageChunk && frame.type === 'event' && isOpenAIUsageOnlyEventShape(frame.event)) continue; - yield completionsProtocolFrameToSSEFrame(frame); + yield openaiCompletionsProtocolFrameToSSEFrame(frame); } })()); @@ -157,17 +157,17 @@ const renderSSE = (frames: CompletionsFrames, wantsUsageChunk: boolean): AsyncIt * A failure is a value: a 429 here is what an earlier stage fails over, and so is a 200 whose * body this protocol cannot read, because the next candidate's path and flags may differ. */ -const callCompletionsUpstream = defineStage< - C<'ingress.completions.wantsStream' | 'request.completions.payload' | 'route.attempt' | 'ingress.http.headers'>, - C<'response.completions.payload' | 'response.completions.streamedUsage' | 'response.usage.billable' +const callOpenAICompletionsUpstream = defineStage< + C<'ingress.openaiCompletions.wantsStream' | 'request.openaiCompletions.payload' | 'route.attempt' | 'ingress.http.headers'>, + C<'response.openaiCompletions.payload' | 'response.openaiCompletions.streamedUsage' | 'response.usage.billable' | 'response.http.status' | 'response.http.headers' | 'response.http.body'>, GatewayServices >({ - name: 'callCompletionsUpstream', + name: 'callOpenAICompletionsUpstream', return: { provides: [ - 'response.completions.payload', - 'response.completions.streamedUsage', + 'response.openaiCompletions.payload', + 'response.openaiCompletions.streamedUsage', 'response.usage.billable', 'response.http.status', 'response.http.headers', @@ -178,14 +178,14 @@ const callCompletionsUpstream = defineStage< const candidate = use.resolveAttempt(facts['route.attempt']); // The provider re-stamps whatever id it resolved upstream, so the id the client // addressed does not travel with the body. - const { model: _addressed, ...body } = facts['request.completions.payload']; + const { model: _addressed, ...body } = facts['request.openaiCompletions.payload']; // Attribution is set before the dial, so an attempt that never completes still names the // candidate it was made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'text_completion'); let result; try { - result = await candidate.provider.instance.callCompletions( + result = await candidate.provider.instance.callOpenAICompletions( providerModelOf(candidate), body, use.gateway.abortSignal, @@ -200,8 +200,8 @@ const callCompletionsUpstream = defineStage< // no headers to carry. What it leaves behind is the performance row settlement writes. return move({ ...facts, - 'response.completions.payload': dialFailure(error), - 'response.completions.streamedUsage': null, + 'response.openaiCompletions.payload': dialFailure(error), + 'response.openaiCompletions.streamedUsage': null, 'response.usage.billable': [], 'response.http.status': 502, 'response.http.headers': [], @@ -212,7 +212,7 @@ const callCompletionsUpstream = defineStage< const answer = await readUpstream( result.response, - facts['ingress.completions.wantsStream'], + facts['ingress.openaiCompletions.wantsStream'], telemetryModelIdentity(candidate, result.modelKey), candidate, use.gateway.abortSignal, @@ -220,8 +220,8 @@ const callCompletionsUpstream = defineStage< ); return move({ ...facts, - 'response.completions.payload': answer.payload, - 'response.completions.streamedUsage': answer.streamedUsage, + 'response.openaiCompletions.payload': answer.payload, + 'response.openaiCompletions.streamedUsage': answer.streamedUsage, 'response.usage.billable': answer.billable, 'response.http.status': result.response.status, 'response.http.headers': [...result.response.headers], @@ -231,8 +231,8 @@ const callCompletionsUpstream = defineStage< }); interface UpstreamAnswer { - readonly payload: CompletionsFacts['response.completions.payload']; - readonly streamedUsage: CompletionsFacts['response.completions.streamedUsage']; + readonly payload: OpenAICompletionsFacts['response.openaiCompletions.payload']; + readonly streamedUsage: OpenAICompletionsFacts['response.openaiCompletions.streamedUsage']; readonly billable: readonly BillableEntity[]; /** Every arm hands one up, this stage's own reading included: the record holds a body as a * stream, and `failover` releases the losing attempts' by consuming that key. */ @@ -247,7 +247,7 @@ const readUpstream = async ( identity: TelemetryModelIdentity, candidate: ModelCandidate, signal: AbortSignal | undefined, - record: (frame: ProtocolFrame) => void, + record: (frame: ProtocolFrame) => void, ): Promise => { // An upstream was called and reported nothing — which is what every arm but a read usage // block leaves standing, and is a different statement from reporting zero. @@ -282,7 +282,7 @@ const readUpstream = async ( // the frames as they pass — so the reading costs one pass and the client's own stream is // what drives it. What it finds arrives with the last frame, long after this stage has // handed up, which is why the entity above carries no quantities. - const metered = meterFrames(parseCompletionsStream(response.body, { signal }), identity, candidate, record); + const metered = meterFrames(parseOpenAICompletionsStream(response.body, { signal }), identity, candidate, record); return { payload: metered.frames, streamedUsage: metered.outcome, @@ -313,24 +313,24 @@ const refusal = async (response: Response): Promise => { /** A body that answered 200 and cannot be read as this protocol is an attempt that failed, * because the gateway serializes what it sends from the value it parsed and has nothing to * serialize. The next candidate may well answer in the protocol it advertised. */ -const readResult = async (response: Response): Promise<{ readonly value: CompletionsResult } | Failure> => { +const readResult = async (response: Response): Promise<{ readonly value: OpenAICompletionsResult } | Failure> => { try { - return { value: parseCompletionsResult(await response.json()) }; + return { value: parseOpenAICompletionsResult(await response.json()) }; } catch (error) { return { status: 502, message: `Upstream answered ${response.status} with a body this endpoint cannot read: ${error instanceof Error ? error.message : String(error)}` }; } }; interface MeteredFrames { - readonly frames: CompletionsFrames; + readonly frames: OpenAICompletionsFrames; readonly outcome: Promise; } const meterFrames = ( - source: AsyncIterable>, + source: AsyncIterable>, identity: TelemetryModelIdentity, candidate: ModelCandidate, - record: (frame: ProtocolFrame) => void, + record: (frame: ProtocolFrame) => void, ): MeteredFrames => { let settle!: (outcome: StreamOutcome) => void; const outcome = new Promise(resolve => { settle = resolve; }); @@ -368,7 +368,7 @@ const meterFrames = ( const billed = (usage: unknown, tier: string | null | undefined, candidate: ModelCandidate): UsageQuantities => { const model = providerModelOf(candidate); - const tokens = tokenUsageFromCompletionsUsage( + const tokens = tokenUsageFromOpenAICompletionsUsage( usage, tier, model.enabledFlags.has('usage-exclusive-cached-tokens'), @@ -389,28 +389,28 @@ const billed = (usage: unknown, tier: string | null | undefined, candidate: Mode const narrowing = { kind: 'chat' as const, reject: (candidate: ModelCandidate): string | null => - candidate.model.endpoints.completions === undefined ? 'the upstream does not expose a completions endpoint' : null, + candidate.model.endpoints.openaiCompletions === undefined ? 'the upstream does not expose an OpenAI Completions endpoint' : null, unsupported: (model: string) => `Model ${model} does not support the /completions endpoint.`, - refuse: (status: number, message: string): C<'response.completions.payload' | 'response.completions.streamedUsage'> => ({ - 'response.completions.payload': { status, message }, - 'response.completions.streamedUsage': null, + refuse: (status: number, message: string): C<'response.openaiCompletions.payload' | 'response.openaiCompletions.streamedUsage'> => ({ + 'response.openaiCompletions.payload': { status, message }, + 'response.openaiCompletions.streamedUsage': null, }), - refuses: ['response.completions.payload', 'response.completions.streamedUsage'] as const, + refuses: ['response.openaiCompletions.payload', 'response.openaiCompletions.streamedUsage'] as const, }; -export const completionsServePipeline: Pipeline< - C<'ingress.http.headers' | 'ingress.completions.wantsStream' | 'ingress.completions.wantsUsageChunk' | 'request.completions.payload' | 'serve.model'>, - C<'response.completions.rendered' | 'response.completions.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> -> = compose('completionsServe', [ - emitCompletions, +export const openaiCompletionsServePipeline: Pipeline< + C<'ingress.http.headers' | 'ingress.openaiCompletions.wantsStream' | 'ingress.openaiCompletions.wantsUsageChunk' | 'request.openaiCompletions.payload' | 'serve.model'>, + C<'response.openaiCompletions.rendered' | 'response.openaiCompletions.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> +> = compose('openaiCompletionsServe', [ + emitOpenAICompletions, writeSettlement( - handedUp => isFailure((handedUp as { 'response.completions.payload'?: unknown })['response.completions.payload']), - handedUp => (handedUp as { 'response.completions.streamedUsage'?: unknown })['response.completions.streamedUsage'] !== null, + handedUp => isFailure((handedUp as { 'response.openaiCompletions.payload'?: unknown })['response.openaiCompletions.payload']), + handedUp => (handedUp as { 'response.openaiCompletions.streamedUsage'?: unknown })['response.openaiCompletions.streamedUsage'] !== null, ), resolveCandidates(narrowing), failover({ - failed: handedUp => isFailure((handedUp as { 'response.completions.payload'?: unknown })['response.completions.payload']), + failed: handedUp => isFailure((handedUp as { 'response.openaiCompletions.payload'?: unknown })['response.openaiCompletions.payload']), owns: ['response.http.body'], }), - callCompletionsUpstream, + callOpenAICompletionsUpstream, ]); diff --git a/packages/gateway/src/data-plane/openai-embeddings/http.ts b/packages/gateway/src/data-plane/openai-embeddings/http.ts index a9183a77e7..2882e60f36 100644 --- a/packages/gateway/src/data-plane/openai-embeddings/http.ts +++ b/packages/gateway/src/data-plane/openai-embeddings/http.ts @@ -1,28 +1,28 @@ // POST /v1/embeddings, served through the pipeline. // -// The handler is a prologue and an epilogue around `embeddingsServePipeline`: parse what +// The handler is a prologue and an epilogue around `openaiEmbeddingsServePipeline`: parse what // the client sent, hand it over, and turn what the run answered with into a response. // Everything between is stages. import type { Context } from 'hono'; -import { embeddingsServePipeline } from './pipeline.ts'; +import { openaiEmbeddingsServePipeline } from './pipeline.ts'; import { openPrologue, readIngress, serveThrough } from '../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; import { move } from '@floway-dev/pipeline'; -import { parseEmbeddingsRequest, type ParsedEmbeddingsRequest } from '@floway-dev/protocols/embeddings'; +import { parseOpenAIEmbeddingsRequest, type ParsedOpenAIEmbeddingsRequest } from '@floway-dev/protocols/openai-embeddings'; // The contract reports a malformed request by throwing; what the client is owed is a 400 // carrying the reason. -const readRequest = (bytes: Uint8Array): { type: 'ok'; parsed: ParsedEmbeddingsRequest } | { type: 'invalid'; message: string } => { +const readRequest = (bytes: Uint8Array): { type: 'ok'; parsed: ParsedOpenAIEmbeddingsRequest } | { type: 'invalid'; message: string } => { try { - return { type: 'ok', parsed: parseEmbeddingsRequest(JSON.parse(new TextDecoder().decode(bytes)) as unknown) }; + return { type: 'ok', parsed: parseOpenAIEmbeddingsRequest(JSON.parse(new TextDecoder().decode(bytes)) as unknown) }; } catch (error) { return { type: 'invalid', message: error instanceof Error ? error.message : String(error) }; } }; -export const embeddings = async (c: Context): Promise => { +export const openaiEmbeddings = async (c: Context): Promise => { const ingress = await readIngress(c); const result = readRequest(ingress.body.bytes); if (result.type === 'invalid') { @@ -42,13 +42,13 @@ export const embeddings = async (c: Context): Promise => { return await serveThrough( c, prologue, - embeddingsServePipeline, + openaiEmbeddingsServePipeline, move({ 'ingress.http.headers': prologue.headers, - 'ingress.embeddings.encodingFormat': request.encodingFormat, - 'request.embeddings.canonical': request, + 'ingress.openaiEmbeddings.encodingFormat': request.encodingFormat, + 'request.openaiEmbeddings.canonical': request, 'serve.model': model, }) as never, - facts => ({ body: JSON.stringify(facts['response.embeddings.rendered']), contentType: 'application/json' }), + facts => ({ body: JSON.stringify(facts['response.openaiEmbeddings.rendered']), contentType: 'application/json' }), ); }; diff --git a/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts b/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts index 011ce5e4af..3ed13760b5 100644 --- a/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts @@ -1,13 +1,14 @@ -// Embeddings as a pipeline. The smallest family there is: one protocol, no translation and -// no stream — so what is left is the four stages every family has, and nothing else. +// OpenAI Embeddings as a pipeline. The smallest family there is: one protocol, no translation +// and no stream — so what is left is the four stages every family has, and nothing else. // -// emitEmbeddings the edge: writes the answer in the encoding the client asked for -// resolveCandidates narrows to the upstreams that can serve this request -// failover runs what follows once per candidate -// callEmbeddingsUpstream the ending: dials, and provides what came back +// emitOpenAIEmbeddings the edge: writes the answer in the encoding the client asked +// for +// resolveCandidates narrows to the upstreams that can serve this request +// failover runs what follows once per candidate +// callOpenAIEmbeddingsUpstream the ending: dials, and provides what came back // // The narrowing is a constant rather than a function of the request, and that is what makes -// this the simple family: an embeddings request carries nothing a candidate could be +// this the simple family: an OpenAI Embeddings request carries nothing a candidate could be // incompatible with, so the only question is whether the upstream exposes the endpoint. import type { UsageQuantities } from '../../repo/types.ts'; @@ -24,31 +25,31 @@ import type { Pipeline } from '@floway-dev/pipeline'; import { compose, defineStage, move } from '@floway-dev/pipeline'; import { parseDecimalString, renderErrorEnvelope, upstreamErrorMessage } from '@floway-dev/protocols/common'; import { - parseEmbeddingsResponse, - renderEmbeddingsResponse, - serializeEmbeddingsRequest, - type CanonicalEmbeddingsRequest, - type CanonicalEmbeddingsResponse, - type CanonicalEmbeddingsUsage, - type EmbeddingsEncodingFormat, -} from '@floway-dev/protocols/embeddings'; + parseOpenAIEmbeddingsResponse, + renderOpenAIEmbeddingsResponse, + serializeOpenAIEmbeddingsRequest, + type CanonicalOpenAIEmbeddingsRequest, + type CanonicalOpenAIEmbeddingsResponse, + type CanonicalOpenAIEmbeddingsUsage, + type OpenAIEmbeddingsEncodingFormat, +} from '@floway-dev/protocols/openai-embeddings'; import { providerModelOf } from '@floway-dev/provider'; -/** Embeddings' own keys. They extend the shared space and never merge into it, so a stage +/** OpenAI Embeddings' own keys. They extend the shared space and never merge into it, so a stage * written against the gateway alone cannot name one. */ -export interface EmbeddingsFacts extends GatewayFacts { +export interface OpenAIEmbeddingsFacts extends GatewayFacts { /** Which encoding the client is able to read. It belongs to the ingress and stays put: * the answer is written in it whichever encoding the upstream chose to answer in, and * the two differ whenever an upstream ignores the field. */ - 'ingress.embeddings.encodingFormat': EmbeddingsEncodingFormat; - 'request.embeddings.canonical': CanonicalEmbeddingsRequest; - 'response.embeddings.canonical': CanonicalEmbeddingsResponse | Failure; + 'ingress.openaiEmbeddings.encodingFormat': OpenAIEmbeddingsEncodingFormat; + 'request.openaiEmbeddings.canonical': CanonicalOpenAIEmbeddingsRequest; + 'response.openaiEmbeddings.canonical': CanonicalOpenAIEmbeddingsResponse | Failure; /** What the client is actually sent. The edge provides it, so a dump shows the bytes the * client received rather than the gateway's canonical form. */ - 'response.embeddings.rendered': Record; + 'response.openaiEmbeddings.rendered': Record; } -type E = { [P in K]: EmbeddingsFacts[P] }; +type E = { [P in K]: OpenAIEmbeddingsFacts[P] }; /** * The outermost edge. Writes the vectors in the encoding the client asked for — which is @@ -56,28 +57,28 @@ type E = { [P in K]: EmbeddingsFacts[P] }; * that answered in the other one. A client on an official OpenAI SDK asked for `base64` * without its caller choosing to, and will decode as base64 whatever it is handed. */ -const emitEmbeddings = defineStage< - E<'ingress.embeddings.encodingFormat'>, - E<'ingress.embeddings.encodingFormat'>, - E<'ingress.embeddings.encodingFormat' | 'response.embeddings.canonical' | 'response.http.headers'>, - E<'response.embeddings.rendered' | 'response.http.status' | 'response.http.headers'> +const emitOpenAIEmbeddings = defineStage< + E<'ingress.openaiEmbeddings.encodingFormat'>, + E<'ingress.openaiEmbeddings.encodingFormat'>, + E<'ingress.openaiEmbeddings.encodingFormat' | 'response.openaiEmbeddings.canonical' | 'response.http.headers'>, + E<'response.openaiEmbeddings.rendered' | 'response.http.status' | 'response.http.headers'> >({ - name: 'emitEmbeddings', + name: 'emitOpenAIEmbeddings', through: { request: { - needs: ['ingress.embeddings.encodingFormat'], + needs: ['ingress.openaiEmbeddings.encodingFormat'], consumes: [], provides: [], }, response: { - needs: ['response.embeddings.canonical', 'response.http.headers'], - consumes: ['response.embeddings.canonical', 'response.http.headers'], - provides: ['response.embeddings.rendered', 'response.http.status', 'response.http.headers'], + needs: ['response.openaiEmbeddings.canonical', 'response.http.headers'], + consumes: ['response.openaiEmbeddings.canonical', 'response.http.headers'], + provides: ['response.openaiEmbeddings.rendered', 'response.http.status', 'response.http.headers'], }, }, execute: async (facts, next) => { const back = await next(facts); - const { 'response.embeddings.canonical': answer, 'response.http.headers': headers, ...rest } = back; + const { 'response.openaiEmbeddings.canonical': answer, 'response.http.headers': headers, ...rest } = back; // Vendor traces and quota state stay visible; what an intermediary must strip, and what // would misdescribe a body this gateway serialized itself, does not. A filter that removed // nothing hands the same array on, so the record shows no change where none happened. @@ -87,7 +88,7 @@ const emitEmbeddings = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.embeddings.rendered': move(renderErrorEnvelope(answer.message, answer.body)), + 'response.openaiEmbeddings.rendered': move(renderErrorEnvelope(answer.message, answer.body)), // The upstream's own status, or the gateway's own when it refused before dialling. // A client is not owed the upstream's exact bytes; it is owed the truth about what // happened, and a 429 arriving as a 200 is not that. @@ -98,7 +99,7 @@ const emitEmbeddings = defineStage< ...rest, 'response.http.headers': forClient, 'response.http.status': 200, - 'response.embeddings.rendered': move(renderEmbeddingsResponse(back['ingress.embeddings.encodingFormat'], answer)), + 'response.openaiEmbeddings.rendered': move(renderOpenAIEmbeddingsResponse(back['ingress.openaiEmbeddings.encodingFormat'], answer)), }; }, }); @@ -108,14 +109,14 @@ const emitEmbeddings = defineStage< * what the call is billable for. A failure is a value: a 429 here is what an earlier stage * fails over, and even a 400 can be, because the next candidate's path and flags may differ. */ -const callEmbeddingsUpstream = defineStage< - E<'request.embeddings.canonical' | 'route.attempt' | 'ingress.http.headers' | 'serve.model'>, - E<'response.embeddings.canonical' | 'response.http.headers' | 'response.usage.billable'>, +const callOpenAIEmbeddingsUpstream = defineStage< + E<'request.openaiEmbeddings.canonical' | 'route.attempt' | 'ingress.http.headers' | 'serve.model'>, + E<'response.openaiEmbeddings.canonical' | 'response.http.headers' | 'response.usage.billable'>, GatewayServices >({ - name: 'callEmbeddingsUpstream', + name: 'callOpenAIEmbeddingsUpstream', return: { - provides: ['response.embeddings.canonical', 'response.http.headers', 'response.usage.billable'], + provides: ['response.openaiEmbeddings.canonical', 'response.http.headers', 'response.usage.billable'], }, execute: async (facts, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); @@ -125,9 +126,9 @@ const callEmbeddingsUpstream = defineStage< let result; try { - result = await candidate.provider.instance.callEmbeddings( + result = await candidate.provider.instance.callOpenAIEmbeddings( providerModelOf(candidate), - serializeEmbeddingsRequest(facts['request.embeddings.canonical']), + serializeOpenAIEmbeddingsRequest(facts['request.openaiEmbeddings.canonical']), use.gateway.abortSignal, // The client's own headers reach the upstream from the record, not from a live // request object: what a provider is allowed to forward is filtered per provider, @@ -140,7 +141,7 @@ const callEmbeddingsUpstream = defineStage< // no headers to carry. What it leaves behind is the performance row settlement writes. return move({ ...facts, - 'response.embeddings.canonical': dialFailure(error), + 'response.openaiEmbeddings.canonical': dialFailure(error), 'response.http.headers': [], 'response.usage.billable': [], }); @@ -152,9 +153,9 @@ const callEmbeddingsUpstream = defineStage< const body = await readUpstreamBody(result.response); // The upstream was called and reported nothing, which is a different situation from // reporting zero — so the entity is present with no quantities. - const answered = (canonical: CanonicalEmbeddingsResponse | Failure, quantities: UsageQuantities) => move({ + const answered = (canonical: CanonicalOpenAIEmbeddingsResponse | Failure, quantities: UsageQuantities) => move({ ...facts, - 'response.embeddings.canonical': canonical, + 'response.openaiEmbeddings.canonical': canonical, 'response.http.headers': headers, 'response.usage.billable': [{ identity, quantities }], }); @@ -169,50 +170,50 @@ const callEmbeddingsUpstream = defineStage< }, reportedNothing); } if (!('json' in body)) { - return answered(unreadableBody(result.response, body, 'the embeddings protocol'), reportedNothing); + return answered(unreadableBody(result.response, body, 'the OpenAI Embeddings protocol'), reportedNothing); } // Every protocol the gateway carries is one it fully understands: the body is parsed // here and written again at the edge, in whichever encoding the client can read. - let canonical: CanonicalEmbeddingsResponse; + let canonical: CanonicalOpenAIEmbeddingsResponse; try { - canonical = parseEmbeddingsResponse(body.json, facts['serve.model']); + canonical = parseOpenAIEmbeddingsResponse(body.json, facts['serve.model']); } catch (error) { - use.log.warn('upstream answered with a body the embeddings protocol cannot read', { error: String(error) }); - return answered(unreadableBody(result.response, body, 'the embeddings protocol'), reportedNothing); + use.log.warn('upstream answered with a body the OpenAI Embeddings protocol cannot read', { error: String(error) }); + return answered(unreadableBody(result.response, body, 'the OpenAI Embeddings protocol'), reportedNothing); } return answered(canonical, billed(canonical.usage)); }, }); -// An embeddings call has no output side, so `prompt_tokens` is the whole of what the +// An OpenAI Embeddings call has no output side, so `prompt_tokens` is the whole of what the // upstream metered and `total_tokens` restates it. An upstream that reported nothing bills // nothing, and says so by leaving the entity's quantities empty. -const billed = (usage: CanonicalEmbeddingsUsage | undefined): UsageQuantities => +const billed = (usage: CanonicalOpenAIEmbeddingsUsage | undefined): UsageQuantities => usage === undefined ? {} : { input_tokens: parseDecimalString(String(usage.promptTokens)) }; /** A candidate that cannot serve *this* request is not a candidate. Saying why is what * turns an empty list into a 400 a client can act on. */ -const narrowing: Narrowing> = { +const narrowing: Narrowing> = { kind: 'embedding', - reject: candidate => candidate.model.endpoints.embeddings === undefined - ? 'the upstream does not expose an embeddings endpoint' + reject: candidate => candidate.model.endpoints.openaiEmbeddings === undefined + ? 'the upstream does not expose an OpenAI Embeddings endpoint' : null, unsupported: model => `Model ${model} does not support the /embeddings endpoint.`, - refuse: (status, message) => ({ 'response.embeddings.canonical': { status, message } }), - refuses: ['response.embeddings.canonical'], + refuse: (status, message) => ({ 'response.openaiEmbeddings.canonical': { status, message } }), + refuses: ['response.openaiEmbeddings.canonical'], }; -export const embeddingsServePipeline: Pipeline< - E<'ingress.http.headers' | 'ingress.embeddings.encodingFormat' | 'request.embeddings.canonical' | 'serve.model'>, - E<'response.embeddings.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> -> = compose('embeddingsServe', [ - emitEmbeddings, - writeSettlement(handedUp => isFailure((handedUp as { 'response.embeddings.canonical'?: unknown })['response.embeddings.canonical'])), +export const openaiEmbeddingsServePipeline: Pipeline< + E<'ingress.http.headers' | 'ingress.openaiEmbeddings.encodingFormat' | 'request.openaiEmbeddings.canonical' | 'serve.model'>, + E<'response.openaiEmbeddings.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> +> = compose('openaiEmbeddingsServe', [ + emitOpenAIEmbeddings, + writeSettlement(handedUp => isFailure((handedUp as { 'response.openaiEmbeddings.canonical'?: unknown })['response.openaiEmbeddings.canonical'])), resolveCandidates(narrowing), failover({ - failed: handedUp => isFailure((handedUp as { 'response.embeddings.canonical'?: unknown })['response.embeddings.canonical']), + failed: handedUp => isFailure((handedUp as { 'response.openaiEmbeddings.canonical'?: unknown })['response.openaiEmbeddings.canonical']), owns: [], }), - callEmbeddingsUpstream, + callOpenAIEmbeddingsUpstream, ]); diff --git a/packages/gateway/src/data-plane/openai-images/http.ts b/packages/gateway/src/data-plane/openai-images/http.ts index 099259e718..6ecac99d63 100644 --- a/packages/gateway/src/data-plane/openai-images/http.ts +++ b/packages/gateway/src/data-plane/openai-images/http.ts @@ -6,21 +6,21 @@ import type { Context } from 'hono'; -import { imagesServePipeline } from './pipeline.ts'; +import { openaiImagesServePipeline } from './pipeline.ts'; import { isFrames, openPrologue, readIngress, serveThrough, type Ingress } from '../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; import { move } from '@floway-dev/pipeline'; -import { imagesRequestWantsStream, parseImagesEditsRequest, parseImagesGenerationsRequest, type ParsedImagesRequest } from '@floway-dev/protocols/images'; +import { openaiImagesRequestWantsStream, parseOpenAIImagesEditsRequest, parseOpenAIImagesGenerationsRequest, type ParsedOpenAIImagesRequest } from '@floway-dev/protocols/openai-images'; /** The half both endpoints share: whatever the contract read, hand it over and turn what the * run answered with into a response. The contract reports a malformed request by throwing; * what the client is owed is a 400 carrying the reason. */ -const serveImages = async ( +const serveOpenAIImages = async ( c: Context, ingress: Ingress, - read: () => ParsedImagesRequest | Promise, + read: () => ParsedOpenAIImagesRequest | Promise, ): Promise => { - let parsed: ParsedImagesRequest; + let parsed: ParsedOpenAIImagesRequest; try { parsed = await read(); } catch (error) { @@ -41,35 +41,35 @@ const serveImages = async ( // Whether the answer streams is written in the request the client sent, and the run has to be // opened knowing it: the abort controller a streaming run cancels its read with is minted // from this flag. - const wantsStream = imagesRequestWantsStream(request); + const wantsStream = openaiImagesRequestWantsStream(request); const prologue = openPrologue(c, ingress, { wantsStream, model }); return await serveThrough( c, prologue, - imagesServePipeline(request), + openaiImagesServePipeline(request), move({ 'ingress.http.headers': prologue.headers, - 'ingress.images.wantsStream': wantsStream, - 'request.images.canonical': request, + 'ingress.openaiImages.wantsStream': wantsStream, + 'request.openaiImages.canonical': request, 'serve.model': model, }) as never, facts => { - const answer = facts['response.images.rendered']; + const answer = facts['response.openaiImages.rendered']; return isFrames(answer) ? { frames: answer } : { body: JSON.stringify(answer), contentType: 'application/json' }; }, - facts => facts['response.images.streamedUsage'], + facts => facts['response.openaiImages.streamedUsage'], ); }; -export const imagesGenerations = async (c: Context): Promise => { +export const openaiImagesGenerations = async (c: Context): Promise => { const ingress = await readIngress(c); - return await serveImages(c, ingress, () => parseImagesGenerationsRequest(ingress.body.bytes)); + return await serveOpenAIImages(c, ingress, () => parseOpenAIImagesGenerationsRequest(ingress.body.bytes)); }; -export const imagesEdits = async (c: Context): Promise => { +export const openaiImagesEdits = async (c: Context): Promise => { const ingress = await readIngress(c); // Which of the two bodies arrived is a header's statement and not the payload's, so the // contract is handed the media type alongside the bytes. - return await serveImages(c, ingress, () => parseImagesEditsRequest(c.req.header('content-type'), ingress.body.bytes)); + return await serveOpenAIImages(c, ingress, () => parseOpenAIImagesEditsRequest(c.req.header('content-type'), ingress.body.bytes)); }; diff --git a/packages/gateway/src/data-plane/openai-images/pipeline.ts b/packages/gateway/src/data-plane/openai-images/pipeline.ts index 25dda9588b..21cc41870c 100644 --- a/packages/gateway/src/data-plane/openai-images/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-images/pipeline.ts @@ -1,15 +1,15 @@ -// Images as a pipeline. Two endpoints and one family: `generations` sends JSON and `edits` -// sends either JSON or a multipart form, so what the operation decides is the request fact and -// the provider method the ending calls, and the four stages are the same either way. Both -// endpoints accept `stream: true`, so the answer is a stream whenever the client asked for one -// — the same request field decides both what the upstream is asked for and what shape comes +// OpenAI Images as a pipeline. Two endpoints and one family: `generations` sends JSON and +// `edits` sends either JSON or a multipart form, so what the operation decides is the request +// fact and the provider method the ending calls, and the four stages are the same either way. +// Both endpoints accept `stream: true`, so the answer is a stream whenever the client asked for +// one — the same request field decides both what the upstream is asked for and what shape comes // back. // -// emitImages the edge: serializes the answer into the images protocol, SSE -// framing included -// resolveCandidates narrows to the upstreams that expose this operation's endpoint -// failover runs what follows once per candidate -// callImagesUpstream the ending: dials, and provides what came back +// emitOpenAIImages the edge: serializes the answer into the OpenAI Images protocol, +// SSE framing included +// resolveCandidates narrows to the upstreams that expose this operation's endpoint +// failover runs what follows once per candidate +// callOpenAIImagesUpstream the ending: dials, and provides what came back import type { UsageQuantities } from '../../repo/types.ts'; import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; @@ -34,74 +34,74 @@ import { type SseFrame, } from '@floway-dev/protocols/common'; import { - IMAGES_MISSING_TERMINAL_MESSAGE, - imagesStreamEventToSSEFrame, - isImagesTerminalEvent, - parseImagesResponse, - parseImagesStream, - parseImagesUsage, - renderImagesResponse, - type CanonicalImagesEditsRequest, - type CanonicalImagesRequest, - type CanonicalImagesResponse, - type CanonicalImagesUsage, - type ImageEditReference, - type ImagesEditImage, - type ImagesOperation, - type ImagesStreamEvent, -} from '@floway-dev/protocols/images'; + OPENAI_IMAGES_MISSING_TERMINAL_MESSAGE, + openaiImagesStreamEventToSSEFrame, + isOpenAIImagesTerminalEvent, + parseOpenAIImagesResponse, + parseOpenAIImagesStream, + parseOpenAIImagesUsage, + renderOpenAIImagesResponse, + type CanonicalOpenAIImagesEditsRequest, + type CanonicalOpenAIImagesRequest, + type CanonicalOpenAIImagesResponse, + type CanonicalOpenAIImagesUsage, + type OpenAIImageEditReference, + type OpenAIImagesEditImage, + type OpenAIImagesOperation, + type OpenAIImagesStreamEvent, +} from '@floway-dev/protocols/openai-images'; import { isBase64ImageDataUrl, providerModelOf } from '@floway-dev/provider'; -import type { ImagesEditsRequest, ImagesEditsSource, ModelCandidate, PerformanceOperation, TelemetryModelIdentity } from '@floway-dev/provider'; +import type { OpenAIImagesEditsRequest, OpenAIImagesEditsSource, ModelCandidate, PerformanceOperation, TelemetryModelIdentity } from '@floway-dev/provider'; /** The answer while it is still the upstream's, one event at a time. There is no transport * sentinel below the events — this protocol ends at its completed event — so what travels is * the events themselves rather than frames with a terminal arm the protocol has not got. */ -export type ImagesFrames = AsyncIterable; +export type OpenAIImagesFrames = AsyncIterable; /** A stream as a value the record can hold, as a wrapper around the generator rather than the * generator itself. What the wrapper says is where the resource is: the one resource in an - * images run is the upstream's body at `response.http.body`, claimed with `own()`, and this - * keeps a frame view from reading as another. */ + * OpenAI Images run is the upstream's body at `response.http.body`, claimed with `own()`, and + * this keeps a frame view from reading as another. */ const view = (frames: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => frames }); -/** Images' own keys. They extend the shared space and never merge into it, so a stage written - * against the gateway alone cannot name one. */ -export interface ImagesFacts extends GatewayFacts { +/** OpenAI Images' own keys. They extend the shared space and never merge into it, so a stage + * written against the gateway alone cannot name one. */ +export interface OpenAIImagesFacts extends GatewayFacts { /** Whether the client asked for the answer as a stream. It stays put, as every `ingress.*` * key does: the same flag travels to the upstream inside the parameters, and the answer is * written back in the shape it asked for. */ - 'ingress.images.wantsStream': boolean; - 'request.images.canonical': CanonicalImagesRequest; + 'ingress.openaiImages.wantsStream': boolean; + 'request.openaiImages.canonical': CanonicalOpenAIImagesRequest; /** The answer, whichever kind it turned out to be. A stream, a value and a failure sit at * one key: telling them apart is reading a value, and each stage does that where it needs * to. */ - 'response.images.canonical': CanonicalImagesResponse | ImagesFrames | Failure; + 'response.openaiImages.canonical': CanonicalOpenAIImagesResponse | OpenAIImagesFrames | Failure; /** What the upstream will have reported by the time the events run out, and `null` on every * path that does not stream. A streamed image states its usage in the completed event, * which is after this run has answered — so the numbers cannot be in * `response.usage.billable`, which says what had been reported when the ending stage handed * up: the entity, and no quantities. Settling from this is the epilogue's job, after the * drain. */ - 'response.images.streamedUsage': Promise | null; - /** What the client is actually sent, in the images protocol — a JSON body, or the SSE frames - * of a stream. The edge provides it, so a dump shows the body the client received rather - * than the gateway's canonical form. */ - 'response.images.rendered': Record | AsyncIterable; + 'response.openaiImages.streamedUsage': Promise | null; + /** What the client is actually sent, in the OpenAI Images protocol — a JSON body, or the SSE + * frames of a stream. The edge provides it, so a dump shows the body the client received + * rather than the gateway's canonical form. */ + 'response.openaiImages.rendered': Record | AsyncIterable; } -type I = { [P in K]: ImagesFacts[P] }; +type I = { [P in K]: OpenAIImagesFacts[P] }; const ENDPOINT = { - generations: 'imagesGenerations', - edits: 'imagesEdits', -} as const satisfies Record; + generations: 'openaiImagesGenerations', + edits: 'openaiImagesEdits', +} as const satisfies Record; const PERFORMANCE_OPERATION = { generations: 'image_generation', edits: 'image_edit', -} as const satisfies Record; +} as const satisfies Record; -const isFrames = (answer: CanonicalImagesResponse | ImagesFrames): answer is ImagesFrames => +const isFrames = (answer: CanonicalOpenAIImagesResponse | OpenAIImagesFrames): answer is OpenAIImagesFrames => Symbol.asyncIterator in answer; /** @@ -114,24 +114,24 @@ const isFrames = (answer: CanonicalImagesResponse | ImagesFrames): answer is Ima * events, so the same assembly would serve another transport by rendering differently at this * one point. */ -const emitImages = defineStage< +const emitOpenAIImages = defineStage< Record, Record, - I<'response.images.canonical' | 'response.http.headers'>, - I<'response.images.rendered' | 'response.http.status' | 'response.http.headers'> + I<'response.openaiImages.canonical' | 'response.http.headers'>, + I<'response.openaiImages.rendered' | 'response.http.status' | 'response.http.headers'> >({ - name: 'emitImages', + name: 'emitOpenAIImages', through: { request: { needs: [], consumes: [], provides: [] }, response: { - needs: ['response.images.canonical', 'response.http.headers'], - consumes: ['response.images.canonical', 'response.http.headers'], - provides: ['response.images.rendered', 'response.http.status', 'response.http.headers'], + needs: ['response.openaiImages.canonical', 'response.http.headers'], + consumes: ['response.openaiImages.canonical', 'response.http.headers'], + provides: ['response.openaiImages.rendered', 'response.http.status', 'response.http.headers'], }, }, execute: async (facts, next) => { const back = await next(facts); - const { 'response.images.canonical': answer, 'response.http.headers': headers, ...rest } = back; + const { 'response.openaiImages.canonical': answer, 'response.http.headers': headers, ...rest } = back; // Vendor traces and quota state stay visible; what an intermediary must strip, and what // would misdescribe a body this gateway serialized itself, does not. A filter that removed // nothing hands the same array on, so the record shows no change where none happened. @@ -140,18 +140,18 @@ const emitImages = defineStage< ...rest, 'response.http.headers': forwardable.length === headers.length ? headers : move(forwardable), 'response.http.status': isFailure(answer) ? answer.status : 200, - 'response.images.rendered': move(rendered(answer)), + 'response.openaiImages.rendered': move(rendered(answer)), }; }, }); -const rendered = (answer: ImagesFacts['response.images.canonical']): ImagesFacts['response.images.rendered'] => +const rendered = (answer: OpenAIImagesFacts['response.openaiImages.canonical']): OpenAIImagesFacts['response.openaiImages.rendered'] => isFailure(answer) ? renderErrorEnvelope(answer.message, answer.body) : isFrames(answer) ? renderSSE(answer) - : renderImagesResponse(answer); + : renderOpenAIImagesResponse(answer); -const renderSSE = (frames: ImagesFrames): AsyncIterable => view((async function* () { - for await (const event of frames) yield imagesStreamEventToSSEFrame(event); +const renderSSE = (frames: OpenAIImagesFrames): AsyncIterable => view((async function* () { + for await (const event of frames) yield openaiImagesStreamEventToSSEFrame(event); })()); /** @@ -161,17 +161,17 @@ const renderSSE = (frames: ImagesFrames): AsyncIterable => view((async * answered with something this protocol cannot read, are both outcomes the fork above can take * to the next candidate rather than faults that end the run. */ -const callImagesUpstream = defineStage< - I<'ingress.images.wantsStream' | 'request.images.canonical' | 'route.attempt' | 'ingress.http.headers'>, - I<'response.images.canonical' | 'response.images.streamedUsage' | 'response.http.headers' +const callOpenAIImagesUpstream = defineStage< + I<'ingress.openaiImages.wantsStream' | 'request.openaiImages.canonical' | 'route.attempt' | 'ingress.http.headers'>, + I<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.headers' | 'response.http.body' | 'response.usage.billable'>, GatewayServices >({ - name: 'callImagesUpstream', + name: 'callOpenAIImagesUpstream', return: { provides: [ - 'response.images.canonical', - 'response.images.streamedUsage', + 'response.openaiImages.canonical', + 'response.openaiImages.streamedUsage', 'response.http.headers', 'response.http.body', 'response.usage.billable', @@ -179,7 +179,7 @@ const callImagesUpstream = defineStage< }, execute: async (facts, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); - const request = facts['request.images.canonical']; + const request = facts['request.openaiImages.canonical']; const options = buildUpstreamCallOptions( candidate, use.gateway, @@ -200,16 +200,16 @@ const callImagesUpstream = defineStage< // call would lose the usage reading and save nothing. Reading a stream is the other half of // that and does take the signal — what it drops there is a connection, not an image. result = request.operation === 'generations' - ? await candidate.provider.instance.callImagesGenerations(model, request.parameters, undefined, options) - : await candidate.provider.instance.callImagesEdits(model, providerEditsRequest(request), undefined, options); + ? await candidate.provider.instance.callOpenAIImagesGenerations(model, request.parameters, undefined, options) + : await candidate.provider.instance.callOpenAIImagesEdits(model, providerEditsRequest(request), undefined, options); } catch (error) { use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); // A dial that never completed reached no upstream, so nothing was billed and there are // no headers to carry. What it leaves behind is the performance row settlement writes. return move({ ...facts, - 'response.images.canonical': dialFailure(error), - 'response.images.streamedUsage': null, + 'response.openaiImages.canonical': dialFailure(error), + 'response.openaiImages.streamedUsage': null, 'response.http.headers': [], 'response.http.body': spentBody(null), 'response.usage.billable': [], @@ -222,10 +222,10 @@ const callImagesUpstream = defineStage< // An entity with no quantities is how "the upstream was called and reported nothing" is // said, which is a different situation from reporting zero. const called: readonly BillableEntity[] = [{ identity, quantities: {} }]; - const read = (canonical: CanonicalImagesResponse | Failure, billable: readonly BillableEntity[]) => move({ + const read = (canonical: CanonicalOpenAIImagesResponse | Failure, billable: readonly BillableEntity[]) => move({ ...facts, - 'response.images.canonical': canonical, - 'response.images.streamedUsage': null, + 'response.openaiImages.canonical': canonical, + 'response.openaiImages.streamedUsage': null, 'response.http.headers': headers, 'response.http.body': spentBody(result.response.body), 'response.usage.billable': billable, @@ -248,7 +248,7 @@ const callImagesUpstream = defineStage< // ignores it answers the single JSON body the arm below reads — so what settles which // shape arrived is the media type, and what settles which shape the client is owed is the // request. - if (facts['ingress.images.wantsStream'] && isEventStreamMediaType(mediaType)) { + if (facts['ingress.openaiImages.wantsStream'] && isEventStreamMediaType(mediaType)) { if (result.response.body === null) { return read({ status: 502, message: 'Upstream returned a streaming response with no body.' }, called); } @@ -264,8 +264,8 @@ const callImagesUpstream = defineStage< ); return move({ ...facts, - 'response.images.canonical': metered.frames, - 'response.images.streamedUsage': metered.outcome, + 'response.openaiImages.canonical': metered.frames, + 'response.openaiImages.streamedUsage': metered.outcome, 'response.http.headers': headers, // Releasing this body is reading those events to the end: they are one reader over one // connection, and a second reader is not something a `ReadableStream` allows. @@ -282,17 +282,17 @@ const callImagesUpstream = defineStage< use.log.warn('upstream answered with a body that is not JSON', { status: result.response.status, mediaType: essence }); return read({ status: 502, - message: `The upstream answered ${result.response.status} with ${essence}, and the images protocol is JSON.`, + message: `The upstream answered ${result.response.status} with ${essence}, and the OpenAI Images protocol is JSON.`, body: body.text, }, called); } - let canonical: CanonicalImagesResponse; + let canonical: CanonicalOpenAIImagesResponse; try { - canonical = parseImagesResponse(body.json); + canonical = parseOpenAIImagesResponse(body.json); } catch (error) { const message = error instanceof Error ? error.message : String(error); - use.log.warn('upstream answered with a body the images protocol cannot read', { message }); + use.log.warn('upstream answered with a body the OpenAI Images protocol cannot read', { message }); return read({ status: 502, message, body: body.json }, called); } return read(canonical, [{ identity, quantities: billed(canonical.usage) }]); @@ -306,7 +306,7 @@ const spentBody = (body: ReadableStream | null): ReadableStream({ start: controller => controller.close() }), (): Promise => Promise.resolve()); interface MeteredFrames { - readonly frames: ImagesFrames; + readonly frames: OpenAIImagesFrames; readonly outcome: Promise; } @@ -314,7 +314,7 @@ const meterFrames = ( body: ReadableStream, identity: TelemetryModelIdentity, signal: AbortSignal | undefined, - record: (event: ImagesStreamEvent) => void, + record: (event: OpenAIImagesStreamEvent) => void, ): MeteredFrames => { let settle!: (outcome: StreamOutcome) => void; const outcome = new Promise(resolve => { settle = resolve; }); @@ -322,14 +322,14 @@ const meterFrames = ( // at the same moment the usage is. let sawTerminal = false; const frames = view((async function* () { - let usage: CanonicalImagesUsage | undefined; + let usage: CanonicalOpenAIImagesUsage | undefined; try { - for await (const event of parseImagesStream(body, { signal })) { + for await (const event of parseOpenAIImagesStream(body, { signal })) { // What the upstream sent, before the edge decides what the client sees — a dump reader // is owed the events that arrived. record(event); - if (isImagesTerminalEvent(event)) { - usage = parseImagesUsage(event); + if (isOpenAIImagesTerminalEvent(event)) { + usage = parseOpenAIImagesUsage(event); sawTerminal = true; yield event; // The image is complete, so there is nothing further to read. An upstream that holds @@ -348,7 +348,7 @@ const meterFrames = ( // Only an upstream that ended its body without ever completing the image reaches here: the // arm above returns on the terminal event. A client has been sent partial images and no // image, which is a failed answer however far it got. - throw new Error(IMAGES_MISSING_TERMINAL_MESSAGE); + throw new Error(OPENAI_IMAGES_MISSING_TERMINAL_MESSAGE); })()); return { frames, outcome }; }; @@ -357,7 +357,7 @@ const meterFrames = ( * or per-size unit, and per-size pricing is a selector coordinate rather than a metric, so * there is nothing else here to record a count against. A reported zero is kept — it says the * upstream reported, which an absent metric would not. */ -const billed = (usage: CanonicalImagesUsage | undefined): UsageQuantities => { +const billed = (usage: CanonicalOpenAIImagesUsage | undefined): UsageQuantities => { const quantities: UsageQuantities = {}; if (usage?.inputTokens !== undefined) quantities.input_tokens = parseDecimalString(String(usage.inputTokens)); if (usage?.inputImageTokens !== undefined) quantities.input_image_tokens = parseDecimalString(String(usage.inputImageTokens)); @@ -370,59 +370,59 @@ const billed = (usage: CanonicalImagesUsage | undefined): UsageQuantities => { * whether the data URL inside it can become a file, because that decides whether the edit can * ride as a multipart form; the canonical fact holds the reference the client wrote and leaves * that question to the serializer that has it. */ -const providerEditsRequest = (request: CanonicalImagesEditsRequest): ImagesEditsRequest => ({ +const providerEditsRequest = (request: CanonicalOpenAIImagesEditsRequest): OpenAIImagesEditsRequest => ({ images: request.images.map(providerEditsSource), ...(request.mask === undefined ? {} : { mask: providerEditsSource(request.mask) }), parameters: request.parameters, }); -const providerEditsSource = (image: ImagesEditImage): ImagesEditsSource => { +const providerEditsSource = (image: OpenAIImagesEditImage): OpenAIImagesEditsSource => { if (image.kind === 'file') { return { type: 'upload', file: new File([image.file.bytes], image.file.fileName, { type: image.file.mediaType }) }; } const { reference } = image; return typeof reference.image_url === 'string' && isBase64ImageDataUrl(reference.image_url) - ? { type: 'inline', reference: reference as ImageEditReference & { image_url: string } } + ? { type: 'inline', reference: reference as OpenAIImageEditReference & { image_url: string } } : { type: 'reference', reference }; }; /** A candidate that cannot serve *this* request is not a candidate. One family covers two * endpoints and an upstream may expose either without the other, so which one is asked for is * what narrows the list. */ -const narrowing = (request: CanonicalImagesRequest) => ({ +const narrowing = (request: CanonicalOpenAIImagesRequest) => ({ kind: 'image' as const, reject: (candidate: ModelCandidate) => candidate.model.endpoints[ENDPOINT[request.operation]] === undefined - ? `the upstream does not expose the images ${request.operation} endpoint` + ? `the upstream does not expose the OpenAI Images ${request.operation} endpoint` : null, unsupported: (model: string) => `Model ${model} does not support the /images/${request.operation} endpoint.`, refuse: (status: number, message: string) => ({ - 'response.images.canonical': { status, message } as Failure, - 'response.images.streamedUsage': null, + 'response.openaiImages.canonical': { status, message } as Failure, + 'response.openaiImages.streamedUsage': null, }), - refuses: ['response.images.canonical', 'response.images.streamedUsage'] as const, + refuses: ['response.openaiImages.canonical', 'response.openaiImages.streamedUsage'] as const, }); -/** What a caller must bring. `ingress.http.headers`, `ingress.images.wantsStream` and - * `request.images.canonical` are in it although `compose` cannot derive them — the ending +/** What a caller must bring. `ingress.http.headers`, `ingress.openaiImages.wantsStream` and + * `request.openaiImages.canonical` are in it although `compose` cannot derive them — the ending * stage reads all three and a return-only stage declares no request side — so this type is the * whole statement and `entryNeeds` is part of it. */ -export type ImagesServeEntry = I<'ingress.http.headers' | 'ingress.images.wantsStream' | 'request.images.canonical' | 'serve.model'>; +export type OpenAIImagesServeEntry = I<'ingress.http.headers' | 'ingress.openaiImages.wantsStream' | 'request.openaiImages.canonical' | 'serve.model'>; -export type ImagesServeExit = I< - 'response.images.rendered' | 'response.images.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable' +export type OpenAIImagesServeExit = I< + 'response.openaiImages.rendered' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable' >; -export const imagesServePipeline = (request: CanonicalImagesRequest): Pipeline => - compose('imagesServe', [ - emitImages, +export const openaiImagesServePipeline = (request: CanonicalOpenAIImagesRequest): Pipeline => + compose('openaiImagesServe', [ + emitOpenAIImages, writeSettlement( - handedUp => isFailure((handedUp as { 'response.images.canonical'?: unknown })['response.images.canonical']), - handedUp => (handedUp as { 'response.images.streamedUsage'?: unknown })['response.images.streamedUsage'] !== null, + handedUp => isFailure((handedUp as { 'response.openaiImages.canonical'?: unknown })['response.openaiImages.canonical']), + handedUp => (handedUp as { 'response.openaiImages.streamedUsage'?: unknown })['response.openaiImages.streamedUsage'] !== null, ), resolveCandidates(narrowing(request)), failover({ - failed: handedUp => isFailure((handedUp as { 'response.images.canonical'?: unknown })['response.images.canonical']), + failed: handedUp => isFailure((handedUp as { 'response.openaiImages.canonical'?: unknown })['response.openaiImages.canonical']), owns: ['response.http.body'], }), - callImagesUpstream, + callOpenAIImagesUpstream, ]); diff --git a/packages/protocols/__tests__/audio/transcription_test.ts b/packages/protocols/__tests__/openai-audio/transcription_test.ts similarity index 69% rename from packages/protocols/__tests__/audio/transcription_test.ts rename to packages/protocols/__tests__/openai-audio/transcription_test.ts index c99eb07872..a7d47725b9 100644 --- a/packages/protocols/__tests__/audio/transcription_test.ts +++ b/packages/protocols/__tests__/openai-audio/transcription_test.ts @@ -1,12 +1,12 @@ import { describe, expect, it } from 'vitest'; import { - parseAudioTranscription, - parseAudioTranscriptionResponseFormat, - parseAudioTranscriptionUsage, - renderAudioTranscription, - type AudioTranscriptionResponseFormat, -} from '../../src/audio/index.ts'; + parseOpenAIAudioTranscription, + parseOpenAIAudioTranscriptionResponseFormat, + parseOpenAIAudioTranscriptionUsage, + renderOpenAIAudioTranscription, + type OpenAIAudioTranscriptionResponseFormat, +} from '../../src/openai-audio/index.ts'; const bytes = (document: string): Uint8Array => new TextEncoder().encode(document); @@ -37,13 +37,13 @@ const VERBOSE = { ], }; -const carries = (format: AudioTranscriptionResponseFormat, document: string): string => - carried(renderAudioTranscription(format, parseAudioTranscription(format, bytes(document)))); +const carries = (format: OpenAIAudioTranscriptionResponseFormat, document: string): string => + carried(renderOpenAIAudioTranscription(format, parseOpenAIAudioTranscription(format, bytes(document)))); describe('the renderings of a transcription', () => { it('is the protocol default when the form left the field out', () => { - expect(parseAudioTranscriptionResponseFormat(undefined)).toBe('json'); - expect(() => parseAudioTranscriptionResponseFormat('yaml')).toThrow('response_format is invalid'); + expect(parseOpenAIAudioTranscriptionResponseFormat(undefined)).toBe('json'); + expect(() => parseOpenAIAudioTranscriptionResponseFormat('yaml')).toThrow('response_format is invalid'); }); // The claim the family rests on: `response_format` travels to the upstream inside the @@ -57,7 +57,7 @@ describe('the renderings of a transcription', () => { ['srt', SRT], ['vtt', VTT], ] as const)('reads and writes %s back unchanged', (format, document) => { - const rendered = renderAudioTranscription(format, parseAudioTranscription(format, bytes(document))); + const rendered = renderOpenAIAudioTranscription(format, parseOpenAIAudioTranscription(format, bytes(document))); expect(rendered instanceof Uint8Array ? carried(rendered) : JSON.stringify(rendered)).toBe(document); }); @@ -79,14 +79,14 @@ describe('the renderings of a transcription', () => { // equally rich. This is the whole of why a rendering is never derived from a value read // out of a different one. it('holds a transcript and its cues where the rendering carried both', () => { - expect(parseAudioTranscription('verbose_json', bytes(JSON.stringify(VERBOSE)))).toMatchObject({ + expect(parseOpenAIAudioTranscription('verbose_json', bytes(JSON.stringify(VERBOSE)))).toMatchObject({ text: VERBOSE.text, cues: [{ start: 0, end: 3.32, text: ' The beach was a popular spot.' }], }); }); it('holds no cues where the rendering carried none', () => { - expect(parseAudioTranscription('json', bytes(JSON.stringify({ text: 'hello' }))).cues).toBeUndefined(); + expect(parseOpenAIAudioTranscription('json', bytes(JSON.stringify({ text: 'hello' }))).cues).toBeUndefined(); }); // A subtitle document states the transcript only as its cues' texts, so the transcript on @@ -94,53 +94,53 @@ describe('the renderings of a transcription', () => { // claimed to be the string a `text` request would have returned. // https://github.com/openai/whisper/blob/v20250625/whisper/utils.py#L109-L116 it('rebuilds the transcript of a subtitle document from its cues', () => { - expect(parseAudioTranscription('srt', bytes(SRT)).text).toBe(' The beach was a popular spot.\n People were swimming in the ocean.'); + expect(parseOpenAIAudioTranscription('srt', bytes(SRT)).text).toBe(' The beach was a popular spot.\n People were swimming in the ocean.'); }); it('refuses a body the rendering cannot be read from', () => { - expect(() => parseAudioTranscription('json', bytes('{not-json'))).toThrow(SyntaxError); - expect(() => parseAudioTranscription('json', bytes(JSON.stringify({ transcript: 'hello' })))).toThrow('text must be a string'); - expect(() => parseAudioTranscription('vtt', bytes(SRT))).toThrow('must open with WEBVTT'); + expect(() => parseOpenAIAudioTranscription('json', bytes('{not-json'))).toThrow(SyntaxError); + expect(() => parseOpenAIAudioTranscription('json', bytes(JSON.stringify({ transcript: 'hello' })))).toThrow('text must be a string'); + expect(() => parseOpenAIAudioTranscription('vtt', bytes(SRT))).toThrow('must open with WEBVTT'); }); // The other half of that refusal: a caller that could not read the body still has the body, // and what it writes back is what arrived. A 2xx nobody could parse is still the upstream's // answer to the client's request. it('writes back a body no reading could open', () => { - expect(carried(renderAudioTranscription('json', { document: bytes('{not-json') }))).toBe('{not-json'); + expect(carried(renderOpenAIAudioTranscription('json', { document: bytes('{not-json') }))).toBe('{not-json'); }); }); describe('what a transcription says it will be billed for', () => { it('splits the audio share out of the input tokens the upstream broke down', () => { - expect(parseAudioTranscriptionUsage({ + expect(parseOpenAIAudioTranscriptionUsage({ usage: { type: 'tokens', input_tokens: 14, input_token_details: { text_tokens: 10, audio_tokens: 4 }, output_tokens: 101, total_tokens: 115 }, })).toEqual({ kind: 'tokens', inputTokens: 14, inputAudioTokens: 4, outputTokens: 101 }); }); it('reads a duration report, and whisper\'s top-level duration where there is no report', () => { - expect(parseAudioTranscriptionUsage({ usage: { type: 'duration', seconds: 43 } })).toEqual({ kind: 'duration', seconds: 43 }); - expect(parseAudioTranscriptionUsage(VERBOSE)).toEqual({ kind: 'duration', seconds: 8.47 }); + expect(parseOpenAIAudioTranscriptionUsage({ usage: { type: 'duration', seconds: 43 } })).toEqual({ kind: 'duration', seconds: 43 }); + expect(parseOpenAIAudioTranscriptionUsage(VERBOSE)).toEqual({ kind: 'duration', seconds: 8.47 }); }); // Nothing was stated, or what was stated is not something this reading names. A metric // invented after this was written is not a malformed one, and the caller bills neither. it('is no report where nothing this reading names was stated', () => { - expect(parseAudioTranscriptionUsage({ text: 'hello' })).toBeUndefined(); - expect(parseAudioTranscriptionUsage({ usage: { type: 'credits', spent: 3 } })).toBeUndefined(); + expect(parseOpenAIAudioTranscriptionUsage({ text: 'hello' })).toBeUndefined(); + expect(parseOpenAIAudioTranscriptionUsage({ usage: { type: 'credits', spent: 3 } })).toBeUndefined(); }); // An upstream that reported under a name this reading does know, in a shape it cannot read, // is a third situation and it says so — which is what lets the caller warn rather than // record the request as though the upstream had metered nothing. it('reports a block it names and cannot read, rather than reading it as nothing', () => { - expect(() => parseAudioTranscriptionUsage({ usage: { type: 'duration', seconds: 'invalid' } })) + expect(() => parseOpenAIAudioTranscriptionUsage({ usage: { type: 'duration', seconds: 'invalid' } })) .toThrow('usage.seconds must be a finite non-negative number'); - expect(() => parseAudioTranscriptionUsage({ usage: { type: 'tokens', input_tokens: -1, output_tokens: 2 } })) + expect(() => parseOpenAIAudioTranscriptionUsage({ usage: { type: 'tokens', input_tokens: -1, output_tokens: 2 } })) .toThrow('usage.input_tokens must be a non-negative safe integer'); - expect(() => parseAudioTranscriptionUsage({ usage: { type: 'tokens', input_tokens: 4, input_token_details: { audio_tokens: 9 }, output_tokens: 2 } })) + expect(() => parseOpenAIAudioTranscriptionUsage({ usage: { type: 'tokens', input_tokens: 4, input_token_details: { audio_tokens: 9 }, output_tokens: 2 } })) .toThrow('audio_tokens must not exceed usage.input_tokens'); - expect(() => parseAudioTranscriptionUsage({ usage: 'billed' })).toThrow('usage must be an object'); - expect(() => parseAudioTranscriptionUsage({ duration: 'a while' })).toThrow('duration must be a finite non-negative number'); + expect(() => parseOpenAIAudioTranscriptionUsage({ usage: 'billed' })).toThrow('usage must be an object'); + expect(() => parseOpenAIAudioTranscriptionUsage({ duration: 'a while' })).toThrow('duration must be a finite non-negative number'); }); }); diff --git a/packages/protocols/__tests__/embeddings/translate_test.ts b/packages/protocols/__tests__/openai-embeddings/translate_test.ts similarity index 61% rename from packages/protocols/__tests__/embeddings/translate_test.ts rename to packages/protocols/__tests__/openai-embeddings/translate_test.ts index b337961926..ac8ca9b273 100644 --- a/packages/protocols/__tests__/embeddings/translate_test.ts +++ b/packages/protocols/__tests__/openai-embeddings/translate_test.ts @@ -1,11 +1,11 @@ import { describe, expect, test } from 'vitest'; import { - parseEmbeddingsRequest, - parseEmbeddingsResponse, - renderEmbeddingsResponse, - serializeEmbeddingsRequest, -} from '../../src/embeddings/translate.ts'; + parseOpenAIEmbeddingsRequest, + parseOpenAIEmbeddingsResponse, + renderOpenAIEmbeddingsResponse, + serializeOpenAIEmbeddingsRequest, +} from '../../src/openai-embeddings/translate.ts'; // The float32 little-endian packing of [0.0023064255, -0.009327292, 1, -0.0028842222], // and the float64 values those bits denote. Produced by Float32Array + Buffer on Node 24, @@ -13,13 +13,13 @@ import { const PACKED = 'ZicXO4DRGLwAAIA/OAU9uw=='; const VECTOR = [0.002306425478309393, -0.009327292442321777, 1, -0.0028842221945524216]; -describe('embeddings request ingress', () => { +describe('OpenAI Embeddings request ingress', () => { test('a single text stays a single text, so the wire shape the upstream sees is unchanged', () => { - const parsed = parseEmbeddingsRequest({ model: 'text-embedding-3-small', input: 'a sentence' }); + const parsed = parseOpenAIEmbeddingsRequest({ model: 'text-embedding-3-small', input: 'a sentence' }); expect(parsed.model).toBe('text-embedding-3-small'); expect(parsed.request).toEqual({ input: 'a sentence' }); - expect(serializeEmbeddingsRequest(parsed.request)).toEqual({ input: 'a sentence' }); + expect(serializeOpenAIEmbeddingsRequest(parsed.request)).toEqual({ input: 'a sentence' }); }); test('the four input arms each survive', () => { @@ -30,23 +30,23 @@ describe('embeddings request ingress', () => { [[1212, 318], [257, 1332]], ]; for (const input of arms) { - expect(parseEmbeddingsRequest({ model: 'm', input }).request.input).toEqual(input); + expect(parseOpenAIEmbeddingsRequest({ model: 'm', input }).request.input).toEqual(input); } }); test('an omitted encoding_format is resolved for the client and left off the wire', () => { - const parsed = parseEmbeddingsRequest({ model: 'm', input: 'a' }); + const parsed = parseOpenAIEmbeddingsRequest({ model: 'm', input: 'a' }); expect(parsed.encodingFormat).toBe('float'); expect(parsed.request.encodingFormat).toBeUndefined(); - expect(serializeEmbeddingsRequest(parsed.request)).not.toHaveProperty('encoding_format'); + expect(serializeOpenAIEmbeddingsRequest(parsed.request)).not.toHaveProperty('encoding_format'); }); test('a chosen encoding_format is both what the client reads and what the upstream is asked for', () => { - const parsed = parseEmbeddingsRequest({ model: 'm', input: 'a', encoding_format: 'base64', dimensions: 256, user: 'u' }); + const parsed = parseOpenAIEmbeddingsRequest({ model: 'm', input: 'a', encoding_format: 'base64', dimensions: 256, user: 'u' }); expect(parsed.encodingFormat).toBe('base64'); - expect(serializeEmbeddingsRequest(parsed.request)).toEqual({ + expect(serializeOpenAIEmbeddingsRequest(parsed.request)).toEqual({ input: 'a', encoding_format: 'base64', dimensions: 256, @@ -55,30 +55,30 @@ describe('embeddings request ingress', () => { }); test('a field the protocol has no place for is named rather than dropped', () => { - expect(() => parseEmbeddingsRequest({ model: 'm', input: 'a', truncate: true, task: 'retrieval' })) - .toThrow('Embeddings does not support truncate, task'); + expect(() => parseOpenAIEmbeddingsRequest({ model: 'm', input: 'a', truncate: true, task: 'retrieval' })) + .toThrow('OpenAI Embeddings does not support truncate, task'); }); // An empty array satisfies three of the specification's four input arms at once, so the // canonical value would be ambiguous even though the JSON is well-formed. test('an empty input is refused, which is what keeps the parsed arms distinguishable', () => { - expect(() => parseEmbeddingsRequest({ model: 'm', input: [] })).toThrow('input must not be empty'); + expect(() => parseOpenAIEmbeddingsRequest({ model: 'm', input: [] })).toThrow('input must not be empty'); }); test('a mixed array is refused rather than read as whichever arm came first', () => { - expect(() => parseEmbeddingsRequest({ model: 'm', input: ['a', 1] })).toThrow('input[1] must be a non-empty string'); - expect(() => parseEmbeddingsRequest({ model: 'm', input: [1, 'a'] })).toThrow('input[1] must be an integer'); + expect(() => parseOpenAIEmbeddingsRequest({ model: 'm', input: ['a', 1] })).toThrow('input[1] must be a non-empty string'); + expect(() => parseOpenAIEmbeddingsRequest({ model: 'm', input: [1, 'a'] })).toThrow('input[1] must be an integer'); }); test('a non-integer dimensions is refused', () => { - expect(() => parseEmbeddingsRequest({ model: 'm', input: 'a', dimensions: 0 })).toThrow('dimensions must be a positive integer'); - expect(() => parseEmbeddingsRequest({ model: 'm', input: 'a', dimensions: 1.5 })).toThrow('dimensions must be an integer'); + expect(() => parseOpenAIEmbeddingsRequest({ model: 'm', input: 'a', dimensions: 0 })).toThrow('dimensions must be a positive integer'); + expect(() => parseOpenAIEmbeddingsRequest({ model: 'm', input: 'a', dimensions: 1.5 })).toThrow('dimensions must be an integer'); }); }); -describe('embeddings response egress', () => { +describe('OpenAI Embeddings response egress', () => { test('a base64 vector is read as the numbers it denotes', () => { - const parsed = parseEmbeddingsResponse({ + const parsed = parseOpenAIEmbeddingsResponse({ object: 'list', model: 'text-embedding-3-small', data: [{ object: 'embedding', index: 0, embedding: PACKED }], @@ -90,7 +90,7 @@ describe('embeddings response egress', () => { }); test('a truncated base64 vector is refused rather than read as a shorter one', () => { - expect(() => parseEmbeddingsResponse({ + expect(() => parseOpenAIEmbeddingsResponse({ model: 'm', data: [{ index: 0, embedding: PACKED.slice(0, 8) }], }, 'm')).toThrow('data[0].embedding is not a whole number of float32 values'); @@ -100,40 +100,40 @@ describe('embeddings response egress', () => { // does not. An answer the gateway understands is one it can write again, so the record is // completed with the model the request named rather than refused. test('an upstream that names no model is read as having answered for the requested one', () => { - const parsed = parseEmbeddingsResponse( + const parsed = parseOpenAIEmbeddingsResponse( { object: 'list', data: [{ object: 'embedding', index: 0, embedding: [0.5] }] }, 'text-embedding-real', ); expect(parsed.model).toBe('text-embedding-real'); - expect(renderEmbeddingsResponse('float', parsed)).toMatchObject({ model: 'text-embedding-real' }); + expect(renderOpenAIEmbeddingsResponse('float', parsed)).toMatchObject({ model: 'text-embedding-real' }); }); // A model the upstream *did* name is its own answer, and the request's id never overrides it. test('a model the upstream named is kept', () => { - expect(parseEmbeddingsResponse({ model: 'upstream-id', data: [] }, 'requested-id').model).toBe('upstream-id'); + expect(parseOpenAIEmbeddingsResponse({ model: 'upstream-id', data: [] }, 'requested-id').model).toBe('upstream-id'); }); // An upstream that reports no usage is a fact the gateway has to be able to carry: the // billed entity is present with no quantities, which is not the same as reporting zero. test('a missing usage block leaves the answer intact and reports nothing', () => { - const parsed = parseEmbeddingsResponse({ model: 'm', data: [{ index: 0, embedding: [0.5] }] }, 'm'); + const parsed = parseOpenAIEmbeddingsResponse({ model: 'm', data: [{ index: 0, embedding: [0.5] }] }, 'm'); expect(parsed.usage).toBeUndefined(); - expect(renderEmbeddingsResponse('float', parsed)).not.toHaveProperty('usage'); + expect(renderOpenAIEmbeddingsResponse('float', parsed)).not.toHaveProperty('usage'); }); // The case the encoding split exists for. An OpenAI SDK client asked for base64 without // saying so and will decode whatever it is handed; an upstream that ignores the field // and answers with float arrays would otherwise hand it noise. test('the answer is written in the encoding the client asked for, not the one it arrived in', () => { - const arrived = parseEmbeddingsResponse({ model: 'm', data: [{ index: 0, embedding: VECTOR }] }, 'm'); + const arrived = parseOpenAIEmbeddingsResponse({ model: 'm', data: [{ index: 0, embedding: VECTOR }] }, 'm'); - expect(renderEmbeddingsResponse('base64', arrived)).toMatchObject({ + expect(renderOpenAIEmbeddingsResponse('base64', arrived)).toMatchObject({ object: 'list', data: [{ object: 'embedding', index: 0, embedding: PACKED }], }); - expect(renderEmbeddingsResponse('float', parseEmbeddingsResponse({ + expect(renderOpenAIEmbeddingsResponse('float', parseOpenAIEmbeddingsResponse({ model: 'm', data: [{ index: 0, embedding: PACKED }], }, 'm'))).toMatchObject({ data: [{ embedding: VECTOR }] }); @@ -143,16 +143,16 @@ describe('embeddings response egress', () => { // from a float32 gives the same bits back. Holding vectors as numbers is therefore exact // and not an approximation, however many times a vector crosses the gateway. test('a vector survives any number of trips through the two encodings', () => { - const once = parseEmbeddingsResponse({ model: 'm', data: [{ index: 0, embedding: PACKED }] }, 'm'); - const twice = parseEmbeddingsResponse(renderEmbeddingsResponse('base64', once), 'm'); - const thrice = parseEmbeddingsResponse(renderEmbeddingsResponse('base64', twice), 'm'); + const once = parseOpenAIEmbeddingsResponse({ model: 'm', data: [{ index: 0, embedding: PACKED }] }, 'm'); + const twice = parseOpenAIEmbeddingsResponse(renderOpenAIEmbeddingsResponse('base64', once), 'm'); + const thrice = parseOpenAIEmbeddingsResponse(renderOpenAIEmbeddingsResponse('base64', twice), 'm'); - expect(renderEmbeddingsResponse('base64', thrice)).toMatchObject({ data: [{ embedding: PACKED }] }); + expect(renderOpenAIEmbeddingsResponse('base64', thrice)).toMatchObject({ data: [{ embedding: PACKED }] }); expect(thrice.embeddings).toEqual(once.embeddings); }); test('object is written by the gateway at both levels rather than repeated back', () => { - const rendered = renderEmbeddingsResponse('float', parseEmbeddingsResponse({ + const rendered = renderOpenAIEmbeddingsResponse('float', parseOpenAIEmbeddingsResponse({ object: 'not-a-list', model: 'm', data: [{ object: 'not-an-embedding', index: 0, embedding: [0.5] }], diff --git a/packages/protocols/__tests__/images/request_test.ts b/packages/protocols/__tests__/openai-images/request_test.ts similarity index 64% rename from packages/protocols/__tests__/images/request_test.ts rename to packages/protocols/__tests__/openai-images/request_test.ts index 60fba7ba2b..eefcc3d2da 100644 --- a/packages/protocols/__tests__/images/request_test.ts +++ b/packages/protocols/__tests__/openai-images/request_test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { parseImagesEditsRequest, parseImagesGenerationsRequest } from '../../src/images/request.ts'; +import { parseOpenAIImagesEditsRequest, parseOpenAIImagesGenerationsRequest } from '../../src/openai-images/request.ts'; const json = (value: unknown): Uint8Array => new TextEncoder().encode(JSON.stringify(value)); @@ -12,9 +12,9 @@ const multipart = async (form: FormData): Promise<{ contentType: string; bytes: }; }; -describe('images generations ingress', () => { +describe('OpenAI Images Generations ingress', () => { test('routing takes the model and everything else stays as the client wrote it', () => { - const parsed = parseImagesGenerationsRequest(json({ + const parsed = parseOpenAIImagesGenerationsRequest(json({ model: 'gpt-image-1', prompt: 'a shiba in space', size: '1024x1024', @@ -29,18 +29,18 @@ describe('images generations ingress', () => { }); test('a body that is not a JSON object, or names no model, is refused with what to fix', () => { - expect(() => parseImagesGenerationsRequest(new TextEncoder().encode('not json'))) - .toThrow('Images generations request body must be valid JSON.'); - expect(() => parseImagesGenerationsRequest(json(['gpt-image-1']))) - .toThrow('Images generations request body must be an object.'); - expect(() => parseImagesGenerationsRequest(json({ prompt: 'hi' }))) - .toThrow('Images generations request body must include a model string.'); + expect(() => parseOpenAIImagesGenerationsRequest(new TextEncoder().encode('not json'))) + .toThrow('OpenAI Images Generations request body must be valid JSON.'); + expect(() => parseOpenAIImagesGenerationsRequest(json(['gpt-image-1']))) + .toThrow('OpenAI Images Generations request body must be an object.'); + expect(() => parseOpenAIImagesGenerationsRequest(json({ prompt: 'hi' }))) + .toThrow('OpenAI Images Generations request body must include a model string.'); }); }); -describe('images edits ingress', () => { +describe('OpenAI Images Edits ingress', () => { test('a JSON edit keeps each reference exactly as it arrived', async () => { - const parsed = await parseImagesEditsRequest('application/json', json({ + const parsed = await parseOpenAIImagesEditsRequest('application/json', json({ model: 'gpt-image-1', prompt: 'replace the background', images: [{ image_url: 'data:image/png;base64,iVBORw0KGgo=' }, { file_id: 'file-source' }], @@ -61,13 +61,13 @@ describe('images edits ingress', () => { }); test('a reference naming both ways, or neither, is refused by position', async () => { - await expect(parseImagesEditsRequest('application/json', json({ + await expect(parseOpenAIImagesEditsRequest('application/json', json({ model: 'gpt-image-1', images: [{ file_id: 'file-source' }, { image_url: 'https://example.com/a.png', file_id: 'file-source' }], - }))).rejects.toThrow('Image edits images[1] must contain exactly one string field: image_url or file_id.'); + }))).rejects.toThrow('OpenAI Images Edits images[1] must contain exactly one string field: image_url or file_id.'); - await expect(parseImagesEditsRequest('application/json', json({ model: 'gpt-image-1', prompt: 'hi' }))) - .rejects.toThrow('Image edits request body must include an images array.'); + await expect(parseOpenAIImagesEditsRequest('application/json', json({ model: 'gpt-image-1', prompt: 'hi' }))) + .rejects.toThrow('OpenAI Images Edits request body must include an images array.'); }); test('a multipart edit holds each file as bytes and every other field as text', async () => { @@ -80,7 +80,7 @@ describe('images edits ingress', () => { form.append('mask', new Blob([new Uint8Array([6])], { type: 'image/png' }), 'mask.png'); const { contentType, bytes } = await multipart(form); - const parsed = await parseImagesEditsRequest(contentType, bytes); + const parsed = await parseOpenAIImagesEditsRequest(contentType, bytes); expect(parsed.model).toBe('gpt-image-1'); expect(parsed.request).toEqual({ @@ -100,8 +100,8 @@ describe('images edits ingress', () => { form.append('prompt', 'replace the sky'); const { contentType, bytes } = await multipart(form); - await expect(parseImagesEditsRequest(contentType, bytes)) - .rejects.toThrow('Image edits request body must include a model field.'); + await expect(parseOpenAIImagesEditsRequest(contentType, bytes)) + .rejects.toThrow('OpenAI Images Edits request body must include a model field.'); }); test('an image field carrying text, and a text field carrying a file, are both refused', async () => { @@ -109,21 +109,21 @@ describe('images edits ingress', () => { textImage.append('model', 'gpt-image-1'); textImage.append('image', 'not a file'); const encodedTextImage = await multipart(textImage); - await expect(parseImagesEditsRequest(encodedTextImage.contentType, encodedTextImage.bytes)) - .rejects.toThrow('Image edits image fields must be files.'); + await expect(parseOpenAIImagesEditsRequest(encodedTextImage.contentType, encodedTextImage.bytes)) + .rejects.toThrow('OpenAI Images Edits image fields must be files.'); const fileParameter = new FormData(); fileParameter.append('model', 'gpt-image-1'); fileParameter.append('prompt', new Blob([new Uint8Array([1])], { type: 'text/plain' }), 'prompt.txt'); const encodedFileParameter = await multipart(fileParameter); - await expect(parseImagesEditsRequest(encodedFileParameter.contentType, encodedFileParameter.bytes)) - .rejects.toThrow('Image edits prompt field must be text.'); + await expect(parseOpenAIImagesEditsRequest(encodedFileParameter.contentType, encodedFileParameter.bytes)) + .rejects.toThrow('OpenAI Images Edits prompt field must be text.'); }); test('a body in neither of the two media types the endpoint takes is refused as such', async () => { - await expect(parseImagesEditsRequest('text/plain', new TextEncoder().encode('hi'))) - .rejects.toThrow('Image edits request body must use application/json or multipart/form-data.'); - await expect(parseImagesEditsRequest(undefined, new TextEncoder().encode('hi'))) - .rejects.toThrow('Image edits request body must use application/json or multipart/form-data.'); + await expect(parseOpenAIImagesEditsRequest('text/plain', new TextEncoder().encode('hi'))) + .rejects.toThrow('OpenAI Images Edits request body must use application/json or multipart/form-data.'); + await expect(parseOpenAIImagesEditsRequest(undefined, new TextEncoder().encode('hi'))) + .rejects.toThrow('OpenAI Images Edits request body must use application/json or multipart/form-data.'); }); }); diff --git a/packages/protocols/__tests__/images/response_test.ts b/packages/protocols/__tests__/openai-images/response_test.ts similarity index 60% rename from packages/protocols/__tests__/images/response_test.ts rename to packages/protocols/__tests__/openai-images/response_test.ts index bf9bf9c173..d1b590b086 100644 --- a/packages/protocols/__tests__/images/response_test.ts +++ b/packages/protocols/__tests__/openai-images/response_test.ts @@ -1,9 +1,9 @@ import { describe, expect, test } from 'vitest'; import { renderErrorEnvelope, upstreamErrorMessage } from '../../src/common/error-envelope.ts'; -import { parseImagesResponse, parseImagesUsage, renderImagesResponse } from '../../src/images/response.ts'; +import { parseOpenAIImagesResponse, parseOpenAIImagesUsage, renderOpenAIImagesResponse } from '../../src/openai-images/response.ts'; -describe('images response ingress', () => { +describe('OpenAI Images response ingress', () => { test('the answer is parsed and rendered back whole, fields the gateway does not model included', () => { const body = { created: 1713833628, @@ -12,26 +12,26 @@ describe('images response ingress', () => { output_format: 'png', }; - const parsed = parseImagesResponse(body); + const parsed = parseOpenAIImagesResponse(body); expect(parsed.images).toEqual([ { base64: 'aGVsbG8=', revisedPrompt: 'a shiba, in space' }, { url: 'https://example.com/a.png' }, ]); - expect(renderImagesResponse(parsed)).toBe(body); + expect(renderOpenAIImagesResponse(parsed)).toBe(body); }); test('an answer carrying no images at all is read as such, and one that cannot be read is an error', () => { - expect(parseImagesResponse({ created: 1 }).images).toEqual([]); - expect(() => parseImagesResponse({ data: 'one image' })).toThrow('Images response data must be an array'); - expect(() => parseImagesResponse({ data: [{ b64_json: 42 }] })).toThrow('Images response data[0].b64_json must be a string'); - expect(() => parseImagesResponse('an image')).toThrow('Images response body must be an object'); + expect(parseOpenAIImagesResponse({ created: 1 }).images).toEqual([]); + expect(() => parseOpenAIImagesResponse({ data: 'one image' })).toThrow('OpenAI Images response data must be an array'); + expect(() => parseOpenAIImagesResponse({ data: [{ b64_json: 42 }] })).toThrow('OpenAI Images response data[0].b64_json must be a string'); + expect(() => parseOpenAIImagesResponse('an image')).toThrow('OpenAI Images response body must be an object'); }); }); -describe('images usage', () => { +describe('OpenAI Images usage', () => { test('what the upstream attributed to images is taken out of the count beside it', () => { - expect(parseImagesUsage({ + expect(parseOpenAIImagesUsage({ usage: { total_tokens: 100, input_tokens: 50, @@ -42,27 +42,27 @@ describe('images usage', () => { }); test('a count with no detail beside it is the whole count, and a detail saying nothing is as good as absent', () => { - expect(parseImagesUsage({ usage: { input_tokens: 10, output_tokens: 50 } })) + expect(parseOpenAIImagesUsage({ usage: { input_tokens: 10, output_tokens: 50 } })) .toEqual({ inputTokens: 10, outputTokens: 50 }); - expect(parseImagesUsage({ usage: { input_tokens: 10, input_tokens_details: {} } })) + expect(parseOpenAIImagesUsage({ usage: { input_tokens: 10, input_tokens_details: {} } })) .toEqual({ inputTokens: 10 }); - expect(parseImagesUsage({ usage: { output_tokens: 50, output_tokens_details: { image_tokens: 50 } } })) + expect(parseOpenAIImagesUsage({ usage: { output_tokens: 50, output_tokens_details: { image_tokens: 50 } } })) .toEqual({ outputTokens: 0, outputImageTokens: 50 }); }); // The distinction the return type exists for: an upstream that reported nothing is a // different fact from one that reported zero, and only the second is a reading. test('nothing readable is no reading, and a reported zero is one', () => { - expect(parseImagesUsage({ data: [] })).toBeUndefined(); - expect(parseImagesUsage({ usage: {} })).toBeUndefined(); - expect(parseImagesUsage({ usage: { input_tokens: '50' } })).toBeUndefined(); - expect(parseImagesUsage({ usage: { input_tokens: 10, input_tokens_details: 'text' } })).toBeUndefined(); - expect(parseImagesUsage({ usage: { input_tokens: 0, output_tokens: 0 } })) + expect(parseOpenAIImagesUsage({ data: [] })).toBeUndefined(); + expect(parseOpenAIImagesUsage({ usage: {} })).toBeUndefined(); + expect(parseOpenAIImagesUsage({ usage: { input_tokens: '50' } })).toBeUndefined(); + expect(parseOpenAIImagesUsage({ usage: { input_tokens: 10, input_tokens_details: 'text' } })).toBeUndefined(); + expect(parseOpenAIImagesUsage({ usage: { input_tokens: 0, output_tokens: 0 } })) .toEqual({ inputTokens: 0, outputTokens: 0 }); }); }); -describe('images failures', () => { +describe('OpenAI Images failures', () => { test('an upstream error object is what the client is sent, and anything else becomes the gateway envelope', () => { const upstream = { error: { message: 'Your request was rejected', type: 'image_generation_user_error', code: 'moderation_blocked' } }; diff --git a/packages/protocols/src/openai-audio/index.ts b/packages/protocols/src/openai-audio/index.ts index ceaf14a7d7..d712dd51ab 100644 --- a/packages/protocols/src/openai-audio/index.ts +++ b/packages/protocols/src/openai-audio/index.ts @@ -3,27 +3,27 @@ // picks between three JSON objects and three text documents, and `stream` picks a sequence // of events instead of any of them. -export type { AudioTranscriptionCue, SubtitleDialect } from './subtitles.ts'; +export type { OpenAIAudioTranscriptionCue, SubtitleDialect } from './subtitles.ts'; export { parseSubtitleDocument } from './subtitles.ts'; export type { - AudioTranscriptionObjectFormat, - AudioTranscriptionResponseFormat, - AudioTranscriptionUsage, - CanonicalAudioTranscription, + OpenAIAudioTranscriptionObjectFormat, + OpenAIAudioTranscriptionResponseFormat, + OpenAIAudioTranscriptionUsage, + CanonicalOpenAIAudioTranscription, } from './transcription.ts'; export { - AUDIO_TRANSCRIPTION_RESPONSE_FORMATS, - isAudioTranscriptionObjectFormat, - parseAudioTranscription, - parseAudioTranscriptionResponseFormat, - parseAudioTranscriptionUsage, - renderAudioTranscription, + OPENAI_AUDIO_TRANSCRIPTION_RESPONSE_FORMATS, + isOpenAIAudioTranscriptionObjectFormat, + parseOpenAIAudioTranscription, + parseOpenAIAudioTranscriptionResponseFormat, + parseOpenAIAudioTranscriptionUsage, + renderOpenAIAudioTranscription, } from './transcription.ts'; -export type { AudioTranscriptionDoneEvent, AudioTranscriptionStreamEvent } from './stream.ts'; +export type { OpenAIAudioTranscriptionDoneEvent, OpenAIAudioTranscriptionStreamEvent } from './stream.ts'; export { - isAudioTranscriptionDoneEvent, - parseAudioTranscriptionStreamEvent, - parseAudioTranscriptionStreamUsage, + isOpenAIAudioTranscriptionDoneEvent, + parseOpenAIAudioTranscriptionStreamEvent, + parseOpenAIAudioTranscriptionStreamUsage, } from './stream.ts'; diff --git a/packages/protocols/src/openai-audio/stream.ts b/packages/protocols/src/openai-audio/stream.ts index 790e02e80f..94e90ff10f 100644 --- a/packages/protocols/src/openai-audio/stream.ts +++ b/packages/protocols/src/openai-audio/stream.ts @@ -4,20 +4,20 @@ // the terminal one and the only place a streamed transcription states its usage. // https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L61796-L61917 -import type { AudioTranscriptionUsage } from './transcription.ts'; -import { parseAudioTranscriptionUsage } from './transcription.ts'; +import type { OpenAIAudioTranscriptionUsage } from './transcription.ts'; +import { parseOpenAIAudioTranscriptionUsage } from './transcription.ts'; -export interface AudioTranscriptionStreamEvent { +export interface OpenAIAudioTranscriptionStreamEvent { type: string; [key: string]: unknown; } -export interface AudioTranscriptionDoneEvent extends AudioTranscriptionStreamEvent { +export interface OpenAIAudioTranscriptionDoneEvent extends OpenAIAudioTranscriptionStreamEvent { type: 'transcript.text.done'; text: string; } -export const isAudioTranscriptionDoneEvent = (event: unknown): event is AudioTranscriptionDoneEvent => +export const isOpenAIAudioTranscriptionDoneEvent = (event: unknown): event is OpenAIAudioTranscriptionDoneEvent => typeof event === 'object' && event !== null && (event as { type?: unknown }).type === 'transcript.text.done' @@ -25,15 +25,15 @@ export const isAudioTranscriptionDoneEvent = (event: unknown): event is AudioTra /** The reading every frame goes through, so the value at the canonical key is a parsed * event and the edge re-serializes it rather than relaying the bytes it arrived in. */ -export const parseAudioTranscriptionStreamEvent = (value: unknown): AudioTranscriptionStreamEvent => { +export const parseOpenAIAudioTranscriptionStreamEvent = (value: unknown): OpenAIAudioTranscriptionStreamEvent => { if (typeof value !== 'object' || value === null || typeof (value as { type?: unknown }).type !== 'string') { - throw new Error('Audio transcription stream event must be an object carrying a string type'); + throw new Error('OpenAI Audio Transcriptions stream event must be an object carrying a string type'); } - return value as AudioTranscriptionStreamEvent; + return value as OpenAIAudioTranscriptionStreamEvent; }; /** A streamed transcription states its usage once, in the terminal event, in the same shape * a JSON body states it. */ -export const parseAudioTranscriptionStreamUsage = ( - event: AudioTranscriptionDoneEvent, -): AudioTranscriptionUsage | undefined => parseAudioTranscriptionUsage(event); +export const parseOpenAIAudioTranscriptionStreamUsage = ( + event: OpenAIAudioTranscriptionDoneEvent, +): OpenAIAudioTranscriptionUsage | undefined => parseOpenAIAudioTranscriptionUsage(event); diff --git a/packages/protocols/src/openai-audio/subtitles.ts b/packages/protocols/src/openai-audio/subtitles.ts index 100ff94419..1938104150 100644 --- a/packages/protocols/src/openai-audio/subtitles.ts +++ b/packages/protocols/src/openai-audio/subtitles.ts @@ -14,7 +14,7 @@ // accept more than Whisper writes. // https://github.com/openai/whisper/blob/v20250625/whisper/utils.py#L238-L262 -export interface AudioTranscriptionCue { +export interface OpenAIAudioTranscriptionCue { /** Seconds from the start of the audio. */ readonly start: number; readonly end: number; @@ -36,7 +36,7 @@ const readTimestamp = (value: string, dialect: SubtitleDialect): number => { return Number(hours ?? '0') * 3600 + Number(minutes) * 60 + Number(seconds) + Number(fraction.padEnd(3, '0')) / 1000; }; -const readCueBlock = (block: readonly string[], dialect: SubtitleDialect): AudioTranscriptionCue => { +const readCueBlock = (block: readonly string[], dialect: SubtitleDialect): OpenAIAudioTranscriptionCue => { // SubRip opens a cue with its ordinal and WebVTT with an optional identifier, so the // timing line is the one holding the arrow and the lines after it are the cue's text. const timingIndex = block.findIndex(line => line.includes('-->')); @@ -50,7 +50,7 @@ const readCueBlock = (block: readonly string[], dialect: SubtitleDialect): Audio }; }; -export const parseSubtitleDocument = (dialect: SubtitleDialect, document: string): readonly AudioTranscriptionCue[] => { +export const parseSubtitleDocument = (dialect: SubtitleDialect, document: string): readonly OpenAIAudioTranscriptionCue[] => { const lines = document.replaceAll('\r\n', '\n').split('\n'); if (dialect === 'vtt') { // The signature may carry a byte order mark and may be followed by header metadata on diff --git a/packages/protocols/src/openai-audio/transcription.ts b/packages/protocols/src/openai-audio/transcription.ts index 274f7ab673..3b1edf7f96 100644 --- a/packages/protocols/src/openai-audio/transcription.ts +++ b/packages/protocols/src/openai-audio/transcription.ts @@ -19,38 +19,38 @@ // all. Those are properties of the renderings, not gaps in the reading. // https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L714-L745 -import type { AudioTranscriptionCue } from './subtitles.ts'; +import type { OpenAIAudioTranscriptionCue } from './subtitles.ts'; import { parseSubtitleDocument } from './subtitles.ts'; // `diarized_json` joined the enum with `gpt-4o-transcribe-diarize`; it is a JSON rendering // and needs nothing of its own here. // https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L28527-L28545 -export const AUDIO_TRANSCRIPTION_RESPONSE_FORMATS = ['json', 'text', 'srt', 'verbose_json', 'vtt', 'diarized_json'] as const; -export type AudioTranscriptionResponseFormat = typeof AUDIO_TRANSCRIPTION_RESPONSE_FORMATS[number]; +export const OPENAI_AUDIO_TRANSCRIPTION_RESPONSE_FORMATS = ['json', 'text', 'srt', 'verbose_json', 'vtt', 'diarized_json'] as const; +export type OpenAIAudioTranscriptionResponseFormat = typeof OPENAI_AUDIO_TRANSCRIPTION_RESPONSE_FORMATS[number]; /** The form field is optional and the protocol's own default is `json`. */ -export const parseAudioTranscriptionResponseFormat = (value: unknown): AudioTranscriptionResponseFormat => { +export const parseOpenAIAudioTranscriptionResponseFormat = (value: unknown): OpenAIAudioTranscriptionResponseFormat => { if (value === undefined) return 'json'; - if (typeof value === 'string' && (AUDIO_TRANSCRIPTION_RESPONSE_FORMATS as readonly string[]).includes(value)) { - return value as AudioTranscriptionResponseFormat; + if (typeof value === 'string' && (OPENAI_AUDIO_TRANSCRIPTION_RESPONSE_FORMATS as readonly string[]).includes(value)) { + return value as OpenAIAudioTranscriptionResponseFormat; } - throw new Error(`Audio transcription response_format is invalid: ${JSON.stringify(value)}`); + throw new Error(`OpenAI Audio Transcriptions response_format is invalid: ${JSON.stringify(value)}`); }; /** A JSON rendering answers with an object; `text`, `srt` and `vtt` answer with a document. * The split decides how the body is read and written, and it is the whole of what the * response format decides for a non-streaming answer. */ -export type AudioTranscriptionObjectFormat = 'json' | 'verbose_json' | 'diarized_json'; +export type OpenAIAudioTranscriptionObjectFormat = 'json' | 'verbose_json' | 'diarized_json'; -export const isAudioTranscriptionObjectFormat = ( - format: AudioTranscriptionResponseFormat, -): format is AudioTranscriptionObjectFormat => +export const isOpenAIAudioTranscriptionObjectFormat = ( + format: OpenAIAudioTranscriptionResponseFormat, +): format is OpenAIAudioTranscriptionObjectFormat => format === 'json' || format === 'verbose_json' || format === 'diarized_json'; /** Token-billed and duration-billed models report usage under one discriminated key. A * transcription answered in `text`, `srt` or `vtt` reports none, because those renderings * cannot express it. */ -export type AudioTranscriptionUsage = +export type OpenAIAudioTranscriptionUsage = | { readonly kind: 'tokens'; readonly inputTokens: number; @@ -61,7 +61,7 @@ export type AudioTranscriptionUsage = } | { readonly kind: 'duration'; readonly seconds: number }; -export interface CanonicalAudioTranscription { +export interface CanonicalOpenAIAudioTranscription { /** What the upstream sent, byte for byte. Every rendering but the objects is written back * from this, and so is a body the reading below could not open. * @@ -77,7 +77,7 @@ export interface CanonicalAudioTranscription { readonly text?: string; /** The timed cues. This is the whole of `srt` and `vtt`; `verbose_json` and * `diarized_json` carry the same thing as `segments`. */ - readonly cues?: readonly AudioTranscriptionCue[]; + readonly cues?: readonly OpenAIAudioTranscriptionCue[]; /** The upstream's own object, kept whole for the renderings that are objects. A JSON * answer is re-serialized from this, so a field this type does not name — a segment's * `avg_logprob`, a `logprobs` array, a `speaker` label — reaches the client unchanged, @@ -91,13 +91,13 @@ const isRecord = (value: unknown): value is Record => const isFiniteNumber = (value: unknown): value is number => typeof value === 'number' && Number.isFinite(value); const seconds = (value: unknown, where: string): number => { - if (!isFiniteNumber(value) || value < 0) throw new Error(`Audio transcription ${where} must be a finite non-negative number`); + if (!isFiniteNumber(value) || value < 0) throw new Error(`OpenAI Audio Transcriptions ${where} must be a finite non-negative number`); return value; }; const count = (value: unknown, where: string): number => { if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) { - throw new Error(`Audio transcription ${where} must be a non-negative safe integer`); + throw new Error(`OpenAI Audio Transcriptions ${where} must be a non-negative safe integer`); } return value; }; @@ -116,26 +116,26 @@ const count = (value: unknown, where: string): number => { * https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L36513-L36545 * https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L61968-L62051 */ -export const parseAudioTranscriptionUsage = (value: unknown): AudioTranscriptionUsage | undefined => { +export const parseOpenAIAudioTranscriptionUsage = (value: unknown): OpenAIAudioTranscriptionUsage | undefined => { if (!isRecord(value)) return undefined; if (value.usage === undefined) { if (value.duration === undefined) return undefined; return { kind: 'duration', seconds: seconds(value.duration, 'duration') }; } const usage = value.usage; - if (!isRecord(usage)) throw new Error('Audio transcription usage must be an object'); + if (!isRecord(usage)) throw new Error('OpenAI Audio Transcriptions usage must be an object'); if (usage.type === 'duration') return { kind: 'duration', seconds: seconds(usage.seconds, 'usage.seconds') }; if (usage.type !== 'tokens') return undefined; const inputTokens = count(usage.input_tokens, 'usage.input_tokens'); const outputTokens = count(usage.output_tokens, 'usage.output_tokens'); const details = usage.input_token_details; - if (details !== undefined && !isRecord(details)) throw new Error('Audio transcription usage.input_token_details must be an object'); + if (details !== undefined && !isRecord(details)) throw new Error('OpenAI Audio Transcriptions usage.input_token_details must be an object'); const audioTokens = details?.audio_tokens === undefined ? undefined : count(details.audio_tokens, 'usage.input_token_details.audio_tokens'); if (audioTokens !== undefined && audioTokens > inputTokens) { - throw new Error('Audio transcription usage.input_token_details.audio_tokens must not exceed usage.input_tokens'); + throw new Error('OpenAI Audio Transcriptions usage.input_token_details.audio_tokens must not exceed usage.input_tokens'); } return { kind: 'tokens', @@ -146,17 +146,17 @@ export const parseAudioTranscriptionUsage = (value: unknown): AudioTranscription }; const requiredText = (value: unknown, where: string): string => { - if (typeof value !== 'string') throw new Error(`Audio transcription ${where} must be a string`); + if (typeof value !== 'string') throw new Error(`OpenAI Audio Transcriptions ${where} must be a string`); return value; }; /** `verbose_json` and `diarized_json` both time their segments with seconds and text; the * fields they do not share stay in `raw` and reach the client through it. */ -const objectCues = (value: Record): readonly AudioTranscriptionCue[] | undefined => { +const objectCues = (value: Record): readonly OpenAIAudioTranscriptionCue[] | undefined => { if (!Array.isArray(value.segments)) return undefined; return value.segments.map((segment: unknown, index) => { if (!isRecord(segment) || !isFiniteNumber(segment.start) || !isFiniteNumber(segment.end)) { - throw new Error(`Audio transcription segment ${index} must carry numeric start and end`); + throw new Error(`OpenAI Audio Transcriptions segment ${index} must carry numeric start and end`); } return { start: segment.start, end: segment.end, text: requiredText(segment.text, `segment ${index} text`) }; }); @@ -169,14 +169,14 @@ const objectCues = (value: Record): readonly AudioTranscription * serializing one from what it read answers anyway: what a failed reading costs is the * transcript in the record and the usage measured beside it, never the answer. */ -export const parseAudioTranscription = ( - format: AudioTranscriptionResponseFormat, +export const parseOpenAIAudioTranscription = ( + format: OpenAIAudioTranscriptionResponseFormat, document: Uint8Array, -): CanonicalAudioTranscription => { +): CanonicalOpenAIAudioTranscription => { const decoded = new TextDecoder().decode(document); - if (isAudioTranscriptionObjectFormat(format)) { + if (isOpenAIAudioTranscriptionObjectFormat(format)) { const body: unknown = JSON.parse(decoded); - if (!isRecord(body)) throw new Error('Audio transcription response body must be an object'); + if (!isRecord(body)) throw new Error('OpenAI Audio Transcriptions response body must be an object'); const cues = objectCues(body); return { document, @@ -198,10 +198,10 @@ export const parseAudioTranscription = ( * arrived, handed back untouched — and so is an object rendering whose body the reading could * not open, because bytes nobody could read are still the answer the upstream gave. */ -export const renderAudioTranscription = ( - format: AudioTranscriptionResponseFormat, - transcription: CanonicalAudioTranscription, +export const renderOpenAIAudioTranscription = ( + format: OpenAIAudioTranscriptionResponseFormat, + transcription: CanonicalOpenAIAudioTranscription, ): Record | Uint8Array => - isAudioTranscriptionObjectFormat(format) && transcription.raw !== undefined + isOpenAIAudioTranscriptionObjectFormat(format) && transcription.raw !== undefined ? transcription.raw : transcription.document; diff --git a/packages/protocols/src/openai-completions/index.ts b/packages/protocols/src/openai-completions/index.ts index e566210200..5051241e5b 100644 --- a/packages/protocols/src/openai-completions/index.ts +++ b/packages/protocols/src/openai-completions/index.ts @@ -1,7 +1,7 @@ // OpenAI text completions (`POST /v1/completions`). The gateway carries this protocol end to -// end: what a client sends is parsed into `CompletionsPayload`, what an upstream answers is -// parsed into a `CompletionsResult` or a frame stream, and what the client receives is -// serialized back from those. No body is forwarded verbatim, so the shapes here are the +// end: what a client sends is parsed into `OpenAICompletionsPayload`, what an upstream answers +// is parsed into an `OpenAICompletionsResult` or a frame stream, and what the client receives +// is serialized back from those. No body is forwarded verbatim, so the shapes here are the // protocol's own rather than a convenience view over bytes in flight. // // Request and response follow OpenAI's `CreateCompletionRequest` and @@ -18,7 +18,7 @@ // opinion about that held here would have to be kept in step with every OpenAI-compatible // upstream we route to. The index signature carries a vendor extension through to the // upstream that understands it. -export interface CompletionsPayload { +export interface OpenAICompletionsPayload { model: string; prompt?: string | readonly string[] | readonly number[] | readonly (readonly number[])[] | null; best_of?: number | null; @@ -32,7 +32,7 @@ export interface CompletionsPayload { seed?: number | null; stop?: string | readonly string[] | null; stream?: boolean | null; - stream_options?: CompletionsStreamOptions | null; + stream_options?: OpenAICompletionsStreamOptions | null; suffix?: string | null; temperature?: number | null; top_p?: number | null; @@ -45,7 +45,7 @@ export interface CompletionsPayload { // https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L31764-L31810 // The index signature is what lets the gateway turn `include_usage` on without dropping the // siblings a client sent. -export interface CompletionsStreamOptions { +export interface OpenAICompletionsStreamOptions { include_usage?: boolean; include_obfuscation?: boolean; [key: string]: unknown; @@ -57,7 +57,7 @@ export interface CompletionsStreamOptions { // `finish_reason`) alongside the usage block — so `text` and // `finish_reason` are optional, matching that shape on the typed surface. // `logprobs` is opaque to the gateway — carried through as-is. -interface CompletionsChoiceStreaming { +interface OpenAICompletionsChoiceStreaming { index: number; text?: string; finish_reason?: string | null; @@ -72,7 +72,7 @@ interface CompletionsChoiceStreaming { // all emit `cached_tokens` here on `/v1/completions`, and Azure mirrors the schema. Billing // reads the split through `openAICacheTokensFromUsage`, which also knows the field names the // wilder forks use, so what is named here is the schema rather than the union of the wild. -export interface CompletionsUsage { +export interface OpenAICompletionsUsage { prompt_tokens: number; completion_tokens: number; total_tokens: number; @@ -84,17 +84,17 @@ export interface CompletionsUsage { // empty or placeholder `choices` array; isOpenAIUsageOnlyEventShape (in // protocols/common) detects that chunk shape without consulting the typed // surface. -export interface CompletionsStreamEvent { +export interface OpenAICompletionsStreamEvent { id: string; object: 'text_completion'; created: number; model: string; - choices: CompletionsChoiceStreaming[]; - usage?: CompletionsUsage; + choices: OpenAICompletionsChoiceStreaming[]; + usage?: OpenAICompletionsUsage; system_fingerprint?: string; } -export interface CompletionsChoice { +export interface OpenAICompletionsChoice { index: number; text: string; finish_reason: string | null; @@ -105,19 +105,19 @@ export interface CompletionsChoice { // endpoint — but a vLLM fork was observed emitting it on the non-streaming // `/v1/completions` body (null, on a Zhipu/GLM build) and billing reads it as the pricing // tier when it is there, so it is named rather than left to the index signature. -export interface CompletionsResult { +export interface OpenAICompletionsResult { id: string; object: 'text_completion'; created: number; model: string; - choices: CompletionsChoice[]; + choices: OpenAICompletionsChoice[]; service_tier?: string | null; - usage?: CompletionsUsage; + usage?: OpenAICompletionsUsage; system_fingerprint?: string; [key: string]: unknown; } -export { parseCompletionsPayload, parseCompletionsResult } from './parse.ts'; -export { reassembleCompletionsEvents } from './reassemble.ts'; -export { parseCompletionsStream, type ParseCompletionsStreamOptions } from './stream.ts'; -export { completionsProtocolFrameToSSEFrame } from './to-sse.ts'; +export { parseOpenAICompletionsPayload, parseOpenAICompletionsResult } from './parse.ts'; +export { reassembleOpenAICompletionsEvents } from './reassemble.ts'; +export { parseOpenAICompletionsStream, type ParseOpenAICompletionsStreamOptions } from './stream.ts'; +export { openaiCompletionsProtocolFrameToSSEFrame } from './to-sse.ts'; diff --git a/packages/protocols/src/openai-completions/parse.ts b/packages/protocols/src/openai-completions/parse.ts index 796c1bd094..5837d1f7eb 100644 --- a/packages/protocols/src/openai-completions/parse.ts +++ b/packages/protocols/src/openai-completions/parse.ts @@ -1,4 +1,4 @@ -import type { CompletionsPayload, CompletionsResult } from './index.ts'; +import type { OpenAICompletionsPayload, OpenAICompletionsResult } from './index.ts'; import { isJsonObject } from '../common/json.ts'; // The two boundaries where a `/v1/completions` body becomes a value. Both check what the @@ -8,15 +8,15 @@ import { isJsonObject } from '../common/json.ts'; // upstream's to decide, and a gateway opinion about the rest would have to be kept in step // with every OpenAI-compatible implementation we route to. -export const parseCompletionsPayload = (value: unknown): CompletionsPayload => { - if (!isJsonObject(value)) throw new Error('Completions request body must be an object.'); +export const parseOpenAICompletionsPayload = (value: unknown): OpenAICompletionsPayload => { + if (!isJsonObject(value)) throw new Error('OpenAI Completions request body must be an object.'); if (typeof value.model !== 'string' || value.model.length === 0) { - throw new Error('Completions request body must include a model string.'); + throw new Error('OpenAI Completions request body must include a model string.'); } - return value as CompletionsPayload; + return value as OpenAICompletionsPayload; }; -export const parseCompletionsResult = (value: unknown): CompletionsResult => { - if (!isJsonObject(value)) throw new Error('Completions response body must be an object.'); - return value as CompletionsResult; +export const parseOpenAICompletionsResult = (value: unknown): OpenAICompletionsResult => { + if (!isJsonObject(value)) throw new Error('OpenAI Completions response body must be an object.'); + return value as OpenAICompletionsResult; }; diff --git a/packages/protocols/src/openai-completions/stream.ts b/packages/protocols/src/openai-completions/stream.ts index 2a6427fe7c..179b5f228a 100644 --- a/packages/protocols/src/openai-completions/stream.ts +++ b/packages/protocols/src/openai-completions/stream.ts @@ -1,21 +1,21 @@ -import type { CompletionsStreamEvent } from './index.ts'; +import type { OpenAICompletionsStreamEvent } from './index.ts'; import { parseTargetStreamFrames } from '../common/parse-events.ts'; import { parseSSEStream } from '../common/parse-sse.ts'; import { doneFrame, eventFrame, type ProtocolFrame } from '../common/sse.ts'; -export interface ParseCompletionsStreamOptions { +export interface ParseOpenAICompletionsStreamOptions { signal?: AbortSignal; } // The upstream's SSE body as protocol frames. Transport framing ends here: what comes out // carries `text_completion` chunks and a terminal frame, and nothing that reads it knows how // those arrived. -export const parseCompletionsStream = ( +export const parseOpenAICompletionsStream = ( body: ReadableStream, - options: ParseCompletionsStreamOptions = {}, -): AsyncGenerator> => (async function* () { - for await (const frame of parseTargetStreamFrames(parseSSEStream(body, options), { - protocol: 'Completions', + options: ParseOpenAICompletionsStreamOptions = {}, +): AsyncGenerator> => (async function* () { + for await (const frame of parseTargetStreamFrames(parseSSEStream(body, options), { + protocol: 'OpenAI Completions', })) { if (frame.type === 'done') { yield doneFrame(); diff --git a/packages/protocols/src/openai-embeddings/index.ts b/packages/protocols/src/openai-embeddings/index.ts index 41524c486a..3555398383 100644 --- a/packages/protocols/src/openai-embeddings/index.ts +++ b/packages/protocols/src/openai-embeddings/index.ts @@ -1,7 +1,7 @@ // OpenAI's `/v1/embeddings` contract, and the canonical form the gateway holds it in. // Specification: https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34214-L34331 // -// There is one embeddings protocol, so the canonical form and the wire differ in one +// There is one OpenAI Embeddings protocol, so the canonical form and the wire differ in one // substantive place: `encoding_format` decides how a vector is *written*, and a vector is // the same vector either way — so the canonical response holds numbers, and the encoding // is a rendering decision the edge makes from what the client asked for. @@ -18,7 +18,7 @@ /** How a vector is written on the wire. Under `base64` an embedding is one string, not an * array of numbers. * https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34272-L34280 */ -export type EmbeddingsEncodingFormat = 'float' | 'base64'; +export type OpenAIEmbeddingsEncodingFormat = 'float' | 'base64'; /** One text, a batch of texts, one pre-tokenized text, or a batch of those — the * specification's own four-armed `oneOf`: @@ -27,13 +27,13 @@ export type EmbeddingsEncodingFormat = 'float' | 'base64'; * The arms are told apart by reading an element, which works only because every array arm * is `minItems: 1` there — an empty array would belong to three of them at once. Parsing * rejects it, so a value of this type is always unambiguous. */ -export type EmbeddingsInput = string | readonly string[] | readonly number[] | readonly (readonly number[])[]; +export type OpenAIEmbeddingsInput = string | readonly string[] | readonly number[] | readonly (readonly number[])[]; -export interface CanonicalEmbeddingsRequest { - input: EmbeddingsInput; +export interface CanonicalOpenAIEmbeddingsRequest { + input: OpenAIEmbeddingsInput; /** What the upstream is asked for, absent included — a client that did not ask must not * grow a field on the wire it never sent. */ - encodingFormat?: EmbeddingsEncodingFormat; + encodingFormat?: OpenAIEmbeddingsEncodingFormat; /** Only `text-embedding-3` and later models honour it * (https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34281-L34286), * and which models those are is the upstream's knowledge and not the catalog's — so it @@ -43,18 +43,18 @@ export interface CanonicalEmbeddingsRequest { user?: string; } -export interface ParsedEmbeddingsRequest { +export interface ParsedOpenAIEmbeddingsRequest { model: string; /** What the client will be able to read: `encoding_format` resolved against the * protocol's own default of `float` * (https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34276). * Concrete, because the answer has to be written in one encoding or the other — and * distinct from the request's own field, which stays absent when the client omitted it. */ - encodingFormat: EmbeddingsEncodingFormat; - request: CanonicalEmbeddingsRequest; + encodingFormat: OpenAIEmbeddingsEncodingFormat; + request: CanonicalOpenAIEmbeddingsRequest; } -export interface CanonicalEmbedding { +export interface CanonicalOpenAIEmbedding { /** Which input this vector answers, carried by the specification as a field rather than * left to array position: * https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L38168-L38170 */ @@ -62,16 +62,16 @@ export interface CanonicalEmbedding { values: readonly number[]; } -export interface CanonicalEmbeddingsUsage { +export interface CanonicalOpenAIEmbeddingsUsage { promptTokens: number; totalTokens: number; } -export interface CanonicalEmbeddingsResponse { +export interface CanonicalOpenAIEmbeddingsResponse { /** What the upstream says it generated with, which need not be the id the client * addressed — an upstream is free to answer under a dated or internal name. */ model: string; - embeddings: readonly CanonicalEmbedding[]; + embeddings: readonly CanonicalOpenAIEmbedding[]; /** The specification requires it * (https://github.com/openai/openai-openapi/blob/2186421dca0cca7c1e67caa7739005e8b1ccc4dd/openapi.yaml#L34327-L34331) * and it is optional here regardless, because an upstream that reports nothing is a @@ -79,22 +79,22 @@ export interface CanonicalEmbeddingsResponse { * no quantities, which is a different fact from reporting zero. Rejecting the body * instead would turn an answered call into a 502 and throw away the vectors it * returned. */ - usage?: CanonicalEmbeddingsUsage; + usage?: CanonicalOpenAIEmbeddingsUsage; } /** The request as it goes out. `model` is the upstream's own id and the provider * substitutes it, so what the gateway hands a provider is everything but that. */ -export interface EmbeddingsPayload { +export interface OpenAIEmbeddingsPayload { model: string; - input: EmbeddingsInput; - encoding_format?: EmbeddingsEncodingFormat; + input: OpenAIEmbeddingsInput; + encoding_format?: OpenAIEmbeddingsEncodingFormat; dimensions?: number; user?: string; } export { - parseEmbeddingsRequest, - parseEmbeddingsResponse, - renderEmbeddingsResponse, - serializeEmbeddingsRequest, + parseOpenAIEmbeddingsRequest, + parseOpenAIEmbeddingsResponse, + renderOpenAIEmbeddingsResponse, + serializeOpenAIEmbeddingsRequest, } from './translate.ts'; diff --git a/packages/protocols/src/openai-embeddings/translate.ts b/packages/protocols/src/openai-embeddings/translate.ts index 6afb334f01..f524d8b8e0 100644 --- a/packages/protocols/src/openai-embeddings/translate.ts +++ b/packages/protocols/src/openai-embeddings/translate.ts @@ -1,12 +1,12 @@ import type { - CanonicalEmbedding, - CanonicalEmbeddingsRequest, - CanonicalEmbeddingsResponse, - CanonicalEmbeddingsUsage, - EmbeddingsEncodingFormat, - EmbeddingsInput, - EmbeddingsPayload, - ParsedEmbeddingsRequest, + CanonicalOpenAIEmbedding, + CanonicalOpenAIEmbeddingsRequest, + CanonicalOpenAIEmbeddingsResponse, + CanonicalOpenAIEmbeddingsUsage, + OpenAIEmbeddingsEncodingFormat, + OpenAIEmbeddingsInput, + OpenAIEmbeddingsPayload, + ParsedOpenAIEmbeddingsRequest, } from './index.ts'; import { decodeForgivingBase64, encodeBase64 } from '../common/base-encoding.ts'; @@ -36,7 +36,7 @@ const optionalPositiveInteger = (value: unknown, field: string): number | undefi return integer; }; -const parseEncodingFormat = (value: unknown): EmbeddingsEncodingFormat | undefined => { +const parseEncodingFormat = (value: unknown): OpenAIEmbeddingsEncodingFormat | undefined => { if (value === undefined) return undefined; if (value !== 'float' && value !== 'base64') throw new Error('encoding_format must be float or base64'); return value; @@ -47,7 +47,7 @@ const parseTokenIds = (value: readonly unknown[], field: string): readonly numbe return value.map((token, index) => requiredInteger(token, `${field}[${index}]`)); }; -const parseInput = (value: unknown): EmbeddingsInput => { +const parseInput = (value: unknown): OpenAIEmbeddingsInput => { if (typeof value === 'string') return value; if (!Array.isArray(value)) throw new Error('input must be a string or an array'); if (value.length === 0) throw new Error('input must not be empty'); @@ -70,12 +70,12 @@ const parseInput = (value: unknown): EmbeddingsInput => { // https://github.com/BerriAI/litellm/blob/bc6e7df05b018eefe6c7293790ca3f4de38709ac/litellm/utils.py#L3259-L3273 // — while copilot-api's request type is narrower still, `input` and `model` alone: // https://github.com/ericc-ch/copilot-api/blob/0ea08febdd7e3e055b03dd298bf57e669500b5c1/src/services/copilot/create-embeddings.ts#L19-L22 -const EMBEDDINGS_REQUEST_FIELDS = ['model', 'input', 'encoding_format', 'dimensions', 'user']; +const OPENAI_EMBEDDINGS_REQUEST_FIELDS = ['model', 'input', 'encoding_format', 'dimensions', 'user']; -export const parseEmbeddingsRequest = (value: unknown): ParsedEmbeddingsRequest => { - if (!isRecord(value)) throw new Error('Embeddings request body must be an object'); - const unsupported = Object.keys(value).filter(field => !EMBEDDINGS_REQUEST_FIELDS.includes(field)); - if (unsupported.length > 0) throw new Error(`Embeddings does not support ${unsupported.join(', ')}`); +export const parseOpenAIEmbeddingsRequest = (value: unknown): ParsedOpenAIEmbeddingsRequest => { + if (!isRecord(value)) throw new Error('OpenAI Embeddings request body must be an object'); + const unsupported = Object.keys(value).filter(field => !OPENAI_EMBEDDINGS_REQUEST_FIELDS.includes(field)); + if (unsupported.length > 0) throw new Error(`OpenAI Embeddings does not support ${unsupported.join(', ')}`); const encodingFormat = parseEncodingFormat(value.encoding_format); const dimensions = optionalPositiveInteger(value.dimensions, 'dimensions'); @@ -92,7 +92,7 @@ export const parseEmbeddingsRequest = (value: unknown): ParsedEmbeddingsRequest }; }; -export const serializeEmbeddingsRequest = (request: CanonicalEmbeddingsRequest): Omit => ({ +export const serializeOpenAIEmbeddingsRequest = (request: CanonicalOpenAIEmbeddingsRequest): Omit => ({ input: request.input, ...(request.encodingFormat === undefined ? {} : { encoding_format: request.encodingFormat }), ...(request.dimensions === undefined ? {} : { dimensions: request.dimensions }), @@ -133,7 +133,7 @@ const renderEmbedding = (values: readonly number[]): string => { return encodeBase64(bytes); }; -const parseUsage = (value: unknown): CanonicalEmbeddingsUsage | undefined => { +const parseUsage = (value: unknown): CanonicalOpenAIEmbeddingsUsage | undefined => { if (value === undefined || value === null) return undefined; if (!isRecord(value)) throw new Error('usage must be an object'); return { @@ -157,12 +157,12 @@ const parseUsage = (value: unknown): CanonicalEmbeddingsUsage | undefined => { * what another Copilot gateway meters under for the same reason — * https://github.com/caozhiyuan/copilot-api/blob/96854ce1e1e741d621b12d4ca844efd77f6e8d90/src/routes/embeddings/route.ts#L16-L23 */ -export const parseEmbeddingsResponse = (value: unknown, requestedModel: string): CanonicalEmbeddingsResponse => { - if (!isRecord(value)) throw new Error('Embeddings response body must be an object'); +export const parseOpenAIEmbeddingsResponse = (value: unknown, requestedModel: string): CanonicalOpenAIEmbeddingsResponse => { + if (!isRecord(value)) throw new Error('OpenAI Embeddings response body must be an object'); const { data } = value; if (!Array.isArray(data)) throw new Error('data must be an array'); const usage = parseUsage(value.usage); - const embeddings: CanonicalEmbedding[] = data.map((entry, position) => { + const embeddings: CanonicalOpenAIEmbedding[] = data.map((entry, position) => { if (!isRecord(entry)) throw new Error(`data[${position}] must be an object`); return { index: requiredInteger(entry.index, `data[${position}].index`), @@ -182,9 +182,9 @@ export const parseEmbeddingsResponse = (value: unknown, requestedModel: string): * protocol rather than something an upstream varies, and it is written here rather than * repeated back from the body. */ -export const renderEmbeddingsResponse = ( - format: EmbeddingsEncodingFormat, - response: CanonicalEmbeddingsResponse, +export const renderOpenAIEmbeddingsResponse = ( + format: OpenAIEmbeddingsEncodingFormat, + response: CanonicalOpenAIEmbeddingsResponse, ): Record => ({ object: 'list', data: response.embeddings.map(embedding => ({ diff --git a/packages/protocols/src/openai-images/index.ts b/packages/protocols/src/openai-images/index.ts index 480c61fdad..e820333e23 100644 --- a/packages/protocols/src/openai-images/index.ts +++ b/packages/protocols/src/openai-images/index.ts @@ -1,10 +1,11 @@ -// The images protocol: POST /v1/images/generations and POST /v1/images/edits. One protocol -// over two endpoints — generations takes JSON, and edits takes either JSON, where every image -// is a URL, a data URL or a file id, or a multipart form carrying the files themselves. +// The OpenAI Images protocol: POST /v1/images/generations and POST /v1/images/edits. One +// protocol over two endpoints — generations takes JSON, and edits takes either JSON, where +// every image is a URL, a data URL or a file id, or a multipart form carrying the files +// themselves. // https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L12858-L12870 // https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L12558-L12620 -export type ImagesOperation = 'generations' | 'edits'; +export type OpenAIImagesOperation = 'generations' | 'edits'; // JSON payload accepted by POST /v1/images/generations. Field set follows // OpenAI's reference for gpt-image-* and legacy dall-e-* (dall-e is @@ -40,7 +41,7 @@ export type OpenAIImageEditReference = /** A file the client sent in a multipart form, held as bytes: a form is a parsed value like * any other, and bytes are what survives being read once and what a dump can show. */ -export interface ImagesUploadedFile { +export interface OpenAIImagesUploadedFile { fileName: string; mediaType: string; bytes: Uint8Array; @@ -49,36 +50,36 @@ export interface ImagesUploadedFile { /** One image an edit reads. Multipart carries the file itself; JSON carries a reference for * the upstream to resolve, kept exactly as the client wrote it — whether a data URL can be * turned back into a file is the upstream serializer's question, not this one's. */ -export type ImagesEditImage = - | { kind: 'file'; file: ImagesUploadedFile } - | { kind: 'reference'; reference: ImageEditReference }; +export type OpenAIImagesEditImage = + | { kind: 'file'; file: OpenAIImagesUploadedFile } + | { kind: 'reference'; reference: OpenAIImageEditReference }; -export interface CanonicalImagesGenerationsRequest { +export interface CanonicalOpenAIImagesGenerationsRequest { operation: 'generations'; /** Everything the client sent but `model`: routing owns the model id, and what is left is * what the upstream is asked for. */ parameters: Record; } -export interface CanonicalImagesEditsRequest { +export interface CanonicalOpenAIImagesEditsRequest { operation: 'edits'; - images: ImagesEditImage[]; - mask?: ImagesEditImage; + images: OpenAIImagesEditImage[]; + mask?: OpenAIImagesEditImage; parameters: Record; } -export type CanonicalImagesRequest = CanonicalImagesGenerationsRequest | CanonicalImagesEditsRequest; +export type CanonicalOpenAIImagesRequest = CanonicalOpenAIImagesGenerationsRequest | CanonicalOpenAIImagesEditsRequest; -export interface ParsedImagesRequest { +export interface ParsedOpenAIImagesRequest { model: string; - request: CanonicalImagesRequest; + request: CanonicalOpenAIImagesRequest; } /** One image the upstream returned. `response_format` decides which arm carries it on the * dall-e models, while the GPT image models answer base64 and do not serve a URL at all, so * neither arm is the one to expect. * https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L51044-L51067 */ -export interface CanonicalImage { +export interface CanonicalOpenAIImage { url?: string; base64?: string; revisedPrompt?: string; @@ -89,19 +90,19 @@ export interface CanonicalImage { * the dall-e models report nothing — which is why the whole reading is optional as well as * each field. * https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L78089-L78115 */ -export interface CanonicalImagesUsage { +export interface CanonicalOpenAIImagesUsage { inputTokens?: number; inputImageTokens?: number; outputTokens?: number; outputImageTokens?: number; } -export interface CanonicalImagesResponse { +export interface CanonicalOpenAIImagesResponse { /** The parsed body as it arrived. One protocol in and one out means rendering is * re-serializing this, so the fields the gateway does not model still reach the client. */ raw: Record; - images: CanonicalImage[]; - usage?: CanonicalImagesUsage; + images: CanonicalOpenAIImage[]; + usage?: CanonicalOpenAIImagesUsage; } /** One event on a streamed answer. A stream carries partial images and then the completed one, @@ -113,7 +114,7 @@ export interface CanonicalImagesResponse { * `type` is an open string rather than the four names the specification lists: an event type * is the upstream's own word, and a shape added later has to reach the client whether or not * this gateway has heard of it. The index signature carries everything not named here. */ -export interface ImagesStreamEvent { +export interface OpenAIImagesStreamEvent { type: string; b64_json?: string; created_at?: number; @@ -126,7 +127,7 @@ export interface ImagesStreamEvent { [key: string]: unknown; } -export { imagesRequestWantsStream, parseImagesEditsRequest, parseImagesGenerationsRequest } from './request.ts'; -export { parseImagesResponse, parseImagesUsage, renderImagesResponse } from './response.ts'; -export { IMAGES_MISSING_TERMINAL_MESSAGE, isImagesTerminalEvent, parseImagesStream, type ParseImagesStreamOptions } from './stream.ts'; -export { imagesStreamEventToSSEFrame } from './to-sse.ts'; +export { openaiImagesRequestWantsStream, parseOpenAIImagesEditsRequest, parseOpenAIImagesGenerationsRequest } from './request.ts'; +export { parseOpenAIImagesResponse, parseOpenAIImagesUsage, renderOpenAIImagesResponse } from './response.ts'; +export { OPENAI_IMAGES_MISSING_TERMINAL_MESSAGE, isOpenAIImagesTerminalEvent, parseOpenAIImagesStream, type ParseOpenAIImagesStreamOptions } from './stream.ts'; +export { openaiImagesStreamEventToSSEFrame } from './to-sse.ts'; diff --git a/packages/protocols/src/openai-images/request.ts b/packages/protocols/src/openai-images/request.ts index 65234b74ba..4a485787b7 100644 --- a/packages/protocols/src/openai-images/request.ts +++ b/packages/protocols/src/openai-images/request.ts @@ -2,7 +2,7 @@ // bodies — generations is JSON, edits is JSON or a multipart form — and each message here is // what the client is told, so they name the endpoint and the field they are about. -import type { CanonicalImagesRequest, ImageEditReference, ImagesEditImage, ImagesUploadedFile, ParsedImagesRequest } from './index.ts'; +import type { CanonicalOpenAIImagesRequest, OpenAIImageEditReference, OpenAIImagesEditImage, OpenAIImagesUploadedFile, ParsedOpenAIImagesRequest } from './index.ts'; import { isJsonMediaType, isMultipartFormDataMediaType } from '../common/media-type.ts'; const isRecord = (value: unknown): value is Record => @@ -17,7 +17,7 @@ const isRecord = (value: unknown): value is Record => * and a JSON edit carry the boolean the specification names. * https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L47542-L47673 */ -export const imagesRequestWantsStream = (request: CanonicalImagesRequest): boolean => +export const openaiImagesRequestWantsStream = (request: CanonicalOpenAIImagesRequest): boolean => request.parameters.stream === true || request.parameters.stream === 'true'; const jsonObject = (body: Uint8Array, endpoint: string): Record => { @@ -41,30 +41,30 @@ const requiredModel = (body: Record, endpoint: string): string return model; }; -export const parseImagesGenerationsRequest = (body: Uint8Array): ParsedImagesRequest => { - const payload = jsonObject(body, 'Images generations'); - const model = requiredModel(payload, 'Images generations'); +export const parseOpenAIImagesGenerationsRequest = (body: Uint8Array): ParsedOpenAIImagesRequest => { + const payload = jsonObject(body, 'OpenAI Images Generations'); + const model = requiredModel(payload, 'OpenAI Images Generations'); const { model: _model, ...parameters } = payload; return { model, request: { operation: 'generations', parameters } }; }; -export const parseImagesEditsRequest = async ( +export const parseOpenAIImagesEditsRequest = async ( contentType: string | null | undefined, body: Uint8Array, -): Promise => { +): Promise => { if (isJsonMediaType(contentType)) return jsonEdits(body); if (isMultipartFormDataMediaType(contentType)) return await multipartEdits(contentType, body); - throw new Error('Image edits request body must use application/json or multipart/form-data.'); + throw new Error('OpenAI Images Edits request body must use application/json or multipart/form-data.'); }; -const jsonEdits = (body: Uint8Array): ParsedImagesRequest => { - const payload = jsonObject(body, 'Image edits'); - const model = requiredModel(payload, 'Image edits'); +const jsonEdits = (body: Uint8Array): ParsedOpenAIImagesRequest => { + const payload = jsonObject(body, 'OpenAI Images Edits'); + const model = requiredModel(payload, 'OpenAI Images Edits'); if (!Array.isArray(payload.images)) { - throw new Error('Image edits request body must include an images array.'); + throw new Error('OpenAI Images Edits request body must include an images array.'); } - const images = payload.images.map((value, index) => referenceImage(value, `Image edits images[${index}]`)); - const mask = payload.mask === undefined ? undefined : referenceImage(payload.mask, 'Image edits mask'); + const images = payload.images.map((value, index) => referenceImage(value, `OpenAI Images Edits images[${index}]`)); + const mask = payload.mask === undefined ? undefined : referenceImage(payload.mask, 'OpenAI Images Edits mask'); const { model: _model, images: _images, mask: _mask, ...parameters } = payload; return { model, @@ -72,42 +72,42 @@ const jsonEdits = (body: Uint8Array): ParsedImagesRequest => { }; }; -const referenceImage = (value: unknown, path: string): ImagesEditImage => { +const referenceImage = (value: unknown, path: string): OpenAIImagesEditImage => { if (!isRecord(value)) throw new Error(`${path} must be an object.`); const { image_url: imageUrl, file_id: fileId } = value; const named = (typeof imageUrl === 'string' && fileId === undefined) || (typeof fileId === 'string' && imageUrl === undefined); if (!named) throw new Error(`${path} must contain exactly one string field: image_url or file_id.`); - return { kind: 'reference', reference: value as ImageEditReference }; + return { kind: 'reference', reference: value as OpenAIImageEditReference }; }; -const multipartEdits = async (contentType: string, body: Uint8Array): Promise => { +const multipartEdits = async (contentType: string, body: Uint8Array): Promise => { let form: FormData; try { // `BodyInit` excludes a view over a `SharedArrayBuffer`, which an inbound body never is. form = await new Response(body as BodyInit, { headers: { 'content-type': contentType } }).formData(); } catch { - throw new Error('Image edits request body must be valid multipart/form-data.'); + throw new Error('OpenAI Images Edits request body must be valid multipart/form-data.'); } const model = form.get('model'); if (typeof model !== 'string' || model.length === 0) { - throw new Error('Image edits request body must include a model field.'); + throw new Error('OpenAI Images Edits request body must include a model field.'); } - const images: ImagesEditImage[] = []; - let mask: ImagesEditImage | undefined; + const images: OpenAIImagesEditImage[] = []; + let mask: OpenAIImagesEditImage | undefined; const parameters: Record = {}; for (const [name, value] of form.entries()) { if (name === 'model') continue; // One image is sent as `image` and several as `image[]`; the upstream serializer picks the // field name back off the count, so the two arrive at one list here. if (name === 'image' || name === 'image[]') { - images.push({ kind: 'file', file: await uploadedFile(value, `Image edits ${name} fields must be files.`) }); + images.push({ kind: 'file', file: await uploadedFile(value, `OpenAI Images Edits ${name} fields must be files.`) }); } else if (name === 'mask') { - mask = { kind: 'file', file: await uploadedFile(value, 'Image edits mask field must be a file.') }; + mask = { kind: 'file', file: await uploadedFile(value, 'OpenAI Images Edits mask field must be a file.') }; } else { - if (typeof value !== 'string') throw new Error(`Image edits ${name} field must be text.`); + if (typeof value !== 'string') throw new Error(`OpenAI Images Edits ${name} field must be text.`); parameters[name] = value; } } @@ -118,7 +118,7 @@ const multipartEdits = async (contentType: string, body: Uint8Array): Promise => { +const uploadedFile = async (value: FormDataEntryValue, message: string): Promise => { if (!(value instanceof File)) throw new Error(message); return { fileName: value.name, mediaType: value.type, bytes: new Uint8Array(await value.arrayBuffer()) }; }; diff --git a/packages/protocols/src/openai-images/response.ts b/packages/protocols/src/openai-images/response.ts index 15da5240c2..38bb2f3b9b 100644 --- a/packages/protocols/src/openai-images/response.ts +++ b/packages/protocols/src/openai-images/response.ts @@ -2,17 +2,17 @@ // the two are the same shape and rendering a success is re-serializing what was parsed — // which is what keeps the fields this file does not name from being dropped on the way out. -import type { CanonicalImage, CanonicalImagesResponse, CanonicalImagesUsage } from './index.ts'; +import type { CanonicalOpenAIImage, CanonicalOpenAIImagesResponse, CanonicalOpenAIImagesUsage } from './index.ts'; const isRecord = (value: unknown): value is Record => typeof value === 'object' && value !== null && !Array.isArray(value); -export const parseImagesResponse = (value: unknown): CanonicalImagesResponse => { - if (!isRecord(value)) throw new Error('Images response body must be an object'); - const usage = parseImagesUsage(value); +export const parseOpenAIImagesResponse = (value: unknown): CanonicalOpenAIImagesResponse => { + if (!isRecord(value)) throw new Error('OpenAI Images response body must be an object'); + const usage = parseOpenAIImagesUsage(value); return { raw: value, - images: parseImages(value.data), + images: parseOpenAIImages(value.data), ...(usage === undefined ? {} : { usage }), }; }; @@ -20,14 +20,14 @@ export const parseImagesResponse = (value: unknown): CanonicalImagesResponse => // An answer that carries no `data` at all is not malformed — an upstream reporting a // moderation refusal in a 200 does exactly that — so it parses to no images, and only a `data` // that is there but cannot be read is an error. -const parseImages = (data: unknown): CanonicalImage[] => { +const parseOpenAIImages = (data: unknown): CanonicalOpenAIImage[] => { if (data === undefined) return []; - if (!Array.isArray(data)) throw new Error('Images response data must be an array'); + if (!Array.isArray(data)) throw new Error('OpenAI Images response data must be an array'); return data.map((entry, index) => { - if (!isRecord(entry)) throw new Error(`Images response data[${index}] must be an object`); - const url = optionalString(entry.url, `Images response data[${index}].url`); - const base64 = optionalString(entry.b64_json, `Images response data[${index}].b64_json`); - const revisedPrompt = optionalString(entry.revised_prompt, `Images response data[${index}].revised_prompt`); + if (!isRecord(entry)) throw new Error(`OpenAI Images response data[${index}] must be an object`); + const url = optionalString(entry.url, `OpenAI Images response data[${index}].url`); + const base64 = optionalString(entry.b64_json, `OpenAI Images response data[${index}].b64_json`); + const revisedPrompt = optionalString(entry.revised_prompt, `OpenAI Images response data[${index}].revised_prompt`); return { ...(url === undefined ? {} : { url }), ...(base64 === undefined ? {} : { base64 }), @@ -47,7 +47,7 @@ const optionalString = (value: unknown, field: string): string | undefined => { * distinction is the whole point of the return type: an upstream that reported no usage is a * different fact from one that reported zero, and only one of them is a reading. */ -export const parseImagesUsage = (value: unknown): CanonicalImagesUsage | undefined => { +export const parseOpenAIImagesUsage = (value: unknown): CanonicalOpenAIImagesUsage | undefined => { if (!isRecord(value) || !isRecord(value.usage)) return undefined; const { input_tokens: inputTotal, @@ -88,4 +88,4 @@ const splitModality = ( return { text: text ?? 0, image: image ?? 0 }; }; -export const renderImagesResponse = (response: CanonicalImagesResponse): Record => response.raw; +export const renderOpenAIImagesResponse = (response: CanonicalOpenAIImagesResponse): Record => response.raw; diff --git a/packages/protocols/src/openai-images/stream.ts b/packages/protocols/src/openai-images/stream.ts index 8d3b3b96f4..0596177163 100644 --- a/packages/protocols/src/openai-images/stream.ts +++ b/packages/protocols/src/openai-images/stream.ts @@ -5,31 +5,31 @@ // https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L13077-L13086 // https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L12846-L12857 -import type { ImagesStreamEvent } from './index.ts'; +import type { OpenAIImagesStreamEvent } from './index.ts'; import { parseTargetStreamFrames } from '../common/parse-events.ts'; import { parseSSEStream } from '../common/parse-sse.ts'; -export interface ParseImagesStreamOptions { +export interface ParseOpenAIImagesStreamOptions { signal?: AbortSignal; } -export const IMAGES_MISSING_TERMINAL_MESSAGE = 'Images stream ended without a completed event.'; +export const OPENAI_IMAGES_MISSING_TERMINAL_MESSAGE = 'OpenAI Images stream ended without a completed event.'; /** The stream is over when the image is. Each endpoint names its own terminal — * `image_generation.completed` and `image_edit.completed` — so the suffix is the whole of * what the two have in common, which is also what makes this reading serve an endpoint added * later without being taught its prefix. */ -export const isImagesTerminalEvent = (event: ImagesStreamEvent): boolean => event.type.endsWith('.completed'); +export const isOpenAIImagesTerminalEvent = (event: OpenAIImagesStreamEvent): boolean => event.type.endsWith('.completed'); /** * The upstream's SSE body as this protocol's events. Transport framing ends here: what comes - * out carries images events, and nothing that reads them knows how they arrived. + * out carries OpenAI Images events, and nothing that reads them knows how they arrived. */ -export const parseImagesStream = ( +export const parseOpenAIImagesStream = ( body: ReadableStream, - options: ParseImagesStreamOptions = {}, -): AsyncGenerator => (async function* () { - for await (const frame of parseTargetStreamFrames>(parseSSEStream(body, options), { protocol: 'Images' })) { + options: ParseOpenAIImagesStreamOptions = {}, +): AsyncGenerator => (async function* () { + for await (const frame of parseTargetStreamFrames>(parseSSEStream(body, options), { protocol: 'OpenAI Images' })) { // `[DONE]` is not part of this protocol. An OpenAI-compatible upstream that appends the // sentinel it writes elsewhere is saying the body is over, which is all it can mean here — // whether the image finished is what the completed event says, and that has already passed. @@ -44,8 +44,8 @@ export const parseImagesStream = ( * reconciled once at the boundary rather than at each reader. An event that carries neither * is not one this protocol can place — it could be the terminal or a partial — so it ends the * read rather than being passed on as an unknown. */ -const named = (event: Record, label: string | undefined): ImagesStreamEvent => { - if (typeof event.type === 'string') return event as ImagesStreamEvent; - if (label === undefined) throw new Error('Images stream event carries no type, on the payload or as an SSE event name'); - return { ...event, type: label } as ImagesStreamEvent; +const named = (event: Record, label: string | undefined): OpenAIImagesStreamEvent => { + if (typeof event.type === 'string') return event as OpenAIImagesStreamEvent; + if (label === undefined) throw new Error('OpenAI Images stream event carries no type, on the payload or as an SSE event name'); + return { ...event, type: label } as OpenAIImagesStreamEvent; }; diff --git a/packages/protocols/src/openai-images/to-sse.ts b/packages/protocols/src/openai-images/to-sse.ts index 0075f9eb50..8f4d2410e2 100644 --- a/packages/protocols/src/openai-images/to-sse.ts +++ b/packages/protocols/src/openai-images/to-sse.ts @@ -1,9 +1,9 @@ -import type { ImagesStreamEvent } from './index.ts'; +import type { OpenAIImagesStreamEvent } from './index.ts'; import { type SseFrame, sseFrame } from '../common/index.ts'; // The specification writes an event's name twice — as the SSE `event:` label and as `type` // inside the payload — and the stream parser reconciles the two on the way in, so writing the // label back off the payload writes the name that arrived. // https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L13077-L13086 -export const imagesStreamEventToSSEFrame = (event: ImagesStreamEvent): SseFrame => +export const openaiImagesStreamEventToSSEFrame = (event: OpenAIImagesStreamEvent): SseFrame => sseFrame(JSON.stringify(event), event.type); From b2ba45fb9bd91936f92ba50f4d29d7b3862d5de6 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 07:31:55 +0800 Subject: [PATCH 48/79] fix(pipeline): record the run that threw, instead of losing it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The record is closed at the seam — `finalizeGatewayResponse` is what writes it — so an exception escaping past `serveThrough` never reached that call. The turn that most needs explaining was the one that left nothing behind: no row in the dashboard, no event stream, no reason. Only the OpenAI Responses WebSocket turn had ever called `dump.failed`, which is why the gap survived: the transport that owns its own response also owns its own catch. So the seam catches. What goes to the client is the same envelope the app's own handler writes, from the same function, stack included — a gateway's own fault is the operator's to read — and the record is closed with the reason on it and every stage the run did get through still in it, which is what says how far the turn got. `run` drains what it opened before it rethrows, so nothing is left outstanding to release here; only the record, and the reason. The audio family's header said it writes its own epilogue rather than going through the seam. It named the two things the seam owed it — a null content type for a document whose upstream declared none, and a reading that carries both what was billed and whether the stream finished — and both were given, at which point the handler became `serveThrough` and the prose did not follow. Verified by mutation: rethrowing instead of recording leaves zero stored records where the test expects one. --- .../openai-completions/http_test.ts | 47 ++++++++++++++++++- .../src/data-plane/openai-audio/http.ts | 19 +++----- .../gateway/src/data-plane/pipeline/serve.ts | 33 +++++++++++++ 3 files changed, 86 insertions(+), 13 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts b/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts index f8df8fcd6e..441177ceaf 100644 --- a/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts @@ -1,4 +1,4 @@ -import { test } from 'vitest'; +import { test, vi } from 'vitest'; import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts'; import { tokenCountsFromUsage } from '../../../src/repo/usage-metrics.ts'; @@ -404,3 +404,48 @@ test('/v1/completions streaming records usage row, performance neutral row (text const stored = new TextDecoder().decode(runRecordOf(dumpStubs.stored[0]!.record).events); assertEquals(stored.includes('"prompt_tokens":4'), true); }); + +// A run that threw is the turn that most needs explaining and used to be the one that left +// nothing behind: the record is closed at the seam, so an exception escaping past it lost the +// whole thing — no row, no events, no reason. It is answered there now, with the same envelope +// the app's own handler writes. +test('/v1/completions a run that threw is recorded, with the reason and a debuggable body', async () => { + const { apiKey, repo } = await setupAppTest(); + await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); + await registerOpenAICompletionsUpstream(repo); + const dumpStubs = installDumpStubs(initDumpStore, initDumpBroker); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + + // Read inside `resolveCandidates`, which is a stage — so the throw happens with the run + // open and a record already accumulating. + repo.modelAliases.getByName = () => Promise.reject(new Error('alias lookup exploded')); + + try { + const response = await requestApp('/v1/completions', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + body: JSON.stringify({ model: 'davinci-002', prompt: 'hello' }), + }); + + assertEquals(response.status, 500); + const body = await response.json() as { error: { type: string; message: string; stack?: string } }; + assertEquals(body.error.type, 'internal_error'); + assertEquals(body.error.message, 'alias lookup exploded'); + assertExists(body.error.stack); + } finally { + errorSpy.mockRestore(); + } + + await flushAsyncWork(); + + assertEquals(dumpStubs.stored.length, 1); + const dump = runRecordOf(dumpStubs.stored[0]?.record); + assertEquals(dump.meta.status, 500); + // The model the client asked for survives an outright failure, and the reason is on the row + // rather than only in the answer the client happened to receive. + assertEquals(dump.meta.model, 'davinci-002'); + assertEquals(dump.meta.error, { kind: 'failed', reason: 'alias lookup exploded' }); + // And the stages the run did get through are in it, which is what makes the record worth + // keeping: it says how far the turn got before it threw. + assertEquals(eventsOf(dump).some(event => event.type === 'stage.entered'), true); +}); diff --git a/packages/gateway/src/data-plane/openai-audio/http.ts b/packages/gateway/src/data-plane/openai-audio/http.ts index 036f15fb99..ee5b286054 100644 --- a/packages/gateway/src/data-plane/openai-audio/http.ts +++ b/packages/gateway/src/data-plane/openai-audio/http.ts @@ -79,18 +79,13 @@ const bodyOf = (rendered: Record | Uint8Array): Uint8Array => /** * The epilogue: what the run answered with, as a response. * - * This family writes its own rather than going through `serveThrough`, on two counts that - * belong to the seam rather than to it. - * - * A carried document goes out under the upstream's own media type, and that includes the - * upstream which declared none — a value `Rendered.contentType` cannot hold. - * - * And a transcription's stream states its own outcome in its last event, which is exactly the - * shape `serveThrough`'s open decision on truncated streams names as the fix: the family's - * meter reports how the stream ended, because it is the one place that knows. `DeferredUsage` - * carries only what was billed, so this family settles its own — the write still registered - * while the request is live, on a promise that resolves with both. Give the seam these two - * and this handler is `serveThrough` again. + * It is `serveThrough` like every other family, and it took two things at the seam to make it + * one. A carried document goes out under the upstream's own media type, including the upstream + * that declared none — so `Rendered.contentType` carries a null and the seam answers without + * the header rather than inventing one. And a transcription's stream states its own outcome in + * its last event, which the family's meter reads because it is the one place that knows — so + * what `DeferredUsage` hands back is both what was billed and whether the stream finished, + * settled from one promise while the request is still live. */ export const openaiAudioTranscriptions = async (c: Context): Promise => { const ingress = await readIngress(c); diff --git a/packages/gateway/src/data-plane/pipeline/serve.ts b/packages/gateway/src/data-plane/pipeline/serve.ts index fc3fcbe2a9..a9306c2906 100644 --- a/packages/gateway/src/data-plane/pipeline/serve.ts +++ b/packages/gateway/src/data-plane/pipeline/serve.ts @@ -15,6 +15,7 @@ import type { GatewayServices } from './services.ts'; import { settleBillable } from './settlement.ts'; import { openRunDump } from '../../dump/run-sink.ts'; import { apiKeyFromContext, type AuthedContext } from '../../middleware/auth.ts'; +import { internalErrorResponse } from '../../middleware/internal-error-response.ts'; import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; import { consoleLogSink } from '../../runtime/log.ts'; import { createGatewayCtxFromHono, finalizeGatewayResponse, type GatewayCtx } from '../shared/gateway-ctx.ts'; @@ -154,6 +155,12 @@ export type DeferredUsage = (facts: Exit) => Promise | null * so draining before returning would consume the frames the client is waiting for. Release * is not cancel — an aborted connection cannot be reused and leaves the upstream's own * billing unsettled — so it still happens, just after the answer is on its way. + * + * A run that threw is answered here rather than by the app's own handler, because the record + * is closed here. Letting the throw out would lose the whole record: nothing above this knows + * a dump is open, so the turn that most needs explaining would be the one that left nothing + * behind. What goes to the client is the same envelope the app's handler writes, from the same + * function — the stack included, which is what a gateway owes an operator for its own fault. */ export const serveThrough = async < Entry extends object, @@ -165,6 +172,32 @@ export const serveThrough = async < entry: Entry, render: (facts: Exit) => Rendered, deferredUsage?: DeferredUsage, +): Promise => { + try { + return await serveRun(c, prologue, pipeline, entry, render, deferredUsage); + } catch (error) { + // `run` drains what it opened before it rethrows, so there is nothing left outstanding to + // release here — only the record to close, and the reason to put on it. + prologue.gateway.dump?.failed(error); + return finalizeGatewayResponse(prologue.gateway, internalErrorResponse(asError(error), c)); + } +}; + +/** Anything can be thrown; only an `Error` carries a stack. A throw that was not one is + * reported as what it was rather than being dressed up as something with a call site. */ +const asError = (thrown: unknown): Error => + thrown instanceof Error ? thrown : new Error(String(thrown)); + +const serveRun = async < + Entry extends object, + Exit extends Slice<'response.http.status' | 'response.http.headers'>, +>( + c: Context, + prologue: Prologue, + pipeline: Pipeline, + entry: Entry, + render: (facts: Exit) => Rendered, + deferredUsage?: DeferredUsage, ): Promise => { const { facts, drain } = await run(pipeline, entry, prologue.services as never); const answer = render(facts); From be228b963a1341a09101dc7ba1acd8249fa7a975 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 07:52:14 +0800 Subject: [PATCH 49/79] feat(dump): give a stream an id, a terminator, and a fact that points at it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The format says a stream sits at its key as `{"$stream": n}`, that its frames arrive afterwards naming that id, and that `stream.end` is when its record is complete. Only the middle third existed. `streamFact` had no production caller, so no fact anywhere carried a reference; frames went out under a hardcoded `streamId: 1`, naming a stream nothing had opened; and `stream.end` had no producer at all. A reader could see the frames a turn served, in order, and could not tell which key they belonged to, could not tell two streams apart, and could not tell a stream that ended from one that was cut off. `recordStream` is the producer, and what it hands back *is* the reference: the value a family puts at its fact is the branded iterable, so the fact encodes as `{"$stream": n}` and every frame that follows names the same n. The id comes from the recording rather than being fixed. `stream.end` is written where the source ran out on its own — a reader that stopped early never reaches it, which is what makes its absence mean "cut short". One stream is usually held at two keys: the answer the ending provided, and the framing the edge built over it for the client. `streamReferenceOf` carries the reference onto the wrapper, so both point at one record — a reframing is not a second stream. Recording moves outside the meter in the two families that had threaded a callback into it. That is what makes "reading is what records" true: the release path drains the stream underneath the recorder, so a losing attempt nobody read now contributes nothing instead of contributing frames the client never saw. Audio recorded nothing at all. Its stream carries bare protocol events rather than protocol frames, which is why it had been skipped and why `recordStream` asks that family how one becomes the other — the record holds frames and a cast there would be the record guessing. A streamed transcription was being stored as stage boundaries with the transcript missing. The edge record has one frame log and no way to name anything in it, so its recording writes no terminator and offers no fact. That is the shape's limit rather than an omission, and it goes away with the shape. Verified by mutation: dropping the terminator empties the `stream.end` assertion, and dropping the reference fails the `$stream` one. --- .../data-plane/openai-audio/http_test.ts | 14 ++++ .../openai-completions/http_test.ts | 11 +++ .../src/data-plane/openai-audio/pipeline.ts | 17 +++-- .../data-plane/openai-completions/pipeline.ts | 30 ++++---- .../src/data-plane/openai-images/pipeline.ts | 27 ++++--- packages/gateway/src/dump/accumulator.ts | 9 +++ packages/gateway/src/dump/run-sink.ts | 31 ++++++-- packages/gateway/src/dump/turn-dump.ts | 74 +++++++++++++++++++ 8 files changed, 175 insertions(+), 38 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/openai-audio/http_test.ts b/packages/gateway/__tests__/data-plane/openai-audio/http_test.ts index 90b97ab392..51a12605bf 100644 --- a/packages/gateway/__tests__/data-plane/openai-audio/http_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-audio/http_test.ts @@ -1,5 +1,7 @@ import { test, vi } from 'vitest'; +import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts'; +import { eventsOf, installDumpStubs, runRecordOf } from '../../dump/test-fixtures.ts'; import type { InMemoryRepo } from '../../repo/memory.ts'; import { flushAsyncWork, MOCKED_FETCH_EGRESS, requestApp, setupAppTest } from '../../test-utils/app.ts'; import type { ModelPricing } from '@floway-dev/protocols/common'; @@ -369,9 +371,11 @@ test('/v1/audio/transcriptions preserves token usage unpriced when the model is test('/v1/audio/transcriptions streams through transcript.text.done without adding Chat termination', async () => { const { apiKey, repo } = await setupAppTest(); + await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); await registerAudioModel(repo, { entries: [{ rates: { input_audio_tokens: '0.000001', output_tokens: '0.000001' } }], }); + const dumpStubs = installDumpStubs(initDumpStore, initDumpBroker); await withMockedFetch( () => new Response([ 'data: {"type":"transcript.text.delta","delta":"hel"}', @@ -400,6 +404,16 @@ test('/v1/audio/transcriptions streams through transcript.text.done without addi const [performance] = await repo.performance.listAll(); assertEquals(performance.neutral, 1); assertEquals(performance.errorsNoOutput, 0); + + // This family's stream is bare protocol events rather than protocol frames, so the record is + // told how one becomes a frame — and until it was, a streamed transcription was recorded as + // stage boundaries with the transcript missing entirely. + const events = eventsOf(runRecordOf(dumpStubs.stored[0]?.record)); + const frames = events + .filter(event => event.type === 'stream.frame') + .flatMap(event => (event as unknown as { frames: readonly unknown[] }).frames); + assertEquals(frames.length, 2); + assertEquals(events.filter(event => event.type === 'stream.end'), [{ type: 'stream.end', streamId: 1 }]); }); test('/v1/audio/transcriptions preserves a terminal stream event with malformed usage', async () => { diff --git a/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts b/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts index 441177ceaf..5edead4868 100644 --- a/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts @@ -403,6 +403,17 @@ test('/v1/completions streaming records usage row, performance neutral row (text // what is asserted is that the numbers are in the stream, not the shape they took in it. const stored = new TextDecoder().decode(runRecordOf(dumpStubs.stored[0]!.record).events); assertEquals(stored.includes('"prompt_tokens":4'), true); + + // The stream is named, and every frame says which stream it belongs to — the id is + // allocated per stream rather than fixed, so a run that opened two would keep them apart. + const streamIds = new Set(events.filter(event => event.type === 'stream.frame').map(event => event.streamId)); + assertEquals([...streamIds], [1]); + // And the record of it is complete. A client that stopped reading would leave the frames it + // did get and no terminator, which is how a reader tells the two apart. + assertEquals(events.filter(event => event.type === 'stream.end'), [{ type: 'stream.end', streamId: 1 }]); + // The facts that hold the stream point at it: the answer the ending provided, and the + // framing the edge built for the client over the same frames. + assertEquals(stored.includes('"$stream":1'), true); }); // A run that threw is the turn that most needs explaining and used to be the one that left diff --git a/packages/gateway/src/data-plane/openai-audio/pipeline.ts b/packages/gateway/src/data-plane/openai-audio/pipeline.ts index 41420398c8..e3ad81c1e3 100644 --- a/packages/gateway/src/data-plane/openai-audio/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-audio/pipeline.ts @@ -13,6 +13,7 @@ // callOpenAIAudioTranscriptionUpstream the ending: dials, reads what came back, and // provides the answer plus what is billable +import { recordStream, streamReferenceOf } from '../../dump/turn-dump.ts'; import type { UsageQuantities } from '../../repo/types.ts'; import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; import { isFailure } from '../pipeline/facts.ts'; @@ -24,7 +25,7 @@ import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/te import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import { compose, defineStage, move, own, type Logger, type Owned, type Pipeline } from '@floway-dev/pipeline'; -import { isEventStreamMediaType, parseDecimalString, parseSSEStream, renderErrorEnvelope, sseFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { eventFrame, isEventStreamMediaType, parseDecimalString, parseSSEStream, renderErrorEnvelope, sseFrame, type SseFrame } from '@floway-dev/protocols/common'; import { isOpenAIAudioTranscriptionDoneEvent, parseOpenAIAudioTranscription, @@ -167,10 +168,14 @@ const emitOpenAIAudioTranscription = defineStage< }, }); -const renderSSE = (events: OpenAIAudioTranscriptionEvents): AsyncIterable => - viewOf((async function* () { +const renderSSE = (events: OpenAIAudioTranscriptionEvents): AsyncIterable => ({ + // The frames the client reads are a reframing of the events the record holds, so this key + // points at that same stream rather than at nothing. + ...streamReferenceOf(events), + [Symbol.asyncIterator]: () => (async function* () { for await (const event of events) yield sseFrame(JSON.stringify(event)); - })()); + })(), +}); /** * The ending. It dials, reads what came back in the rendering the request asked for, and @@ -280,7 +285,9 @@ const callOpenAIAudioTranscriptionUpstream = defineStage< const metered = meterEvents(result.response.body, identity, use.gateway.abortSignal, use.log); return move({ ...facts, - 'response.openaiAudioTranscription.canonical': metered.events, + // This protocol's stream is bare events rather than protocol frames, so the record is + // told how one becomes a frame instead of being left to assume. + 'response.openaiAudioTranscription.canonical': recordStream(metered.events, use.gateway.dump, eventFrame), 'response.openaiAudioTranscription.mediaType': mediaType, 'response.openaiAudioTranscription.streamedOutcome': metered.outcome, 'response.usage.billable': called, diff --git a/packages/gateway/src/data-plane/openai-completions/pipeline.ts b/packages/gateway/src/data-plane/openai-completions/pipeline.ts index aa9818432f..7140e006e3 100644 --- a/packages/gateway/src/data-plane/openai-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-completions/pipeline.ts @@ -13,6 +13,7 @@ // answer plus what is billable import { tokenUsageFromOpenAICompletionsUsage } from './usage.ts'; +import { recordStream, streamReferenceOf, type TurnDump } from '../../dump/turn-dump.ts'; import type { UsageQuantities } from '../../repo/types.ts'; import { tokenUsageQuantities } from '../../repo/usage-metrics.ts'; import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; @@ -144,12 +145,17 @@ const rendered = ( : isFrames(answer) ? renderSSE(answer, wantsUsageChunk) : answer; -const renderSSE = (frames: OpenAICompletionsFrames, wantsUsageChunk: boolean): AsyncIterable => view((async function* () { - for await (const frame of frames) { - if (!wantsUsageChunk && frame.type === 'event' && isOpenAIUsageOnlyEventShape(frame.event)) continue; - yield openaiCompletionsProtocolFrameToSSEFrame(frame); - } -})()); +const renderSSE = (frames: OpenAICompletionsFrames, wantsUsageChunk: boolean): AsyncIterable => ({ + // The frames the client reads are a reframing of the ones the record holds, so this key + // points at that same stream rather than at nothing. + ...streamReferenceOf(frames), + [Symbol.asyncIterator]: () => (async function* () { + for await (const frame of frames) { + if (!wantsUsageChunk && frame.type === 'event' && isOpenAIUsageOnlyEventShape(frame.event)) continue; + yield openaiCompletionsProtocolFrameToSSEFrame(frame); + } + })(), +}); /** * The ending. It dials, reads what came back on the shape the request asked for, and @@ -216,7 +222,7 @@ const callOpenAICompletionsUpstream = defineStage< telemetryModelIdentity(candidate, result.modelKey), candidate, use.gateway.abortSignal, - frame => { use.gateway.dump?.frame(frame); }, + use.gateway.dump, ); return move({ ...facts, @@ -247,7 +253,7 @@ const readUpstream = async ( identity: TelemetryModelIdentity, candidate: ModelCandidate, signal: AbortSignal | undefined, - record: (frame: ProtocolFrame) => void, + dump: TurnDump | null, ): Promise => { // An upstream was called and reported nothing — which is what every arm but a read usage // block leaves standing, and is a different statement from reporting zero. @@ -282,9 +288,9 @@ const readUpstream = async ( // the frames as they pass — so the reading costs one pass and the client's own stream is // what drives it. What it finds arrives with the last frame, long after this stage has // handed up, which is why the entity above carries no quantities. - const metered = meterFrames(parseOpenAICompletionsStream(response.body, { signal }), identity, candidate, record); + const metered = meterFrames(parseOpenAICompletionsStream(response.body, { signal }), identity, candidate); return { - payload: metered.frames, + payload: recordStream(metered.frames, dump), streamedUsage: metered.outcome, billable: called, // Releasing this body is draining those frames: they are one reader over one connection, @@ -330,7 +336,6 @@ const meterFrames = ( source: AsyncIterable>, identity: TelemetryModelIdentity, candidate: ModelCandidate, - record: (frame: ProtocolFrame) => void, ): MeteredFrames => { let settle!: (outcome: StreamOutcome) => void; const outcome = new Promise(resolve => { settle = resolve; }); @@ -349,11 +354,8 @@ const meterFrames = ( if (root.service_tier !== undefined) tier = root.service_tier; if (isOpenAIUsageOnlyEventShape(frame.event)) usage = root.usage; } - // What the upstream sent, before the edge decides what the client sees — a dump - // reader is owed the frames that arrived, including the one metering asked for. // The transport's own terminator is this protocol's end-of-turn. if (frame.type === 'done') sawTerminal = true; - record(frame); yield frame; } } finally { diff --git a/packages/gateway/src/data-plane/openai-images/pipeline.ts b/packages/gateway/src/data-plane/openai-images/pipeline.ts index 21cc41870c..e8897ad7a7 100644 --- a/packages/gateway/src/data-plane/openai-images/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-images/pipeline.ts @@ -11,6 +11,7 @@ // failover runs what follows once per candidate // callOpenAIImagesUpstream the ending: dials, and provides what came back +import { recordStream, streamReferenceOf } from '../../dump/turn-dump.ts'; import type { UsageQuantities } from '../../repo/types.ts'; import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; import { isFailure } from '../pipeline/facts.ts'; @@ -150,9 +151,14 @@ const rendered = (answer: OpenAIImagesFacts['response.openaiImages.canonical']): : isFrames(answer) ? renderSSE(answer) : renderOpenAIImagesResponse(answer); -const renderSSE = (frames: OpenAIImagesFrames): AsyncIterable => view((async function* () { - for await (const event of frames) yield openaiImagesStreamEventToSSEFrame(event); -})()); +const renderSSE = (frames: OpenAIImagesFrames): AsyncIterable => ({ + // The frames the client reads are a reframing of the ones the record holds, so this key + // points at that same stream rather than at nothing. + ...streamReferenceOf(frames), + [Symbol.asyncIterator]: () => (async function* () { + for await (const event of frames) yield openaiImagesStreamEventToSSEFrame(event); + })(), +}); /** * The ending. It dials, reads what came back on the shape the request asked for, and provides @@ -256,15 +262,12 @@ const callOpenAIImagesUpstream = defineStage< // folding the events as they pass — so the reading costs one pass and the client's own // stream is what drives it. What it finds arrives with the completed event, long after // this stage has handed up, which is why the entity above carries no quantities. - const metered = meterFrames( - result.response.body, - identity, - use.gateway.abortSignal, - event => { use.gateway.dump?.frame(eventFrame(event)); }, - ); + const metered = meterFrames(result.response.body, identity, use.gateway.abortSignal); return move({ ...facts, - 'response.openaiImages.canonical': metered.frames, + // This protocol's stream is bare events rather than protocol frames, so the record is + // told how one becomes a frame instead of being left to assume. + 'response.openaiImages.canonical': recordStream(metered.frames, use.gateway.dump, eventFrame), 'response.openaiImages.streamedUsage': metered.outcome, 'response.http.headers': headers, // Releasing this body is reading those events to the end: they are one reader over one @@ -314,7 +317,6 @@ const meterFrames = ( body: ReadableStream, identity: TelemetryModelIdentity, signal: AbortSignal | undefined, - record: (event: OpenAIImagesStreamEvent) => void, ): MeteredFrames => { let settle!: (outcome: StreamOutcome) => void; const outcome = new Promise(resolve => { settle = resolve; }); @@ -325,9 +327,6 @@ const meterFrames = ( let usage: CanonicalOpenAIImagesUsage | undefined; try { for await (const event of parseOpenAIImagesStream(body, { signal })) { - // What the upstream sent, before the edge decides what the client sees — a dump reader - // is owed the events that arrived. - record(event); if (isOpenAIImagesTerminalEvent(event)) { usage = parseOpenAIImagesUsage(event); sawTerminal = true; diff --git a/packages/gateway/src/dump/accumulator.ts b/packages/gateway/src/dump/accumulator.ts index 4a3a8b7e53..deebb7e391 100644 --- a/packages/gateway/src/dump/accumulator.ts +++ b/packages/gateway/src/dump/accumulator.ts @@ -12,6 +12,7 @@ import type { Context } from 'hono'; import { DumpAttribution, oneLineError, streamReadError } from './attribution.ts'; import { getDumpBroker, getDumpStore } from './registry.ts'; +import type { StreamRecording } from './turn-dump.ts'; import type { DumpMetadata, DumpStreamEvent, @@ -93,6 +94,14 @@ export class DumpAccumulator { this.events.push({ frame, ts: Date.now() - this.startedAt }); } + /** This shape has one frame log and no way to name anything in it, so every stream lands in + * the same place, there is no terminator to write, and no fact can point at one. A turn + * that opened two would have them interleaved — which the endpoints on this shape do not do, + * and which the run shape is what fixes. */ + openStream(): StreamRecording { + return { frame: frame => { this.frame(frame); }, end: () => {}, fact: null }; + } + recordSentPayloadBytes(byteLength: number): void { this.sentPayloadBytes += byteLength; } diff --git a/packages/gateway/src/dump/run-sink.ts b/packages/gateway/src/dump/run-sink.ts index 0b2d4bd905..2e9c36489f 100644 --- a/packages/gateway/src/dump/run-sink.ts +++ b/packages/gateway/src/dump/run-sink.ts @@ -15,11 +15,12 @@ import { DumpAttribution, oneLineError, streamReadError } from './attribution.ts'; import { getDumpBroker, getDumpStore } from './registry.ts'; +import type { StreamRecording } from './turn-dump.ts'; import type { DumpMetadata } from './types.ts'; import type { RequestBody } from '../data-plane/shared/request-body.ts'; import type { ApiKey, TokenUsage } from '../repo/types.ts'; import { ulid } from '../shared/ulid.ts'; -import { createRunEncoder, toNdjson, type DumpEvent, type Event } from '@floway-dev/pipeline'; +import { createRunEncoder, streamFact, toNdjson, type DumpEvent, type Event } from '@floway-dev/pipeline'; import type { BackgroundScheduler } from '@floway-dev/platform'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { TelemetryModelIdentity } from '@floway-dev/provider'; @@ -39,6 +40,8 @@ export class RunDump { private readonly encode = createRunEncoder(); private readonly events: DumpEvent[] = []; private sentPayloadBytes = 0; + private streams = 0; + private answerStream: StreamRecording | undefined; constructor( private readonly apiKey: ApiKey, @@ -71,12 +74,30 @@ export class RunDump { * A frame the client was sent, as the event the format names for one. * * The edge dump kept a frame log of its own; here a frame is content about a stream, so it - * is `stream.frame` and it folds through the same encoder as everything else. One stream - * per turn is what the endpoints this carries produce, so the id is fixed — a family that - * opens two would need to say which, and none does. + * is `stream.frame` and it folds through the same encoder as everything else. A frame pushed + * without opening a stream first belongs to the run's first one, which is what a transport + * writing single synthesized frames alongside its answer is doing. */ frame(frame: ProtocolFrame): void { - this.sink({ type: 'stream.frame', streamId: 1, frames: [frame] }); + this.answerStream ??= this.openStream(); + this.answerStream.frame(frame); + } + + /** + * Begins recording one stream, under an id of its own. + * + * The id is what makes the frames resolvable: the fact holding the stream carries + * `{"$stream": n}` and every frame event names the same `n`, so a run that opened two — a + * sub-request's stream beside the answer's — keeps them apart. `end` says the record of that + * stream is complete, and a client that stopped reading never reaches it. + */ + openStream(): StreamRecording { + const streamId = ++this.streams; + return { + frame: frame => { this.sink({ type: 'stream.frame', streamId, frames: [frame] }); }, + end: () => { this.sink({ type: 'stream.end', streamId }); }, + fact: streamFact(streamId), + }; } success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void { diff --git a/packages/gateway/src/dump/turn-dump.ts b/packages/gateway/src/dump/turn-dump.ts index 5e8483a4e6..4037b368d3 100644 --- a/packages/gateway/src/dump/turn-dump.ts +++ b/packages/gateway/src/dump/turn-dump.ts @@ -6,15 +6,38 @@ // and the stages in between neither know nor care which they are stamping. import type { TokenUsage } from '../repo/types.ts'; +import { isStreamFact, type StreamFact } from '@floway-dev/pipeline'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { TelemetryModelIdentity } from '@floway-dev/provider'; +/** + * One stream, as the recording knows it. + * + * A run record identifies its streams, because their content arrives over time and after the + * fact that holds them: the fact carries `{"$stream": n}` and the frames arrive afterwards + * naming that id. `end` is what says the record of this stream is complete — a client that + * stopped reading leaves it short, and the absence of the terminator is how a reader tells a + * stream that ended from one that was cut off. + * + * The edge record has one frame log and no way to name anything in it, so its `fact` is null + * and its terminator has nothing to write. That is the shape's limit rather than an omission, + * and it goes away with the shape. + */ +export interface StreamRecording { + frame(frame: ProtocolFrame): void; + end(): void; + readonly fact: StreamFact | null; +} + export interface TurnDump { requestedModel(model: string): void; success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void; error(kind: 'upstream' | 'gateway', upstream?: string): void; failed(reason: unknown): void; frame(frame: ProtocolFrame): void; + /** Begins recording one stream. Every call is a new one, which is what lets a turn that + * opens two — a sub-request beside the answer — keep them apart. */ + openStream(): StreamRecording; /** How much of the answer actually went out. A transport that writes its own frames counts * them itself, because nothing downstream of it can. */ recordSentPayloadBytes(byteLength: number): void; @@ -23,3 +46,54 @@ export interface TurnDump { finalize(status: number | null, responseBytes: number | readonly unknown[]): void; finalize(response: Response): Response; } + +/** + * Records a stream's frames as they are read, and hands them on untouched. + * + * What the record holds is the stream as the client is served it, so the tee goes outside + * whatever shapes the frames and inside whatever frames them for a transport. Reading is what + * records: a losing attempt nobody read contributes nothing, because the release path drains + * the stream underneath this rather than through it. + * + * The value handed back *is* the stream reference, so the fact that holds it encodes as + * `{"$stream": n}` and the frames that arrive afterwards say which stream they belong to. + * + * A record holds protocol frames, which is what most families' streams already carry. The + * family whose stream is bare protocol events says how one becomes a frame, because the record + * cannot guess and a cast would be it guessing. + */ +export function recordStream(stream: AsyncIterable>, dump: TurnDump | null): AsyncIterable>; +export function recordStream(stream: AsyncIterable, dump: TurnDump | null, asFrame: (value: T) => ProtocolFrame): AsyncIterable; +export function recordStream( + stream: AsyncIterable, + dump: TurnDump | null, + asFrame: (value: T) => ProtocolFrame = value => value as ProtocolFrame, +): AsyncIterable { + // No recording configured hands the same iterable back, so a record shows no step where + // nothing happened and the stream is not wrapped for nobody. + if (dump === null) return stream; + + const recording = dump.openStream(); + return { + ...recording.fact, + [Symbol.asyncIterator]: () => (async function* () { + for await (const value of stream) { + recording.frame(asFrame(value)); + yield value; + } + // Reached only where the source ran out on its own, which is what makes the record of + // this stream complete. A reader that stopped early never gets here. + recording.end(); + })(), + }; +} + +/** + * The reference a value carries to the stream the record holds, for a wrapper to carry across. + * + * A stream is framed again on its way out — protocol frames become SSE — and what the client + * is handed is a different object over the same frames. Carrying the reference onto it is what + * lets the fact that produced the stream and the fact that framed it point at one record. + */ +export const streamReferenceOf = (value: unknown): StreamFact | Record => + isStreamFact(value) ? { ...value } : {}; From aaa961df278661227b5222af357c03cb634e71ce Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 08:03:17 +0800 Subject: [PATCH 50/79] feat(pipeline): declare the streamed reading as work the run started MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `defer` had no production caller. The type existed, the runner swept for it at teardown, `deferral_test.ts` exercised the whole mechanism — and no fact in the gateway was ever marked, so the sweep never had anything to find. The streamed usage reading is exactly what it names: work a stage starts and does not finish, whose value arrives with the stream's last chunk, long after the run has answered. Three families hand one up and all three now declare it. What that buys is not the settlement — the epilogue already composed the promise — but the runner seeing it: a reading that never settles is reported at teardown instead of silently never billing, and it is legible from the run's own record rather than from a capability someone remembered to pass. `Deferred` is a `Promise` rather than a `PromiseLike`. Branding is all `defer` does, so what comes back is the same object, and the narrower interface only made a caller reach for `Promise.resolve` to get back what it already had. --- .../data-plane/openai-completions-pipeline_test.ts | 6 +++++- .../gateway/src/data-plane/openai-audio/pipeline.ts | 10 ++++++---- .../src/data-plane/openai-completions/pipeline.ts | 10 ++++++---- .../gateway/src/data-plane/openai-images/pipeline.ts | 10 ++++++---- packages/gateway/src/data-plane/pipeline/serve.ts | 10 ++++++---- packages/pipeline/src/run.ts | 10 +++++++--- 6 files changed, 36 insertions(+), 20 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts index cc0a9af162..db633e25ed 100644 --- a/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts @@ -10,7 +10,7 @@ import { openaiCompletionsServePipeline } from '../../src/data-plane/openai-comp import { enumerateModelCandidates } from '../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../src/repo/index.ts'; import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts'; -import { move, run } from '@floway-dev/pipeline'; +import { isDeferred, move, run } from '@floway-dev/pipeline'; import type { SseFrame } from '@floway-dev/protocols/common'; import { directFetcher, type ModelCandidate, type ProviderCallResult, type UpstreamCallOptions } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; @@ -162,6 +162,10 @@ describe('the OpenAI Completions pipeline', () => { expect(facts['response.usage.billable']).toEqual([ { identity: { model: 'text-model', upstream: 'up_a', modelKey: 'text-model-key', pricing: null }, quantities: {} }, ]); + // And the reading is declared as this run's own unfinished work rather than started and + // forgotten, which is what puts it in front of teardown: a family that hands up a reading + // that never settles is reported instead of silently never billing. + expect(isDeferred(facts['response.openaiCompletions.streamedUsage'])).toBe(true); await collect(facts['response.openaiCompletions.rendered']); expect(await facts['response.openaiCompletions.streamedUsage']).toMatchObject({ failed: false, diff --git a/packages/gateway/src/data-plane/openai-audio/pipeline.ts b/packages/gateway/src/data-plane/openai-audio/pipeline.ts index e3ad81c1e3..88407b021c 100644 --- a/packages/gateway/src/data-plane/openai-audio/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-audio/pipeline.ts @@ -24,7 +24,7 @@ import { dialFailure } from '../pipeline/upstream-body.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; -import { compose, defineStage, move, own, type Logger, type Owned, type Pipeline } from '@floway-dev/pipeline'; +import { compose, defer, defineStage, move, own, type Deferred, type Logger, type Owned, type Pipeline } from '@floway-dev/pipeline'; import { eventFrame, isEventStreamMediaType, parseDecimalString, parseSSEStream, renderErrorEnvelope, sseFrame, type SseFrame } from '@floway-dev/protocols/common'; import { isOpenAIAudioTranscriptionDoneEvent, @@ -90,7 +90,7 @@ export interface OpenAIAudioTranscriptionFacts extends GatewayFacts { * `response.usage.billable`, which says what had been reported when the ending stage * handed up: the entity, and no quantities. Settling from this is the epilogue's job, * after the drain. */ - 'response.openaiAudioTranscription.streamedOutcome': Promise | null; + 'response.openaiAudioTranscription.streamedOutcome': Deferred | null; /** What the client is actually sent — a JSON object, the upstream's own document, or the * SSE frames of a stream. The edge provides it, so a dump shows what the client received * rather than the gateway's own reading of it. */ @@ -376,7 +376,7 @@ const readUsage = (read: () => OpenAIAudioTranscriptionUsage | undefined, log: L interface MeteredEvents { readonly events: OpenAIAudioTranscriptionEvents; - readonly outcome: Promise; + readonly outcome: Deferred; } const meterEvents = ( @@ -386,7 +386,9 @@ const meterEvents = ( log: Logger, ): MeteredEvents => { let settle!: (outcome: OpenAIAudioTranscriptionStreamOutcome) => void; - const outcome = new Promise(resolve => { settle = resolve; }); + // Declared as this run's own unfinished work, so the runner waits for it at teardown where + // it can see it rather than the reading being started and forgotten. + const outcome = defer(new Promise(resolve => { settle = resolve; })); const events = viewOf((async function* () { let usage: OpenAIAudioTranscriptionUsage | undefined; let completed = false; diff --git a/packages/gateway/src/data-plane/openai-completions/pipeline.ts b/packages/gateway/src/data-plane/openai-completions/pipeline.ts index 7140e006e3..1468139696 100644 --- a/packages/gateway/src/data-plane/openai-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-completions/pipeline.ts @@ -26,7 +26,7 @@ import { dialFailure } from '../pipeline/upstream-body.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; -import { compose, defineStage, move, own, type Owned, type Pipeline } from '@floway-dev/pipeline'; +import { compose, defer, defineStage, move, own, type Deferred, type Owned, type Pipeline } from '@floway-dev/pipeline'; import { isOpenAIUsageOnlyEventShape, renderErrorEnvelope, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; import { openaiCompletionsProtocolFrameToSSEFrame, @@ -71,7 +71,7 @@ export interface OpenAICompletionsFacts extends GatewayFacts { * after this run has answered — so the numbers cannot be in `response.usage.billable`, * which says what had been reported when the ending stage handed up: the entity, and no * quantities. Settling billing from this is the prologue's job, after the drain. */ - 'response.openaiCompletions.streamedUsage': Promise | null; + 'response.openaiCompletions.streamedUsage': Deferred | null; /** What the client is actually sent, in its own protocol — a JSON body, or the SSE frames * of a stream. The edge provides it, so a dump shows what the client received rather than * the gateway's own reading of it. */ @@ -329,7 +329,7 @@ const readResult = async (response: Response): Promise<{ readonly value: OpenAIC interface MeteredFrames { readonly frames: OpenAICompletionsFrames; - readonly outcome: Promise; + readonly outcome: Deferred; } const meterFrames = ( @@ -338,7 +338,9 @@ const meterFrames = ( candidate: ModelCandidate, ): MeteredFrames => { let settle!: (outcome: StreamOutcome) => void; - const outcome = new Promise(resolve => { settle = resolve; }); + // Declared as this run's own unfinished work, so the runner waits for it at teardown where + // it can see it rather than the reading being started and forgotten. + const outcome = defer(new Promise(resolve => { settle = resolve; })); // Running out without the terminal frame is what "it did not finish" means, and it is known // at the same moment the usage is. let sawTerminal = false; diff --git a/packages/gateway/src/data-plane/openai-images/pipeline.ts b/packages/gateway/src/data-plane/openai-images/pipeline.ts index e8897ad7a7..4151a93971 100644 --- a/packages/gateway/src/data-plane/openai-images/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-images/pipeline.ts @@ -23,7 +23,7 @@ import { dialFailure, readUpstreamBody } from '../pipeline/upstream-body.ts'; import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; -import { compose, defineStage, move, own, type Owned, type Pipeline } from '@floway-dev/pipeline'; +import { compose, defer, defineStage, move, own, type Deferred, type Owned, type Pipeline } from '@floway-dev/pipeline'; import { eventFrame, isEventStreamMediaType, @@ -83,7 +83,7 @@ export interface OpenAIImagesFacts extends GatewayFacts { * `response.usage.billable`, which says what had been reported when the ending stage handed * up: the entity, and no quantities. Settling from this is the epilogue's job, after the * drain. */ - 'response.openaiImages.streamedUsage': Promise | null; + 'response.openaiImages.streamedUsage': Deferred | null; /** What the client is actually sent, in the OpenAI Images protocol — a JSON body, or the SSE * frames of a stream. The edge provides it, so a dump shows the body the client received * rather than the gateway's canonical form. */ @@ -310,7 +310,7 @@ const spentBody = (body: ReadableStream | null): ReadableStream; + readonly outcome: Deferred; } const meterFrames = ( @@ -319,7 +319,9 @@ const meterFrames = ( signal: AbortSignal | undefined, ): MeteredFrames => { let settle!: (outcome: StreamOutcome) => void; - const outcome = new Promise(resolve => { settle = resolve; }); + // Declared as this run's own unfinished work, so the runner waits for it at teardown where + // it can see it rather than the reading being started and forgotten. + const outcome = defer(new Promise(resolve => { settle = resolve; })); // Running out without the completed event is what "it did not finish" means, and it is known // at the same moment the usage is. let sawTerminal = false; diff --git a/packages/gateway/src/data-plane/pipeline/serve.ts b/packages/gateway/src/data-plane/pipeline/serve.ts index a9306c2906..45b03e5c56 100644 --- a/packages/gateway/src/data-plane/pipeline/serve.ts +++ b/packages/gateway/src/data-plane/pipeline/serve.ts @@ -21,8 +21,7 @@ import { consoleLogSink } from '../../runtime/log.ts'; import { createGatewayCtxFromHono, finalizeGatewayResponse, type GatewayCtx } from '../shared/gateway-ctx.ts'; import { readRequestBody, takeRequestBody, type RequestBody } from '../shared/request-body.ts'; import { writeSSEFrames } from '../shared/sse.ts'; -import type { Pipeline } from '@floway-dev/pipeline'; -import { run } from '@floway-dev/pipeline'; +import { run, type Deferred, type Pipeline } from '@floway-dev/pipeline'; import { sseCommentFrame, type SseFrame } from '@floway-dev/protocols/common'; import type { ModelCandidate } from '@floway-dev/provider'; @@ -145,8 +144,11 @@ export interface StreamOutcome { /** What a streaming family will have been billed, and whether it got there. A stream's usage * arrives with its last chunk, which is after the run has answered — so the run hands up a - * promise and settlement of it belongs here, after the answer is on its way. */ -export type DeferredUsage = (facts: Exit) => Promise | null; + * reading it has started and not finished, and settlement of it belongs here, after the answer + * is on its way. It is `Deferred` because that is what it is: the runner sees it in the record + * and waits for it at teardown, so a family that hands up a reading that never settles is + * reported rather than silently never billed. */ +export type DeferredUsage = (facts: Exit) => Deferred | null; /** * Runs a family's pipeline and turns what it answered with into a response. diff --git a/packages/pipeline/src/run.ts b/packages/pipeline/src/run.ts index cca2116ec5..2b2a5ef0ea 100644 --- a/packages/pipeline/src/run.ts +++ b/packages/pipeline/src/run.ts @@ -52,8 +52,12 @@ export const isOwned = (value: unknown): value is Owned => const DEFERRED = Symbol('floway.deferred'); /** A value the run has started and has not finished. It is a property of the value, not a - * capability a stage was handed: what a run must wait for is legible from its own record. */ -export type Deferred = PromiseLike & { readonly [DEFERRED]: true }; + * capability a stage was handed: what a run must wait for is legible from its own record. + * + * A `Promise` rather than a `PromiseLike`, because branding one is all `defer` does — what + * comes back is the same object, and narrowing it to the smaller interface would make a + * caller reach for `Promise.resolve` to get back what it already had. */ +export type Deferred = Promise & { readonly [DEFERRED]: true }; /** * Marks a promise as this run's to finish. @@ -66,7 +70,7 @@ export type Deferred = PromiseLike & { readonly [DEFERRED]: true }; * "stages have no fire-and-forget" means in practice: work a stage starts becomes a fact * with a name, and the runner waits for it where it can see it. */ -export const defer = (promise: PromiseLike): Deferred => +export const defer = (promise: Promise): Deferred => Object.assign(promise, { [DEFERRED]: true as const }); export const isDeferred = (value: unknown): value is Deferred => From f611a7df72bc91a0f91c7bd7387e29f2ca52c37b Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Aug 2026 08:16:37 +0800 Subject: [PATCH 51/79] feat(search): serve /alpha/search through the pipeline written for it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `searchServePipeline` was complete and had no consumer. The endpoint still ran the handler it was written to replace, so the family that had been given an edge, a settlement, two endings and a fact space was reachable only from its own tests. The seam is `http.ts`, on the shape every other family uses. What it decides before the run is which ending the chain gets: `passthroughOpenAiSearch` pins an upstream and everything else runs the commands locally. That is configuration rather than request, so it is read once and the chain assembled around it — which is why the family has one pipeline builder and no stage that branches. The configured backend reaches the chain as a service and never as a fact: it holds live handles built from the operator's credential, so none of it is dumpable. The family reads its own body, as every pipelined family does. A validator middleware would have consumed the bytes the run is given, and those bytes are what the record holds. The refusal it writes is the same status and the same `{error: message}` shape the validator wrote. The pinned ending gains `response.http.headers`, filtered at the edge by the rule every other family filters by. That replaces the relay's own blocklist, which differed only in also forwarding `content-type` — a header the seam sets from what the gateway actually serialized. `relay-response.ts` had no other caller and is gone. What the record shows is the proof: the handler this replaced opened no run at all, so a turn on this path produced no dump however much retention the key had. It now records the four stages the local ending composes. The note about pinning `SearchRequest.id` to one provider account travelled with the dispatch it describes; the port had dropped it. --- .../data-plane/alpha-search/routes_test.ts | 33 ++++++- .../alpha-search/relay-response_test.ts | 35 ------- .../src/data-plane/alpha-search/http.ts | 95 +++++++++++++++++++ .../src/data-plane/alpha-search/pipeline.ts | 53 +++++++---- .../src/data-plane/alpha-search/routes.ts | 86 ++--------------- .../web-search/alpha-search/relay-response.ts | 36 ------- 6 files changed, 171 insertions(+), 167 deletions(-) delete mode 100644 packages/gateway/__tests__/data-plane/tools/web-search/alpha-search/relay-response_test.ts create mode 100644 packages/gateway/src/data-plane/alpha-search/http.ts delete mode 100644 packages/gateway/src/data-plane/tools/web-search/alpha-search/relay-response.ts diff --git a/packages/gateway/__tests__/data-plane/alpha-search/routes_test.ts b/packages/gateway/__tests__/data-plane/alpha-search/routes_test.ts index f10fb3bf80..947a24c4bd 100644 --- a/packages/gateway/__tests__/data-plane/alpha-search/routes_test.ts +++ b/packages/gateway/__tests__/data-plane/alpha-search/routes_test.ts @@ -4,9 +4,11 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { mountAlphaSearchRoutes } from '../../../src/data-plane/alpha-search/routes.ts'; import { resolveConfiguredWebSearchProvider } from '../../../src/data-plane/tools/web-search/provider.ts'; import type { WebSearchConfig, WebSearchFetchPageRequest, WebSearchFetchPageResult, WebSearchProvider, WebSearchProviderRequest, WebSearchProviderResult } from '../../../src/data-plane/tools/web-search/types.ts'; +import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts'; import { type AuthVars, authMiddleware } from '../../../src/middleware/auth.ts'; import { internalErrorResponse } from '../../../src/middleware/internal-error-response.ts'; -import { buildCustomUpstreamRecord, setupAppTest } from '../../test-utils/app.ts'; +import { eventsOf, installDumpStubs, runRecordOf } from '../../dump/test-fixtures.ts'; +import { buildCustomUpstreamRecord, flushAsyncWork, setupAppTest } from '../../test-utils/app.ts'; import { withMockedFetch } from '@floway-dev/test-utils'; // Real provider construction (`createTavilyWebSearchProvider` etc.) hits the @@ -333,6 +335,35 @@ describe('/alpha/search data plane', () => { }); }); + // The endpoint is served by the pipeline that was written for it, which is what a run record + // says and nothing else does: the handler it replaced opened no run at all, so a turn on this + // path produced no record however much retention the key had. + describe('the run it is served by', () => { + it('records the whole run, stage by stage', async () => { + const { apiKey, repo } = await setupAppTest({ webSearchConfig: TAVILY_CONFIG }); + await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); + const stub = makeStubProvider(); + mockResolveConfigured.mockReturnValue({ type: 'enabled', provider: 'tavily', impl: stub.provider }); + const dumpStubs = installDumpStubs(initDumpStore, initDumpBroker); + + const response = await postSearch(buildAlphaSearchApp(), apiKey.key, { commands: { search_query: [{ q: 'Floway' }] } }); + expect(response.status).toBe(200); + await response.json(); + await flushAsyncWork(); + + expect(dumpStubs.stored).toHaveLength(1); + const record = runRecordOf(dumpStubs.stored[0]?.record); + expect(record.meta.path).toBe(SEARCH_PATH); + expect(record.meta.status).toBe(200); + // The chain the operator's configuration assembled: the edge, settlement, and the local + // ending's two stages. + const entered = eventsOf(record) + .filter(event => event.type === 'stage.entered') + .map(event => event.name); + expect(entered).toEqual(['emitAlphaSearch', 'writeSettlement', 'parseSearchOperations', 'executeSearchOperations']); + }); + }); + describe('provider not configured', () => { it('surfaces disabled search as in-band output text (contract-shaped 200)', async () => { const { apiKey, repo } = await setupAppTest(); diff --git a/packages/gateway/__tests__/data-plane/tools/web-search/alpha-search/relay-response_test.ts b/packages/gateway/__tests__/data-plane/tools/web-search/alpha-search/relay-response_test.ts deleted file mode 100644 index 5e601b11e6..0000000000 --- a/packages/gateway/__tests__/data-plane/tools/web-search/alpha-search/relay-response_test.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import { relayFetchedResponse } from '../../../../../src/data-plane/tools/web-search/alpha-search/relay-response.ts'; - -describe('relayFetchedResponse', () => { - it('keeps decoded bytes and representation-safe upstream headers', async () => { - const body = JSON.stringify({ output: 'decoded' }); - const upstream = new Response(body, { - status: 202, - statusText: 'Accepted upstream', - headers: { - connection: 'close', - 'content-encoding': 'gzip', - 'content-length': '999', - 'content-type': 'application/json', - 'set-cookie': 'session=upstream-secret', - 'transfer-encoding': 'chunked', - 'x-oai-request-id': 'req_search_1', - }, - }); - - const relayed = relayFetchedResponse(upstream); - - expect(relayed.status).toBe(202); - expect(relayed.statusText).toBe('Accepted upstream'); - expect(await relayed.text()).toBe(body); - expect(relayed.headers.get('content-type')).toBe('application/json'); - expect(relayed.headers.get('x-oai-request-id')).toBe('req_search_1'); - expect(relayed.headers.get('connection')).toBeNull(); - expect(relayed.headers.get('content-encoding')).toBeNull(); - expect(relayed.headers.get('content-length')).toBeNull(); - expect(relayed.headers.get('set-cookie')).toBeNull(); - expect(relayed.headers.get('transfer-encoding')).toBeNull(); - }); -}); diff --git a/packages/gateway/src/data-plane/alpha-search/http.ts b/packages/gateway/src/data-plane/alpha-search/http.ts new file mode 100644 index 0000000000..ad130eb635 --- /dev/null +++ b/packages/gateway/src/data-plane/alpha-search/http.ts @@ -0,0 +1,95 @@ +// POST /alpha/search, served through the pipeline. +// +// What the handler decides before the run is which ending the chain gets: the operator's +// `passthroughOpenAiSearch` setting pins an upstream, and everything else runs the commands +// here. That is a property of configuration rather than of the request, so it is read once and +// the chain is assembled around it — which is why the family has one pipeline builder rather +// than a stage that branches. +// +// The configured search backend reaches the chain as a service and never as a fact: it holds +// live handles built from the operator's provider credential, so none of it is dumpable and +// none of it belongs in the record. + +import type { Context } from 'hono'; + +import { searchServePipeline, type SearchExecution, type SearchServices } from './pipeline.ts'; +import { alphaSearchRequestSchema, type AlphaSearchRequest } from './protocol.ts'; +import type { AuthedContext } from '../../middleware/auth.ts'; +import { openPrologue, readIngress, serveThrough, type Prologue } from '../pipeline/serve.ts'; +import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; +import { loadWebSearchConfig } from '../tools/web-search/config.ts'; +import { resolveConfiguredWebSearchProvider } from '../tools/web-search/provider.ts'; +import type { ConfiguredWebSearchProvider, WebSearchConfig } from '../tools/web-search/types.ts'; +import { move } from '@floway-dev/pipeline'; + +/** Which ending this gateway's configuration asks for. The word "passthrough" is about whose + * search results the client is given, not about carrying a protocol the gateway has not + * parsed — what comes back is read and written again either way. */ +const executionFor = (config: WebSearchConfig): SearchExecution => + config.passthroughOpenAiSearch.enabled + ? { kind: 'upstream', upstreamId: config.passthroughOpenAiSearch.upstreamId, model: config.passthroughOpenAiSearch.model } + : { kind: 'local' }; + +/** Resolved once per turn and shared by every operation in it: one turn's commands run against + * one backend, and resolving per operation would build the same handles again. */ +const searchProviderFor = (config: WebSearchConfig): SearchServices['searchProvider'] => { + let resolved: Promise | undefined; + return () => { + resolved ??= Promise.resolve(resolveConfiguredWebSearchProvider(config)); + return resolved; + }; +}; + +/** The request as this protocol, or the sentence saying why it is not. The family reads its own + * body because the run is given the bytes the client sent, and a validator middleware would + * have consumed them first. */ +type Read = { readonly ok: true; readonly request: AlphaSearchRequest } | { readonly ok: false; readonly message: string }; + +const readAlphaSearchRequest = (bytes: Uint8Array): Read => { + let body: unknown; + try { + body = JSON.parse(new TextDecoder().decode(bytes)) as unknown; + } catch (error) { + return { ok: false, message: error instanceof Error ? error.message : String(error) }; + } + const parsed = alphaSearchRequestSchema.safeParse(body); + // The first issue's message, verbatim: a schema attaches a field-aware message where it wants + // one, and prepending the path would name the field twice. + return parsed.success + ? { ok: true, request: parsed.data } + : { ok: false, message: parsed.error.issues[0]?.message ?? 'Invalid input' }; +}; + +export const alphaSearch = async (c: Context): Promise => { + const ingress = await readIngress(c); + const read = readAlphaSearchRequest(ingress.body.bytes); + if (!read.ok) { + // A request the gateway could not read never reaches a pipeline: there is no backend to + // reach and no attempt to make, so there is nothing for a run to record. + const refused = openPrologue(c as AuthedContext, ingress, { wantsStream: false }); + refused.gateway.dump?.error('gateway'); + return finalizeGatewayResponse(refused.gateway, Response.json({ error: read.message }, { status: 400 })); + } + + const config = await loadWebSearchConfig(); + const execution = executionFor(config); + // A pinned turn is attributed to the model the operator pinned, because that is the one that + // will be called: the id the caller sent names a model of its own that never travels. + const prologue = openPrologue(c as AuthedContext, ingress, { + wantsStream: false, + ...(execution.kind === 'upstream' ? { model: execution.model } : {}), + }); + const services: SearchServices = { ...prologue.services, searchProvider: searchProviderFor(config) }; + const withBackend: Prologue = { ...prologue, services }; + + return await serveThrough( + c, + withBackend, + searchServePipeline(execution), + move({ + 'ingress.http.headers': prologue.headers, + 'request.search.alphaSearch': read.request, + }) as never, + facts => ({ body: JSON.stringify(facts['response.search.rendered']), contentType: 'application/json' }), + ); +}; diff --git a/packages/gateway/src/data-plane/alpha-search/pipeline.ts b/packages/gateway/src/data-plane/alpha-search/pipeline.ts index 2a2e52893b..0f6c116d11 100644 --- a/packages/gateway/src/data-plane/alpha-search/pipeline.ts +++ b/packages/gateway/src/data-plane/alpha-search/pipeline.ts @@ -31,6 +31,7 @@ import { enumerateModelCandidates } from '../providers/resolution.ts'; import type { GatewayCtx } from '../shared/gateway-ctx.ts'; import { filterInboundHeadersForProvider } from '../shared/inbound-headers.ts'; import { telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; +import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import { assertLocalWebSearchSupport, executeOperationToText, @@ -80,27 +81,33 @@ export interface SearchServices extends GatewayServices { const emitAlphaSearch = defineStage< S, S, - S<'response.search.alphaSearch' | 'response.usage.billable'>, - S<'response.search.rendered' | 'response.http.status'> + S<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>, + S<'response.search.rendered' | 'response.http.status' | 'response.http.headers'> >({ name: 'emitAlphaSearch', through: { request: { needs: [], consumes: [], provides: [] }, response: { - needs: ['response.search.alphaSearch', 'response.usage.billable'], - consumes: ['response.search.alphaSearch'], - provides: ['response.search.rendered', 'response.http.status'], + needs: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers'], + consumes: ['response.search.alphaSearch', 'response.http.headers'], + provides: ['response.search.rendered', 'response.http.status', 'response.http.headers'], }, }, execute: async (facts, next) => { const back = await next(facts); - const { 'response.search.alphaSearch': answer, ...rest } = back; + const { 'response.search.alphaSearch': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); if (isFailure(answer)) { // An upstream error body is JSON like any other: it was parsed below and is serialized // again here. A body that was not an object is one this protocol cannot carry, so what // goes out instead is the gateway's own envelope. return { ...rest, + 'response.http.headers': forClient, 'response.search.rendered': move(isJsonObject(answer.body) ? answer.body : { error: { message: answer.message, type: 'api_error' } }), @@ -109,6 +116,7 @@ const emitAlphaSearch = defineStage< } return { ...rest, + 'response.http.headers': forClient, 'response.search.rendered': move(renderAlphaSearchResponse(answer)), 'response.http.status': 200, }; @@ -121,6 +129,8 @@ const inBandOutput = (facts: S<'request.search.alphaSearch'>, output: string) => ...facts, 'response.search.alphaSearch': { encryptedOutput: null, output }, 'response.usage.billable': [], + // Nothing was called, so there are no upstream headers to carry. + 'response.http.headers': [], }); /** @@ -131,9 +141,9 @@ const inBandOutput = (facts: S<'request.search.alphaSearch'>, output: string) => const parseSearchOperations = defineStage< S<'request.search.alphaSearch'>, S<'request.search.operations' | 'request.search.filters'>, - S<'response.search.alphaSearch' | 'response.usage.billable'>, - S<'response.search.alphaSearch' | 'response.usage.billable'>, - S<'response.search.alphaSearch' | 'response.usage.billable'> + S<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>, + S<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>, + S<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'> >({ name: 'parseSearchOperations', through: { @@ -144,7 +154,7 @@ const parseSearchOperations = defineStage< }, response: { needs: [], consumes: [], provides: [] }, }, - return: { provides: ['response.search.alphaSearch', 'response.usage.billable'] }, + return: { provides: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers'] }, execute: async (facts, next) => { const { 'request.search.alphaSearch': request, ...rest } = facts; const commands = request.commands ?? {}; @@ -177,11 +187,11 @@ const parseSearchOperations = defineStage< */ const executeSearchOperations = defineStage< S<'request.search.operations' | 'request.search.filters'>, - S<'response.search.alphaSearch' | 'response.usage.billable'>, + S<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>, SearchServices >({ name: 'executeSearchOperations', - return: { provides: ['response.search.alphaSearch', 'response.usage.billable'] }, + return: { provides: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers'] }, execute: async (facts, use) => { const session: WebSearchExecutionSession = { getProvider: use.searchProvider, @@ -206,6 +216,8 @@ const executeSearchOperations = defineStage< ...facts, 'response.search.alphaSearch': { encryptedOutput: null, output: blocks.join('\n\n') }, 'response.usage.billable': [], + // Nothing was called, so there are no upstream headers to carry. + 'response.http.headers': [], }); }, }); @@ -272,15 +284,17 @@ const asJson = (raw: string): unknown => { */ const callSearchUpstream = (pinned: PinnedSearchUpstream) => defineStage< S<'request.search.alphaSearch' | 'ingress.http.headers'>, - S<'response.search.alphaSearch' | 'response.usage.billable'>, + S<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>, GatewayServices >({ name: 'callSearchUpstream', - return: { provides: ['response.search.alphaSearch', 'response.usage.billable'] }, + return: { provides: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers'] }, execute: async (facts, use) => { const candidate = await resolvePinnedUpstream(pinned, use.gateway); // The caller's model is dropped: this endpoint is pinned to the operator's model, and the // provider stamps that one on the way out. + // TODO: pin SearchRequest.id to one provider account when Codex upstreams support account + // pools. The current Codex provider has one active account. const { model: _named, ...request } = facts['request.search.alphaSearch']; const result = await candidate.provider.instance.callAlphaSearch( providerModelOf(candidate), @@ -320,6 +334,7 @@ const callSearchUpstream = (pinned: PinnedSearchUpstream) => defineStage< ...(body === undefined ? {} : { body }), }, 'response.usage.billable': billable, + 'response.http.headers': [...result.response.headers], }); } @@ -335,9 +350,15 @@ const callSearchUpstream = (pinned: PinnedSearchUpstream) => defineStage< body: raw, }, 'response.usage.billable': billable, + 'response.http.headers': [...result.response.headers], }); } - return move({ ...facts, 'response.search.alphaSearch': verdict.response, 'response.usage.billable': billable }); + return move({ + ...facts, + 'response.search.alphaSearch': verdict.response, + 'response.usage.billable': billable, + 'response.http.headers': [...result.response.headers], + }); }, }); @@ -348,7 +369,7 @@ export type SearchExecution = { readonly kind: 'local' } | PinnedSearchUpstream; export const searchServePipeline = (execution: SearchExecution): Pipeline< S<'request.search.alphaSearch'>, - S<'response.search.rendered' | 'response.http.status' | 'response.usage.billable'> + S<'response.search.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> > => compose('searchServe', [ emitAlphaSearch, // Unconditional, as everywhere: a run that searched locally reached no upstream and bills diff --git a/packages/gateway/src/data-plane/alpha-search/routes.ts b/packages/gateway/src/data-plane/alpha-search/routes.ts index 0658361ff8..0f3cbd9029 100644 --- a/packages/gateway/src/data-plane/alpha-search/routes.ts +++ b/packages/gateway/src/data-plane/alpha-search/routes.ts @@ -1,98 +1,26 @@ // Codex `/alpha/search` compatibility endpoint. The protocol itself — the request // schema, the response shape, and the projection of Codex's settings onto the -// gateway's search filters — lives in `protocol.ts`. +// gateway's search filters — lives in `protocol.ts`; the chain that serves it is +// `pipeline.ts`, and `http.ts` is the seam between the two. // // Clients append `alpha/search` to an OpenAI-compatible provider base. The // aliases below cover Floway's general root and `/v1` base conventions. // https://github.com/openai/codex/blob/2e1607ee2fa8099a233df7437adee5f16a741905/codex-rs/codex-api/src/endpoint/search.rs#L31-L47 // -// In the default mode, Floway executes supported commands through the general -// configured search provider and renders a local `{ encrypted_output: null, -// output }` response. Passthrough mode instead returns the selected Codex or -// Custom provider response verbatim, preserving its optional structured data. -// -// The shared data-plane auth middleware guards every alias; this handler reads -// the resolved API key for per-key search-usage accounting. +// The shared data-plane auth middleware guards every alias; the handler reads the +// resolved API key through the run's own prologue. import type { Hono } from 'hono'; -import { alphaSearchRequestSchema, webSearchFiltersFromSettings } from './protocol.ts'; -import { type AuthVars, apiKeyFromContext, effectiveUpstreamIdsFromContext } from '../../middleware/auth.ts'; -import { type CtxWithJson, zValidator } from '../../middleware/zod-validator.ts'; -import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; -import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; +import { alphaSearch } from './http.ts'; +import type { AuthVars } from '../../middleware/auth.ts'; import { mountPublicRoute } from '../public-route.ts'; -import { relayFetchedResponse } from '../tools/web-search/alpha-search/relay-response.ts'; -import { resolveAlphaSearchDispatcher } from '../tools/web-search/alpha-search/upstream.ts'; -import { loadWebSearchConfig } from '../tools/web-search/config.ts'; -import { assertLocalWebSearchSupport, executeOperationToText, parseWebSearchOperations, startBatchFetch, UnsupportedLocalWebSearchFeatureError, type WebSearchExecutionSession } from '../tools/web-search/operations.ts'; -import { resolveConfiguredWebSearchProvider } from '../tools/web-search/provider.ts'; -import type { ConfiguredWebSearchProvider } from '../tools/web-search/types.ts'; import { PUBLIC_DATA_PLANE_ROUTES } from '@floway-dev/protocols/common'; -const alphaSearch = async (c: CtxWithJson): Promise => { - const body = c.req.valid('json'); - const webSearchConfig = await loadWebSearchConfig(); - if (webSearchConfig.passthroughOpenAiSearch.enabled) { - const dispatcher = await resolveAlphaSearchDispatcher({ - config: webSearchConfig.passthroughOpenAiSearch, - upstreamIds: effectiveUpstreamIdsFromContext(c), - scheduler: backgroundSchedulerFromContext(c), - runtimeLocation: getRuntimeLocation(c.req.raw), - }); - const headers = new Headers(); - const turnMetadata = c.req.header('x-codex-turn-metadata'); - if (turnMetadata !== undefined) headers.set('x-codex-turn-metadata', turnMetadata); - const response = await dispatcher(body, c.req.raw.signal, headers); - return relayFetchedResponse(response); - } - - try { - assertLocalWebSearchSupport(body.commands ?? {}); - } catch (error) { - if (error instanceof UnsupportedLocalWebSearchFeatureError) { - return c.json({ encrypted_output: null, output: error.message }); - } - throw error; - } - - let configuredProvider: Promise | undefined; - const session: WebSearchExecutionSession = { - getProvider: () => { - configuredProvider ??= Promise.resolve(resolveConfiguredWebSearchProvider(webSearchConfig)); - return configuredProvider; - }, - filters: webSearchFiltersFromSettings(body.settings), - apiKeyId: apiKeyFromContext(c).id, - pageCache: new Map(), - // Codex renders `output` as plain text; the search-action sources list - // is an OpenAI Responses wire concern with no place here. - includeSearchActionSources: false, - signal: c.req.raw.signal, - }; - - const parsed = parseWebSearchOperations(body.commands ?? {}); - if (parsed.kind !== 'ops' || parsed.ops.length === 0) { - return c.json({ - encrypted_output: null, - output: 'No web search commands were provided. Populate at least one of `search_query`, `open`, or `find`.', - }); - } - - // One batched provider.fetchPage covers every open/find URL; each op then - // renders its own text block. The shared parser's canonical order is - // search_query → open → find, preserving array order within each command - // kind. - const batch = await startBatchFetch(parsed, session); - const blocks = await Promise.all(parsed.ops.map(op => executeOperationToText(op, session, batch))); - - return c.json({ encrypted_output: null, output: blocks.join('\n\n') }); -}; - type AlphaSearchRoute = typeof PUBLIC_DATA_PLANE_ROUTES.alphaSearch | typeof PUBLIC_DATA_PLANE_ROUTES.codexAlphaSearch; export const mountAlphaSearchRoute = (app: Hono<{ Variables: AuthVars }>, route: AlphaSearchRoute) => { - mountPublicRoute(route, (method, path) => app.on(method, path, zValidator('json', alphaSearchRequestSchema), alphaSearch)); + mountPublicRoute(route, (method, path) => app.on(method, path, alphaSearch)); }; export const mountAlphaSearchRoutes = (app: Hono<{ Variables: AuthVars }>) => { diff --git a/packages/gateway/src/data-plane/tools/web-search/alpha-search/relay-response.ts b/packages/gateway/src/data-plane/tools/web-search/alpha-search/relay-response.ts deleted file mode 100644 index d990344a7a..0000000000 --- a/packages/gateway/src/data-plane/tools/web-search/alpha-search/relay-response.ts +++ /dev/null @@ -1,36 +0,0 @@ -// Alpha-search upstream Fetch decodes a response's content coding before exposing its body stream, -// but keeps the upstream Content-Encoding and Content-Length headers. Relaying -// that stream with the stale representation headers makes the next Fetch -// consumer decode plain bytes a second time. Rebuild the response around the -// decoded stream and preserve every header that still describes it. - -const BLOCKED_RELAY_HEADERS: ReadonlySet = new Set([ - // Hop-by-hop headers (RFC 9110 §7.6.1). - // https://www.rfc-editor.org/rfc/rfc9110#section-7.6.1 - 'connection', - 'keep-alive', - 'proxy-authenticate', - 'proxy-authorization', - 'te', - 'trailer', - 'transfer-encoding', - 'upgrade', - // Fetch owns the decoded body's representation framing. - 'content-encoding', - 'content-length', - // Upstream session cookies must not bind a gateway client. - 'set-cookie', - 'set-cookie2', -]); - -export const relayFetchedResponse = (response: Response): Response => { - const headers = new Headers(); - for (const [name, value] of response.headers) { - if (!BLOCKED_RELAY_HEADERS.has(name.toLowerCase())) headers.set(name, value); - } - return new Response(response.body, { - status: response.status, - statusText: response.statusText, - headers, - }); -}; From b85006882d855aaecbe7c923c93a83d818953495 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 20 Aug 2026 02:36:06 +0800 Subject: [PATCH 52/79] Merge branch 'pr1-core' into feat/pipeline-non-chat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Brings main's #472 down the stack. No conflicts here — the three files it added merge clean — but one of them fails against this branch, which is the point of merging it down. `/v1/completions reaches a Completions upstream` sent a non-streaming request and answered it with `text/event-stream`. The replaced handler served that; the pipeline reads the upstream's body against what the request asked for, so a non-streaming turn handed a stream has nothing it can parse and says 502 rather than serving a body it never read. The fixture is what is wrong, not the reading: the case exists to prove the ingress header rules reach the upstream on this route, and it now asks to stream, which is what its own upstream answers with. --- .../data-plane/shared/custom-ingress-header-rules_test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/gateway/__tests__/data-plane/shared/custom-ingress-header-rules_test.ts b/packages/gateway/__tests__/data-plane/shared/custom-ingress-header-rules_test.ts index 7d4f37ec33..d88e4551ad 100644 --- a/packages/gateway/__tests__/data-plane/shared/custom-ingress-header-rules_test.ts +++ b/packages/gateway/__tests__/data-plane/shared/custom-ingress-header-rules_test.ts @@ -152,7 +152,11 @@ const CASES: RouteCase[] = [ name: '/v1/completions reaches a Completions upstream', path: '/v1/completions', contentType: 'application/json', - body: () => JSON.stringify({ model: 'completions-model', prompt: 'hi' }), + // Asks to stream, because the upstream below answers with one. A + // non-streaming request whose upstream replies `text/event-stream` has no + // body this protocol can read, and the pipeline answers 502 rather than + // serving something it never parsed. + body: () => JSON.stringify({ model: 'completions-model', prompt: 'hi', stream: true }), upstreamPath: '/v1/completions', upstreamResponse: () => new Response( 'data: {"id":"cmpl_1","object":"text_completion","created":1,"model":"completions-model","choices":[{"index":0,"text":"ok","finish_reason":"stop"}]}\n\ndata: [DONE]\n\n', From 89fe58090475244582733fd51cca085770087eb4 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 20 Aug 2026 04:07:33 +0800 Subject: [PATCH 53/79] refactor(gateway): say why, not what the code used to be MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six comments in this PR's own files justified a decision by naming what the surface this replaced did. That reads as a citation and resolves to nothing once this ships. Each now states the property itself, and where the point really is a contrast, contrasts with the alternative a reader can still evaluate. These were swept on #476 by mistake — the files are this PR's, and the sweep belongs where they live. The final state was already right either way, which is exactly why a per-PR review catches this and an end-state sweep does not. --- .../data-plane/openai-completions-pipeline_test.ts | 6 +++--- .../data-plane/openai-embeddings/failures_test.ts | 7 ++++--- .../gateway/__tests__/data-plane/rerank-status_test.ts | 3 ++- packages/gateway/__tests__/data-plane/rerank/serve_test.ts | 6 +++--- packages/gateway/src/data-plane/openai-audio/pipeline.ts | 2 +- packages/gateway/src/data-plane/pipeline/upstream-body.ts | 4 ++-- 6 files changed, 15 insertions(+), 13 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts index db633e25ed..2bdc7e6a9a 100644 --- a/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts @@ -285,9 +285,9 @@ describe('the OpenAI Completions pipeline', () => { expect(outcome.failed).toBe(false); }); - // A run that reached no upstream bills nothing and samples nothing, which is what the - // replaced surface did: `recordPerformance` returns early without an attempt's telemetry, - // and there is no attempt. Settlement still runs — it is unconditional — and finds an + // A run that reached no upstream bills nothing and samples nothing: + // `recordPerformance` returns early without an attempt's telemetry, and there is no + // attempt. Settlement still runs — it is unconditional — and finds an // empty billed set, which is how "we did not call an upstream" is said. it('bills nothing and samples nothing when no upstream was reached', async () => { vi.mocked(enumerateModelCandidates).mockResolvedValue({ candidates: [], sawModel: false, failedUpstreams: [] }); diff --git a/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts b/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts index 3d812f671f..b06b87e761 100644 --- a/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts @@ -76,9 +76,10 @@ test('a usage write that fails leaves the answer alone and still reports', async } }); -// The replaced surface forwarded such a body verbatim with the upstream's 200. A protocol -// that requires JSON and did not get JSON has no answer to serve, so the gateway says so -// itself — it cannot claim to have served a request whose answer it never read. +// A protocol that requires JSON and did not get JSON has no answer to serve, so the gateway +// says so itself — it cannot claim to have served a request whose answer it never read. +// Forwarding the body verbatim under the upstream's 200 would report success for a turn +// nothing in this gateway could read. test('a 2xx body the OpenAI Embeddings protocol cannot read is refused rather than forwarded', async () => { const { apiKey, repo } = await setupAppTest(); await registerOpenAIEmbeddingsUpstream(repo); diff --git a/packages/gateway/__tests__/data-plane/rerank-status_test.ts b/packages/gateway/__tests__/data-plane/rerank-status_test.ts index e7507b3c15..0294b84032 100644 --- a/packages/gateway/__tests__/data-plane/rerank-status_test.ts +++ b/packages/gateway/__tests__/data-plane/rerank-status_test.ts @@ -1,7 +1,8 @@ // The two families that read their answer to the end, driven end to end. Until the review, // neither could express a status at all: an upstream 429, a resolver's 404 and a 400 all // reached the client as a 200 carrying an error envelope, which is not a difference the -// no-passthrough ruling asks for — the replaced surface forwarded the upstream's status. +// no-passthrough ruling asks for — declining to forward a body is not declining to forward +// a status. import { beforeEach, describe, expect, it, vi } from 'vitest'; diff --git a/packages/gateway/__tests__/data-plane/rerank/serve_test.ts b/packages/gateway/__tests__/data-plane/rerank/serve_test.ts index ea74bed97a..b1bd422dbc 100644 --- a/packages/gateway/__tests__/data-plane/rerank/serve_test.ts +++ b/packages/gateway/__tests__/data-plane/rerank/serve_test.ts @@ -516,9 +516,9 @@ test('cross-protocol success still validates result items before rendering', asy assertEquals(performance.errorsNoOutput, 1); }); -// The replaced surface forwarded these bytes verbatim with the upstream's 200. Rerank is a -// JSON protocol, so a body that is not JSON at all is no answer to serve and the gateway says -// so itself rather than passing on something it never read. +// Rerank is a JSON protocol, so a body that is not JSON at all is no answer to serve and the +// gateway says so itself rather than passing on something it never read. Forwarding those +// bytes verbatim under the upstream's 200 would report success for a turn nothing could read. test('same-protocol answer that is not JSON is refused rather than forwarded', async () => { const { apiKey, repo } = await setupAppTest(); await saveRerankUpstream(repo, { protocol: 'jina-v1' }); diff --git a/packages/gateway/src/data-plane/openai-audio/pipeline.ts b/packages/gateway/src/data-plane/openai-audio/pipeline.ts index 88407b021c..de1d0c12cf 100644 --- a/packages/gateway/src/data-plane/openai-audio/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-audio/pipeline.ts @@ -57,7 +57,7 @@ export interface OpenAIAudioTranscriptionStreamOutcome { readonly billable: readonly BillableEntity[]; /** An upstream that stopped before `transcript.text.done` answered 200 and then did not * finish what it started, which is a failed request however much of the transcript - * reached the client — the same reading the replaced surface made. */ + * reached the client. */ readonly failed: boolean; } diff --git a/packages/gateway/src/data-plane/pipeline/upstream-body.ts b/packages/gateway/src/data-plane/pipeline/upstream-body.ts index 068b4cce65..84f5484345 100644 --- a/packages/gateway/src/data-plane/pipeline/upstream-body.ts +++ b/packages/gateway/src/data-plane/pipeline/upstream-body.ts @@ -43,8 +43,8 @@ export const unreadableBody = (response: Response, body: UpstreamBody, protocolN * Nothing in the domain throws. A connection that was refused, timed out or was reset is an * outcome failover has to be able to see — a run that ends there has more candidates to try — * so the ending catches whatever the platform raised and hands it up like any other failure. - * The status is the one the replaced surface answered with: the gateway reached no upstream, - * which is what 502 says. + * The status says what happened rather than what the error was: the gateway reached no + * upstream at all, which is what 502 states. */ export const dialFailure = (error: unknown): Failure => ({ status: 502, From b0b3910e9a3df967ad53cc3840f9ce1aacde9319 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 30 Sep 2026 16:14:03 +0800 Subject: [PATCH 54/79] style(gateway): format refreshed catalog fixtures --- .../__tests__/data-plane/openai-embeddings/failures_test.ts | 4 ++-- .../gateway/__tests__/data-plane/openai-images/stream_test.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts b/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts index f8fd5ee1f1..4a6156d31b 100644 --- a/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts @@ -14,7 +14,7 @@ const registerOpenAIEmbeddingsUpstream = async ( ): Promise => { await repo.upstreams.deleteAll(); clearInProcessCopilotTokenCache(); - await saveUpstreamForTest(repo.upstreams,buildCustomUpstreamRecord({ + await saveUpstreamForTest(repo.upstreams, buildCustomUpstreamRecord({ id: 'up_embeddings', name: 'Embedding Provider', sortOrder: 100, @@ -124,7 +124,7 @@ test('when every candidate refuses the client gets the last upstream-s own refus await repo.upstreams.deleteAll(); clearInProcessCopilotTokenCache(); for (const [id, host, order] of [['up_a', 'up-a.example.com', 100], ['up_b', 'up-b.example.com', 200]] as const) { - await saveUpstreamForTest(repo.upstreams,buildCustomUpstreamRecord({ + await saveUpstreamForTest(repo.upstreams, buildCustomUpstreamRecord({ id, name: id, sortOrder: order, config: { baseUrl: `https://${host}`, authStyle: 'bearer', ingressHeadersRules: [], apiKey: 'sk-x', endpoints: {} }, })); diff --git a/packages/gateway/__tests__/data-plane/openai-images/stream_test.ts b/packages/gateway/__tests__/data-plane/openai-images/stream_test.ts index 1a5707d566..acae744d47 100644 --- a/packages/gateway/__tests__/data-plane/openai-images/stream_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-images/stream_test.ts @@ -12,7 +12,7 @@ import { withMockedFetch, assertEquals, assertExists } from '@floway-dev/test-ut const registerOpenAIImagesModel = async (repo: InMemoryRepo): Promise => { await repo.upstreams.deleteAll(); clearInProcessCopilotTokenCache(); - await saveUpstreamForTest(repo.upstreams,buildCustomUpstreamRecord({ + await saveUpstreamForTest(repo.upstreams, buildCustomUpstreamRecord({ id: 'up_images', name: 'Image Provider', sortOrder: 100, From 871851441a92de893fd212c61f5f51331c3023c0 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 30 Sep 2026 16:24:09 +0800 Subject: [PATCH 55/79] refactor(gateway): keep pipeline recording in facts Record only immutable facts and protocol frames for migrated endpoints. Retain searchable events, timing, and redacted exports; allocate candidate selectors without reusing earlier identities. --- .../components/requests/detail_test.tsx | 15 +++- apps/web/src/components/requests/detail.tsx | 83 ++++++++++++------- apps/web/src/components/requests/export.ts | 4 +- .../data-plane/openai-audio-pipeline_test.ts | 3 +- .../openai-completions-pipeline_test.ts | 3 +- .../data-plane/pipeline/candidates_test.ts | 18 ++++ .../data-plane/rerank-status_test.ts | 7 +- .../gateway/__tests__/dump/run-sink_test.ts | 21 ++--- .../gateway/__tests__/repo/dump-store_test.ts | 2 +- .../src/data-plane/pipeline/candidates.ts | 23 +++++ .../gateway/src/data-plane/pipeline/serve.ts | 20 ++--- .../src/data-plane/pipeline/services.ts | 2 +- .../gateway/src/data-plane/pipeline/stages.ts | 16 +--- .../shared/upstream-call-options.ts | 2 +- packages/gateway/src/dump/attribution.ts | 3 +- packages/gateway/src/dump/run-sink.ts | 35 ++------ packages/gateway/src/dump/schemas.ts | 15 +--- packages/gateway/src/dump/turn-dump.ts | 2 +- packages/gateway/src/dump/types.ts | 5 +- packages/gateway/src/dump/wire.ts | 1 - packages/gateway/src/repo/dump-store.ts | 15 ++-- 21 files changed, 150 insertions(+), 145 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/pipeline/candidates_test.ts create mode 100644 packages/gateway/src/data-plane/pipeline/candidates.ts diff --git a/apps/web/__tests__/components/requests/detail_test.tsx b/apps/web/__tests__/components/requests/detail_test.tsx index f6af1d7b81..895307d22f 100644 --- a/apps/web/__tests__/components/requests/detail_test.tsx +++ b/apps/web/__tests__/components/requests/detail_test.tsx @@ -4,7 +4,7 @@ import { describe, expect, it, vi } from 'vitest'; import { RequestDetailPanel } from '../../../src/components/requests/detail'; import { renderInApp } from '../../render'; -import type { DumpEdgeRecord } from '@floway-dev/gateway/dump-types'; +import type { DumpEdgeRecord, DumpRunRecord } from '@floway-dev/gateway/dump-types'; vi.mock('../../../src/components/ui/body-editor', () => ({ default: ({ text, toolbarStart }: { text: string; toolbarStart?: ReactNode }) => <>{toolbarStart}
{text}
})); @@ -16,6 +16,19 @@ const record: DumpEdgeRecord = { capture: { exchanges: [{ upstreamId: 'u', request: { url: 'https://upstream.test', method: 'POST', headers: [], body: { encoding: 'utf8', data: '{"upstream":"translated request"}' } }, response: { status: 200, headers: [], body: { encoding: 'utf8', data: 'data: {broken\n' }, complete: false, error: null }, error: null }], response: { body: { encoding: 'utf8', data: 'data: downstream\n' }, complete: true, error: null } }, }; +it('shows pipeline facts through the searchable event viewer with timing and export controls', () => { + const run: DumpRunRecord = { + shape: 'run', meta: { ...record.meta, ttftMs: 125 }, + events: '{"type":"stage.entered","stageId":1,"name":"serve","parentStageId":null,"facts":{"serve.model":"m"}}\n', + }; + renderInApp(); + expect(screen.getByRole('button', { name: '#1 stage.entered serve' })).toBeTruthy(); + expect(screen.getByRole('button', { name: 'Search events' })).toBeTruthy(); + expect(screen.getByRole('button', { name: 'Export record' })).toBeTruthy(); + expect(screen.getByText('125ms')).toBeTruthy(); + expect(screen.queryByRole('combobox', { name: 'Request details' })).toBeNull(); +}); + describe('request detail navigation', () => { it('opens each body directly and preserves malformed raw response text', async () => { renderInApp(); diff --git a/apps/web/src/components/requests/detail.tsx b/apps/web/src/components/requests/detail.tsx index 12c2f0f40d..7df33c054d 100644 --- a/apps/web/src/components/requests/detail.tsx +++ b/apps/web/src/components/requests/detail.tsx @@ -19,12 +19,12 @@ import { OutcomeMessageBar } from '../ui/outcome-message-bar'; import { PANEL_BAND_CLASS } from '../ui/panel'; import { TooltipIconButton } from '../ui/tooltip-icon-button'; import { copyOutcomeIcon, useCopyLabel, useCopyToClipboard } from '../ui/use-copy-to-clipboard'; -import type { DumpRecord, DumpResponseBody } from '@floway-dev/gateway/dump-types'; +import type { DumpMetadata, DumpRecord, DumpResponseBody, DumpRunRecord, DumpEdgeRecord } from '@floway-dev/gateway/dump-types'; const BodyEditor = lazy(() => import('../ui/body-editor')); const { Button, DialogActions, DialogTitle, Option, Spinner, Text, Tooltip } = fluentComponents; -type Source = 'run' | 'request' | 'upstreamRequest' | 'upstreamResponse' | 'response'; +type Source = 'request' | 'upstreamRequest' | 'upstreamResponse' | 'response'; function CopyButton({ text }: { text: string }) { const { t } = useTranslation(); @@ -66,12 +66,13 @@ export function RequestDetailPanel({ collected, upstreamCollected, error, record if (!shown.recordId) return {t('dashboard.requests.selectPrompt')}; if (shown.error) return {shown.error}; if (!shown.record) return null; + if (shown.record.shape === 'run') return ; return ; } -function RecordDetail({ record, collected, upstreamCollected }: { record: DumpRecord; collected: CollectedStream | null; upstreamCollected: CollectedStream | null }) { +function RecordDetail({ record, collected, upstreamCollected }: { record: DumpEdgeRecord; collected: CollectedStream | null; upstreamCollected: CollectedStream | null }) { const { t } = useTranslation(); - const [source, setSource] = useState(record.shape === 'run' ? 'run' : 'response'); + const [source, setSource] = useState('response'); const [view, setView] = useState('collected'); const [detailsOpen, setDetailsOpen] = useState(false); const exchanges = record.capture?.exchanges ?? []; @@ -79,22 +80,19 @@ function RecordDetail({ record, collected, upstreamCollected }: { record: DumpRe const upstream = source === 'upstreamRequest' || source === 'upstreamResponse'; const request = source === 'request' || source === 'upstreamRequest'; const exchange = upstream ? exchanges[index] : undefined; - const clientRequest = record.shape === 'edge' ? record.request : record.capture?.request; - const clientResponse = useMemo(() => record.shape === 'edge' ? record.response : { status: record.meta.status, headers: record.capture?.response?.headers ?? [], body: record.capture?.response ? { type: 'bytes' as const, body: record.capture.response.body } : { type: 'none' as const } }, [record]); - const legacy = record.shape === 'edge' ? record.response.upstream : undefined; - const runEvents = useMemo(() => record.shape === 'run' ? renderRunEvents(record.events).map(event => ({ event: `${event.type} ${event.subject ?? ''}`.trim(), text: event.text, parseError: event.parseError })) : [], [record]); + const legacy = record.response.upstream; const headers = useMemo(() => upstream ? (request ? exchange?.request.headers : exchange?.response?.headers ?? legacy?.headers) ?? [] - : request ? clientRequest?.headers ?? [] : clientResponse.headers, [exchange, legacy, clientRequest, clientResponse, request, upstream]); - const status = upstream ? exchange?.response?.status ?? legacy?.status ?? null : clientResponse.status; - const endpoint = upstream ? exchange?.request.url : record.meta.path; - const method = upstream ? exchange?.request.method : record.meta.method; + : request ? record.request.headers : record.response.headers, [exchange, legacy, record, request, upstream]); + const status = upstream ? exchange?.response?.status ?? legacy?.status ?? null : record.response.status; + const endpoint = upstream ? exchange?.request.url : record.request.path; + const method = upstream ? exchange?.request.method : record.request.method; const raw = request ? undefined : upstream ? exchange?.response ?? undefined : record.capture?.response; const result = request ? null : upstream ? (!exchange || index === exchanges.length - 1 ? upstreamCollected : null) : collected; const kind = upstream ? collectKindFromTargetApi(record.meta.targetApi) : detectCollectKind(record.meta.path); const body = useMemo(() => upstream ? request ? exchange ? { type: 'bytes', body: exchange.request.body } : { type: 'none' } : (!exchange || index === exchanges.length - 1) && legacy ? legacy.body : raw ? { type: 'bytes', body: raw.body } : { type: 'none' } - : request ? clientRequest ? { type: 'bytes', body: clientRequest.body } : { type: 'none' } : clientResponse.body, [upstream, request, exchange, index, exchanges.length, legacy, raw, clientRequest, clientResponse]); + : request ? { type: 'bytes', body: record.request.body } : record.response.body, [upstream, request, exchange, index, exchanges.length, legacy, raw, record]); const displayed = useMemo(() => { if (view === 'raw' && raw) return { text: raw.body.data, isJson: false, decodeError: null }; if (body.type === 'bytes') return renderBody(body.body, contentTypeOf(headers)); @@ -107,7 +105,7 @@ function RecordDetail({ record, collected, upstreamCollected }: { record: DumpRe displayed.decodeError ? t('dashboard.requests.decodeError', { error: displayed.decodeError }) : null, ].filter((value): value is string => Boolean(value)))]; const labels: Record = { - run: t('dashboard.requests.run'), request: t('dashboard.requests.clientRequest'), upstreamRequest: t('dashboard.requests.upstreamRequest'), + request: t('dashboard.requests.clientRequest'), upstreamRequest: t('dashboard.requests.upstreamRequest'), upstreamResponse: t('dashboard.requests.upstreamResponse'), response: t('dashboard.requests.clientResponse'), }; const viewLabels: Record = { @@ -132,28 +130,17 @@ function RecordDetail({ record, collected, upstreamCollected }: { record: DumpRe return
chooseSource(data.optionValue)}> - {Object.entries(labels).filter(([value]) => value !== 'run' || record.shape === 'run').map(([value, label]) => )} + {Object.entries(labels).map(([value, label]) => )} {status ?? t('dashboard.requests.noStatus')} - - - - - {record.meta.ttftMs != null && ( - - - - - )} + } label={t('dashboard.requests.exportRecord')} onClick={() => downloadRecords([record])} />
- {source === 'run' && record.shape === 'run' ? {t('dashboard.requests.run')}} /> : view === 'events' && body.type === 'stream' + {view === 'events' && body.type === 'stream' ? : }>}
setDetailsOpen(data.open)} title={{labels[source]}} actions={}> -
{endpoint ?? record.meta.path}
+
{endpoint ?? record.request.path}
{diagnostics.map((message, i) => {message})} {upstream && !exchange && !legacy && {t('dashboard.requests.noUpstreamCapture')}}
{t('dashboard.requests.headers', { count: String(headers.length) })} `${name}: ${value}`).join('\n')} />
@@ -170,3 +157,41 @@ function RecordDetail({ record, collected, upstreamCollected }: { record: DumpRe
; } + +function RecordTiming({ meta }: { meta: DumpMetadata }) { + const { t } = useTranslation(); + return <> + + + + + {meta.ttftMs != null && ( + + + + + )} + ; +} + +function RunRecordDetail({ record }: { record: DumpRunRecord }) { + const { t } = useTranslation(); + const events = useMemo(() => renderRunEvents(record.events).map(event => ({ + event: `${event.type} ${event.subject ?? ''}`.trim(), text: event.text, parseError: event.parseError, + })), [record.events]); + const failure = errorLabel(record.meta.error); + return
+
+ {t('dashboard.requests.run')} + {record.meta.status ?? t('dashboard.requests.noStatus')} + + } label={t('dashboard.requests.exportRecord')} onClick={() => downloadRecords([record])} /> +
+ {failure && {failure}} +
+ {t('dashboard.requests.events', { count: events.length })}} /> +
+
; +} diff --git a/apps/web/src/components/requests/export.ts b/apps/web/src/components/requests/export.ts index 497a5834d6..ec61f64aa5 100644 --- a/apps/web/src/components/requests/export.ts +++ b/apps/web/src/components/requests/export.ts @@ -9,17 +9,15 @@ const redactHeaders = (headers: Array<[string, string]>): Array<[string, string] headers.map(([name, value]) => isSensitiveHeader(name) ? [name, redactHeaderValue(value)] as [string, string] : [name, value]); const redactRecord = (record: DumpRecord): DumpRecord => { + if (record.shape === 'run') return { ...record, events: redactRunHeaders(record.events) }; const capture = record.capture === undefined ? undefined : { ...record.capture, - ...(record.capture.request === undefined ? {} : { request: { ...record.capture.request, headers: redactHeaders(record.capture.request.headers) } }), - ...(record.capture.response?.headers === undefined ? {} : { response: { ...record.capture.response, headers: redactHeaders(record.capture.response.headers) } }), exchanges: record.capture.exchanges.map(exchange => ({ ...exchange, request: { ...exchange.request, headers: redactHeaders(exchange.request.headers) }, response: exchange.response === null ? null : { ...exchange.response, headers: redactHeaders(exchange.response.headers) }, })), }; - if (record.shape === 'run') return { ...record, events: redactRunHeaders(record.events), ...(capture === undefined ? {} : { capture }) }; return { ...record, request: { ...record.request, headers: redactHeaders(record.request.headers) }, diff --git a/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts index 3bf5712372..8d4344ace9 100644 --- a/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts @@ -7,6 +7,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { openaiAudioTranscriptionServePipeline } from '../../src/data-plane/openai-audio/pipeline.ts'; +import { createCandidateRegistry } from '../../src/data-plane/pipeline/candidates.ts'; import { enumerateModelCandidates } from '../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../src/repo/index.ts'; import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts'; @@ -57,7 +58,7 @@ const serve = async (responseFormat = 'json') => await run( { gateway: mockGatewayCtx({ wantsStream: responseFormat === 'stream' }), background: () => {}, - rememberCandidates: () => {}, + ...createCandidateRegistry(), resolveAttempt: (selector: { readonly upstreamId: string }) => { const found = live.find(c => c.provider.upstreamId === selector.upstreamId); if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); diff --git a/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts index 2bdc7e6a9a..9577ef34f8 100644 --- a/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts @@ -7,6 +7,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { openaiCompletionsServePipeline } from '../../src/data-plane/openai-completions/pipeline.ts'; +import { createCandidateRegistry } from '../../src/data-plane/pipeline/candidates.ts'; import { enumerateModelCandidates } from '../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../src/repo/index.ts'; import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts'; @@ -87,7 +88,7 @@ const serve = async (facts: Record) => await run( { gateway: mockGatewayCtx({ wantsStream: facts['ingress.openaiCompletions.wantsStream'] === true }), background: () => {}, - rememberCandidates: () => {}, + ...createCandidateRegistry(), resolveAttempt, } as never, ); diff --git a/packages/gateway/__tests__/data-plane/pipeline/candidates_test.ts b/packages/gateway/__tests__/data-plane/pipeline/candidates_test.ts new file mode 100644 index 0000000000..9416a94d15 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/pipeline/candidates_test.ts @@ -0,0 +1,18 @@ +import { expect, it } from 'vitest'; + +import { createCandidateRegistry } from '../../../src/data-plane/pipeline/candidates.ts'; +import { stubModelCandidate } from '@floway-dev/test-utils'; + +it('preserves exact candidate identity across same-upstream variants and later registrations', () => { + const registry = createCandidateRegistry(); + const first = stubModelCandidate({ model: { id: 'first' } }); + const second = { ...first, rules: { verbosity: 'high' } }; + const later = stubModelCandidate({ model: { id: 'later' } }); + const [firstSelector, secondSelector] = registry.rememberCandidates([first, second]); + const [laterSelector] = registry.rememberCandidates([later]); + expect([firstSelector!.candidateId, secondSelector!.candidateId, laterSelector!.candidateId]).toEqual([0, 1, 2]); + expect(registry.resolveAttempt(firstSelector!)).toBe(first); + expect(registry.resolveAttempt(secondSelector!)).toBe(second); + expect(registry.resolveAttempt(laterSelector!)).toBe(later); + expect(() => registry.resolveAttempt({ ...firstSelector!, candidateId: 99 })).toThrow('nothing live for candidate 99'); +}); diff --git a/packages/gateway/__tests__/data-plane/rerank-status_test.ts b/packages/gateway/__tests__/data-plane/rerank-status_test.ts index 0294b84032..80ca9acc89 100644 --- a/packages/gateway/__tests__/data-plane/rerank-status_test.ts +++ b/packages/gateway/__tests__/data-plane/rerank-status_test.ts @@ -1,11 +1,12 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + // The two families that read their answer to the end, driven end to end. Until the review, // neither could express a status at all: an upstream 429, a resolver's 404 and a 400 all // reached the client as a 200 carrying an error envelope, which is not a difference the // no-passthrough ruling asks for — declining to forward a body is not declining to forward // a status. -import { beforeEach, describe, expect, it, vi } from 'vitest'; - +import { createCandidateRegistry } from '../../src/data-plane/pipeline/candidates.ts'; import { enumerateModelCandidates } from '../../src/data-plane/providers/resolution.ts'; import { rerankServePipeline } from '../../src/data-plane/rerank/pipeline.ts'; import { initRepo } from '../../src/repo/index.ts'; @@ -57,7 +58,7 @@ const serve = async () => await run( { gateway: mockGatewayCtx({ wantsStream: false }), background: () => {}, - rememberCandidates: () => {}, + ...createCandidateRegistry(), resolveAttempt: (selector: { readonly upstreamId: string }) => { const found = live.find(c => c.provider.upstreamId === selector.upstreamId); if (found === undefined) throw new Error(`no live candidate for ${selector.upstreamId}`); diff --git a/packages/gateway/__tests__/dump/run-sink_test.ts b/packages/gateway/__tests__/dump/run-sink_test.ts index ec86d4a1b8..f6a93e931c 100644 --- a/packages/gateway/__tests__/dump/run-sink_test.ts +++ b/packages/gateway/__tests__/dump/run-sink_test.ts @@ -4,9 +4,7 @@ import { installDumpStubs } from './test-fixtures.ts'; import { initDumpBroker, initDumpStore } from '../../src/dump/registry.ts'; import { openRunDump } from '../../src/dump/run-sink.ts'; import type { StoredDumpRecord, StoredDumpRunRecord } from '../../src/dump/types.ts'; -import { initRepo } from '../../src/repo/index.ts'; import type { ApiKey } from '../../src/repo/types.ts'; -import { InMemoryRepo } from '../repo/memory.ts'; import { flushBackground, trackBackground } from '../test-utils/background-tracker.ts'; import { compose, defineStage, move, run, type DumpEvent, type Event } from '@floway-dev/pipeline'; import { assertEquals } from '@floway-dev/test-utils'; @@ -169,29 +167,20 @@ test('finalizing on a response measures what the client reads and leaves it inta const record = runRecordOf(stubs.stored[0]); assertEquals(record.meta.responseBytes, 22); - assertEquals(record.capture?.response?.body, { encoding: 'utf8', data: 'data: one\n\ndata: two\n\n' }); - assertEquals(record.capture?.response?.complete, true); + }); -test('a run retains raw upstream exchanges, client bytes and the stable timing reading beside its stage history', async () => { - initRepo(new InMemoryRepo()); +test('a run reads the stable timing state at completion alongside its stage history', async () => { const stubs = installDumpStubs(initDumpStore, initDumpBroker); const timing = { upstreamCallStartedAt: 100, firstOutputTokenAt: null as number | null }; - const dump = openRunDump(apiKey(3600), { ...turn, headers: [['content-type', 'application/json']] }, trackBackground, true, timing); + const dump = openRunDump(apiKey(3600), turn, trackBackground, true, timing); if (dump === null) throw new Error('a key with retention must open a run dump'); await run(pipeline, move({ 'in.text': 'hey' }), { dump: dump.sink }); - const bytes = new Uint8Array([0xFF, 0, 0x80]); - const fetcher = dump.http.wrapFetcher(async () => new Response(bytes, { headers: { 'content-type': 'application/octet-stream' } }), 'up_run'); - const response = await fetcher('https://upstream.test/embeddings', { method: 'POST', body: '{"model":"upstream"}' }); - assertEquals([...new Uint8Array(await response.arrayBuffer())], [...bytes]); timing.firstOutputTokenAt = 225; - assertEquals(await dump.finalize(new Response('client', { headers: { 'content-type': 'text/plain' } })).text(), 'client'); + assertEquals(await dump.finalize(new Response('client')).text(), 'client'); await flushBackground(); const record = runRecordOf(stubs.stored[0]); assertEquals(record.meta.ttftMs, 125); - assertEquals(record.capture?.request?.body, { encoding: 'utf8', data: '{"input":"hi"}' }); - assertEquals(record.capture?.exchanges[0]?.response?.body, { encoding: 'base64', data: '/wCA' }); - assertEquals(record.capture?.exchanges[0]?.response?.complete, true); - assertEquals(record.capture?.response?.body, { encoding: 'utf8', data: 'client' }); + assertEquals('capture' in record, false); assertEquals(lines(record).some(event => event.type === 'stage.entered'), true); }); diff --git a/packages/gateway/__tests__/repo/dump-store_test.ts b/packages/gateway/__tests__/repo/dump-store_test.ts index f487c44c5f..5b10d2e4fd 100644 --- a/packages/gateway/__tests__/repo/dump-store_test.ts +++ b/packages/gateway/__tests__/repo/dump-store_test.ts @@ -638,7 +638,7 @@ test('FileDumpStore round-trips raw exchanges and parsed events in one owned spi record.response = { ...record.response, upstream: { status: 200, headers: [['x-trace', 'upstream']], body: { type: 'stream', events: [{ ts: 0, frame: { type: 'done' } }] } } }; await store.put('key_x', record); const stored = await store.get('key_x', record.meta.id); - expect(stored?.capture).toEqual(record.capture); + expect(edgeOf(stored).capture).toEqual(record.capture); expect(edgeOf(stored!).response.upstream).toEqual(record.response.upstream); const spill = await db.prepare("SELECT file_key, state FROM spilled_files WHERE owner_kind = 'dump-response-upstream'").first<{ file_key: string; state: string }>(); expect(spill?.state).toBe('owned'); diff --git a/packages/gateway/src/data-plane/pipeline/candidates.ts b/packages/gateway/src/data-plane/pipeline/candidates.ts new file mode 100644 index 0000000000..d195850f32 --- /dev/null +++ b/packages/gateway/src/data-plane/pipeline/candidates.ts @@ -0,0 +1,23 @@ +import type { AttemptSelector } from './facts.ts'; +import { providerModelOf, type ModelCandidate } from '@floway-dev/provider'; + +export const createCandidateRegistry = () => { + const live: ModelCandidate[] = []; + return { + rememberCandidates: (candidates: readonly ModelCandidate[]): readonly AttemptSelector[] => candidates.map(candidate => { + const candidateId = live.length; + live.push(candidate); + return { + candidateId, + upstreamId: candidate.provider.upstreamId, + modelId: candidate.model.id, + flags: [...providerModelOf(candidate).enabledFlags], + }; + }), + resolveAttempt: (selector: AttemptSelector): ModelCandidate => { + const candidate = live[selector.candidateId]; + if (candidate === undefined) throw new Error(`resolveAttempt: nothing live for candidate ${selector.candidateId}; the selector did not come from this run`); + return candidate; + }, + }; +}; diff --git a/packages/gateway/src/data-plane/pipeline/serve.ts b/packages/gateway/src/data-plane/pipeline/serve.ts index ca17b601b9..a7e5ee6195 100644 --- a/packages/gateway/src/data-plane/pipeline/serve.ts +++ b/packages/gateway/src/data-plane/pipeline/serve.ts @@ -10,7 +10,8 @@ import type { Context } from 'hono'; import { streamSSE } from 'hono/streaming'; import type { ContentfulStatusCode } from 'hono/utils/http-status'; -import type { AttemptSelector, BillableEntity, GatewayFacts } from './facts.ts'; +import { createCandidateRegistry } from './candidates.ts'; +import type { BillableEntity, GatewayFacts } from './facts.ts'; import type { GatewayServices } from './services.ts'; import { settleBillable } from './settlement.ts'; import { openRunDump } from '../../dump/run-sink.ts'; @@ -23,7 +24,6 @@ import { readRequestBody, takeRequestBody, type RequestBody } from '../shared/re import { writeSSEFrames } from '../shared/sse.ts'; import { run, type Deferred, type Pipeline } from '@floway-dev/pipeline'; import { sseCommentFrame, type SseFrame } from '@floway-dev/protocols/common'; -import type { ModelCandidate } from '@floway-dev/provider'; type Slice = { [P in K]: GatewayFacts[P] }; @@ -76,7 +76,7 @@ export const openPrologue = ( // no turn is ever written twice. const runDump = openRunDump( apiKeyFromContext(c), - { method: c.req.method, path: new URL(c.req.raw.url).pathname, body: ingress.body, headers: ingress.headers }, + { method: c.req.method, path: new URL(c.req.raw.url).pathname, body: ingress.body }, backgroundScheduler, options.wantsStream, attempt.timing, @@ -90,8 +90,6 @@ export const openPrologue = ( dump: runDump, }); - const live = new Map(); - return { gateway, headers: ingress.headers, @@ -99,19 +97,11 @@ export const openPrologue = ( gateway, log: consoleLogSink, background: work => { gateway.backgroundScheduler(work); }, - rememberCandidates: candidates => { - for (const [candidateId, candidate] of candidates.entries()) live.set(candidateId, candidate); - }, + ...createCandidateRegistry(), // Absent when this key has no retention configured, which is what keeps recording // conditional: the runner does none of it rather than doing it and discarding. ...(runDump === null ? {} : { dump: runDump.sink }), - resolveAttempt: (selector: AttemptSelector) => { - const candidate = live.get(selector.candidateId); - if (candidate === undefined) { - throw new Error(`resolveAttempt: nothing live for candidate ${selector.candidateId}; the selector did not come from this run`); - } - return candidate; - }, + }, }; }; diff --git a/packages/gateway/src/data-plane/pipeline/services.ts b/packages/gateway/src/data-plane/pipeline/services.ts index 000ce02049..9a429d8f49 100644 --- a/packages/gateway/src/data-plane/pipeline/services.ts +++ b/packages/gateway/src/data-plane/pipeline/services.ts @@ -28,7 +28,7 @@ export interface GatewayServices { readonly resolveAttempt: (selector: AttemptSelector) => ModelCandidate; /** The other half of the resolver: the stage that enumerates hands the live candidates * here, so the ones that travel can be selectors. Per run, because a candidate list is. */ - readonly rememberCandidates: (candidates: readonly ModelCandidate[]) => void; + readonly rememberCandidates: (candidates: readonly ModelCandidate[]) => readonly AttemptSelector[]; /** Binds a promise to the request's lifetime: `waitUntil` on Workers, the event loop on * Node. What the drain is handed to, so an answer is not held up by it. */ readonly background: (work: Promise) => void; diff --git a/packages/gateway/src/data-plane/pipeline/stages.ts b/packages/gateway/src/data-plane/pipeline/stages.ts index 5d97bcf22b..f4c434ea09 100644 --- a/packages/gateway/src/data-plane/pipeline/stages.ts +++ b/packages/gateway/src/data-plane/pipeline/stages.ts @@ -10,28 +10,17 @@ // them compose into a pipeline over any family's larger space with no variance question to // lose: assembly reasons over declarations, which are strings. -import type { AttemptSelector, GatewayFacts } from './facts.ts'; +import type { GatewayFacts } from './facts.ts'; import type { GatewayServices } from './services.ts'; import { enumerateModelCandidates } from '../providers/resolution.ts'; import { appendFailedUpstreams } from '../shared/failed-upstreams.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import type { Facts } from '@floway-dev/pipeline'; import type { ModelKind } from '@floway-dev/protocols/common'; -import { providerModelOf } from '@floway-dev/provider'; import type { ModelCandidate } from '@floway-dev/provider'; type Slice = { [P in K]: GatewayFacts[P] }; -/** Everything about a candidate that is data. The live half — the provider instance, the - * fetcher, the models cache — stays out of the record and is looked back up by the - * resolver service at the moment of the call. */ -const selectorFor = (candidate: ModelCandidate, candidateId: number): AttemptSelector => ({ - candidateId, - upstreamId: candidate.provider.upstreamId, - modelId: candidate.model.id, - flags: [...providerModelOf(candidate).enabledFlags], -}); - /** What a family narrows its candidates by, and what it says when nothing is left. A * candidate that resolves but cannot serve this request — no endpoint for the kind, or a * payload the target protocol cannot express — is not a candidate, and saying why is what @@ -110,14 +99,13 @@ export const resolveCandidates = (narrowing: Narrowing ({ - fetcher: ctx.dump?.http.wrapFetcher(candidate.fetcher, candidate.provider.upstreamId) ?? candidate.fetcher, + fetcher: ctx.dump?.http?.wrapFetcher(candidate.fetcher, candidate.provider.upstreamId) ?? candidate.fetcher, waitUntil: ctx.backgroundScheduler, headers: filterInboundHeadersForProvider(headers, candidate.provider), wrapUpstreamCall: stampUpstreamCallStart(ctx.attempt.timing), diff --git a/packages/gateway/src/dump/attribution.ts b/packages/gateway/src/dump/attribution.ts index a2e68bfec5..c240c82f4e 100644 --- a/packages/gateway/src/dump/attribution.ts +++ b/packages/gateway/src/dump/attribution.ts @@ -70,7 +70,6 @@ export interface DumpTurnOutcome { readonly requestBytes: number; readonly responseBytes: number; readonly ttftMs: number | null; - readonly fallbackUpstreamId: string | null; // Applied only when no hook stamped an error, so an explicit stamp from the // respond path always outranks a transport-level read failure. readonly fallbackError: DumpErrorMeta | null; @@ -120,7 +119,7 @@ export class DumpAttribution { method: outcome.method, path: outcome.path, status: outcome.status, - upstream: await resolveUpstreamRef(this.upstreamId ?? outcome.fallbackUpstreamId), + upstream: await resolveUpstreamRef(this.upstreamId), model: this.model, inputTokens: this.inputTokens, outputTokens: this.outputTokens, diff --git a/packages/gateway/src/dump/run-sink.ts b/packages/gateway/src/dump/run-sink.ts index 7ef26c401b..4bd50d0629 100644 --- a/packages/gateway/src/dump/run-sink.ts +++ b/packages/gateway/src/dump/run-sink.ts @@ -14,11 +14,9 @@ // the result away. import { DumpAttribution, oneLineError, streamReadError } from './attribution.ts'; -import { HttpCapture } from './http-capture.ts'; import { getDumpBroker, getDumpStore } from './registry.ts'; import type { StreamRecording } from './turn-dump.ts'; -import type { DumpCapture, DumpMetadata } from './types.ts'; -import { encodeBodyForWire } from './wire.ts'; +import type { DumpMetadata } from './types.ts'; import { attemptTtftMs, type AttemptTiming } from '../data-plane/shared/attempt-timing.ts'; import type { RequestBody } from '../data-plane/shared/request-body.ts'; import type { ApiKey, TokenUsage } from '../repo/types.ts'; @@ -28,9 +26,8 @@ import type { BackgroundScheduler } from '@floway-dev/platform'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { TelemetryModelIdentity } from '@floway-dev/provider'; -// Raw ingress preserves the bytes before protocol parsing or serialization. +// Only metadata is snapshotted here; request contents are facts recorded by the run. interface RequestSnapshot { - readonly request: NonNullable; readonly method: string; readonly path: string; readonly bodyByteLength: number; @@ -38,7 +35,6 @@ interface RequestSnapshot { } export class RunDump { - readonly http = new HttpCapture(); private readonly attribution = new DumpAttribution(); private readonly encode = createRunEncoder(); private readonly events: DumpEvent[] = []; @@ -115,7 +111,7 @@ export class RunDump { // // • `(status, responseBytes)` — the caller already knows what it wrote. // • `(response)` — tees the answer so the client gets bytes flowing while a - // background reader records the raw client bytes alongside the stage history. + // background reader measures the other half. Protocol contents live in the run events. // // The drain → encode → store put → broker publish runs on the runtime's // BackgroundScheduler so a dump write failure cannot turn a served answer @@ -146,7 +142,6 @@ export class RunDump { const [forClient, forMeasure] = response.body.tee(); this.backgroundScheduler((async () => { const reader = forMeasure.getReader(); - const chunks: Uint8Array[] = []; let payloadBytes = 0; let streamError: string | null = null; try { @@ -154,23 +149,13 @@ export class RunDump { const { value, done } = await reader.read(); if (done) break; payloadBytes += value.byteLength; - chunks.push(value); } } catch (err) { streamError = oneLineError(err); } finally { reader.releaseLock(); } - const bytes = new Uint8Array(payloadBytes); - let offset = 0; - for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.byteLength; } - await this.write(response.status, payloadBytes, streamError, { - status: response.status, - headers: [...response.headers], - body: encodeBodyForWire(bytes, response.headers.get('content-type') ?? ''), - complete: streamError === null, - error: streamError, - }); + await this.write(response.status, payloadBytes, streamError); })()); return new Response(forClient, { @@ -180,7 +165,7 @@ export class RunDump { }); } - private async write(status: number | null, responseBytes: number, responseStreamError: string | null, response?: DumpCapture['response']): Promise { + private async write(status: number | null, responseBytes: number, responseStreamError: string | null): Promise { // ULID-from-completedAt keeps ids increasing with row creation time; the // random tail provides the deterministic tie-breaker for one millisecond. const completedAt = Date.now(); @@ -195,7 +180,6 @@ export class RunDump { requestBytes: this.requestSnapshot.bodyByteLength, responseBytes, ttftMs: this.wantsStream ? attemptTtftMs(this.timing) : null, - fallbackUpstreamId: this.http.exchanges.at(-1)?.upstreamId ?? null, fallbackError: streamReadError(this.requestSnapshot.streamError, responseStreamError), }); @@ -203,7 +187,6 @@ export class RunDump { try { await getDumpStore().put(this.apiKey.id, { shape: 'run', - capture: { request: this.requestSnapshot.request, exchanges: this.http.exchanges, ...(response === undefined ? {} : { response }) }, meta, events: new TextEncoder().encode(toNdjson(this.events)), }); @@ -225,7 +208,7 @@ export class RunDump { */ export const openRunDump = ( apiKey: ApiKey, - turn: { readonly method: string; readonly path: string; readonly body: RequestBody; readonly headers: readonly (readonly [string, string])[] }, + turn: { readonly method: string; readonly path: string; readonly body: RequestBody }, backgroundScheduler: BackgroundScheduler, wantsStream: boolean, timing: AttemptTiming, @@ -234,12 +217,6 @@ export const openRunDump = ( return new RunDump( apiKey, { - request: { - method: turn.method, - path: turn.path, - headers: turn.headers.map(([name, value]): [string, string] => [name, value]), - body: encodeBodyForWire(turn.body.bytes, turn.headers.find(([name]) => name.toLowerCase() === 'content-type')?.[1] ?? ''), - }, method: turn.method, path: turn.path, bodyByteLength: turn.body.bytes.byteLength, diff --git a/packages/gateway/src/dump/schemas.ts b/packages/gateway/src/dump/schemas.ts index e9a2b6b9f9..e4b51f03fc 100644 --- a/packages/gateway/src/dump/schemas.ts +++ b/packages/gateway/src/dump/schemas.ts @@ -43,10 +43,6 @@ export const persistedDumpMetadataSchema = dumpMetadataSchema.omit({ upstream: t export const dumpHeadersSchema = z.array(z.tuple([z.string(), z.string()])); -// `type` says how to read the file the descriptor points at: raw bytes, the -// JSON array of captured protocol frames, or a run's NDJSON event stream. It is -// also what tells the reader which shape the record is — a run's stream is one -// more body file under the same contract, so the row needs nothing else. export const dumpBodyDescriptorSchema = z.object({ key: z.string(), type: z.enum(['bytes', 'events', 'run', 'capture']), @@ -76,12 +72,6 @@ const rawCaptureSchema = z.object({ }).strict(); export const dumpCaptureSchema = z.object({ - request: z.object({ - method: z.string(), - path: z.string(), - headers: dumpHeadersSchema, - body: rawBodySchema, - }).strict().optional(), exchanges: z.array(z.object({ upstreamId: z.string(), request: z.object({ @@ -93,10 +83,7 @@ export const dumpCaptureSchema = z.object({ response: rawCaptureSchema.extend({ status: z.number(), headers: dumpHeadersSchema }).nullable(), error: z.string().nullable(), }).strict()), - response: rawCaptureSchema.extend({ - status: z.number().nullable().optional(), - headers: dumpHeadersSchema.optional(), - }).optional(), + response: rawCaptureSchema.optional(), }).strict(); export const dumpCaptureEnvelopeSchema = z.object({ diff --git a/packages/gateway/src/dump/turn-dump.ts b/packages/gateway/src/dump/turn-dump.ts index c6da6c520f..2ce3277344 100644 --- a/packages/gateway/src/dump/turn-dump.ts +++ b/packages/gateway/src/dump/turn-dump.ts @@ -31,7 +31,7 @@ export interface StreamRecording { } export interface TurnDump { - readonly http: HttpCapture; + readonly http?: HttpCapture; requestedModel(model: string): void; success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void; error(kind: 'upstream' | 'gateway', upstream?: string): void; diff --git a/packages/gateway/src/dump/types.ts b/packages/gateway/src/dump/types.ts index cd3c95bf4f..5585403f54 100644 --- a/packages/gateway/src/dump/types.ts +++ b/packages/gateway/src/dump/types.ts @@ -159,18 +159,15 @@ export type DumpEdgeRecord = { response: DumpResponse; }; -// A run's event stream records every stage and both directions. Raw HTTP -// exchanges remain parallel diagnostic data, captured at the actual fetch boundary. +// A run's event stream records every stage and both directions. export type StoredDumpRunRecord = { shape: 'run'; - capture?: DumpCapture; meta: DumpMetadata; events: Uint8Array; }; export type DumpRunRecord = { shape: 'run'; - capture?: DumpCapture; meta: DumpMetadata; events: string; }; diff --git a/packages/gateway/src/dump/wire.ts b/packages/gateway/src/dump/wire.ts index d496d79659..fe74a0320e 100644 --- a/packages/gateway/src/dump/wire.ts +++ b/packages/gateway/src/dump/wire.ts @@ -50,7 +50,6 @@ export const dumpRecordToWire = (record: StoredDumpRecord): DumpRecord => { if (record.shape === 'run') { return { shape: 'run', - ...(record.capture === undefined ? {} : { capture: record.capture }), meta: record.meta, events: new TextDecoder('utf-8', { fatal: true }).decode(record.events), }; diff --git a/packages/gateway/src/repo/dump-store.ts b/packages/gateway/src/repo/dump-store.ts index b24ae60b11..5b88ed72c7 100644 --- a/packages/gateway/src/repo/dump-store.ts +++ b/packages/gateway/src/repo/dump-store.ts @@ -20,6 +20,7 @@ import type { DumpWriteRecord, PreparedDumpRequestBody, StoredDumpRecord, + StoredDumpEdgeRecord, StoredDumpRequest, StoredDumpResponse, StoredDumpResponseBody, @@ -143,9 +144,10 @@ export class FileDumpStore implements DumpStore { // The pre-translation upstream body, when present. Same descriptor shape // as the downstream response body (`{key, type}`), spilled under a // distinct `resp.up` side so the sweep can own it independently. + const capture = record.shape === 'edge' ? record.capture : undefined; const upstream = record.shape === 'edge' ? record.response.upstream : undefined; const upstreamBody = upstream?.body; - const upstreamFileKey = record.capture !== undefined + const upstreamFileKey = capture !== undefined ? bodyPath(keyId, bucket, record.meta.id, 'resp.up') : upstreamBody === undefined ? null @@ -198,8 +200,8 @@ export class FileDumpStore implements DumpStore { } let upstreamDescriptor: DumpBodyDescriptor | null = null; - if (record.capture !== undefined) { - const envelope = dumpCaptureEnvelopeSchema.parse({ version: 1, capture: record.capture, upstream: upstreamResponseToWire(upstream) }); + if (capture !== undefined) { + const envelope = dumpCaptureEnvelopeSchema.parse({ version: 1, capture, upstream: upstreamResponseToWire(upstream) }); upstreamDescriptor = await putRawBody(this.files, upstreamFileKey!, new TextEncoder().encode(JSON.stringify(envelope)), 'capture'); } else if (upstreamBody !== undefined) { if (upstreamBody.type === 'bytes') { @@ -309,10 +311,7 @@ export class FileDumpStore implements DumpStore { // The body kind is the shape: a run was written as one NDJSON stream and // has no edge halves to rebuild. if (responseDescriptor?.type === 'run') { - const capture = upstreamDescriptor?.type === 'capture' - ? dumpCaptureEnvelopeSchema.parse(JSON.parse(new TextDecoder().decode(await fetchBody(this.files, upstreamDescriptor)))).capture - : undefined; - return { shape: 'run', meta, events: await fetchBody(this.files, responseDescriptor), ...(capture === undefined ? {} : { capture }) }; + return { shape: 'run', meta, events: await fetchBody(this.files, responseDescriptor) }; } const requestHeaders = decodeDumpHeaders(row.request_headers_json, `dump record ${recordId} request headers`); @@ -349,7 +348,7 @@ export class FileDumpStore implements DumpStore { // Same rehydration rules as the downstream body; absent on native turns // and on records written before the upstream column existed (NULL). let upstream: StoredDumpUpstreamResponse | undefined; - let capture: StoredDumpRecord['capture']; + let capture: StoredDumpEdgeRecord['capture']; if (upstreamDescriptor?.type === 'capture') { const envelope = dumpCaptureEnvelopeSchema.parse(JSON.parse(new TextDecoder().decode(await fetchBody(this.files, upstreamDescriptor)))); capture = envelope.capture; From 68d1a8a97d501c97ef2ab9495aa6ff9832e3ac62 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 30 Sep 2026 16:33:23 +0800 Subject: [PATCH 56/79] fix(web): distinguish empty pipeline recordings from search misses --- apps/web/src/components/requests/detail.tsx | 2 +- apps/web/src/components/requests/events.tsx | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/web/src/components/requests/detail.tsx b/apps/web/src/components/requests/detail.tsx index 7df33c054d..55a368d023 100644 --- a/apps/web/src/components/requests/detail.tsx +++ b/apps/web/src/components/requests/detail.tsx @@ -191,7 +191,7 @@ function RunRecordDetail({ record }: { record: DumpRunRecord }) { {failure && {failure}}
- {t('dashboard.requests.events', { count: events.length })}} /> + {t('dashboard.requests.events', { count: events.length })}} emptyText={t('dashboard.requests.noRunEvents')} />
; } diff --git a/apps/web/src/components/requests/events.tsx b/apps/web/src/components/requests/events.tsx index b561a50d2c..fd54317fed 100644 --- a/apps/web/src/components/requests/events.tsx +++ b/apps/web/src/components/requests/events.tsx @@ -54,7 +54,7 @@ export function EventList({ events, kind, toolbarStart }: { events: DumpStreamEv return ; } -export function RenderedEventList({ events, copyText, toolbarStart }: { events: RenderedEvent[]; copyText: string; toolbarStart: ReactNode }) { +export function RenderedEventList({ events, copyText, toolbarStart, emptyText }: { events: RenderedEvent[]; copyText: string; toolbarStart: ReactNode; emptyText?: string }) { const { t } = useTranslation(); const [query, setQuery] = useState(''); const [searchOpen, setSearchOpen] = useState(false); @@ -103,7 +103,7 @@ export function RenderedEventList({ events, copyText, toolbarStart }: { events: {filtered.length} / {events.length} } label={t('common.dismiss')} onClick={() => { setSearchOpen(false); changeQuery(''); }} /> } - {filtered.length === 0 ? {t('dashboard.requests.noEventMatches')} :
+ {filtered.length === 0 ? {query.trim() ? t('dashboard.requests.noEventMatches') : emptyText ?? t('dashboard.requests.noEventMatches')} :
Date: Wed, 30 Sep 2026 18:37:25 +0800 Subject: [PATCH 57/79] fix: mask credential headers in run diagnostics --- .../__tests__/components/requests/detail_test.tsx | 12 ++++++++++++ apps/web/src/components/requests/detail.tsx | 8 +++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/apps/web/__tests__/components/requests/detail_test.tsx b/apps/web/__tests__/components/requests/detail_test.tsx index 895307d22f..0add7c1963 100644 --- a/apps/web/__tests__/components/requests/detail_test.tsx +++ b/apps/web/__tests__/components/requests/detail_test.tsx @@ -79,3 +79,15 @@ it('switches upstream attempts without showing the last attempt’s payload', as fireEvent.click(screen.getAllByRole('option')[0]!); expect((await screen.findByTestId('body-content')).textContent).toContain('first attempt'); }); + +it('masks credential headers in rendered run facts', () => { + const secret = 'Bearer private-request-token-1234567890'; + const events = [ + { type: 'object', fromObjectId: 1, nodes: [[['authorization', secret]]] }, + { type: 'stage.entered', stageId: 1, name: 'serve', parentStageId: null, facts: { 'ingress.http.headers': { $: 1 } } }, + ].map(event => `${JSON.stringify(event)}\n`).join(''); + const view = renderInApp(); + expect(view.container.textContent).not.toContain(secret); + expect(view.container.textContent).toContain('authorization'); + expect(view.container.textContent).toContain('••••'); +}); diff --git a/apps/web/src/components/requests/detail.tsx b/apps/web/src/components/requests/detail.tsx index 55a368d023..b6339b36d0 100644 --- a/apps/web/src/components/requests/detail.tsx +++ b/apps/web/src/components/requests/detail.tsx @@ -6,6 +6,7 @@ import { EventList, RenderedEventList } from './events'; import { downloadRecords } from './export'; import { errorLabel, requestSeverity } from './format'; import { isSensitiveHeader, redactHeaderValue } from './header-redact'; +import { redactRunHeaders } from './run-redact'; import { renderRunEvents } from './run-render'; import { collectKindFromTargetApi, detectCollectKind, type CollectedStream } from './stream-render'; import { fluentComponents } from '../../fluent'; @@ -178,9 +179,10 @@ function RecordTiming({ meta }: { meta: DumpMetadata }) { function RunRecordDetail({ record }: { record: DumpRunRecord }) { const { t } = useTranslation(); - const events = useMemo(() => renderRunEvents(record.events).map(event => ({ + const redacted = useMemo(() => redactRunHeaders(record.events), [record.events]); + const events = useMemo(() => renderRunEvents(redacted).map(event => ({ event: `${event.type} ${event.subject ?? ''}`.trim(), text: event.text, parseError: event.parseError, - })), [record.events]); + })), [redacted]); const failure = errorLabel(record.meta.error); return
@@ -191,7 +193,7 @@ function RunRecordDetail({ record }: { record: DumpRunRecord }) {
{failure && {failure}}
- {t('dashboard.requests.events', { count: events.length })}} emptyText={t('dashboard.requests.noRunEvents')} /> + {t('dashboard.requests.events', { count: events.length })}} emptyText={t('dashboard.requests.noRunEvents')} />
; } From b9e4c1c565ff6d006f34ebb2b43539fcacd39954 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 30 Sep 2026 20:04:33 +0800 Subject: [PATCH 58/79] refactor: give non-chat stages their own modules Keep protocol pipeline files focused on assembly. Move emit and upstream-call stages beside their facts and target policy, separate shared candidate resolution from failover, and update consumers to import each owning module. --- .../data-plane/openai-images-pipeline_test.ts | 3 +- .../data-plane/rerank-pipeline_test.ts | 2 +- .../data-plane/alpha-search/call-upstream.ts | 141 ++++++ .../src/data-plane/alpha-search/emit.ts | 59 +++ .../alpha-search/execute-operations.ts | 47 ++ .../src/data-plane/alpha-search/facts.ts | 32 ++ .../src/data-plane/alpha-search/http.ts | 4 +- .../alpha-search/parse-operations.ts | 60 +++ .../src/data-plane/alpha-search/pipeline.ts | 376 +------------- .../src/data-plane/alpha-search/services.ts | 9 + .../data-plane/openai-audio/call-upstream.ts | 265 ++++++++++ .../src/data-plane/openai-audio/emit.ts | 85 ++++ .../src/data-plane/openai-audio/facts.ts | 62 +++ .../src/data-plane/openai-audio/pipeline.ts | 462 +----------------- .../src/data-plane/openai-audio/target.ts | 24 + .../openai-completions/call-upstream.ts | 256 ++++++++++ .../src/data-plane/openai-completions/emit.ts | 84 ++++ .../data-plane/openai-completions/facts.ts | 35 ++ .../data-plane/openai-completions/pipeline.ts | 412 +--------------- .../data-plane/openai-completions/target.ts | 17 + .../openai-embeddings/call-upstream.ts | 99 ++++ .../src/data-plane/openai-embeddings/emit.ts | 59 +++ .../src/data-plane/openai-embeddings/facts.ts | 18 + .../data-plane/openai-embeddings/pipeline.ts | 214 +------- .../data-plane/openai-embeddings/target.ts | 14 + .../data-plane/openai-images/call-upstream.ts | 251 ++++++++++ .../src/data-plane/openai-images/emit.ts | 65 +++ .../src/data-plane/openai-images/facts.ts | 47 ++ .../src/data-plane/openai-images/pipeline.ts | 420 +--------------- .../src/data-plane/openai-images/target.ts | 25 + .../src/data-plane/pipeline/failover.ts | 62 +++ .../{stages.ts => resolve-candidates.ts} | 76 +-- .../data-plane/pipeline/stage-contracts.ts | 3 + .../src/data-plane/rerank/call-upstream.ts | 155 ++++++ .../gateway/src/data-plane/rerank/emit.ts | 78 +++ .../gateway/src/data-plane/rerank/facts.ts | 22 + .../gateway/src/data-plane/rerank/pipeline.ts | 299 +----------- .../gateway/src/data-plane/rerank/target.ts | 20 + 38 files changed, 2155 insertions(+), 2207 deletions(-) create mode 100644 packages/gateway/src/data-plane/alpha-search/call-upstream.ts create mode 100644 packages/gateway/src/data-plane/alpha-search/emit.ts create mode 100644 packages/gateway/src/data-plane/alpha-search/execute-operations.ts create mode 100644 packages/gateway/src/data-plane/alpha-search/facts.ts create mode 100644 packages/gateway/src/data-plane/alpha-search/parse-operations.ts create mode 100644 packages/gateway/src/data-plane/alpha-search/services.ts create mode 100644 packages/gateway/src/data-plane/openai-audio/call-upstream.ts create mode 100644 packages/gateway/src/data-plane/openai-audio/emit.ts create mode 100644 packages/gateway/src/data-plane/openai-audio/facts.ts create mode 100644 packages/gateway/src/data-plane/openai-audio/target.ts create mode 100644 packages/gateway/src/data-plane/openai-completions/call-upstream.ts create mode 100644 packages/gateway/src/data-plane/openai-completions/emit.ts create mode 100644 packages/gateway/src/data-plane/openai-completions/facts.ts create mode 100644 packages/gateway/src/data-plane/openai-completions/target.ts create mode 100644 packages/gateway/src/data-plane/openai-embeddings/call-upstream.ts create mode 100644 packages/gateway/src/data-plane/openai-embeddings/emit.ts create mode 100644 packages/gateway/src/data-plane/openai-embeddings/facts.ts create mode 100644 packages/gateway/src/data-plane/openai-embeddings/target.ts create mode 100644 packages/gateway/src/data-plane/openai-images/call-upstream.ts create mode 100644 packages/gateway/src/data-plane/openai-images/emit.ts create mode 100644 packages/gateway/src/data-plane/openai-images/facts.ts create mode 100644 packages/gateway/src/data-plane/openai-images/target.ts create mode 100644 packages/gateway/src/data-plane/pipeline/failover.ts rename packages/gateway/src/data-plane/pipeline/{stages.ts => resolve-candidates.ts} (56%) create mode 100644 packages/gateway/src/data-plane/pipeline/stage-contracts.ts create mode 100644 packages/gateway/src/data-plane/rerank/call-upstream.ts create mode 100644 packages/gateway/src/data-plane/rerank/emit.ts create mode 100644 packages/gateway/src/data-plane/rerank/facts.ts create mode 100644 packages/gateway/src/data-plane/rerank/target.ts diff --git a/packages/gateway/__tests__/data-plane/openai-images-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-images-pipeline_test.ts index 08f1d8aff5..a08cdadcc7 100644 --- a/packages/gateway/__tests__/data-plane/openai-images-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-images-pipeline_test.ts @@ -5,7 +5,8 @@ import { describe, expect, it } from 'vitest'; -import { openaiImagesServePipeline, type OpenAIImagesServeEntry } from '../../src/data-plane/openai-images/pipeline.ts'; +import { type OpenAIImagesServeEntry } from '../../src/data-plane/openai-images/facts.ts'; +import { openaiImagesServePipeline } from '../../src/data-plane/openai-images/pipeline.ts'; import type { CanonicalOpenAIImagesRequest } from '@floway-dev/protocols/openai-images'; const generations: CanonicalOpenAIImagesRequest = { diff --git a/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts b/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts index ec18347f71..63ef01032f 100644 --- a/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts @@ -5,7 +5,7 @@ import { describe, expect, it } from 'vitest'; -import { failover } from '../../src/data-plane/pipeline/stages.ts'; +import { failover } from '../../src/data-plane/pipeline/failover.ts'; import { rerankServePipeline } from '../../src/data-plane/rerank/pipeline.ts'; import { move, run } from '@floway-dev/pipeline'; import type { CanonicalRerankRequest } from '@floway-dev/protocols/rerank'; diff --git a/packages/gateway/src/data-plane/alpha-search/call-upstream.ts b/packages/gateway/src/data-plane/alpha-search/call-upstream.ts new file mode 100644 index 0000000000..72238ed4ee --- /dev/null +++ b/packages/gateway/src/data-plane/alpha-search/call-upstream.ts @@ -0,0 +1,141 @@ +import type { PinnedSearchUpstream, Fields } from './facts.ts'; +import { parseAlphaSearchResponse } from './protocol.ts'; +import type { GatewayServices } from '../pipeline/services.ts'; +import { enumerateModelCandidates } from '../providers/resolution.ts'; +import type { GatewayCtx } from '../shared/gateway-ctx.ts'; +import { filterInboundHeadersForProvider } from '../shared/inbound-headers.ts'; +import { telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { providerModelOf, identityWrapUpstreamCall, type ModelCandidate } from '@floway-dev/provider'; + +/** + * A misconfigured pin is the operator's to see with its stack, the way the gateway surfaces + * any internal failure. It is not a failure value: a failure value exists so an earlier stage + * can fail over it, and there is no earlier stage here with anywhere to go. + */ +const resolvePinnedUpstream = async (pinned: PinnedSearchUpstream, gateway: GatewayCtx): Promise => { + if (gateway.upstreamIds !== null && !gateway.upstreamIds.includes(pinned.upstreamId)) { + throw new Error('Selected OpenAI search upstream is outside this API key scope'); + } + const { candidates } = await enumerateModelCandidates({ + upstreamIds: [pinned.upstreamId], + model: pinned.model, + kind: 'chat', + scheduler: gateway.backgroundScheduler, + runtimeLocation: gateway.runtimeLocation, + }); + const candidate = candidates.find(value => value.provider.upstreamId === pinned.upstreamId); + if (candidate === undefined) { + throw new Error(`Selected OpenAI search model ${pinned.model} is unavailable`); + } + if (candidate.provider.kind !== 'codex' && candidate.provider.kind !== 'custom') { + throw new Error('Selected upstream does not support OpenAI search passthrough'); + } + return candidate; +}; + +/** Codex projects one per-turn metadata snapshot onto several surfaces, and `SearchRequest` + * carries no field for it — so on this endpoint the header is the whole of that surface, and + * it is the only inbound header this call reads. + * https://github.com/openai/codex/blob/2e1607ee2fa8099a233df7437adee5f16a741905/codex-rs/codex-api/src/search.rs#L8-L29 */ +const turnMetadataHeaders = (ingress: readonly (readonly [string, string])[]): Headers => { + const headers = new Headers(); + const metadata = ingress.find(([name]) => name.toLowerCase() === 'x-codex-turn-metadata'); + if (metadata !== undefined) headers.set('x-codex-turn-metadata', metadata[1]); + return headers; +}; + +/** The body as JSON, or nothing when it was not JSON at all. Which of the two happened is + * what the caller reports; here it is only the question being asked. */ +const asJson = (raw: string): unknown => { + try { + return JSON.parse(raw) as unknown; + } catch { + return undefined; + } +}; + +/** + * The pinned ending. Dials the operator's search upstream and provides what came back plus + * what the call is billable for. A failure is a value here as everywhere, even though this + * family has nothing that fails over it: the edge is what turns one into a status and a body. + */ +export const callSearchUpstream = (pinned: PinnedSearchUpstream) => defineStage< + Fields<'request.search.alphaSearch' | 'ingress.http.headers'>, + Fields<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>, + GatewayServices +>({ + name: 'callSearchUpstream', + return: { provides: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers'] }, + execute: async (facts, use) => { + const candidate = await resolvePinnedUpstream(pinned, use.gateway); + // The caller's model is dropped: this endpoint is pinned to the operator's model, and the + // provider stamps that one on the way out. + // TODO: pin SearchRequest.id to one provider account when Codex upstreams support account + // pools. The current Codex provider has one active account. + const { model: _named, ...request } = facts['request.search.alphaSearch']; + const result = await candidate.provider.instance.callAlphaSearch( + providerModelOf(candidate), + request, + use.gateway.abortSignal, + { + fetcher: candidate.fetcher, + waitUntil: use.gateway.backgroundScheduler, + // The client's own headers reach the upstream from the record, not from a live + // request object: what a provider is allowed to forward is filtered per provider, + // and the dump shows what was there to filter. + headers: filterInboundHeadersForProvider(turnMetadataHeaders(facts['ingress.http.headers']), candidate.provider), + // No `PerformanceOperation` names search, so there is no performance row for a + // stamping wrapper's interval to land on. + wrapUpstreamCall: identityWrapUpstreamCall, + }, + ); + // The alpha-search protocol reports no usage at all, so the entity is present with no + // quantities — the upstream was called and reported nothing, which is a different + // situation from reporting zero. + const billable = [{ identity: telemetryModelIdentity(candidate, result.modelKey), quantities: {} }]; + + // Every protocol the gateway carries is one it fully understands: the body is read here + // and serialized again at the edge, an error body included. + const raw = await result.response.text(); + if (!result.response.ok) { + use.log.warn('upstream refused', { status: result.response.status }); + // The message is what came back as text and the body is the same thing parsed: a dump + // reader gets the upstream's own words either way, and only the parsed form is + // something the edge can serialize back out. + const body = asJson(raw); + return move({ + ...facts, + 'response.search.alphaSearch': { + status: result.response.status, + message: raw, + ...(body === undefined ? {} : { body }), + }, + 'response.usage.billable': billable, + 'response.http.headers': [...result.response.headers], + }); + } + + const verdict = parseAlphaSearchResponse(asJson(raw)); + if (!verdict.ok) { + // A protocol that requires JSON and receives something else synthesizes its own error, + // which is also why the raw text rides along: a dump reader is owed what came back. + return move({ + ...facts, + 'response.search.alphaSearch': { + status: 502, + message: `The search upstream answered ${result.response.status} but not in the search protocol: ${verdict.reason}.`, + body: raw, + }, + 'response.usage.billable': billable, + 'response.http.headers': [...result.response.headers], + }); + } + return move({ + ...facts, + 'response.search.alphaSearch': verdict.response, + 'response.usage.billable': billable, + 'response.http.headers': [...result.response.headers], + }); + }, +}); diff --git a/packages/gateway/src/data-plane/alpha-search/emit.ts b/packages/gateway/src/data-plane/alpha-search/emit.ts new file mode 100644 index 0000000000..addd33d040 --- /dev/null +++ b/packages/gateway/src/data-plane/alpha-search/emit.ts @@ -0,0 +1,59 @@ +import type { Fields } from './facts.ts'; +import { renderAlphaSearchResponse } from './protocol.ts'; +import { isJsonObject } from '../../shared/json-helpers.ts'; +import { isFailure } from '../pipeline/facts.ts'; +import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; + +/** + * The outermost edge. Renders the answer into Codex's protocol and says what status the + * client is owed — the one thing a failure value carries that a rendered body cannot. + * + * It declares needing `response.usage.billable` without reading it, which is this family's + * statement that every path accounts for usage: assembly then rejects an ending, or a + * short-circuit, that does not provide it. + */ +export const emitAlphaSearch = defineStage< + Fields, + Fields, + Fields<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>, + Fields<'response.search.rendered' | 'response.http.status' | 'response.http.headers'> +>({ + name: 'emitAlphaSearch', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { + needs: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers'], + consumes: ['response.search.alphaSearch', 'response.http.headers'], + provides: ['response.search.rendered', 'response.http.status', 'response.http.headers'], + }, + }, + execute: async (facts, next) => { + const back = await next(facts); + const { 'response.search.alphaSearch': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); + if (isFailure(answer)) { + // An upstream error body is JSON like any other: it was parsed below and is serialized + // again here. A body that was not an object is one this protocol cannot carry, so what + // goes out instead is the gateway's own envelope. + return { + ...rest, + 'response.http.headers': forClient, + 'response.search.rendered': move(isJsonObject(answer.body) + ? answer.body + : { error: { message: answer.message, type: 'api_error' } }), + 'response.http.status': answer.status, + }; + } + return { + ...rest, + 'response.http.headers': forClient, + 'response.search.rendered': move(renderAlphaSearchResponse(answer)), + 'response.http.status': 200, + }; + }, +}); diff --git a/packages/gateway/src/data-plane/alpha-search/execute-operations.ts b/packages/gateway/src/data-plane/alpha-search/execute-operations.ts new file mode 100644 index 0000000000..17439847c7 --- /dev/null +++ b/packages/gateway/src/data-plane/alpha-search/execute-operations.ts @@ -0,0 +1,47 @@ +import type { Fields } from './facts.ts'; +import type { SearchServices } from './services.ts'; +import { startBatchFetch, executeOperationToText, type WebSearchExecutionSession } from '../tools/web-search/operations.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; + +/** + * The local ending. Runs every operation against the configured backend and provides the + * answer plus what the run is billable for — which is nothing, because no upstream model was + * called. What the search backend itself charges is accounted per api key by the operations + * as they run, in units no model prices. + */ +export const executeSearchOperations = defineStage< + Fields<'request.search.operations' | 'request.search.filters'>, + Fields<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>, + SearchServices +>({ + name: 'executeSearchOperations', + return: { provides: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers'] }, + execute: async (facts, use) => { + const session: WebSearchExecutionSession = { + getProvider: use.searchProvider, + filters: facts['request.search.filters'], + apiKeyId: use.gateway.apiKeyId, + pageCache: new Map(), + // Codex renders `output` as plain text; the search-action sources list is a Responses + // protocol concern with no place here. + includeSearchActionSources: false, + ...(use.gateway.abortSignal === undefined ? {} : { signal: use.gateway.abortSignal }), + }; + + // One batched fetchPage covers every open and find URL; each operation then renders its + // own text block, in the parser's canonical order — search_query, open, find, preserving + // array order within each command kind. + const ops = [...facts['request.search.operations']]; + const batch = await startBatchFetch({ kind: 'ops', ops }, session); + const blocks = await Promise.all(ops.map(op => executeOperationToText(op, session, batch))); + use.log.debug('ran the search operations', { operations: ops.length }); + + return move({ + ...facts, + 'response.search.alphaSearch': { encryptedOutput: null, output: blocks.join('\n\n') }, + 'response.usage.billable': [], + // Nothing was called, so there are no upstream headers to carry. + 'response.http.headers': [], + }); + }, +}); diff --git a/packages/gateway/src/data-plane/alpha-search/facts.ts b/packages/gateway/src/data-plane/alpha-search/facts.ts new file mode 100644 index 0000000000..aa06f1a5dc --- /dev/null +++ b/packages/gateway/src/data-plane/alpha-search/facts.ts @@ -0,0 +1,32 @@ +import type { AlphaSearchRequest, AlphaSearchResponse } from './protocol.ts'; +import type { GatewayFacts, Failure } from '../pipeline/facts.ts'; +import type { WebSearchOperation, WebSearchFilters } from '../tools/web-search/operations.ts'; + +/** Search's own keys. They extend the shared space and never merge into it, so a stage + * written against the gateway alone cannot name one. */ +export interface SearchFacts extends GatewayFacts { + 'request.search.alphaSearch': AlphaSearchRequest; + /** What Codex's commands mean to this gateway. Local execution runs on these and never + * reads Codex's request again, which is why the stage that provides them consumes it. */ + 'request.search.operations': readonly WebSearchOperation[]; + 'request.search.filters': WebSearchFilters; + 'response.search.alphaSearch': AlphaSearchResponse | Failure; + /** What the client is actually sent, in Codex's protocol. The edge provides it, so a dump + * shows the bytes the client received rather than the gateway's own form. */ + 'response.search.rendered': Record; +} + +export type Fields = { [P in K]: SearchFacts[P] }; + +/** The operator's pinned search upstream. One upstream and one model, both from + * configuration, which is why nothing here narrows a candidate list. */ +export interface PinnedSearchUpstream { + readonly kind: 'upstream'; + readonly upstreamId: string; + readonly model: string; +} + +/** Which ending the chain gets. `upstream` is the operator's `passthroughOpenAiSearch` + * setting: the word there is about whose search results the client is given, not about + * carrying a protocol the gateway has not parsed. */ +export type SearchExecution = { readonly kind: 'local' } | PinnedSearchUpstream; diff --git a/packages/gateway/src/data-plane/alpha-search/http.ts b/packages/gateway/src/data-plane/alpha-search/http.ts index ad130eb635..7d1eebe842 100644 --- a/packages/gateway/src/data-plane/alpha-search/http.ts +++ b/packages/gateway/src/data-plane/alpha-search/http.ts @@ -12,8 +12,10 @@ import type { Context } from 'hono'; -import { searchServePipeline, type SearchExecution, type SearchServices } from './pipeline.ts'; +import { type SearchExecution } from './facts.ts'; +import { searchServePipeline } from './pipeline.ts'; import { alphaSearchRequestSchema, type AlphaSearchRequest } from './protocol.ts'; +import { type SearchServices } from './services.ts'; import type { AuthedContext } from '../../middleware/auth.ts'; import { openPrologue, readIngress, serveThrough, type Prologue } from '../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; diff --git a/packages/gateway/src/data-plane/alpha-search/parse-operations.ts b/packages/gateway/src/data-plane/alpha-search/parse-operations.ts new file mode 100644 index 0000000000..77da35444d --- /dev/null +++ b/packages/gateway/src/data-plane/alpha-search/parse-operations.ts @@ -0,0 +1,60 @@ +import type { Fields } from './facts.ts'; +import { webSearchFiltersFromSettings } from './protocol.ts'; +import { assertLocalWebSearchSupport, UnsupportedLocalWebSearchFeatureError, parseWebSearchOperations } from '../tools/web-search/operations.ts'; +import { move, defineStage } from '@floway-dev/pipeline'; + +/** Both in-band answers a local run can give: Codex reads `output`, so a command it asked for + * and this gateway cannot run is text the model sees rather than an HTTP failure. */ +const inBandOutput = (facts: Fields<'request.search.alphaSearch'>, output: string) => move({ + ...facts, + 'response.search.alphaSearch': { encryptedOutput: null, output }, + 'response.usage.billable': [], + // Nothing was called, so there are no upstream headers to carry. + 'response.http.headers': [], +}); + +/** + * Codex's commands become the gateway's own operations, and its settings become the filters + * they run under. Nothing below reads Codex's request afterwards, so it is consumed here and + * assembly is what holds that: a stage placed under this one cannot need it back. + */ +export const parseSearchOperations = defineStage< + Fields<'request.search.alphaSearch'>, + Fields<'request.search.operations' | 'request.search.filters'>, + Fields<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>, + Fields<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>, + Fields<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'> +>({ + name: 'parseSearchOperations', + through: { + request: { + needs: ['request.search.alphaSearch'], + consumes: ['request.search.alphaSearch'], + provides: ['request.search.operations', 'request.search.filters'], + }, + response: { needs: [], consumes: [], provides: [] }, + }, + return: { provides: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers'] }, + execute: async (facts, next) => { + const { 'request.search.alphaSearch': request, ...rest } = facts; + const commands = request.commands ?? {}; + + try { + assertLocalWebSearchSupport(commands); + } catch (error) { + if (!(error instanceof UnsupportedLocalWebSearchFeatureError)) throw error; + return inBandOutput(facts, error.message); + } + + const parsed = parseWebSearchOperations(commands); + if (parsed.kind !== 'ops' || parsed.ops.length === 0) { + return inBandOutput(facts, 'No web search commands were provided. Populate at least one of `search_query`, `open`, or `find`.'); + } + + return await next({ + ...rest, + 'request.search.operations': move(parsed.ops), + 'request.search.filters': move(webSearchFiltersFromSettings(request.settings)), + }); + }, +}); diff --git a/packages/gateway/src/data-plane/alpha-search/pipeline.ts b/packages/gateway/src/data-plane/alpha-search/pipeline.ts index 0f6c116d11..79254ed12e 100644 --- a/packages/gateway/src/data-plane/alpha-search/pipeline.ts +++ b/packages/gateway/src/data-plane/alpha-search/pipeline.ts @@ -1,375 +1,15 @@ -// Search as a pipeline. The family with no upstream model and no candidate list: by default -// it runs the request's commands here, through the configured search backend, and when the -// operator has pinned a Codex or Custom upstream it asks that upstream's own search endpoint -// instead. -// -// That is an ending, not a missing family. What this shares with the other families is the -// edge; what varies is where the chain stops: -// -// emitAlphaSearch the edge: serializes the answer into Codex's protocol -// parseSearchOperations local only: Codex's commands become the gateway's own -// executeSearchOperations the local ending: runs them, and provides the answer -// callSearchUpstream the pinned ending: dials, and provides what came back -// -// `resolveCandidates` and `failover` have nothing to range over here. Local execution reaches -// no upstream at all, and the pinned mode names exactly one upstream and one model in -// operator configuration, so a run is its own only attempt. - -import { - parseAlphaSearchResponse, - renderAlphaSearchResponse, - webSearchFiltersFromSettings, - type AlphaSearchRequest, - type AlphaSearchResponse, -} from './protocol.ts'; -import { isJsonObject } from '../../shared/json-helpers.ts'; -import type { Failure, GatewayFacts } from '../pipeline/facts.ts'; +import { callSearchUpstream } from './call-upstream.ts'; +import { emitAlphaSearch } from './emit.ts'; +import { executeSearchOperations } from './execute-operations.ts'; +import type { SearchExecution, Fields } from './facts.ts'; +import { parseSearchOperations } from './parse-operations.ts'; import { isFailure } from '../pipeline/facts.ts'; -import type { GatewayServices } from '../pipeline/services.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; -import { enumerateModelCandidates } from '../providers/resolution.ts'; -import type { GatewayCtx } from '../shared/gateway-ctx.ts'; -import { filterInboundHeadersForProvider } from '../shared/inbound-headers.ts'; -import { telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; -import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; -import { - assertLocalWebSearchSupport, - executeOperationToText, - parseWebSearchOperations, - startBatchFetch, - UnsupportedLocalWebSearchFeatureError, - type WebSearchExecutionSession, - type WebSearchFilters, - type WebSearchOperation, -} from '../tools/web-search/operations.ts'; -import type { ConfiguredWebSearchProvider } from '../tools/web-search/types.ts'; -import type { Pipeline } from '@floway-dev/pipeline'; -import { compose, defineStage, move } from '@floway-dev/pipeline'; -import { identityWrapUpstreamCall, providerModelOf, type ModelCandidate } from '@floway-dev/provider'; - -/** Search's own keys. They extend the shared space and never merge into it, so a stage - * written against the gateway alone cannot name one. */ -export interface SearchFacts extends GatewayFacts { - 'request.search.alphaSearch': AlphaSearchRequest; - /** What Codex's commands mean to this gateway. Local execution runs on these and never - * reads Codex's request again, which is why the stage that provides them consumes it. */ - 'request.search.operations': readonly WebSearchOperation[]; - 'request.search.filters': WebSearchFilters; - 'response.search.alphaSearch': AlphaSearchResponse | Failure; - /** What the client is actually sent, in Codex's protocol. The edge provides it, so a dump - * shows the bytes the client received rather than the gateway's own form. */ - 'response.search.rendered': Record; -} - -type S = { [P in K]: SearchFacts[P] }; - -/** The configured search backend is reached through a resolver rather than a fact: it holds - * live handles, and the operator's provider credential is what builds one — so nothing about - * it is dumpable and none of it belongs in the record. */ -export interface SearchServices extends GatewayServices { - readonly searchProvider: () => Promise; -} - -/** - * The outermost edge. Renders the answer into Codex's protocol and says what status the - * client is owed — the one thing a failure value carries that a rendered body cannot. - * - * It declares needing `response.usage.billable` without reading it, which is this family's - * statement that every path accounts for usage: assembly then rejects an ending, or a - * short-circuit, that does not provide it. - */ -const emitAlphaSearch = defineStage< - S, - S, - S<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>, - S<'response.search.rendered' | 'response.http.status' | 'response.http.headers'> ->({ - name: 'emitAlphaSearch', - through: { - request: { needs: [], consumes: [], provides: [] }, - response: { - needs: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers'], - consumes: ['response.search.alphaSearch', 'response.http.headers'], - provides: ['response.search.rendered', 'response.http.status', 'response.http.headers'], - }, - }, - execute: async (facts, next) => { - const back = await next(facts); - const { 'response.search.alphaSearch': answer, 'response.http.headers': headers, ...rest } = back; - // Vendor traces and quota state stay visible; what an intermediary must strip, and what - // would misdescribe a body this gateway serialized itself, does not. A filter that removed - // nothing hands the same array on, so the record shows no change where none happened. - const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); - const forClient = forwardable.length === headers.length ? headers : move(forwardable); - if (isFailure(answer)) { - // An upstream error body is JSON like any other: it was parsed below and is serialized - // again here. A body that was not an object is one this protocol cannot carry, so what - // goes out instead is the gateway's own envelope. - return { - ...rest, - 'response.http.headers': forClient, - 'response.search.rendered': move(isJsonObject(answer.body) - ? answer.body - : { error: { message: answer.message, type: 'api_error' } }), - 'response.http.status': answer.status, - }; - } - return { - ...rest, - 'response.http.headers': forClient, - 'response.search.rendered': move(renderAlphaSearchResponse(answer)), - 'response.http.status': 200, - }; - }, -}); - -/** Both in-band answers a local run can give: Codex reads `output`, so a command it asked for - * and this gateway cannot run is text the model sees rather than an HTTP failure. */ -const inBandOutput = (facts: S<'request.search.alphaSearch'>, output: string) => move({ - ...facts, - 'response.search.alphaSearch': { encryptedOutput: null, output }, - 'response.usage.billable': [], - // Nothing was called, so there are no upstream headers to carry. - 'response.http.headers': [], -}); - -/** - * Codex's commands become the gateway's own operations, and its settings become the filters - * they run under. Nothing below reads Codex's request afterwards, so it is consumed here and - * assembly is what holds that: a stage placed under this one cannot need it back. - */ -const parseSearchOperations = defineStage< - S<'request.search.alphaSearch'>, - S<'request.search.operations' | 'request.search.filters'>, - S<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>, - S<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>, - S<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'> ->({ - name: 'parseSearchOperations', - through: { - request: { - needs: ['request.search.alphaSearch'], - consumes: ['request.search.alphaSearch'], - provides: ['request.search.operations', 'request.search.filters'], - }, - response: { needs: [], consumes: [], provides: [] }, - }, - return: { provides: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers'] }, - execute: async (facts, next) => { - const { 'request.search.alphaSearch': request, ...rest } = facts; - const commands = request.commands ?? {}; - - try { - assertLocalWebSearchSupport(commands); - } catch (error) { - if (!(error instanceof UnsupportedLocalWebSearchFeatureError)) throw error; - return inBandOutput(facts, error.message); - } - - const parsed = parseWebSearchOperations(commands); - if (parsed.kind !== 'ops' || parsed.ops.length === 0) { - return inBandOutput(facts, 'No web search commands were provided. Populate at least one of `search_query`, `open`, or `find`.'); - } - - return await next({ - ...rest, - 'request.search.operations': move(parsed.ops), - 'request.search.filters': move(webSearchFiltersFromSettings(request.settings)), - }); - }, -}); - -/** - * The local ending. Runs every operation against the configured backend and provides the - * answer plus what the run is billable for — which is nothing, because no upstream model was - * called. What the search backend itself charges is accounted per api key by the operations - * as they run, in units no model prices. - */ -const executeSearchOperations = defineStage< - S<'request.search.operations' | 'request.search.filters'>, - S<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>, - SearchServices ->({ - name: 'executeSearchOperations', - return: { provides: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers'] }, - execute: async (facts, use) => { - const session: WebSearchExecutionSession = { - getProvider: use.searchProvider, - filters: facts['request.search.filters'], - apiKeyId: use.gateway.apiKeyId, - pageCache: new Map(), - // Codex renders `output` as plain text; the search-action sources list is a Responses - // protocol concern with no place here. - includeSearchActionSources: false, - ...(use.gateway.abortSignal === undefined ? {} : { signal: use.gateway.abortSignal }), - }; - - // One batched fetchPage covers every open and find URL; each operation then renders its - // own text block, in the parser's canonical order — search_query, open, find, preserving - // array order within each command kind. - const ops = [...facts['request.search.operations']]; - const batch = await startBatchFetch({ kind: 'ops', ops }, session); - const blocks = await Promise.all(ops.map(op => executeOperationToText(op, session, batch))); - use.log.debug('ran the search operations', { operations: ops.length }); - - return move({ - ...facts, - 'response.search.alphaSearch': { encryptedOutput: null, output: blocks.join('\n\n') }, - 'response.usage.billable': [], - // Nothing was called, so there are no upstream headers to carry. - 'response.http.headers': [], - }); - }, -}); - -/** The operator's pinned search upstream. One upstream and one model, both from - * configuration, which is why nothing here narrows a candidate list. */ -export interface PinnedSearchUpstream { - readonly kind: 'upstream'; - readonly upstreamId: string; - readonly model: string; -} - -/** - * A misconfigured pin is the operator's to see with its stack, the way the gateway surfaces - * any internal failure. It is not a failure value: a failure value exists so an earlier stage - * can fail over it, and there is no earlier stage here with anywhere to go. - */ -const resolvePinnedUpstream = async (pinned: PinnedSearchUpstream, gateway: GatewayCtx): Promise => { - if (gateway.upstreamIds !== null && !gateway.upstreamIds.includes(pinned.upstreamId)) { - throw new Error('Selected OpenAI search upstream is outside this API key scope'); - } - const { candidates } = await enumerateModelCandidates({ - upstreamIds: [pinned.upstreamId], - model: pinned.model, - kind: 'chat', - scheduler: gateway.backgroundScheduler, - runtimeLocation: gateway.runtimeLocation, - }); - const candidate = candidates.find(value => value.provider.upstreamId === pinned.upstreamId); - if (candidate === undefined) { - throw new Error(`Selected OpenAI search model ${pinned.model} is unavailable`); - } - if (candidate.provider.kind !== 'codex' && candidate.provider.kind !== 'custom') { - throw new Error('Selected upstream does not support OpenAI search passthrough'); - } - return candidate; -}; - -/** Codex projects one per-turn metadata snapshot onto several surfaces, and `SearchRequest` - * carries no field for it — so on this endpoint the header is the whole of that surface, and - * it is the only inbound header this call reads. - * https://github.com/openai/codex/blob/2e1607ee2fa8099a233df7437adee5f16a741905/codex-rs/codex-api/src/search.rs#L8-L29 */ -const turnMetadataHeaders = (ingress: readonly (readonly [string, string])[]): Headers => { - const headers = new Headers(); - const metadata = ingress.find(([name]) => name.toLowerCase() === 'x-codex-turn-metadata'); - if (metadata !== undefined) headers.set('x-codex-turn-metadata', metadata[1]); - return headers; -}; - -/** The body as JSON, or nothing when it was not JSON at all. Which of the two happened is - * what the caller reports; here it is only the question being asked. */ -const asJson = (raw: string): unknown => { - try { - return JSON.parse(raw) as unknown; - } catch { - return undefined; - } -}; - -/** - * The pinned ending. Dials the operator's search upstream and provides what came back plus - * what the call is billable for. A failure is a value here as everywhere, even though this - * family has nothing that fails over it: the edge is what turns one into a status and a body. - */ -const callSearchUpstream = (pinned: PinnedSearchUpstream) => defineStage< - S<'request.search.alphaSearch' | 'ingress.http.headers'>, - S<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>, - GatewayServices ->({ - name: 'callSearchUpstream', - return: { provides: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers'] }, - execute: async (facts, use) => { - const candidate = await resolvePinnedUpstream(pinned, use.gateway); - // The caller's model is dropped: this endpoint is pinned to the operator's model, and the - // provider stamps that one on the way out. - // TODO: pin SearchRequest.id to one provider account when Codex upstreams support account - // pools. The current Codex provider has one active account. - const { model: _named, ...request } = facts['request.search.alphaSearch']; - const result = await candidate.provider.instance.callAlphaSearch( - providerModelOf(candidate), - request, - use.gateway.abortSignal, - { - fetcher: candidate.fetcher, - waitUntil: use.gateway.backgroundScheduler, - // The client's own headers reach the upstream from the record, not from a live - // request object: what a provider is allowed to forward is filtered per provider, - // and the dump shows what was there to filter. - headers: filterInboundHeadersForProvider(turnMetadataHeaders(facts['ingress.http.headers']), candidate.provider), - // No `PerformanceOperation` names search, so there is no performance row for a - // stamping wrapper's interval to land on. - wrapUpstreamCall: identityWrapUpstreamCall, - }, - ); - // The alpha-search protocol reports no usage at all, so the entity is present with no - // quantities — the upstream was called and reported nothing, which is a different - // situation from reporting zero. - const billable = [{ identity: telemetryModelIdentity(candidate, result.modelKey), quantities: {} }]; - - // Every protocol the gateway carries is one it fully understands: the body is read here - // and serialized again at the edge, an error body included. - const raw = await result.response.text(); - if (!result.response.ok) { - use.log.warn('upstream refused', { status: result.response.status }); - // The message is what came back as text and the body is the same thing parsed: a dump - // reader gets the upstream's own words either way, and only the parsed form is - // something the edge can serialize back out. - const body = asJson(raw); - return move({ - ...facts, - 'response.search.alphaSearch': { - status: result.response.status, - message: raw, - ...(body === undefined ? {} : { body }), - }, - 'response.usage.billable': billable, - 'response.http.headers': [...result.response.headers], - }); - } - - const verdict = parseAlphaSearchResponse(asJson(raw)); - if (!verdict.ok) { - // A protocol that requires JSON and receives something else synthesizes its own error, - // which is also why the raw text rides along: a dump reader is owed what came back. - return move({ - ...facts, - 'response.search.alphaSearch': { - status: 502, - message: `The search upstream answered ${result.response.status} but not in the search protocol: ${verdict.reason}.`, - body: raw, - }, - 'response.usage.billable': billable, - 'response.http.headers': [...result.response.headers], - }); - } - return move({ - ...facts, - 'response.search.alphaSearch': verdict.response, - 'response.usage.billable': billable, - 'response.http.headers': [...result.response.headers], - }); - }, -}); - -/** Which ending the chain gets. `upstream` is the operator's `passthroughOpenAiSearch` - * setting: the word there is about whose search results the client is given, not about - * carrying a protocol the gateway has not parsed. */ -export type SearchExecution = { readonly kind: 'local' } | PinnedSearchUpstream; +import { compose, type Pipeline } from '@floway-dev/pipeline'; export const searchServePipeline = (execution: SearchExecution): Pipeline< - S<'request.search.alphaSearch'>, - S<'response.search.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> + Fields<'request.search.alphaSearch'>, + Fields<'response.search.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> > => compose('searchServe', [ emitAlphaSearch, // Unconditional, as everywhere: a run that searched locally reached no upstream and bills diff --git a/packages/gateway/src/data-plane/alpha-search/services.ts b/packages/gateway/src/data-plane/alpha-search/services.ts new file mode 100644 index 0000000000..21cf82f1f8 --- /dev/null +++ b/packages/gateway/src/data-plane/alpha-search/services.ts @@ -0,0 +1,9 @@ +import type { GatewayServices } from '../pipeline/services.ts'; +import type { ConfiguredWebSearchProvider } from '../tools/web-search/types.ts'; + +/** The configured search backend is reached through a resolver rather than a fact: it holds + * live handles, and the operator's provider credential is what builds one — so nothing about + * it is dumpable and none of it belongs in the record. */ +export interface SearchServices extends GatewayServices { + readonly searchProvider: () => Promise; +} diff --git a/packages/gateway/src/data-plane/openai-audio/call-upstream.ts b/packages/gateway/src/data-plane/openai-audio/call-upstream.ts new file mode 100644 index 0000000000..0f10a1a72a --- /dev/null +++ b/packages/gateway/src/data-plane/openai-audio/call-upstream.ts @@ -0,0 +1,265 @@ +import type { Fields, OpenAIAudioTranscriptionEvents, OpenAIAudioTranscriptionStreamOutcome } from './facts.ts'; +import { recordStream } from '../../dump/turn-dump.ts'; +import type { UsageQuantities } from '../../repo/types.ts'; +import type { BillableEntity, Failure } from '../pipeline/facts.ts'; +import type { GatewayServices } from '../pipeline/services.ts'; +import { dialFailure } from '../pipeline/upstream-body.ts'; +import { upstreamPerformanceContext, telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; +import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; +import { defineStage, move, own, defer, type Owned, type Logger, type Deferred } from '@floway-dev/pipeline'; +import { isEventStreamMediaType, eventFrame, parseSSEStream, parseDecimalString } from '@floway-dev/protocols/common'; +import { parseOpenAIAudioTranscription, parseOpenAIAudioTranscriptionUsage, parseOpenAIAudioTranscriptionStreamEvent, isOpenAIAudioTranscriptionDoneEvent, parseOpenAIAudioTranscriptionStreamUsage, type OpenAIAudioTranscriptionResponseFormat, type CanonicalOpenAIAudioTranscription, type OpenAIAudioTranscriptionUsage } from '@floway-dev/protocols/openai-audio'; +import { providerModelOf, type TelemetryModelIdentity } from '@floway-dev/provider'; + +const viewOf = (events: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => events }); + +/** + * The ending. It dials, reads what came back in the rendering the request asked for, and + * provides the answer, the raw HTTP response beneath it, and what the call is billable for. + * A failure is a value: a 429 here is what an earlier stage fails over, and so is a dial that + * never reached anyone. A 200 is never one of them — this endpoint carries the document the + * upstream sent rather than serializing one from what it read, so a body no reading could + * open is still that upstream's answer and there is nothing to try the next candidate for. + */ +export const callOpenAIAudioTranscriptionUpstream = defineStage< + Fields<'ingress.openaiAudioTranscription.responseFormat' | 'request.openaiAudioTranscription.form' | 'route.attempt' | 'ingress.http.headers'>, + Fields<'response.openaiAudioTranscription.canonical' | 'response.openaiAudioTranscription.mediaType' | 'response.openaiAudioTranscription.streamedOutcome'> + & { 'response.usage.billable': readonly BillableEntity[]; 'response.http.status': number; + 'response.http.headers': readonly (readonly [string, string])[]; + 'response.http.body': ReadableStream & Owned; }, + GatewayServices +>({ + name: 'callOpenAIAudioTranscriptionUpstream', + return: { + provides: [ + 'response.openaiAudioTranscription.canonical', + 'response.openaiAudioTranscription.mediaType', + 'response.openaiAudioTranscription.streamedOutcome', + 'response.usage.billable', + 'response.http.status', + 'response.http.headers', + 'response.http.body', + ], + }, + execute: async (facts, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + // Attribution is set before the dial, so an attempt that never completes still names the + // candidate it was made against rather than the one tried before it. + use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'audio_transcription'); + + let result; + try { + result = await candidate.provider.instance.callOpenAIAudioTranscriptions( + providerModelOf(candidate), + { entries: facts['request.openaiAudioTranscription.form'] }, + use.gateway.abortSignal, + // The client's own headers reach the upstream from the record, not from a live request + // object: what a provider is allowed to forward is filtered per provider, and the dump + // shows what was there to filter. + buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), + ); + } catch (error) { + use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); + // A dial that never completed reached no upstream, so nothing was billed and there are + // no headers to carry. What it leaves behind is the performance row settlement writes. + return move({ + ...facts, + 'response.openaiAudioTranscription.canonical': dialFailure(error), + 'response.openaiAudioTranscription.mediaType': null, + 'response.openaiAudioTranscription.streamedOutcome': null, + 'response.usage.billable': [], + 'response.http.status': 502, + 'response.http.headers': [], + 'response.http.body': spentBody(null), + }); + } + const identity = telemetryModelIdentity(candidate, result.modelKey); + const format = facts['ingress.openaiAudioTranscription.responseFormat']; + const status = result.response.status; + const mediaType = result.response.headers.get('content-type'); + const headers = move([...result.response.headers] as readonly (readonly [string, string])[]); + // An upstream that was called and reported nothing is a different situation from one + // that reported zero, so the entity is present with no quantities. + const called: readonly BillableEntity[] = [{ identity, quantities: {} }]; + + if (!result.response.ok) { + use.log.warn('upstream refused', { status }); + // An upstream error body is JSON like any other body, and reading it here is also what + // leaves a losing attempt with nothing open behind it. + return move({ + ...facts, + 'response.openaiAudioTranscription.canonical': await refusal(status, result.response), + 'response.openaiAudioTranscription.mediaType': mediaType, + 'response.openaiAudioTranscription.streamedOutcome': null, + 'response.usage.billable': called, + 'response.http.status': status, + 'response.http.headers': headers, + 'response.http.body': spentBody(result.response.body), + }); + } + + // A streamed answer is the one the client asked for with `stream`, and it is the media + // type that says one arrived: an upstream that ignores `stream` — `whisper-1` does — + // answers in the rendering `response_format` named instead. + // https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L36325-L36336 + if (isEventStreamMediaType(mediaType)) { + if (result.response.body === null) { + return move({ + ...facts, + 'response.openaiAudioTranscription.canonical': { status: 502, message: 'Upstream returned a streaming response with no body.' }, + 'response.openaiAudioTranscription.mediaType': mediaType, + 'response.openaiAudioTranscription.streamedOutcome': null, + 'response.usage.billable': called, + 'response.http.status': status, + 'response.http.headers': headers, + 'response.http.body': spentBody(null), + }); + } + // What the upstream metered, and whether the transcript finished, are both observed + // here — closest to the upstream and on the protocol it spoke — by folding the events + // as they pass, so the reading costs one pass and the client's own stream drives it. + const metered = meterEvents(result.response.body, identity, use.gateway.abortSignal, use.log); + return move({ + ...facts, + // This protocol's stream is bare events rather than protocol frames, so the record is + // told how one becomes a frame instead of being left to assume. + 'response.openaiAudioTranscription.canonical': recordStream(metered.events, use.gateway.dump, eventFrame), + 'response.openaiAudioTranscription.mediaType': mediaType, + 'response.openaiAudioTranscription.streamedOutcome': metered.outcome, + 'response.usage.billable': called, + 'response.http.status': status, + 'response.http.headers': headers, + // Releasing this body is reading those events to the end: they are one reader over + // one connection, and a second reader is not something a `ReadableStream` allows. + 'response.http.body': own(result.response.body, async (): Promise => { for await (const _event of metered.events) { /* to end of stream */ } }), + }); + } + + // A 2xx body is the answer whether or not this endpoint could read it, because what the + // client is sent is the document that arrived and not something serialized from a parse. + // So there is nothing here to fail over from: the reading feeds the record and the usage + // row, and the bytes travel either way. + const read = readTranscription(format, new Uint8Array(await result.response.arrayBuffer()), use.log); + return move({ + ...facts, + 'response.openaiAudioTranscription.canonical': read.canonical, + 'response.openaiAudioTranscription.mediaType': mediaType, + 'response.openaiAudioTranscription.streamedOutcome': null, + 'response.usage.billable': [{ identity, quantities: billed(read.usage) }], + 'response.http.status': status, + 'response.http.headers': headers, + 'response.http.body': spentBody(result.response.body), + }); + }, +}); + +/** A body this stage has already read to the end, or one the upstream never sent. The record + * holds a body as a stream and `failover` releases the losing attempts', so every path hands + * one up; what says an answer was unusable is the failure at the canonical key, not this. */ +const spentBody = (body: ReadableStream | null): ReadableStream & Owned => + own(body ?? new ReadableStream({ start: controller => controller.close() }), (): Promise => Promise.resolve()); + +const refusal = async (status: number, response: Response): Promise => { + const text = await response.text(); + try { + return { status, message: text, body: JSON.parse(text) as unknown }; + } catch { + // A refusal that is not JSON is still a refusal, and its text is what the client is + // told; there is simply no parsed body for a dump reader to open. + return { status, message: text }; + } +}; + +/** + * What a 2xx body was worth to this endpoint, in two readings that do not depend on each + * other. + * + * Neither can cost the client the answer: the document is carried, so a body no reading + * could open is still what goes back, and what a failed reading costs is the transcript in + * the record and whatever usage the body would have stated. Nor can either cost the other — + * an upstream whose usage block this gateway cannot model still had its transcript read, and + * one whose document it could not open is still billed for nothing rather than mis-billed. + */ +const readTranscription = ( + format: OpenAIAudioTranscriptionResponseFormat, + document: Uint8Array, + log: Logger, +): { readonly canonical: CanonicalOpenAIAudioTranscription; readonly usage: OpenAIAudioTranscriptionUsage | undefined } => { + let canonical: CanonicalOpenAIAudioTranscription; + try { + canonical = parseOpenAIAudioTranscription(format, document); + } catch (error) { + log.warn('failed to parse 2xx upstream body for /audio/transcriptions; forwarding it as it arrived', { error: String(error) }); + return { canonical: { document }, usage: undefined }; + } + // Only an object rendering states usage: `text`, `srt` and `vtt` have nowhere to put it, + // and asking them for one is what would warn about every subtitle a client requests. + return { canonical, usage: canonical.raw === undefined ? undefined : readUsage(() => parseOpenAIAudioTranscriptionUsage(canonical.raw), log) }; +}; + +/** A transcription states its usage in two places — the body of an object rendering and the + * terminal event of a stream — and either can state it in a shape this gateway cannot read. + * That upstream metered something, and the request is recorded saying exactly that: the + * entity, and no quantities. */ +const readUsage = (read: () => OpenAIAudioTranscriptionUsage | undefined, log: Logger): OpenAIAudioTranscriptionUsage | undefined => { + try { + return read(); + } catch (error) { + log.warn('invalid usage in 2xx upstream response; recording the request only', { error: String(error) }); + return undefined; + } +}; + +interface MeteredEvents { + readonly events: OpenAIAudioTranscriptionEvents; + readonly outcome: Deferred; +} + +const meterEvents = ( + body: ReadableStream, + identity: TelemetryModelIdentity, + signal: AbortSignal | undefined, + log: Logger, +): MeteredEvents => { + let settle!: (outcome: OpenAIAudioTranscriptionStreamOutcome) => void; + // Declared as this run's own unfinished work, so the runner waits for it at teardown where + // it can see it rather than the reading being started and forgotten. + const outcome = defer(new Promise(resolve => { settle = resolve; })); + const events = viewOf((async function* () { + let usage: OpenAIAudioTranscriptionUsage | undefined; + let completed = false; + try { + for await (const frame of parseSSEStream(body, { signal })) { + const event = parseOpenAIAudioTranscriptionStreamEvent(JSON.parse(frame.data) as unknown); + if (isOpenAIAudioTranscriptionDoneEvent(event)) { + usage = readUsage(() => parseOpenAIAudioTranscriptionStreamUsage(event), log); + completed = true; + yield event; + // The transcript is complete, so there is nothing further to read. An upstream that + // holds the connection open past this point would otherwise keep the client's own + // stream open with it; returning here closes the read, which cancels the upstream. + return; + } + yield event; + } + } finally { + // Reached however the events ended — the terminal one, a client that stopped reading, + // or a broken upstream — because what the upstream already metered is billable + // whatever happened to the downstream half. + settle({ billable: [{ identity, quantities: billed(usage) }], failed: !completed }); + } + })()); + return { events, outcome }; +}; + +const billed = (usage: OpenAIAudioTranscriptionUsage | undefined): UsageQuantities => { + if (usage === undefined) return {}; + if (usage.kind === 'duration') return { input_audio_seconds: parseDecimalString(String(usage.seconds)) }; + // Audio input is priced apart from text input, so what stays on the general input metric is + // what the upstream did not attribute to audio. + return { + input_tokens: parseDecimalString(String(usage.inputTokens - (usage.inputAudioTokens ?? 0))), + ...(usage.inputAudioTokens === undefined ? {} : { input_audio_tokens: parseDecimalString(String(usage.inputAudioTokens)) }), + output_tokens: parseDecimalString(String(usage.outputTokens)), + }; +}; diff --git a/packages/gateway/src/data-plane/openai-audio/emit.ts b/packages/gateway/src/data-plane/openai-audio/emit.ts new file mode 100644 index 0000000000..a51093eea6 --- /dev/null +++ b/packages/gateway/src/data-plane/openai-audio/emit.ts @@ -0,0 +1,85 @@ +import type { OpenAIAudioTranscriptionEvents, Fields } from './facts.ts'; +import { streamReferenceOf } from '../../dump/turn-dump.ts'; +import { isFailure } from '../pipeline/facts.ts'; +import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { renderErrorEnvelope, sseFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { renderOpenAIAudioTranscription, type CanonicalOpenAIAudioTranscription } from '@floway-dev/protocols/openai-audio'; + +const isEvents = (answer: CanonicalOpenAIAudioTranscription | OpenAIAudioTranscriptionEvents): answer is OpenAIAudioTranscriptionEvents => + Symbol.asyncIterator in answer; + +/** + * The outermost edge. Writes the answer back in the rendering the client asked for, and names + * a media type for the one body the gateway wrote out of nothing the upstream sent — its + * error envelope. Every other answer goes out under the media type it arrived with, because + * for a document that is carried rather than rewritten the upstream's label is the only true + * description there is. + * + * SSE framing is produced here and nowhere else — below this stage the answer is parsed + * events, so the same assembly would serve another transport by rendering differently at + * this one point. The upstream's own `event:` label is not carried: it is transport, and + * OpenAI's clients read this endpoint's frames by their payload's `type` rather than by the + * label. https://github.com/openai/openai-python/blob/10ee3f0da2ac6f93345c1204bd7bb1a2faa79ff2/src/openai/_streaming.py#L61-L107 + */ +export const emitOpenAIAudioTranscription = defineStage< + Fields<'ingress.openaiAudioTranscription.responseFormat'>, + Fields<'ingress.openaiAudioTranscription.responseFormat'>, + Fields<'ingress.openaiAudioTranscription.responseFormat' | 'response.openaiAudioTranscription.canonical' | 'response.openaiAudioTranscription.mediaType'> + & { 'response.http.headers': readonly (readonly [string, string])[] }, + Fields<'response.openaiAudioTranscription.rendered' | 'response.openaiAudioTranscription.mediaType'> + & { 'response.http.status': number; 'response.http.headers': readonly (readonly [string, string])[] } +>({ + name: 'emitOpenAIAudioTranscription', + through: { + request: { + needs: ['ingress.openaiAudioTranscription.responseFormat'], + consumes: [], + provides: [], + }, + response: { + needs: ['response.openaiAudioTranscription.canonical', 'response.openaiAudioTranscription.mediaType', 'response.http.headers'], + consumes: ['response.openaiAudioTranscription.canonical', 'response.http.headers'], + provides: ['response.openaiAudioTranscription.rendered', 'response.openaiAudioTranscription.mediaType', 'response.http.status', 'response.http.headers'], + }, + }, + execute: async (facts, next) => { + const back = await next(facts); + const { 'response.openaiAudioTranscription.canonical': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); + + if (isFailure(answer)) { + return { + ...rest, + 'response.http.headers': forClient, + 'response.http.status': answer.status, + 'response.openaiAudioTranscription.mediaType': 'application/json', + 'response.openaiAudioTranscription.rendered': move(renderErrorEnvelope(answer.message, answer.body)), + }; + } + // Everything that reaches here answered. The same key carried the upstream's own status + // further down; re-providing it is what makes the top of the record the response the + // client gets rather than the one the upstream gave. + return { + ...rest, + 'response.http.headers': forClient, + 'response.http.status': 200, + 'response.openaiAudioTranscription.rendered': move(isEvents(answer) + ? renderSSE(answer) + : renderOpenAIAudioTranscription(back['ingress.openaiAudioTranscription.responseFormat'], answer)), + }; + }, +}); + +const renderSSE = (events: OpenAIAudioTranscriptionEvents): AsyncIterable => ({ + // The frames the client reads are a reframing of the events the record holds, so this key + // points at that same stream rather than at nothing. + ...streamReferenceOf(events), + [Symbol.asyncIterator]: () => (async function* () { + for await (const event of events) yield sseFrame(JSON.stringify(event)); + })(), +}); diff --git a/packages/gateway/src/data-plane/openai-audio/facts.ts b/packages/gateway/src/data-plane/openai-audio/facts.ts new file mode 100644 index 0000000000..61bcf473f5 --- /dev/null +++ b/packages/gateway/src/data-plane/openai-audio/facts.ts @@ -0,0 +1,62 @@ +import type { BillableEntity, GatewayFacts, Failure } from '../pipeline/facts.ts'; +import type { Deferred } from '@floway-dev/pipeline'; +import type { SseFrame } from '@floway-dev/protocols/common'; +import type { OpenAIAudioTranscriptionStreamEvent, OpenAIAudioTranscriptionResponseFormat, CanonicalOpenAIAudioTranscription } from '@floway-dev/protocols/openai-audio'; +import type { OpenAIAudioTranscriptionFormEntry } from '@floway-dev/provider'; + +/** The answer while it is still the upstream's, one event at a time. It is a view and not a + * resource: what owns the connection is `response.http.body`, which is where release and + * failover's ownership both read. + * + * A view is a wrapper around the generator rather than the generator itself, which is what + * says where the resource is: the upstream's body at `response.http.body`, claimed with + * `own()`, and nothing else here. */ +export type OpenAIAudioTranscriptionEvents = AsyncIterable; + +/** What settling this run will be told once the events run out: what the upstream metered, + * and whether the transcript ever finished. */ +export interface OpenAIAudioTranscriptionStreamOutcome { + readonly billable: readonly BillableEntity[]; + /** An upstream that stopped before `transcript.text.done` answered 200 and then did not + * finish what it started, which is a failed request however much of the transcript + * reached the client. */ + readonly failed: boolean; +} + +/** OpenAI Audio Transcriptions' own keys, extending the shared space by intersection. */ +export interface OpenAIAudioTranscriptionFacts extends GatewayFacts { + /** Which of the six renderings the client asked for. It belongs to the ingress and stays + * put: the same value travels to the upstream inside the form, so the rendering the + * answer arrives in is the rendering the answer is written back in — and a text document + * does not say which of the three it is, so nothing else could decide. */ + 'ingress.openaiAudioTranscription.responseFormat': OpenAIAudioTranscriptionResponseFormat; + /** The multipart form as ordered semantic entries. The body is parsed before routing + * because field order is unconstrained, and every candidate builds a fresh body from + * these, so a retry never reuses a consumed one. The bytes the client sent are recorded + * at `ingress.http.body`, which is where a dump reads the upload itself. */ + 'request.openaiAudioTranscription.form': readonly OpenAIAudioTranscriptionFormEntry[]; + /** The one transcription, whichever rendering carried it — or the events it is arriving + * as, or the failure that came instead. A stream, a value and a failure sit at one key: + * telling them apart is reading a value, and each stage does that where it needs to. */ + 'response.openaiAudioTranscription.canonical': CanonicalOpenAIAudioTranscription | OpenAIAudioTranscriptionEvents | Failure; + /** What the answer goes out under. A media type is upstream-owned — OpenAI answers + * `text`, `srt` and `vtt` under one media type and other upstreams label them apart, and + * the document beneath it is the upstream's own either way — so the upstream's travels, + * and the edge names one only where the gateway wrote the body out of nothing the + * upstream sent. `null` is an upstream that declared none, and it stays `null`: labelling + * a body nobody described is a statement this gateway has no grounds to make. */ + 'response.openaiAudioTranscription.mediaType': string | null; + /** What the stream will have come to by the time the events run out, and `null` on every + * path that does not stream. A streamed transcription states its usage in the terminal + * event, which is after this run has answered — so the numbers cannot be in + * `response.usage.billable`, which says what had been reported when the ending stage + * handed up: the entity, and no quantities. Settling from this is the epilogue's job, + * after the drain. */ + 'response.openaiAudioTranscription.streamedOutcome': Deferred | null; + /** What the client is actually sent — a JSON object, the upstream's own document, or the + * SSE frames of a stream. The edge provides it, so a dump shows what the client received + * rather than the gateway's own reading of it. */ + 'response.openaiAudioTranscription.rendered': Record | Uint8Array | AsyncIterable; +} + +export type Fields = { [P in K]: OpenAIAudioTranscriptionFacts[P] }; diff --git a/packages/gateway/src/data-plane/openai-audio/pipeline.ts b/packages/gateway/src/data-plane/openai-audio/pipeline.ts index de1d0c12cf..1f3eac7caf 100644 --- a/packages/gateway/src/data-plane/openai-audio/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-audio/pipeline.ts @@ -1,458 +1,16 @@ -// OpenAI Audio Transcriptions as a pipeline. The family whose answer is not one document but -// six: `response_format` picks between three JSON objects and three text documents, `stream` -// picks a sequence of events instead of any of them, and each is its own media type. That -// is the whole reason this endpoint needed a response strategy of its own, and it is what -// one canonical fact plus one media-type fact dissolve. -// -// The shape: -// -// emitOpenAIAudioTranscription the edge: writes the answer back in the rendering -// the client asked for, SSE framing included -// resolveCandidates narrows to the upstreams that expose the endpoint -// failover runs what follows once per candidate -// callOpenAIAudioTranscriptionUpstream the ending: dials, reads what came back, and -// provides the answer plus what is billable - -import { recordStream, streamReferenceOf } from '../../dump/turn-dump.ts'; -import type { UsageQuantities } from '../../repo/types.ts'; -import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; -import { isFailure } from '../pipeline/facts.ts'; -import type { GatewayServices } from '../pipeline/services.ts'; +import { callOpenAIAudioTranscriptionUpstream } from './call-upstream.ts'; +import { emitOpenAIAudioTranscription } from './emit.ts'; +import type { Fields } from './facts.ts'; +import { narrowing } from './target.ts'; +import { isFailure, type BillableEntity } from '../pipeline/facts.ts'; +import { failover } from '../pipeline/failover.ts'; +import { resolveCandidates } from '../pipeline/resolve-candidates.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; -import { failover, resolveCandidates } from '../pipeline/stages.ts'; -import { dialFailure } from '../pipeline/upstream-body.ts'; -import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; -import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; -import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; -import { compose, defer, defineStage, move, own, type Deferred, type Logger, type Owned, type Pipeline } from '@floway-dev/pipeline'; -import { eventFrame, isEventStreamMediaType, parseDecimalString, parseSSEStream, renderErrorEnvelope, sseFrame, type SseFrame } from '@floway-dev/protocols/common'; -import { - isOpenAIAudioTranscriptionDoneEvent, - parseOpenAIAudioTranscription, - parseOpenAIAudioTranscriptionStreamEvent, - parseOpenAIAudioTranscriptionStreamUsage, - parseOpenAIAudioTranscriptionUsage, - renderOpenAIAudioTranscription, - type OpenAIAudioTranscriptionResponseFormat, - type OpenAIAudioTranscriptionStreamEvent, - type OpenAIAudioTranscriptionUsage, - type CanonicalOpenAIAudioTranscription, -} from '@floway-dev/protocols/openai-audio'; -import { providerModelOf, type OpenAIAudioTranscriptionFormEntry, type ModelCandidate, type TelemetryModelIdentity } from '@floway-dev/provider'; - -/** The answer while it is still the upstream's, one event at a time. It is a view and not a - * resource: what owns the connection is `response.http.body`, which is where release and - * failover's ownership both read. - * - * A view is a wrapper around the generator rather than the generator itself, which is what - * says where the resource is: the upstream's body at `response.http.body`, claimed with - * `own()`, and nothing else here. */ -export type OpenAIAudioTranscriptionEvents = AsyncIterable; - -const viewOf = (events: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => events }); - -/** What settling this run will be told once the events run out: what the upstream metered, - * and whether the transcript ever finished. */ -export interface OpenAIAudioTranscriptionStreamOutcome { - readonly billable: readonly BillableEntity[]; - /** An upstream that stopped before `transcript.text.done` answered 200 and then did not - * finish what it started, which is a failed request however much of the transcript - * reached the client. */ - readonly failed: boolean; -} - -/** OpenAI Audio Transcriptions' own keys, extending the shared space by intersection. */ -export interface OpenAIAudioTranscriptionFacts extends GatewayFacts { - /** Which of the six renderings the client asked for. It belongs to the ingress and stays - * put: the same value travels to the upstream inside the form, so the rendering the - * answer arrives in is the rendering the answer is written back in — and a text document - * does not say which of the three it is, so nothing else could decide. */ - 'ingress.openaiAudioTranscription.responseFormat': OpenAIAudioTranscriptionResponseFormat; - /** The multipart form as ordered semantic entries. The body is parsed before routing - * because field order is unconstrained, and every candidate builds a fresh body from - * these, so a retry never reuses a consumed one. The bytes the client sent are recorded - * at `ingress.http.body`, which is where a dump reads the upload itself. */ - 'request.openaiAudioTranscription.form': readonly OpenAIAudioTranscriptionFormEntry[]; - /** The one transcription, whichever rendering carried it — or the events it is arriving - * as, or the failure that came instead. A stream, a value and a failure sit at one key: - * telling them apart is reading a value, and each stage does that where it needs to. */ - 'response.openaiAudioTranscription.canonical': CanonicalOpenAIAudioTranscription | OpenAIAudioTranscriptionEvents | Failure; - /** What the answer goes out under. A media type is upstream-owned — OpenAI answers - * `text`, `srt` and `vtt` under one media type and other upstreams label them apart, and - * the document beneath it is the upstream's own either way — so the upstream's travels, - * and the edge names one only where the gateway wrote the body out of nothing the - * upstream sent. `null` is an upstream that declared none, and it stays `null`: labelling - * a body nobody described is a statement this gateway has no grounds to make. */ - 'response.openaiAudioTranscription.mediaType': string | null; - /** What the stream will have come to by the time the events run out, and `null` on every - * path that does not stream. A streamed transcription states its usage in the terminal - * event, which is after this run has answered — so the numbers cannot be in - * `response.usage.billable`, which says what had been reported when the ending stage - * handed up: the entity, and no quantities. Settling from this is the epilogue's job, - * after the drain. */ - 'response.openaiAudioTranscription.streamedOutcome': Deferred | null; - /** What the client is actually sent — a JSON object, the upstream's own document, or the - * SSE frames of a stream. The edge provides it, so a dump shows what the client received - * rather than the gateway's own reading of it. */ - 'response.openaiAudioTranscription.rendered': Record | Uint8Array | AsyncIterable; -} - -type A = { [P in K]: OpenAIAudioTranscriptionFacts[P] }; - -const isEvents = (answer: CanonicalOpenAIAudioTranscription | OpenAIAudioTranscriptionEvents): answer is OpenAIAudioTranscriptionEvents => - Symbol.asyncIterator in answer; - -/** - * The outermost edge. Writes the answer back in the rendering the client asked for, and names - * a media type for the one body the gateway wrote out of nothing the upstream sent — its - * error envelope. Every other answer goes out under the media type it arrived with, because - * for a document that is carried rather than rewritten the upstream's label is the only true - * description there is. - * - * SSE framing is produced here and nowhere else — below this stage the answer is parsed - * events, so the same assembly would serve another transport by rendering differently at - * this one point. The upstream's own `event:` label is not carried: it is transport, and - * OpenAI's clients read this endpoint's frames by their payload's `type` rather than by the - * label. https://github.com/openai/openai-python/blob/10ee3f0da2ac6f93345c1204bd7bb1a2faa79ff2/src/openai/_streaming.py#L61-L107 - */ -const emitOpenAIAudioTranscription = defineStage< - A<'ingress.openaiAudioTranscription.responseFormat'>, - A<'ingress.openaiAudioTranscription.responseFormat'>, - A<'ingress.openaiAudioTranscription.responseFormat' | 'response.openaiAudioTranscription.canonical' | 'response.openaiAudioTranscription.mediaType'> - & { 'response.http.headers': readonly (readonly [string, string])[] }, - A<'response.openaiAudioTranscription.rendered' | 'response.openaiAudioTranscription.mediaType'> - & { 'response.http.status': number; 'response.http.headers': readonly (readonly [string, string])[] } ->({ - name: 'emitOpenAIAudioTranscription', - through: { - request: { - needs: ['ingress.openaiAudioTranscription.responseFormat'], - consumes: [], - provides: [], - }, - response: { - needs: ['response.openaiAudioTranscription.canonical', 'response.openaiAudioTranscription.mediaType', 'response.http.headers'], - consumes: ['response.openaiAudioTranscription.canonical', 'response.http.headers'], - provides: ['response.openaiAudioTranscription.rendered', 'response.openaiAudioTranscription.mediaType', 'response.http.status', 'response.http.headers'], - }, - }, - execute: async (facts, next) => { - const back = await next(facts); - const { 'response.openaiAudioTranscription.canonical': answer, 'response.http.headers': headers, ...rest } = back; - // Vendor traces and quota state stay visible; what an intermediary must strip, and what - // would misdescribe a body this gateway serialized itself, does not. A filter that removed - // nothing hands the same array on, so the record shows no change where none happened. - const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); - const forClient = forwardable.length === headers.length ? headers : move(forwardable); - - if (isFailure(answer)) { - return { - ...rest, - 'response.http.headers': forClient, - 'response.http.status': answer.status, - 'response.openaiAudioTranscription.mediaType': 'application/json', - 'response.openaiAudioTranscription.rendered': move(renderErrorEnvelope(answer.message, answer.body)), - }; - } - // Everything that reaches here answered. The same key carried the upstream's own status - // further down; re-providing it is what makes the top of the record the response the - // client gets rather than the one the upstream gave. - return { - ...rest, - 'response.http.headers': forClient, - 'response.http.status': 200, - 'response.openaiAudioTranscription.rendered': move(isEvents(answer) - ? renderSSE(answer) - : renderOpenAIAudioTranscription(back['ingress.openaiAudioTranscription.responseFormat'], answer)), - }; - }, -}); - -const renderSSE = (events: OpenAIAudioTranscriptionEvents): AsyncIterable => ({ - // The frames the client reads are a reframing of the events the record holds, so this key - // points at that same stream rather than at nothing. - ...streamReferenceOf(events), - [Symbol.asyncIterator]: () => (async function* () { - for await (const event of events) yield sseFrame(JSON.stringify(event)); - })(), -}); - -/** - * The ending. It dials, reads what came back in the rendering the request asked for, and - * provides the answer, the raw HTTP response beneath it, and what the call is billable for. - * A failure is a value: a 429 here is what an earlier stage fails over, and so is a dial that - * never reached anyone. A 200 is never one of them — this endpoint carries the document the - * upstream sent rather than serializing one from what it read, so a body no reading could - * open is still that upstream's answer and there is nothing to try the next candidate for. - */ -const callOpenAIAudioTranscriptionUpstream = defineStage< - A<'ingress.openaiAudioTranscription.responseFormat' | 'request.openaiAudioTranscription.form' | 'route.attempt' | 'ingress.http.headers'>, - A<'response.openaiAudioTranscription.canonical' | 'response.openaiAudioTranscription.mediaType' | 'response.openaiAudioTranscription.streamedOutcome'> - & { 'response.usage.billable': readonly BillableEntity[]; 'response.http.status': number; - 'response.http.headers': readonly (readonly [string, string])[]; - 'response.http.body': ReadableStream & Owned; }, - GatewayServices ->({ - name: 'callOpenAIAudioTranscriptionUpstream', - return: { - provides: [ - 'response.openaiAudioTranscription.canonical', - 'response.openaiAudioTranscription.mediaType', - 'response.openaiAudioTranscription.streamedOutcome', - 'response.usage.billable', - 'response.http.status', - 'response.http.headers', - 'response.http.body', - ], - }, - execute: async (facts, use) => { - const candidate = use.resolveAttempt(facts['route.attempt']); - // Attribution is set before the dial, so an attempt that never completes still names the - // candidate it was made against rather than the one tried before it. - use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'audio_transcription'); - - let result; - try { - result = await candidate.provider.instance.callOpenAIAudioTranscriptions( - providerModelOf(candidate), - { entries: facts['request.openaiAudioTranscription.form'] }, - use.gateway.abortSignal, - // The client's own headers reach the upstream from the record, not from a live request - // object: what a provider is allowed to forward is filtered per provider, and the dump - // shows what was there to filter. - buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), - ); - } catch (error) { - use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); - // A dial that never completed reached no upstream, so nothing was billed and there are - // no headers to carry. What it leaves behind is the performance row settlement writes. - return move({ - ...facts, - 'response.openaiAudioTranscription.canonical': dialFailure(error), - 'response.openaiAudioTranscription.mediaType': null, - 'response.openaiAudioTranscription.streamedOutcome': null, - 'response.usage.billable': [], - 'response.http.status': 502, - 'response.http.headers': [], - 'response.http.body': spentBody(null), - }); - } - const identity = telemetryModelIdentity(candidate, result.modelKey); - const format = facts['ingress.openaiAudioTranscription.responseFormat']; - const status = result.response.status; - const mediaType = result.response.headers.get('content-type'); - const headers = move([...result.response.headers] as readonly (readonly [string, string])[]); - // An upstream that was called and reported nothing is a different situation from one - // that reported zero, so the entity is present with no quantities. - const called: readonly BillableEntity[] = [{ identity, quantities: {} }]; - - if (!result.response.ok) { - use.log.warn('upstream refused', { status }); - // An upstream error body is JSON like any other body, and reading it here is also what - // leaves a losing attempt with nothing open behind it. - return move({ - ...facts, - 'response.openaiAudioTranscription.canonical': await refusal(status, result.response), - 'response.openaiAudioTranscription.mediaType': mediaType, - 'response.openaiAudioTranscription.streamedOutcome': null, - 'response.usage.billable': called, - 'response.http.status': status, - 'response.http.headers': headers, - 'response.http.body': spentBody(result.response.body), - }); - } - - // A streamed answer is the one the client asked for with `stream`, and it is the media - // type that says one arrived: an upstream that ignores `stream` — `whisper-1` does — - // answers in the rendering `response_format` named instead. - // https://github.com/openai/openai-openapi/blob/db3e53198a66732cfe161339ea63bf36fc0137ad/openapi.yaml#L36325-L36336 - if (isEventStreamMediaType(mediaType)) { - if (result.response.body === null) { - return move({ - ...facts, - 'response.openaiAudioTranscription.canonical': { status: 502, message: 'Upstream returned a streaming response with no body.' }, - 'response.openaiAudioTranscription.mediaType': mediaType, - 'response.openaiAudioTranscription.streamedOutcome': null, - 'response.usage.billable': called, - 'response.http.status': status, - 'response.http.headers': headers, - 'response.http.body': spentBody(null), - }); - } - // What the upstream metered, and whether the transcript finished, are both observed - // here — closest to the upstream and on the protocol it spoke — by folding the events - // as they pass, so the reading costs one pass and the client's own stream drives it. - const metered = meterEvents(result.response.body, identity, use.gateway.abortSignal, use.log); - return move({ - ...facts, - // This protocol's stream is bare events rather than protocol frames, so the record is - // told how one becomes a frame instead of being left to assume. - 'response.openaiAudioTranscription.canonical': recordStream(metered.events, use.gateway.dump, eventFrame), - 'response.openaiAudioTranscription.mediaType': mediaType, - 'response.openaiAudioTranscription.streamedOutcome': metered.outcome, - 'response.usage.billable': called, - 'response.http.status': status, - 'response.http.headers': headers, - // Releasing this body is reading those events to the end: they are one reader over - // one connection, and a second reader is not something a `ReadableStream` allows. - 'response.http.body': own(result.response.body, async (): Promise => { for await (const _event of metered.events) { /* to end of stream */ } }), - }); - } - - // A 2xx body is the answer whether or not this endpoint could read it, because what the - // client is sent is the document that arrived and not something serialized from a parse. - // So there is nothing here to fail over from: the reading feeds the record and the usage - // row, and the bytes travel either way. - const read = readTranscription(format, new Uint8Array(await result.response.arrayBuffer()), use.log); - return move({ - ...facts, - 'response.openaiAudioTranscription.canonical': read.canonical, - 'response.openaiAudioTranscription.mediaType': mediaType, - 'response.openaiAudioTranscription.streamedOutcome': null, - 'response.usage.billable': [{ identity, quantities: billed(read.usage) }], - 'response.http.status': status, - 'response.http.headers': headers, - 'response.http.body': spentBody(result.response.body), - }); - }, -}); - -/** A body this stage has already read to the end, or one the upstream never sent. The record - * holds a body as a stream and `failover` releases the losing attempts', so every path hands - * one up; what says an answer was unusable is the failure at the canonical key, not this. */ -const spentBody = (body: ReadableStream | null): ReadableStream & Owned => - own(body ?? new ReadableStream({ start: controller => controller.close() }), (): Promise => Promise.resolve()); - -const refusal = async (status: number, response: Response): Promise => { - const text = await response.text(); - try { - return { status, message: text, body: JSON.parse(text) as unknown }; - } catch { - // A refusal that is not JSON is still a refusal, and its text is what the client is - // told; there is simply no parsed body for a dump reader to open. - return { status, message: text }; - } -}; - -/** - * What a 2xx body was worth to this endpoint, in two readings that do not depend on each - * other. - * - * Neither can cost the client the answer: the document is carried, so a body no reading - * could open is still what goes back, and what a failed reading costs is the transcript in - * the record and whatever usage the body would have stated. Nor can either cost the other — - * an upstream whose usage block this gateway cannot model still had its transcript read, and - * one whose document it could not open is still billed for nothing rather than mis-billed. - */ -const readTranscription = ( - format: OpenAIAudioTranscriptionResponseFormat, - document: Uint8Array, - log: Logger, -): { readonly canonical: CanonicalOpenAIAudioTranscription; readonly usage: OpenAIAudioTranscriptionUsage | undefined } => { - let canonical: CanonicalOpenAIAudioTranscription; - try { - canonical = parseOpenAIAudioTranscription(format, document); - } catch (error) { - log.warn('failed to parse 2xx upstream body for /audio/transcriptions; forwarding it as it arrived', { error: String(error) }); - return { canonical: { document }, usage: undefined }; - } - // Only an object rendering states usage: `text`, `srt` and `vtt` have nowhere to put it, - // and asking them for one is what would warn about every subtitle a client requests. - return { canonical, usage: canonical.raw === undefined ? undefined : readUsage(() => parseOpenAIAudioTranscriptionUsage(canonical.raw), log) }; -}; - -/** A transcription states its usage in two places — the body of an object rendering and the - * terminal event of a stream — and either can state it in a shape this gateway cannot read. - * That upstream metered something, and the request is recorded saying exactly that: the - * entity, and no quantities. */ -const readUsage = (read: () => OpenAIAudioTranscriptionUsage | undefined, log: Logger): OpenAIAudioTranscriptionUsage | undefined => { - try { - return read(); - } catch (error) { - log.warn('invalid usage in 2xx upstream response; recording the request only', { error: String(error) }); - return undefined; - } -}; - -interface MeteredEvents { - readonly events: OpenAIAudioTranscriptionEvents; - readonly outcome: Deferred; -} - -const meterEvents = ( - body: ReadableStream, - identity: TelemetryModelIdentity, - signal: AbortSignal | undefined, - log: Logger, -): MeteredEvents => { - let settle!: (outcome: OpenAIAudioTranscriptionStreamOutcome) => void; - // Declared as this run's own unfinished work, so the runner waits for it at teardown where - // it can see it rather than the reading being started and forgotten. - const outcome = defer(new Promise(resolve => { settle = resolve; })); - const events = viewOf((async function* () { - let usage: OpenAIAudioTranscriptionUsage | undefined; - let completed = false; - try { - for await (const frame of parseSSEStream(body, { signal })) { - const event = parseOpenAIAudioTranscriptionStreamEvent(JSON.parse(frame.data) as unknown); - if (isOpenAIAudioTranscriptionDoneEvent(event)) { - usage = readUsage(() => parseOpenAIAudioTranscriptionStreamUsage(event), log); - completed = true; - yield event; - // The transcript is complete, so there is nothing further to read. An upstream that - // holds the connection open past this point would otherwise keep the client's own - // stream open with it; returning here closes the read, which cancels the upstream. - return; - } - yield event; - } - } finally { - // Reached however the events ended — the terminal one, a client that stopped reading, - // or a broken upstream — because what the upstream already metered is billable - // whatever happened to the downstream half. - settle({ billable: [{ identity, quantities: billed(usage) }], failed: !completed }); - } - })()); - return { events, outcome }; -}; - -const billed = (usage: OpenAIAudioTranscriptionUsage | undefined): UsageQuantities => { - if (usage === undefined) return {}; - if (usage.kind === 'duration') return { input_audio_seconds: parseDecimalString(String(usage.seconds)) }; - // Audio input is priced apart from text input, so what stays on the general input metric is - // what the upstream did not attribute to audio. - return { - input_tokens: parseDecimalString(String(usage.inputTokens - (usage.inputAudioTokens ?? 0))), - ...(usage.inputAudioTokens === undefined ? {} : { input_audio_tokens: parseDecimalString(String(usage.inputAudioTokens)) }), - output_tokens: parseDecimalString(String(usage.outputTokens)), - }; -}; - -/** Nothing about this request narrows a candidate further. A transcription's parameters are - * form fields the upstream reads for itself, and no endpoint metadata says which renderings - * a model can write — so exposing the endpoint is the whole of the test. */ -const narrowing = { - kind: 'transcription' as const, - reject: (candidate: ModelCandidate): string | null => - candidate.model.endpoints.openaiAudioTranscriptions === undefined - ? 'the upstream does not expose an OpenAI Audio Transcriptions endpoint' - : null, - unsupported: (model: string) => `Model ${model} does not support the /audio/transcriptions endpoint.`, - refuse: (status: number, message: string) => ({ - 'response.openaiAudioTranscription.canonical': { status, message } as Failure, - 'response.openaiAudioTranscription.mediaType': null, - 'response.openaiAudioTranscription.streamedOutcome': null, - }), - refuses: [ - 'response.openaiAudioTranscription.canonical', - 'response.openaiAudioTranscription.mediaType', - 'response.openaiAudioTranscription.streamedOutcome', - ] as const, -}; +import { compose, type Pipeline } from '@floway-dev/pipeline'; export const openaiAudioTranscriptionServePipeline: Pipeline< - A<'ingress.http.headers' | 'ingress.openaiAudioTranscription.responseFormat' | 'request.openaiAudioTranscription.form' | 'serve.model'>, - A<'response.openaiAudioTranscription.rendered' | 'response.openaiAudioTranscription.mediaType' | 'response.openaiAudioTranscription.streamedOutcome'> + Fields<'ingress.http.headers' | 'ingress.openaiAudioTranscription.responseFormat' | 'request.openaiAudioTranscription.form' | 'serve.model'>, + Fields<'response.openaiAudioTranscription.rendered' | 'response.openaiAudioTranscription.mediaType' | 'response.openaiAudioTranscription.streamedOutcome'> & { 'response.http.status': number; 'response.usage.billable': readonly BillableEntity[]; 'response.http.headers': readonly (readonly [string, string])[]; } > = compose('openaiAudioTranscriptionServe', [ diff --git a/packages/gateway/src/data-plane/openai-audio/target.ts b/packages/gateway/src/data-plane/openai-audio/target.ts new file mode 100644 index 0000000000..820b6d004a --- /dev/null +++ b/packages/gateway/src/data-plane/openai-audio/target.ts @@ -0,0 +1,24 @@ +import type { Failure } from '../pipeline/facts.ts'; +import type { ModelCandidate } from '@floway-dev/provider'; + +/** Nothing about this request narrows a candidate further. A transcription's parameters are + * form fields the upstream reads for itself, and no endpoint metadata says which renderings + * a model can write — so exposing the endpoint is the whole of the test. */ +export const narrowing = { + kind: 'transcription' as const, + reject: (candidate: ModelCandidate): string | null => + candidate.model.endpoints.openaiAudioTranscriptions === undefined + ? 'the upstream does not expose an OpenAI Audio Transcriptions endpoint' + : null, + unsupported: (model: string) => `Model ${model} does not support the /audio/transcriptions endpoint.`, + refuse: (status: number, message: string) => ({ + 'response.openaiAudioTranscription.canonical': { status, message } as Failure, + 'response.openaiAudioTranscription.mediaType': null, + 'response.openaiAudioTranscription.streamedOutcome': null, + }), + refuses: [ + 'response.openaiAudioTranscription.canonical', + 'response.openaiAudioTranscription.mediaType', + 'response.openaiAudioTranscription.streamedOutcome', + ] as const, +}; diff --git a/packages/gateway/src/data-plane/openai-completions/call-upstream.ts b/packages/gateway/src/data-plane/openai-completions/call-upstream.ts new file mode 100644 index 0000000000..9b1d5018ed --- /dev/null +++ b/packages/gateway/src/data-plane/openai-completions/call-upstream.ts @@ -0,0 +1,256 @@ +import type { Fields, OpenAICompletionsFacts, OpenAICompletionsFrames } from './facts.ts'; +import { tokenUsageFromOpenAICompletionsUsage } from './usage.ts'; +import { recordStream, type TurnDump } from '../../dump/turn-dump.ts'; +import type { UsageQuantities } from '../../repo/types.ts'; +import { tokenUsageQuantities } from '../../repo/usage-metrics.ts'; +import { isFailure, type BillableEntity, type Failure } from '../pipeline/facts.ts'; +import type { StreamOutcome } from '../pipeline/serve.ts'; +import type { GatewayServices } from '../pipeline/services.ts'; +import { dialFailure } from '../pipeline/upstream-body.ts'; +import { upstreamPerformanceContext, telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; +import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; +import { defineStage, move, own, defer, type Owned, type Deferred } from '@floway-dev/pipeline'; +import { isOpenAIUsageOnlyEventShape, type ProtocolFrame } from '@floway-dev/protocols/common'; +import { parseOpenAICompletionsStream, parseOpenAICompletionsResult, type OpenAICompletionsResult, type OpenAICompletionsStreamEvent } from '@floway-dev/protocols/openai-completions'; +import { providerModelOf, type TelemetryModelIdentity, type ModelCandidate } from '@floway-dev/provider'; + +/** + * A stream as a value the record can hold: a wrapper around the generator rather than the + * generator itself, which is what says where the resource is. There is exactly one resource in + * an OpenAI Completions run — the upstream's body, claimed with `own()` — and the wrapper keeps + * a frame view from reading as another. + * + * What comes back is single-shot on purpose: a stream that has been read is a stream that is + * over, and a second reader learns that rather than being lied to. + */ +const view = (frames: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => frames }); + +/** + * The ending. It dials, reads what came back on the shape the request asked for, and + * provides the answer, the raw HTTP response beneath it, and what the call is billable for. + * A failure is a value: a 429 here is what an earlier stage fails over, and so is a 200 whose + * body this protocol cannot read, because the next candidate's path and flags may differ. + */ +export const callOpenAICompletionsUpstream = defineStage< + Fields<'ingress.openaiCompletions.wantsStream' | 'request.openaiCompletions.payload' | 'route.attempt' | 'ingress.http.headers'>, + Fields<'response.openaiCompletions.payload' | 'response.openaiCompletions.streamedUsage' | 'response.usage.billable' + | 'response.http.status' | 'response.http.headers' | 'response.http.body'>, + GatewayServices +>({ + name: 'callOpenAICompletionsUpstream', + return: { + provides: [ + 'response.openaiCompletions.payload', + 'response.openaiCompletions.streamedUsage', + 'response.usage.billable', + 'response.http.status', + 'response.http.headers', + 'response.http.body', + ], + }, + execute: async (facts, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + // The provider re-stamps whatever id it resolved upstream, so the id the client + // addressed does not travel with the body. + const { model: _addressed, ...body } = facts['request.openaiCompletions.payload']; + // Attribution is set before the dial, so an attempt that never completes still names the + // candidate it was made against rather than the one tried before it. + use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'text_completion'); + + let result; + try { + result = await candidate.provider.instance.callOpenAICompletions( + providerModelOf(candidate), + body, + use.gateway.abortSignal, + // The client's own headers reach the upstream from the record, not from a live request + // object: what a provider is allowed to forward is filtered per provider, and the dump + // shows what was there to filter. + buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), + ); + } catch (error) { + use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); + // A dial that never completed reached no upstream, so nothing was billed and there are + // no headers to carry. What it leaves behind is the performance row settlement writes. + return move({ + ...facts, + 'response.openaiCompletions.payload': dialFailure(error), + 'response.openaiCompletions.streamedUsage': null, + 'response.usage.billable': [], + 'response.http.status': 502, + 'response.http.headers': [], + 'response.http.body': spentBody(null), + }); + } + if (!result.response.ok) use.log.warn('upstream refused', { status: result.response.status }); + + const answer = await readUpstream( + result.response, + facts['ingress.openaiCompletions.wantsStream'], + telemetryModelIdentity(candidate, result.modelKey), + candidate, + use.gateway.abortSignal, + use.gateway.dump, + ); + return move({ + ...facts, + 'response.openaiCompletions.payload': answer.payload, + 'response.openaiCompletions.streamedUsage': answer.streamedUsage, + 'response.usage.billable': answer.billable, + 'response.http.status': result.response.status, + 'response.http.headers': [...result.response.headers], + 'response.http.body': answer.body, + }); + }, +}); + +interface UpstreamAnswer { + readonly payload: OpenAICompletionsFacts['response.openaiCompletions.payload']; + readonly streamedUsage: OpenAICompletionsFacts['response.openaiCompletions.streamedUsage']; + readonly billable: readonly BillableEntity[]; + /** Every arm hands one up, this stage's own reading included: the record holds a body as a + * stream, and `failover` releases the losing attempts' by consuming that key. */ + readonly body: ReadableStream & Owned; +} + +/** What the upstream said, on the shape the request asked for. Which of the four this is is + * read from the response, never declared: a stream, a value and a failure sit at one key. */ +const readUpstream = async ( + response: Response, + wantsStream: boolean, + identity: TelemetryModelIdentity, + candidate: ModelCandidate, + signal: AbortSignal | undefined, + dump: TurnDump | null, +): Promise => { + // An upstream was called and reported nothing — which is what every arm but a read usage + // block leaves standing, and is a different statement from reporting zero. + const called: readonly BillableEntity[] = [{ identity, quantities: {} }]; + + if (!response.ok) { + // An upstream error body is JSON like any other body. Reading it here is also what + // leaves a losing attempt with nothing open behind it. + return { payload: await refusal(response), streamedUsage: null, billable: called, body: spentBody(response.body) }; + } + + if (!wantsStream) { + const read = await readResult(response); + return { + payload: isFailure(read) ? read : read.value, + streamedUsage: null, + billable: isFailure(read) ? called : [{ identity, quantities: billed(read.value.usage, read.value.service_tier, candidate) }], + body: spentBody(response.body), + }; + } + + if (response.body === null) { + return { + payload: { status: 502, message: 'Upstream returned a streaming response with no body.' }, + streamedUsage: null, + billable: called, + body: spentBody(null), + }; + } + + // Usage is observed here, closest to the upstream and on the protocol it spoke, by folding + // the frames as they pass — so the reading costs one pass and the client's own stream is + // what drives it. What it finds arrives with the last frame, long after this stage has + // handed up, which is why the entity above carries no quantities. + const metered = meterFrames(parseOpenAICompletionsStream(response.body, { signal }), identity, candidate); + return { + payload: recordStream(metered.frames, dump), + streamedUsage: metered.outcome, + billable: called, + // Releasing this body is draining those frames: they are one reader over one connection, + // and a second reader is not something a `ReadableStream` allows. + body: own(response.body, async (): Promise => { for await (const _frame of metered.frames) { /* to end of stream */ } }), + }; +}; + +/** A body this stage has already read to the end, or one the upstream never sent. The record + * holds a body as a stream and `failover` releases the losing attempts', so every path hands + * one up; what says an answer was unusable is the failure at the payload key, not this. */ +const spentBody = (body: ReadableStream | null): ReadableStream & Owned => + own(body ?? new ReadableStream({ start: controller => controller.close() }), (): Promise => Promise.resolve()); + +const refusal = async (response: Response): Promise => { + const text = await response.text(); + try { + return { status: response.status, message: text, body: JSON.parse(text) as unknown }; + } catch { + // A refusal that is not JSON is still a refusal, and its text is what the client is + // told; there is simply no parsed body for a dump reader to open. + return { status: response.status, message: text }; + } +}; + +/** A body that answered 200 and cannot be read as this protocol is an attempt that failed, + * because the gateway serializes what it sends from the value it parsed and has nothing to + * serialize. The next candidate may well answer in the protocol it advertised. */ +const readResult = async (response: Response): Promise<{ readonly value: OpenAICompletionsResult } | Failure> => { + try { + return { value: parseOpenAICompletionsResult(await response.json()) }; + } catch (error) { + return { status: 502, message: `Upstream answered ${response.status} with a body this endpoint cannot read: ${error instanceof Error ? error.message : String(error)}` }; + } +}; + +interface MeteredFrames { + readonly frames: OpenAICompletionsFrames; + readonly outcome: Deferred; +} + +const meterFrames = ( + source: AsyncIterable>, + identity: TelemetryModelIdentity, + candidate: ModelCandidate, +): MeteredFrames => { + let settle!: (outcome: StreamOutcome) => void; + // Declared as this run's own unfinished work, so the runner waits for it at teardown where + // it can see it rather than the reading being started and forgotten. + const outcome = defer(new Promise(resolve => { settle = resolve; })); + // Running out without the terminal frame is what "it did not finish" means, and it is known + // at the same moment the usage is. + let sawTerminal = false; + const frames = view((async function* () { + let usage: unknown; + let tier: string | null | undefined; + try { + for await (const frame of source) { + if (frame.type === 'event') { + // `service_tier` can ride on any chunk's root while the totals only land on the + // usage-only one, so the two are tracked apart and settled together. + const root = frame.event as { service_tier?: string | null; usage?: unknown }; + if (root.service_tier !== undefined) tier = root.service_tier; + if (isOpenAIUsageOnlyEventShape(frame.event)) usage = root.usage; + } + // The transport's own terminator is this protocol's end-of-turn. + if (frame.type === 'done') sawTerminal = true; + yield frame; + } + } finally { + // Reached however the frames ended — the terminal chunk, a client that stopped + // reading, or a broken upstream — because tokens the upstream already metered are + // billable whatever happened to the downstream half. + settle({ billable: [{ identity, quantities: billed(usage, tier, candidate) }], failed: !sawTerminal }); + } + })()); + return { frames, outcome }; +}; + +const billed = (usage: unknown, tier: string | null | undefined, candidate: ModelCandidate): UsageQuantities => { + const model = providerModelOf(candidate); + const tokens = tokenUsageFromOpenAICompletionsUsage( + usage, + tier, + model.enabledFlags.has('usage-exclusive-cached-tokens'), + `${candidate.provider.upstreamId}/${model.id}`, + ); + // An upstream that reported nothing leaves no quantities at all, which is a different + // statement from reporting zero. + // + // The service tier survives as far as `TokenUsage.tier` and no further: a billed entity is + // an identity and a bag of quantities, and the pricing selector the tier feeds has no seat + // there. It comes back when settlement does. + return tokens === null ? {} : tokenUsageQuantities(tokens); +}; diff --git a/packages/gateway/src/data-plane/openai-completions/emit.ts b/packages/gateway/src/data-plane/openai-completions/emit.ts new file mode 100644 index 0000000000..481290b227 --- /dev/null +++ b/packages/gateway/src/data-plane/openai-completions/emit.ts @@ -0,0 +1,84 @@ +import type { OpenAICompletionsFacts, OpenAICompletionsFrames, Fields } from './facts.ts'; +import { streamReferenceOf } from '../../dump/turn-dump.ts'; +import { isFailure } from '../pipeline/facts.ts'; +import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { renderErrorEnvelope, isOpenAIUsageOnlyEventShape, type SseFrame } from '@floway-dev/protocols/common'; +import { openaiCompletionsProtocolFrameToSSEFrame } from '@floway-dev/protocols/openai-completions'; + +const isFrames = (answer: OpenAICompletionsFacts['response.openaiCompletions.payload']): answer is OpenAICompletionsFrames => + Symbol.asyncIterator in answer; + +/** + * The outermost edge, and the only place where what the client asked for and what the + * upstream is asked for differ: billing needs the usage chunk on every stream, so it is + * turned on going down and taken back out coming up unless the client asked to see it. + * + * Rendering the answer is the other half. SSE framing is produced here and nowhere else — + * below this stage a stream carries protocol frames, so the same assembly would serve + * another transport by rendering differently at this one point. + */ +export const emitOpenAICompletions = defineStage< + Fields<'ingress.openaiCompletions.wantsStream' | 'ingress.openaiCompletions.wantsUsageChunk' | 'request.openaiCompletions.payload'>, + Fields<'ingress.openaiCompletions.wantsStream' | 'ingress.openaiCompletions.wantsUsageChunk' | 'request.openaiCompletions.payload'>, + Fields<'ingress.openaiCompletions.wantsUsageChunk' | 'response.openaiCompletions.payload' | 'response.http.headers'>, + Fields<'response.openaiCompletions.rendered' | 'response.http.status' | 'response.http.headers'> +>({ + name: 'emitOpenAICompletions', + through: { + request: { + needs: ['ingress.openaiCompletions.wantsStream', 'ingress.openaiCompletions.wantsUsageChunk', 'request.openaiCompletions.payload'], + consumes: [], + provides: ['request.openaiCompletions.payload'], + }, + response: { + needs: ['response.openaiCompletions.payload', 'response.http.headers'], + consumes: ['response.openaiCompletions.payload', 'response.http.headers'], + provides: ['response.openaiCompletions.rendered', 'response.http.status', 'response.http.headers'], + }, + }, + execute: async (facts, next) => { + const asked = facts['request.openaiCompletions.payload']; + const back = await next({ + ...facts, + 'request.openaiCompletions.payload': move(facts['ingress.openaiCompletions.wantsStream'] + ? { ...asked, stream_options: { ...asked.stream_options, include_usage: true } } + : asked), + }); + + const { 'response.openaiCompletions.payload': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + // A refusal keeps the status the upstream gave it. Anything that answered is a 200 the + // gateway says itself, because what the client receives is serialized here rather than + // relayed — the upstream's own status is a fact further down for whoever wants it. + return { + ...rest, + 'response.http.headers': forwardable.length === headers.length ? headers : move(forwardable), + 'response.http.status': isFailure(answer) ? answer.status : 200, + 'response.openaiCompletions.rendered': move(rendered(answer, back['ingress.openaiCompletions.wantsUsageChunk'])), + }; + }, +}); + +const rendered = ( + answer: OpenAICompletionsFacts['response.openaiCompletions.payload'], + wantsUsageChunk: boolean, +): OpenAICompletionsFacts['response.openaiCompletions.rendered'] => + isFailure(answer) ? renderErrorEnvelope(answer.message, answer.body) + : isFrames(answer) ? renderSSE(answer, wantsUsageChunk) + : answer; + +const renderSSE = (frames: OpenAICompletionsFrames, wantsUsageChunk: boolean): AsyncIterable => ({ + // The frames the client reads are a reframing of the ones the record holds, so this key + // points at that same stream rather than at nothing. + ...streamReferenceOf(frames), + [Symbol.asyncIterator]: () => (async function* () { + for await (const frame of frames) { + if (!wantsUsageChunk && frame.type === 'event' && isOpenAIUsageOnlyEventShape(frame.event)) continue; + yield openaiCompletionsProtocolFrameToSSEFrame(frame); + } + })(), +}); diff --git a/packages/gateway/src/data-plane/openai-completions/facts.ts b/packages/gateway/src/data-plane/openai-completions/facts.ts new file mode 100644 index 0000000000..5c4e7c9b72 --- /dev/null +++ b/packages/gateway/src/data-plane/openai-completions/facts.ts @@ -0,0 +1,35 @@ +import type { GatewayFacts, Failure } from '../pipeline/facts.ts'; +import type { StreamOutcome } from '../pipeline/serve.ts'; +import type { Deferred } from '@floway-dev/pipeline'; +import type { ProtocolFrame, SseFrame } from '@floway-dev/protocols/common'; +import type { OpenAICompletionsStreamEvent, OpenAICompletionsPayload, OpenAICompletionsResult } from '@floway-dev/protocols/openai-completions'; + +/** The answer while it is still the upstream's, one frame at a time. */ +export type OpenAICompletionsFrames = AsyncIterable>; + +/** OpenAI Completions' own keys, extending the shared space by intersection. */ +export interface OpenAICompletionsFacts extends GatewayFacts { + /** What the client asked for, which is not what the upstream is asked for: the gateway + * meters every stream and so always turns the usage chunk on. These stay put for the same + * reason every `ingress.*` key does — they describe the request that arrived, and the + * answer is rendered back into it. */ + 'ingress.openaiCompletions.wantsStream': boolean; + 'ingress.openaiCompletions.wantsUsageChunk': boolean; + 'request.openaiCompletions.payload': OpenAICompletionsPayload; + /** The answer, whichever kind it turned out to be. A stream, a value and a failure sit at + * one key: telling them apart is reading a value, and each stage does that where it needs + * to. */ + 'response.openaiCompletions.payload': OpenAICompletionsResult | OpenAICompletionsFrames | Failure; + /** What the upstream will have reported by the time the frames run out, and `null` on + * every path that does not stream. A stream's usage arrives with its last chunk, which is + * after this run has answered — so the numbers cannot be in `response.usage.billable`, + * which says what had been reported when the ending stage handed up: the entity, and no + * quantities. Settling billing from this is the prologue's job, after the drain. */ + 'response.openaiCompletions.streamedUsage': Deferred | null; + /** What the client is actually sent, in its own protocol — a JSON body, or the SSE frames + * of a stream. The edge provides it, so a dump shows what the client received rather than + * the gateway's own reading of it. */ + 'response.openaiCompletions.rendered': Record | AsyncIterable; +} + +export type Fields = { [P in K]: OpenAICompletionsFacts[P] }; diff --git a/packages/gateway/src/data-plane/openai-completions/pipeline.ts b/packages/gateway/src/data-plane/openai-completions/pipeline.ts index 1468139696..993ae8fba2 100644 --- a/packages/gateway/src/data-plane/openai-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-completions/pipeline.ts @@ -1,410 +1,16 @@ -// Text completions as a pipeline. One protocol, no translation, and an answer that is a -// stream whenever the client asked for one — the same request field decides both what the -// upstream is asked for and what shape comes back. -// -// The shape: -// -// emitOpenAICompletions the edge: asks the upstream for the usage chunk on the way -// down, and renders the answer into the client's protocol on -// the way back, SSE framing included -// resolveCandidates narrows to the upstreams that expose the endpoint -// failover runs what follows once per candidate -// callOpenAICompletionsUpstream the ending: dials, parses what came back, and provides the -// answer plus what is billable - -import { tokenUsageFromOpenAICompletionsUsage } from './usage.ts'; -import { recordStream, streamReferenceOf, type TurnDump } from '../../dump/turn-dump.ts'; -import type { UsageQuantities } from '../../repo/types.ts'; -import { tokenUsageQuantities } from '../../repo/usage-metrics.ts'; -import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; +import { callOpenAICompletionsUpstream } from './call-upstream.ts'; +import { emitOpenAICompletions } from './emit.ts'; +import type { Fields } from './facts.ts'; +import { narrowing } from './target.ts'; import { isFailure } from '../pipeline/facts.ts'; -import type { StreamOutcome } from '../pipeline/serve.ts'; -import type { GatewayServices } from '../pipeline/services.ts'; +import { failover } from '../pipeline/failover.ts'; +import { resolveCandidates } from '../pipeline/resolve-candidates.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; -import { failover, resolveCandidates } from '../pipeline/stages.ts'; -import { dialFailure } from '../pipeline/upstream-body.ts'; -import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; -import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; -import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; -import { compose, defer, defineStage, move, own, type Deferred, type Owned, type Pipeline } from '@floway-dev/pipeline'; -import { isOpenAIUsageOnlyEventShape, renderErrorEnvelope, type ProtocolFrame, type SseFrame } from '@floway-dev/protocols/common'; -import { - openaiCompletionsProtocolFrameToSSEFrame, - parseOpenAICompletionsResult, - parseOpenAICompletionsStream, - type OpenAICompletionsPayload, - type OpenAICompletionsResult, - type OpenAICompletionsStreamEvent, -} from '@floway-dev/protocols/openai-completions'; -import { providerModelOf } from '@floway-dev/provider'; -import type { ModelCandidate, TelemetryModelIdentity } from '@floway-dev/provider'; - -/** The answer while it is still the upstream's, one frame at a time. */ -export type OpenAICompletionsFrames = AsyncIterable>; - -/** - * A stream as a value the record can hold: a wrapper around the generator rather than the - * generator itself, which is what says where the resource is. There is exactly one resource in - * an OpenAI Completions run — the upstream's body, claimed with `own()` — and the wrapper keeps - * a frame view from reading as another. - * - * What comes back is single-shot on purpose: a stream that has been read is a stream that is - * over, and a second reader learns that rather than being lied to. - */ -const view = (frames: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => frames }); - -/** OpenAI Completions' own keys, extending the shared space by intersection. */ -export interface OpenAICompletionsFacts extends GatewayFacts { - /** What the client asked for, which is not what the upstream is asked for: the gateway - * meters every stream and so always turns the usage chunk on. These stay put for the same - * reason every `ingress.*` key does — they describe the request that arrived, and the - * answer is rendered back into it. */ - 'ingress.openaiCompletions.wantsStream': boolean; - 'ingress.openaiCompletions.wantsUsageChunk': boolean; - 'request.openaiCompletions.payload': OpenAICompletionsPayload; - /** The answer, whichever kind it turned out to be. A stream, a value and a failure sit at - * one key: telling them apart is reading a value, and each stage does that where it needs - * to. */ - 'response.openaiCompletions.payload': OpenAICompletionsResult | OpenAICompletionsFrames | Failure; - /** What the upstream will have reported by the time the frames run out, and `null` on - * every path that does not stream. A stream's usage arrives with its last chunk, which is - * after this run has answered — so the numbers cannot be in `response.usage.billable`, - * which says what had been reported when the ending stage handed up: the entity, and no - * quantities. Settling billing from this is the prologue's job, after the drain. */ - 'response.openaiCompletions.streamedUsage': Deferred | null; - /** What the client is actually sent, in its own protocol — a JSON body, or the SSE frames - * of a stream. The edge provides it, so a dump shows what the client received rather than - * the gateway's own reading of it. */ - 'response.openaiCompletions.rendered': Record | AsyncIterable; -} - -type C = { [P in K]: OpenAICompletionsFacts[P] }; - -const isFrames = (answer: OpenAICompletionsFacts['response.openaiCompletions.payload']): answer is OpenAICompletionsFrames => - Symbol.asyncIterator in answer; - -/** - * The outermost edge, and the only place where what the client asked for and what the - * upstream is asked for differ: billing needs the usage chunk on every stream, so it is - * turned on going down and taken back out coming up unless the client asked to see it. - * - * Rendering the answer is the other half. SSE framing is produced here and nowhere else — - * below this stage a stream carries protocol frames, so the same assembly would serve - * another transport by rendering differently at this one point. - */ -const emitOpenAICompletions = defineStage< - C<'ingress.openaiCompletions.wantsStream' | 'ingress.openaiCompletions.wantsUsageChunk' | 'request.openaiCompletions.payload'>, - C<'ingress.openaiCompletions.wantsStream' | 'ingress.openaiCompletions.wantsUsageChunk' | 'request.openaiCompletions.payload'>, - C<'ingress.openaiCompletions.wantsUsageChunk' | 'response.openaiCompletions.payload' | 'response.http.headers'>, - C<'response.openaiCompletions.rendered' | 'response.http.status' | 'response.http.headers'> ->({ - name: 'emitOpenAICompletions', - through: { - request: { - needs: ['ingress.openaiCompletions.wantsStream', 'ingress.openaiCompletions.wantsUsageChunk', 'request.openaiCompletions.payload'], - consumes: [], - provides: ['request.openaiCompletions.payload'], - }, - response: { - needs: ['response.openaiCompletions.payload', 'response.http.headers'], - consumes: ['response.openaiCompletions.payload', 'response.http.headers'], - provides: ['response.openaiCompletions.rendered', 'response.http.status', 'response.http.headers'], - }, - }, - execute: async (facts, next) => { - const asked = facts['request.openaiCompletions.payload']; - const back = await next({ - ...facts, - 'request.openaiCompletions.payload': move(facts['ingress.openaiCompletions.wantsStream'] - ? { ...asked, stream_options: { ...asked.stream_options, include_usage: true } } - : asked), - }); - - const { 'response.openaiCompletions.payload': answer, 'response.http.headers': headers, ...rest } = back; - // Vendor traces and quota state stay visible; what an intermediary must strip, and what - // would misdescribe a body this gateway serialized itself, does not. A filter that removed - // nothing hands the same array on, so the record shows no change where none happened. - const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); - // A refusal keeps the status the upstream gave it. Anything that answered is a 200 the - // gateway says itself, because what the client receives is serialized here rather than - // relayed — the upstream's own status is a fact further down for whoever wants it. - return { - ...rest, - 'response.http.headers': forwardable.length === headers.length ? headers : move(forwardable), - 'response.http.status': isFailure(answer) ? answer.status : 200, - 'response.openaiCompletions.rendered': move(rendered(answer, back['ingress.openaiCompletions.wantsUsageChunk'])), - }; - }, -}); - -const rendered = ( - answer: OpenAICompletionsFacts['response.openaiCompletions.payload'], - wantsUsageChunk: boolean, -): OpenAICompletionsFacts['response.openaiCompletions.rendered'] => - isFailure(answer) ? renderErrorEnvelope(answer.message, answer.body) - : isFrames(answer) ? renderSSE(answer, wantsUsageChunk) - : answer; - -const renderSSE = (frames: OpenAICompletionsFrames, wantsUsageChunk: boolean): AsyncIterable => ({ - // The frames the client reads are a reframing of the ones the record holds, so this key - // points at that same stream rather than at nothing. - ...streamReferenceOf(frames), - [Symbol.asyncIterator]: () => (async function* () { - for await (const frame of frames) { - if (!wantsUsageChunk && frame.type === 'event' && isOpenAIUsageOnlyEventShape(frame.event)) continue; - yield openaiCompletionsProtocolFrameToSSEFrame(frame); - } - })(), -}); - -/** - * The ending. It dials, reads what came back on the shape the request asked for, and - * provides the answer, the raw HTTP response beneath it, and what the call is billable for. - * A failure is a value: a 429 here is what an earlier stage fails over, and so is a 200 whose - * body this protocol cannot read, because the next candidate's path and flags may differ. - */ -const callOpenAICompletionsUpstream = defineStage< - C<'ingress.openaiCompletions.wantsStream' | 'request.openaiCompletions.payload' | 'route.attempt' | 'ingress.http.headers'>, - C<'response.openaiCompletions.payload' | 'response.openaiCompletions.streamedUsage' | 'response.usage.billable' - | 'response.http.status' | 'response.http.headers' | 'response.http.body'>, - GatewayServices ->({ - name: 'callOpenAICompletionsUpstream', - return: { - provides: [ - 'response.openaiCompletions.payload', - 'response.openaiCompletions.streamedUsage', - 'response.usage.billable', - 'response.http.status', - 'response.http.headers', - 'response.http.body', - ], - }, - execute: async (facts, use) => { - const candidate = use.resolveAttempt(facts['route.attempt']); - // The provider re-stamps whatever id it resolved upstream, so the id the client - // addressed does not travel with the body. - const { model: _addressed, ...body } = facts['request.openaiCompletions.payload']; - // Attribution is set before the dial, so an attempt that never completes still names the - // candidate it was made against rather than the one tried before it. - use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'text_completion'); - - let result; - try { - result = await candidate.provider.instance.callOpenAICompletions( - providerModelOf(candidate), - body, - use.gateway.abortSignal, - // The client's own headers reach the upstream from the record, not from a live request - // object: what a provider is allowed to forward is filtered per provider, and the dump - // shows what was there to filter. - buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), - ); - } catch (error) { - use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); - // A dial that never completed reached no upstream, so nothing was billed and there are - // no headers to carry. What it leaves behind is the performance row settlement writes. - return move({ - ...facts, - 'response.openaiCompletions.payload': dialFailure(error), - 'response.openaiCompletions.streamedUsage': null, - 'response.usage.billable': [], - 'response.http.status': 502, - 'response.http.headers': [], - 'response.http.body': spentBody(null), - }); - } - if (!result.response.ok) use.log.warn('upstream refused', { status: result.response.status }); - - const answer = await readUpstream( - result.response, - facts['ingress.openaiCompletions.wantsStream'], - telemetryModelIdentity(candidate, result.modelKey), - candidate, - use.gateway.abortSignal, - use.gateway.dump, - ); - return move({ - ...facts, - 'response.openaiCompletions.payload': answer.payload, - 'response.openaiCompletions.streamedUsage': answer.streamedUsage, - 'response.usage.billable': answer.billable, - 'response.http.status': result.response.status, - 'response.http.headers': [...result.response.headers], - 'response.http.body': answer.body, - }); - }, -}); - -interface UpstreamAnswer { - readonly payload: OpenAICompletionsFacts['response.openaiCompletions.payload']; - readonly streamedUsage: OpenAICompletionsFacts['response.openaiCompletions.streamedUsage']; - readonly billable: readonly BillableEntity[]; - /** Every arm hands one up, this stage's own reading included: the record holds a body as a - * stream, and `failover` releases the losing attempts' by consuming that key. */ - readonly body: ReadableStream & Owned; -} - -/** What the upstream said, on the shape the request asked for. Which of the four this is is - * read from the response, never declared: a stream, a value and a failure sit at one key. */ -const readUpstream = async ( - response: Response, - wantsStream: boolean, - identity: TelemetryModelIdentity, - candidate: ModelCandidate, - signal: AbortSignal | undefined, - dump: TurnDump | null, -): Promise => { - // An upstream was called and reported nothing — which is what every arm but a read usage - // block leaves standing, and is a different statement from reporting zero. - const called: readonly BillableEntity[] = [{ identity, quantities: {} }]; - - if (!response.ok) { - // An upstream error body is JSON like any other body. Reading it here is also what - // leaves a losing attempt with nothing open behind it. - return { payload: await refusal(response), streamedUsage: null, billable: called, body: spentBody(response.body) }; - } - - if (!wantsStream) { - const read = await readResult(response); - return { - payload: isFailure(read) ? read : read.value, - streamedUsage: null, - billable: isFailure(read) ? called : [{ identity, quantities: billed(read.value.usage, read.value.service_tier, candidate) }], - body: spentBody(response.body), - }; - } - - if (response.body === null) { - return { - payload: { status: 502, message: 'Upstream returned a streaming response with no body.' }, - streamedUsage: null, - billable: called, - body: spentBody(null), - }; - } - - // Usage is observed here, closest to the upstream and on the protocol it spoke, by folding - // the frames as they pass — so the reading costs one pass and the client's own stream is - // what drives it. What it finds arrives with the last frame, long after this stage has - // handed up, which is why the entity above carries no quantities. - const metered = meterFrames(parseOpenAICompletionsStream(response.body, { signal }), identity, candidate); - return { - payload: recordStream(metered.frames, dump), - streamedUsage: metered.outcome, - billable: called, - // Releasing this body is draining those frames: they are one reader over one connection, - // and a second reader is not something a `ReadableStream` allows. - body: own(response.body, async (): Promise => { for await (const _frame of metered.frames) { /* to end of stream */ } }), - }; -}; - -/** A body this stage has already read to the end, or one the upstream never sent. The record - * holds a body as a stream and `failover` releases the losing attempts', so every path hands - * one up; what says an answer was unusable is the failure at the payload key, not this. */ -const spentBody = (body: ReadableStream | null): ReadableStream & Owned => - own(body ?? new ReadableStream({ start: controller => controller.close() }), (): Promise => Promise.resolve()); - -const refusal = async (response: Response): Promise => { - const text = await response.text(); - try { - return { status: response.status, message: text, body: JSON.parse(text) as unknown }; - } catch { - // A refusal that is not JSON is still a refusal, and its text is what the client is - // told; there is simply no parsed body for a dump reader to open. - return { status: response.status, message: text }; - } -}; - -/** A body that answered 200 and cannot be read as this protocol is an attempt that failed, - * because the gateway serializes what it sends from the value it parsed and has nothing to - * serialize. The next candidate may well answer in the protocol it advertised. */ -const readResult = async (response: Response): Promise<{ readonly value: OpenAICompletionsResult } | Failure> => { - try { - return { value: parseOpenAICompletionsResult(await response.json()) }; - } catch (error) { - return { status: 502, message: `Upstream answered ${response.status} with a body this endpoint cannot read: ${error instanceof Error ? error.message : String(error)}` }; - } -}; - -interface MeteredFrames { - readonly frames: OpenAICompletionsFrames; - readonly outcome: Deferred; -} - -const meterFrames = ( - source: AsyncIterable>, - identity: TelemetryModelIdentity, - candidate: ModelCandidate, -): MeteredFrames => { - let settle!: (outcome: StreamOutcome) => void; - // Declared as this run's own unfinished work, so the runner waits for it at teardown where - // it can see it rather than the reading being started and forgotten. - const outcome = defer(new Promise(resolve => { settle = resolve; })); - // Running out without the terminal frame is what "it did not finish" means, and it is known - // at the same moment the usage is. - let sawTerminal = false; - const frames = view((async function* () { - let usage: unknown; - let tier: string | null | undefined; - try { - for await (const frame of source) { - if (frame.type === 'event') { - // `service_tier` can ride on any chunk's root while the totals only land on the - // usage-only one, so the two are tracked apart and settled together. - const root = frame.event as { service_tier?: string | null; usage?: unknown }; - if (root.service_tier !== undefined) tier = root.service_tier; - if (isOpenAIUsageOnlyEventShape(frame.event)) usage = root.usage; - } - // The transport's own terminator is this protocol's end-of-turn. - if (frame.type === 'done') sawTerminal = true; - yield frame; - } - } finally { - // Reached however the frames ended — the terminal chunk, a client that stopped - // reading, or a broken upstream — because tokens the upstream already metered are - // billable whatever happened to the downstream half. - settle({ billable: [{ identity, quantities: billed(usage, tier, candidate) }], failed: !sawTerminal }); - } - })()); - return { frames, outcome }; -}; - -const billed = (usage: unknown, tier: string | null | undefined, candidate: ModelCandidate): UsageQuantities => { - const model = providerModelOf(candidate); - const tokens = tokenUsageFromOpenAICompletionsUsage( - usage, - tier, - model.enabledFlags.has('usage-exclusive-cached-tokens'), - `${candidate.provider.upstreamId}/${model.id}`, - ); - // An upstream that reported nothing leaves no quantities at all, which is a different - // statement from reporting zero. - // - // The service tier survives as far as `TokenUsage.tier` and no further: a billed entity is - // an identity and a bag of quantities, and the pricing selector the tier feeds has no seat - // there. It comes back when settlement does. - return tokens === null ? {} : tokenUsageQuantities(tokens); -}; - -/** A candidate that cannot serve this endpoint is not a candidate. The resolver's own filter - * is by kind, and a text-completion model is a chat-kind model, so what is left to say is - * whether the upstream exposes the endpoint at all. */ -const narrowing = { - kind: 'chat' as const, - reject: (candidate: ModelCandidate): string | null => - candidate.model.endpoints.openaiCompletions === undefined ? 'the upstream does not expose an OpenAI Completions endpoint' : null, - unsupported: (model: string) => `Model ${model} does not support the /completions endpoint.`, - refuse: (status: number, message: string): C<'response.openaiCompletions.payload' | 'response.openaiCompletions.streamedUsage'> => ({ - 'response.openaiCompletions.payload': { status, message }, - 'response.openaiCompletions.streamedUsage': null, - }), - refuses: ['response.openaiCompletions.payload', 'response.openaiCompletions.streamedUsage'] as const, -}; +import { compose, type Pipeline } from '@floway-dev/pipeline'; export const openaiCompletionsServePipeline: Pipeline< - C<'ingress.http.headers' | 'ingress.openaiCompletions.wantsStream' | 'ingress.openaiCompletions.wantsUsageChunk' | 'request.openaiCompletions.payload' | 'serve.model'>, - C<'response.openaiCompletions.rendered' | 'response.openaiCompletions.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> + Fields<'ingress.http.headers' | 'ingress.openaiCompletions.wantsStream' | 'ingress.openaiCompletions.wantsUsageChunk' | 'request.openaiCompletions.payload' | 'serve.model'>, + Fields<'response.openaiCompletions.rendered' | 'response.openaiCompletions.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> > = compose('openaiCompletionsServe', [ emitOpenAICompletions, writeSettlement( diff --git a/packages/gateway/src/data-plane/openai-completions/target.ts b/packages/gateway/src/data-plane/openai-completions/target.ts new file mode 100644 index 0000000000..d12b85364e --- /dev/null +++ b/packages/gateway/src/data-plane/openai-completions/target.ts @@ -0,0 +1,17 @@ +import type { Fields } from './facts.ts'; +import type { ModelCandidate } from '@floway-dev/provider'; + +/** A candidate that cannot serve this endpoint is not a candidate. The resolver's own filter + * is by kind, and a text-completion model is a chat-kind model, so what is left to say is + * whether the upstream exposes the endpoint at all. */ +export const narrowing = { + kind: 'chat' as const, + reject: (candidate: ModelCandidate): string | null => + candidate.model.endpoints.openaiCompletions === undefined ? 'the upstream does not expose an OpenAI Completions endpoint' : null, + unsupported: (model: string) => `Model ${model} does not support the /completions endpoint.`, + refuse: (status: number, message: string): Fields<'response.openaiCompletions.payload' | 'response.openaiCompletions.streamedUsage'> => ({ + 'response.openaiCompletions.payload': { status, message }, + 'response.openaiCompletions.streamedUsage': null, + }), + refuses: ['response.openaiCompletions.payload', 'response.openaiCompletions.streamedUsage'] as const, +}; diff --git a/packages/gateway/src/data-plane/openai-embeddings/call-upstream.ts b/packages/gateway/src/data-plane/openai-embeddings/call-upstream.ts new file mode 100644 index 0000000000..a23f438eec --- /dev/null +++ b/packages/gateway/src/data-plane/openai-embeddings/call-upstream.ts @@ -0,0 +1,99 @@ +import type { Fields } from './facts.ts'; +import type { UsageQuantities } from '../../repo/types.ts'; +import type { Failure } from '../pipeline/facts.ts'; +import type { GatewayServices } from '../pipeline/services.ts'; +import { dialFailure, readUpstreamBody, unreadableBody } from '../pipeline/upstream-body.ts'; +import { upstreamPerformanceContext, telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; +import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { upstreamErrorMessage, parseDecimalString } from '@floway-dev/protocols/common'; +import { serializeOpenAIEmbeddingsRequest, parseOpenAIEmbeddingsResponse, type CanonicalOpenAIEmbeddingsResponse, type CanonicalOpenAIEmbeddingsUsage } from '@floway-dev/protocols/openai-embeddings'; +import { providerModelOf } from '@floway-dev/provider'; + +/** + * The ending. It dials, reads the upstream's body, and provides the canonical answer and + * what the call is billable for. A failure is a value: a 429 here is what an earlier stage + * fails over, and even a 400 can be, because the next candidate's path and flags may differ. + */ +export const callOpenAIEmbeddingsUpstream = defineStage< + Fields<'request.openaiEmbeddings.canonical' | 'route.attempt' | 'ingress.http.headers' | 'serve.model'>, + Fields<'response.openaiEmbeddings.canonical' | 'response.http.headers' | 'response.usage.billable'>, + GatewayServices +>({ + name: 'callOpenAIEmbeddingsUpstream', + return: { + provides: ['response.openaiEmbeddings.canonical', 'response.http.headers', 'response.usage.billable'], + }, + execute: async (facts, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + // Attribution is set before the dial, so an attempt that never completes still names the + // candidate it was made against rather than the one tried before it. + use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'embeddings'); + + let result; + try { + result = await candidate.provider.instance.callOpenAIEmbeddings( + providerModelOf(candidate), + serializeOpenAIEmbeddingsRequest(facts['request.openaiEmbeddings.canonical']), + use.gateway.abortSignal, + // The client's own headers reach the upstream from the record, not from a live + // request object: what a provider is allowed to forward is filtered per provider, + // and the dump shows what was there to filter. + buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), + ); + } catch (error) { + use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); + // A dial that never completed reached no upstream, so nothing was billed and there are + // no headers to carry. What it leaves behind is the performance row settlement writes. + return move({ + ...facts, + 'response.openaiEmbeddings.canonical': dialFailure(error), + 'response.http.headers': [], + 'response.usage.billable': [], + }); + } + + const identity = telemetryModelIdentity(candidate, result.modelKey); + // What came back, unfiltered: the edge is where a client's view of it is decided. + const headers = [...result.response.headers]; + const body = await readUpstreamBody(result.response); + // The upstream was called and reported nothing, which is a different situation from + // reporting zero — so the entity is present with no quantities. + const answered = (canonical: CanonicalOpenAIEmbeddingsResponse | Failure, quantities: UsageQuantities) => move({ + ...facts, + 'response.openaiEmbeddings.canonical': canonical, + 'response.http.headers': headers, + 'response.usage.billable': [{ identity, quantities }], + }); + const reportedNothing: UsageQuantities = {}; + + if (!result.response.ok) { + use.log.warn('upstream refused', { status: result.response.status }); + return answered({ + status: result.response.status, + message: upstreamErrorMessage(body.json) ?? body.text, + ...('json' in body ? { body: body.json } : {}), + }, reportedNothing); + } + if (!('json' in body)) { + return answered(unreadableBody(result.response, body, 'the OpenAI Embeddings protocol'), reportedNothing); + } + + // Every protocol the gateway carries is one it fully understands: the body is parsed + // here and written again at the edge, in whichever encoding the client can read. + let canonical: CanonicalOpenAIEmbeddingsResponse; + try { + canonical = parseOpenAIEmbeddingsResponse(body.json, facts['serve.model']); + } catch (error) { + use.log.warn('upstream answered with a body the OpenAI Embeddings protocol cannot read', { error: String(error) }); + return answered(unreadableBody(result.response, body, 'the OpenAI Embeddings protocol'), reportedNothing); + } + return answered(canonical, billed(canonical.usage)); + }, +}); + +// An OpenAI Embeddings call has no output side, so `prompt_tokens` is the whole of what the +// upstream metered and `total_tokens` restates it. An upstream that reported nothing bills +// nothing, and says so by leaving the entity's quantities empty. +const billed = (usage: CanonicalOpenAIEmbeddingsUsage | undefined): UsageQuantities => + usage === undefined ? {} : { input_tokens: parseDecimalString(String(usage.promptTokens)) }; diff --git a/packages/gateway/src/data-plane/openai-embeddings/emit.ts b/packages/gateway/src/data-plane/openai-embeddings/emit.ts new file mode 100644 index 0000000000..0671161d47 --- /dev/null +++ b/packages/gateway/src/data-plane/openai-embeddings/emit.ts @@ -0,0 +1,59 @@ +import type { Fields } from './facts.ts'; +import { isFailure } from '../pipeline/facts.ts'; +import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { renderErrorEnvelope } from '@floway-dev/protocols/common'; +import { renderOpenAIEmbeddingsResponse } from '@floway-dev/protocols/openai-embeddings'; + +/** + * The outermost edge. Writes the vectors in the encoding the client asked for — which is + * why the encoding is an ingress fact and not a request one: it has to survive an upstream + * that answered in the other one. A client on an official OpenAI SDK asked for `base64` + * without its caller choosing to, and will decode as base64 whatever it is handed. + */ +export const emitOpenAIEmbeddings = defineStage< + Fields<'ingress.openaiEmbeddings.encodingFormat'>, + Fields<'ingress.openaiEmbeddings.encodingFormat'>, + Fields<'ingress.openaiEmbeddings.encodingFormat' | 'response.openaiEmbeddings.canonical' | 'response.http.headers'>, + Fields<'response.openaiEmbeddings.rendered' | 'response.http.status' | 'response.http.headers'> +>({ + name: 'emitOpenAIEmbeddings', + through: { + request: { + needs: ['ingress.openaiEmbeddings.encodingFormat'], + consumes: [], + provides: [], + }, + response: { + needs: ['response.openaiEmbeddings.canonical', 'response.http.headers'], + consumes: ['response.openaiEmbeddings.canonical', 'response.http.headers'], + provides: ['response.openaiEmbeddings.rendered', 'response.http.status', 'response.http.headers'], + }, + }, + execute: async (facts, next) => { + const back = await next(facts); + const { 'response.openaiEmbeddings.canonical': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); + if (isFailure(answer)) { + return { + ...rest, + 'response.http.headers': forClient, + 'response.openaiEmbeddings.rendered': move(renderErrorEnvelope(answer.message, answer.body)), + // The upstream's own status, or the gateway's own when it refused before dialling. + // A client is not owed the upstream's exact bytes; it is owed the truth about what + // happened, and a 429 arriving as a 200 is not that. + 'response.http.status': answer.status, + }; + } + return { + ...rest, + 'response.http.headers': forClient, + 'response.http.status': 200, + 'response.openaiEmbeddings.rendered': move(renderOpenAIEmbeddingsResponse(back['ingress.openaiEmbeddings.encodingFormat'], answer)), + }; + }, +}); diff --git a/packages/gateway/src/data-plane/openai-embeddings/facts.ts b/packages/gateway/src/data-plane/openai-embeddings/facts.ts new file mode 100644 index 0000000000..382dc5b6c3 --- /dev/null +++ b/packages/gateway/src/data-plane/openai-embeddings/facts.ts @@ -0,0 +1,18 @@ +import type { GatewayFacts, Failure } from '../pipeline/facts.ts'; +import type { OpenAIEmbeddingsEncodingFormat, CanonicalOpenAIEmbeddingsRequest, CanonicalOpenAIEmbeddingsResponse } from '@floway-dev/protocols/openai-embeddings'; + +/** OpenAI Embeddings' own keys. They extend the shared space and never merge into it, so a stage + * written against the gateway alone cannot name one. */ +export interface OpenAIEmbeddingsFacts extends GatewayFacts { + /** Which encoding the client is able to read. It belongs to the ingress and stays put: + * the answer is written in it whichever encoding the upstream chose to answer in, and + * the two differ whenever an upstream ignores the field. */ + 'ingress.openaiEmbeddings.encodingFormat': OpenAIEmbeddingsEncodingFormat; + 'request.openaiEmbeddings.canonical': CanonicalOpenAIEmbeddingsRequest; + 'response.openaiEmbeddings.canonical': CanonicalOpenAIEmbeddingsResponse | Failure; + /** What the client is actually sent. The edge provides it, so a dump shows the bytes the + * client received rather than the gateway's canonical form. */ + 'response.openaiEmbeddings.rendered': Record; +} + +export type Fields = { [P in K]: OpenAIEmbeddingsFacts[P] }; diff --git a/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts b/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts index 3ed13760b5..7ee6f970fa 100644 --- a/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts @@ -1,212 +1,16 @@ -// OpenAI Embeddings as a pipeline. The smallest family there is: one protocol, no translation -// and no stream — so what is left is the four stages every family has, and nothing else. -// -// emitOpenAIEmbeddings the edge: writes the answer in the encoding the client asked -// for -// resolveCandidates narrows to the upstreams that can serve this request -// failover runs what follows once per candidate -// callOpenAIEmbeddingsUpstream the ending: dials, and provides what came back -// -// The narrowing is a constant rather than a function of the request, and that is what makes -// this the simple family: an OpenAI Embeddings request carries nothing a candidate could be -// incompatible with, so the only question is whether the upstream exposes the endpoint. - -import type { UsageQuantities } from '../../repo/types.ts'; -import type { Failure, GatewayFacts } from '../pipeline/facts.ts'; +import { callOpenAIEmbeddingsUpstream } from './call-upstream.ts'; +import { emitOpenAIEmbeddings } from './emit.ts'; +import type { Fields } from './facts.ts'; +import { narrowing } from './target.ts'; import { isFailure } from '../pipeline/facts.ts'; -import type { GatewayServices } from '../pipeline/services.ts'; +import { failover } from '../pipeline/failover.ts'; +import { resolveCandidates } from '../pipeline/resolve-candidates.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; -import { failover, resolveCandidates, type Narrowing } from '../pipeline/stages.ts'; -import { dialFailure, readUpstreamBody, unreadableBody } from '../pipeline/upstream-body.ts'; -import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; -import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; -import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; -import type { Pipeline } from '@floway-dev/pipeline'; -import { compose, defineStage, move } from '@floway-dev/pipeline'; -import { parseDecimalString, renderErrorEnvelope, upstreamErrorMessage } from '@floway-dev/protocols/common'; -import { - parseOpenAIEmbeddingsResponse, - renderOpenAIEmbeddingsResponse, - serializeOpenAIEmbeddingsRequest, - type CanonicalOpenAIEmbeddingsRequest, - type CanonicalOpenAIEmbeddingsResponse, - type CanonicalOpenAIEmbeddingsUsage, - type OpenAIEmbeddingsEncodingFormat, -} from '@floway-dev/protocols/openai-embeddings'; -import { providerModelOf } from '@floway-dev/provider'; - -/** OpenAI Embeddings' own keys. They extend the shared space and never merge into it, so a stage - * written against the gateway alone cannot name one. */ -export interface OpenAIEmbeddingsFacts extends GatewayFacts { - /** Which encoding the client is able to read. It belongs to the ingress and stays put: - * the answer is written in it whichever encoding the upstream chose to answer in, and - * the two differ whenever an upstream ignores the field. */ - 'ingress.openaiEmbeddings.encodingFormat': OpenAIEmbeddingsEncodingFormat; - 'request.openaiEmbeddings.canonical': CanonicalOpenAIEmbeddingsRequest; - 'response.openaiEmbeddings.canonical': CanonicalOpenAIEmbeddingsResponse | Failure; - /** What the client is actually sent. The edge provides it, so a dump shows the bytes the - * client received rather than the gateway's canonical form. */ - 'response.openaiEmbeddings.rendered': Record; -} - -type E = { [P in K]: OpenAIEmbeddingsFacts[P] }; - -/** - * The outermost edge. Writes the vectors in the encoding the client asked for — which is - * why the encoding is an ingress fact and not a request one: it has to survive an upstream - * that answered in the other one. A client on an official OpenAI SDK asked for `base64` - * without its caller choosing to, and will decode as base64 whatever it is handed. - */ -const emitOpenAIEmbeddings = defineStage< - E<'ingress.openaiEmbeddings.encodingFormat'>, - E<'ingress.openaiEmbeddings.encodingFormat'>, - E<'ingress.openaiEmbeddings.encodingFormat' | 'response.openaiEmbeddings.canonical' | 'response.http.headers'>, - E<'response.openaiEmbeddings.rendered' | 'response.http.status' | 'response.http.headers'> ->({ - name: 'emitOpenAIEmbeddings', - through: { - request: { - needs: ['ingress.openaiEmbeddings.encodingFormat'], - consumes: [], - provides: [], - }, - response: { - needs: ['response.openaiEmbeddings.canonical', 'response.http.headers'], - consumes: ['response.openaiEmbeddings.canonical', 'response.http.headers'], - provides: ['response.openaiEmbeddings.rendered', 'response.http.status', 'response.http.headers'], - }, - }, - execute: async (facts, next) => { - const back = await next(facts); - const { 'response.openaiEmbeddings.canonical': answer, 'response.http.headers': headers, ...rest } = back; - // Vendor traces and quota state stay visible; what an intermediary must strip, and what - // would misdescribe a body this gateway serialized itself, does not. A filter that removed - // nothing hands the same array on, so the record shows no change where none happened. - const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); - const forClient = forwardable.length === headers.length ? headers : move(forwardable); - if (isFailure(answer)) { - return { - ...rest, - 'response.http.headers': forClient, - 'response.openaiEmbeddings.rendered': move(renderErrorEnvelope(answer.message, answer.body)), - // The upstream's own status, or the gateway's own when it refused before dialling. - // A client is not owed the upstream's exact bytes; it is owed the truth about what - // happened, and a 429 arriving as a 200 is not that. - 'response.http.status': answer.status, - }; - } - return { - ...rest, - 'response.http.headers': forClient, - 'response.http.status': 200, - 'response.openaiEmbeddings.rendered': move(renderOpenAIEmbeddingsResponse(back['ingress.openaiEmbeddings.encodingFormat'], answer)), - }; - }, -}); - -/** - * The ending. It dials, reads the upstream's body, and provides the canonical answer and - * what the call is billable for. A failure is a value: a 429 here is what an earlier stage - * fails over, and even a 400 can be, because the next candidate's path and flags may differ. - */ -const callOpenAIEmbeddingsUpstream = defineStage< - E<'request.openaiEmbeddings.canonical' | 'route.attempt' | 'ingress.http.headers' | 'serve.model'>, - E<'response.openaiEmbeddings.canonical' | 'response.http.headers' | 'response.usage.billable'>, - GatewayServices ->({ - name: 'callOpenAIEmbeddingsUpstream', - return: { - provides: ['response.openaiEmbeddings.canonical', 'response.http.headers', 'response.usage.billable'], - }, - execute: async (facts, use) => { - const candidate = use.resolveAttempt(facts['route.attempt']); - // Attribution is set before the dial, so an attempt that never completes still names the - // candidate it was made against rather than the one tried before it. - use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'embeddings'); - - let result; - try { - result = await candidate.provider.instance.callOpenAIEmbeddings( - providerModelOf(candidate), - serializeOpenAIEmbeddingsRequest(facts['request.openaiEmbeddings.canonical']), - use.gateway.abortSignal, - // The client's own headers reach the upstream from the record, not from a live - // request object: what a provider is allowed to forward is filtered per provider, - // and the dump shows what was there to filter. - buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), - ); - } catch (error) { - use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); - // A dial that never completed reached no upstream, so nothing was billed and there are - // no headers to carry. What it leaves behind is the performance row settlement writes. - return move({ - ...facts, - 'response.openaiEmbeddings.canonical': dialFailure(error), - 'response.http.headers': [], - 'response.usage.billable': [], - }); - } - - const identity = telemetryModelIdentity(candidate, result.modelKey); - // What came back, unfiltered: the edge is where a client's view of it is decided. - const headers = [...result.response.headers]; - const body = await readUpstreamBody(result.response); - // The upstream was called and reported nothing, which is a different situation from - // reporting zero — so the entity is present with no quantities. - const answered = (canonical: CanonicalOpenAIEmbeddingsResponse | Failure, quantities: UsageQuantities) => move({ - ...facts, - 'response.openaiEmbeddings.canonical': canonical, - 'response.http.headers': headers, - 'response.usage.billable': [{ identity, quantities }], - }); - const reportedNothing: UsageQuantities = {}; - - if (!result.response.ok) { - use.log.warn('upstream refused', { status: result.response.status }); - return answered({ - status: result.response.status, - message: upstreamErrorMessage(body.json) ?? body.text, - ...('json' in body ? { body: body.json } : {}), - }, reportedNothing); - } - if (!('json' in body)) { - return answered(unreadableBody(result.response, body, 'the OpenAI Embeddings protocol'), reportedNothing); - } - - // Every protocol the gateway carries is one it fully understands: the body is parsed - // here and written again at the edge, in whichever encoding the client can read. - let canonical: CanonicalOpenAIEmbeddingsResponse; - try { - canonical = parseOpenAIEmbeddingsResponse(body.json, facts['serve.model']); - } catch (error) { - use.log.warn('upstream answered with a body the OpenAI Embeddings protocol cannot read', { error: String(error) }); - return answered(unreadableBody(result.response, body, 'the OpenAI Embeddings protocol'), reportedNothing); - } - return answered(canonical, billed(canonical.usage)); - }, -}); - -// An OpenAI Embeddings call has no output side, so `prompt_tokens` is the whole of what the -// upstream metered and `total_tokens` restates it. An upstream that reported nothing bills -// nothing, and says so by leaving the entity's quantities empty. -const billed = (usage: CanonicalOpenAIEmbeddingsUsage | undefined): UsageQuantities => - usage === undefined ? {} : { input_tokens: parseDecimalString(String(usage.promptTokens)) }; - -/** A candidate that cannot serve *this* request is not a candidate. Saying why is what - * turns an empty list into a 400 a client can act on. */ -const narrowing: Narrowing> = { - kind: 'embedding', - reject: candidate => candidate.model.endpoints.openaiEmbeddings === undefined - ? 'the upstream does not expose an OpenAI Embeddings endpoint' - : null, - unsupported: model => `Model ${model} does not support the /embeddings endpoint.`, - refuse: (status, message) => ({ 'response.openaiEmbeddings.canonical': { status, message } }), - refuses: ['response.openaiEmbeddings.canonical'], -}; +import { compose, type Pipeline } from '@floway-dev/pipeline'; export const openaiEmbeddingsServePipeline: Pipeline< - E<'ingress.http.headers' | 'ingress.openaiEmbeddings.encodingFormat' | 'request.openaiEmbeddings.canonical' | 'serve.model'>, - E<'response.openaiEmbeddings.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> + Fields<'ingress.http.headers' | 'ingress.openaiEmbeddings.encodingFormat' | 'request.openaiEmbeddings.canonical' | 'serve.model'>, + Fields<'response.openaiEmbeddings.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> > = compose('openaiEmbeddingsServe', [ emitOpenAIEmbeddings, writeSettlement(handedUp => isFailure((handedUp as { 'response.openaiEmbeddings.canonical'?: unknown })['response.openaiEmbeddings.canonical'])), diff --git a/packages/gateway/src/data-plane/openai-embeddings/target.ts b/packages/gateway/src/data-plane/openai-embeddings/target.ts new file mode 100644 index 0000000000..d592f5b111 --- /dev/null +++ b/packages/gateway/src/data-plane/openai-embeddings/target.ts @@ -0,0 +1,14 @@ +import type { Fields } from './facts.ts'; +import type { Narrowing } from '../pipeline/resolve-candidates.ts'; + +/** A candidate that cannot serve *this* request is not a candidate. Saying why is what + * turns an empty list into a 400 a client can act on. */ +export const narrowing: Narrowing> = { + kind: 'embedding', + reject: candidate => candidate.model.endpoints.openaiEmbeddings === undefined + ? 'the upstream does not expose an OpenAI Embeddings endpoint' + : null, + unsupported: model => `Model ${model} does not support the /embeddings endpoint.`, + refuse: (status, message) => ({ 'response.openaiEmbeddings.canonical': { status, message } }), + refuses: ['response.openaiEmbeddings.canonical'], +}; diff --git a/packages/gateway/src/data-plane/openai-images/call-upstream.ts b/packages/gateway/src/data-plane/openai-images/call-upstream.ts new file mode 100644 index 0000000000..2c7f3df8de --- /dev/null +++ b/packages/gateway/src/data-plane/openai-images/call-upstream.ts @@ -0,0 +1,251 @@ +import type { Fields, OpenAIImagesFrames } from './facts.ts'; +import { recordStream } from '../../dump/turn-dump.ts'; +import type { UsageQuantities } from '../../repo/types.ts'; +import type { BillableEntity, Failure } from '../pipeline/facts.ts'; +import type { StreamOutcome } from '../pipeline/serve.ts'; +import type { GatewayServices } from '../pipeline/services.ts'; +import { dialFailure, readUpstreamBody } from '../pipeline/upstream-body.ts'; +import { upstreamPerformanceContext, telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; +import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; +import { defineStage, move, own, defer, type Owned, type Deferred } from '@floway-dev/pipeline'; +import { upstreamErrorMessage, isEventStreamMediaType, eventFrame, mediaTypeEssence, parseDecimalString } from '@floway-dev/protocols/common'; +import { parseOpenAIImagesResponse, parseOpenAIImagesStream, isOpenAIImagesTerminalEvent, parseOpenAIImagesUsage, OPENAI_IMAGES_MISSING_TERMINAL_MESSAGE, type OpenAIImagesOperation, type CanonicalOpenAIImagesResponse, type CanonicalOpenAIImagesUsage, type CanonicalOpenAIImagesEditsRequest, type OpenAIImagesEditImage, type OpenAIImageEditReference } from '@floway-dev/protocols/openai-images'; +import { providerModelOf, isBase64ImageDataUrl, type PerformanceOperation, type TelemetryModelIdentity, type OpenAIImagesEditsRequest, type OpenAIImagesEditsSource } from '@floway-dev/provider'; + +/** A stream as a value the record can hold, as a wrapper around the generator rather than the + * generator itself. What the wrapper says is where the resource is: the one resource in an + * OpenAI Images run is the upstream's body at `response.http.body`, claimed with `own()`, and + * this keeps a frame view from reading as another. */ +const view = (frames: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => frames }); + +const PERFORMANCE_OPERATION = { + generations: 'image_generation', + edits: 'image_edit', +} as const satisfies Record; + +/** + * The ending. It dials, reads what came back on the shape the request asked for, and provides + * the canonical answer, the headers that came with it, the raw HTTP body beneath it, and what + * the call is billable for. A failure is a value: an upstream that refused, and one that + * answered with something this protocol cannot read, are both outcomes the fork above can take + * to the next candidate rather than faults that end the run. + */ +export const callOpenAIImagesUpstream = defineStage< + Fields<'ingress.openaiImages.wantsStream' | 'request.openaiImages.canonical' | 'route.attempt' | 'ingress.http.headers'>, + Fields<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.headers' + | 'response.http.body' | 'response.usage.billable'>, + GatewayServices +>({ + name: 'callOpenAIImagesUpstream', + return: { + provides: [ + 'response.openaiImages.canonical', + 'response.openaiImages.streamedUsage', + 'response.http.headers', + 'response.http.body', + 'response.usage.billable', + ], + }, + execute: async (facts, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + const request = facts['request.openaiImages.canonical']; + const options = buildUpstreamCallOptions( + candidate, + use.gateway, + // The client's own headers reach the upstream from the record, not from a live request + // object: what a provider is allowed to forward is filtered per provider, and the dump + // shows what was there to filter. + new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value])), + ); + const model = providerModelOf(candidate); + // Attribution is set before the dial, so an attempt that never completes still names the + // candidate it was made against rather than the one tried before it. + use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, PERFORMANCE_OPERATION[request.operation]); + + let result; + try { + // No abort signal, as on this family's endpoints from the beginning: an image the upstream + // has already begun is charged for whether or not the client waited for it, so dropping the + // call would lose the usage reading and save nothing. Reading a stream is the other half of + // that and does take the signal — what it drops there is a connection, not an image. + result = request.operation === 'generations' + ? await candidate.provider.instance.callOpenAIImagesGenerations(model, request.parameters, undefined, options) + : await candidate.provider.instance.callOpenAIImagesEdits(model, providerEditsRequest(request), undefined, options); + } catch (error) { + use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); + // A dial that never completed reached no upstream, so nothing was billed and there are + // no headers to carry. What it leaves behind is the performance row settlement writes. + return move({ + ...facts, + 'response.openaiImages.canonical': dialFailure(error), + 'response.openaiImages.streamedUsage': null, + 'response.http.headers': [], + 'response.http.body': spentBody(null), + 'response.usage.billable': [], + }); + } + + const identity = telemetryModelIdentity(candidate, result.modelKey); + // What came back, unfiltered: the edge is where a client's view of it is decided. + const headers = [...result.response.headers]; + // An entity with no quantities is how "the upstream was called and reported nothing" is + // said, which is a different situation from reporting zero. + const called: readonly BillableEntity[] = [{ identity, quantities: {} }]; + const read = (canonical: CanonicalOpenAIImagesResponse | Failure, billable: readonly BillableEntity[]) => move({ + ...facts, + 'response.openaiImages.canonical': canonical, + 'response.openaiImages.streamedUsage': null, + 'response.http.headers': headers, + 'response.http.body': spentBody(result.response.body), + 'response.usage.billable': billable, + }); + + if (!result.response.ok) { + use.log.warn('upstream refused', { status: result.response.status }); + // An upstream error body is JSON like any other body. Reading it here is also what + // leaves a losing attempt with nothing open behind it. + const body = await readUpstreamBody(result.response); + return read({ + status: result.response.status, + message: upstreamErrorMessage(body.json) ?? body.text, + ...('json' in body ? { body: body.json } : {}), + }, called); + } + + const mediaType = result.response.headers.get('content-type'); + // Both halves have to hold. `stream` is what the client asked for, and an upstream that + // ignores it answers the single JSON body the arm below reads — so what settles which + // shape arrived is the media type, and what settles which shape the client is owed is the + // request. + if (facts['ingress.openaiImages.wantsStream'] && isEventStreamMediaType(mediaType)) { + if (result.response.body === null) { + return read({ status: 502, message: 'Upstream returned a streaming response with no body.' }, called); + } + // Usage is observed here, closest to the upstream and on the protocol it spoke, by + // folding the events as they pass — so the reading costs one pass and the client's own + // stream is what drives it. What it finds arrives with the completed event, long after + // this stage has handed up, which is why the entity above carries no quantities. + const metered = meterFrames(result.response.body, identity, use.gateway.abortSignal); + return move({ + ...facts, + // This protocol's stream is bare events rather than protocol frames, so the record is + // told how one becomes a frame instead of being left to assume. + 'response.openaiImages.canonical': recordStream(metered.frames, use.gateway.dump, eventFrame), + 'response.openaiImages.streamedUsage': metered.outcome, + 'response.http.headers': headers, + // Releasing this body is reading those events to the end: they are one reader over one + // connection, and a second reader is not something a `ReadableStream` allows. + 'response.http.body': own(result.response.body, async (): Promise => { for await (const _event of metered.frames) { /* to end of stream */ } }), + 'response.usage.billable': called, + }); + } + + const body = await readUpstreamBody(result.response); + if (!('json' in body)) { + // Every protocol the gateway carries is one it fully understands, so a body it cannot + // read is not handed on unread. + const essence = mediaTypeEssence(mediaType) ?? 'no media type'; + use.log.warn('upstream answered with a body that is not JSON', { status: result.response.status, mediaType: essence }); + return read({ + status: 502, + message: `The upstream answered ${result.response.status} with ${essence}, and the OpenAI Images protocol is JSON.`, + body: body.text, + }, called); + } + + let canonical: CanonicalOpenAIImagesResponse; + try { + canonical = parseOpenAIImagesResponse(body.json); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + use.log.warn('upstream answered with a body the OpenAI Images protocol cannot read', { message }); + return read({ status: 502, message, body: body.json }, called); + } + return read(canonical, [{ identity, quantities: billed(canonical.usage) }]); + }, +}); + +/** A body this stage has already read to the end, or one the upstream never sent. The record + * holds a body as a stream and `failover` releases the losing attempts', so every path hands + * one up; what says an answer was unusable is the failure at the canonical key, not this. */ +const spentBody = (body: ReadableStream | null): ReadableStream & Owned => + own(body ?? new ReadableStream({ start: controller => controller.close() }), (): Promise => Promise.resolve()); + +interface MeteredFrames { + readonly frames: OpenAIImagesFrames; + readonly outcome: Deferred; +} + +const meterFrames = ( + body: ReadableStream, + identity: TelemetryModelIdentity, + signal: AbortSignal | undefined, +): MeteredFrames => { + let settle!: (outcome: StreamOutcome) => void; + // Declared as this run's own unfinished work, so the runner waits for it at teardown where + // it can see it rather than the reading being started and forgotten. + const outcome = defer(new Promise(resolve => { settle = resolve; })); + // Running out without the completed event is what "it did not finish" means, and it is known + // at the same moment the usage is. + let sawTerminal = false; + const frames = view((async function* () { + let usage: CanonicalOpenAIImagesUsage | undefined; + try { + for await (const event of parseOpenAIImagesStream(body, { signal })) { + if (isOpenAIImagesTerminalEvent(event)) { + usage = parseOpenAIImagesUsage(event); + sawTerminal = true; + yield event; + // The image is complete, so there is nothing further to read. An upstream that holds + // the connection open past this point would otherwise keep the client's own stream + // open with it; returning here closes the read, which cancels the upstream. + return; + } + yield event; + } + } finally { + // Reached however the events ended — the completed one, a client that stopped reading, + // or a broken upstream — because what the upstream already metered is billable whatever + // happened to the downstream half. + settle({ billable: [{ identity, quantities: billed(usage) }], failed: !sawTerminal }); + } + // Only an upstream that ended its body without ever completing the image reaches here: the + // arm above returns on the terminal event. A client has been sent partial images and no + // image, which is a failed answer however far it got. + throw new Error(OPENAI_IMAGES_MISSING_TERMINAL_MESSAGE); + })()); + return { frames, outcome }; +}; + +/** An image is billed by the tokens its upstream reports: `BILLING_METRICS` names no per-image + * or per-size unit, and per-size pricing is a selector coordinate rather than a metric, so + * there is nothing else here to record a count against. A reported zero is kept — it says the + * upstream reported, which an absent metric would not. */ +const billed = (usage: CanonicalOpenAIImagesUsage | undefined): UsageQuantities => { + const quantities: UsageQuantities = {}; + if (usage?.inputTokens !== undefined) quantities.input_tokens = parseDecimalString(String(usage.inputTokens)); + if (usage?.inputImageTokens !== undefined) quantities.input_image_tokens = parseDecimalString(String(usage.inputImageTokens)); + if (usage?.outputTokens !== undefined) quantities.output_tokens = parseDecimalString(String(usage.outputTokens)); + if (usage?.outputImageTokens !== undefined) quantities.output_image_tokens = parseDecimalString(String(usage.outputImageTokens)); + return quantities; +}; + +/** The provider's own shape, built where the dial happens. What it needs from a reference is + * whether the data URL inside it can become a file, because that decides whether the edit can + * ride as a multipart form; the canonical fact holds the reference the client wrote and leaves + * that question to the serializer that has it. */ +const providerEditsRequest = (request: CanonicalOpenAIImagesEditsRequest): OpenAIImagesEditsRequest => ({ + images: request.images.map(providerEditsSource), + ...(request.mask === undefined ? {} : { mask: providerEditsSource(request.mask) }), + parameters: request.parameters, +}); + +const providerEditsSource = (image: OpenAIImagesEditImage): OpenAIImagesEditsSource => { + if (image.kind === 'file') { + return { type: 'upload', file: new File([image.file.bytes], image.file.fileName, { type: image.file.mediaType }) }; + } + const { reference } = image; + return typeof reference.image_url === 'string' && isBase64ImageDataUrl(reference.image_url) + ? { type: 'inline', reference: reference as OpenAIImageEditReference & { image_url: string } } + : { type: 'reference', reference }; +}; diff --git a/packages/gateway/src/data-plane/openai-images/emit.ts b/packages/gateway/src/data-plane/openai-images/emit.ts new file mode 100644 index 0000000000..9cee0e6d4d --- /dev/null +++ b/packages/gateway/src/data-plane/openai-images/emit.ts @@ -0,0 +1,65 @@ +import type { OpenAIImagesFrames, Fields, OpenAIImagesFacts } from './facts.ts'; +import { streamReferenceOf } from '../../dump/turn-dump.ts'; +import { isFailure } from '../pipeline/facts.ts'; +import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { renderErrorEnvelope, type SseFrame } from '@floway-dev/protocols/common'; +import { renderOpenAIImagesResponse, openaiImagesStreamEventToSSEFrame, type CanonicalOpenAIImagesResponse } from '@floway-dev/protocols/openai-images'; + +const isFrames = (answer: CanonicalOpenAIImagesResponse | OpenAIImagesFrames): answer is OpenAIImagesFrames => + Symbol.asyncIterator in answer; + +/** + * The outermost edge. It names nothing on the way down — the family has one protocol, so there + * is no request to read to know how to answer — and coming back it renders the answer, keeps + * the upstream headers a client may see, and decides the status. The status is decided here + * rather than carried up because a refusal that never reached an upstream has none to carry. + * + * SSE framing is produced here and nowhere else: below this stage a stream carries protocol + * events, so the same assembly would serve another transport by rendering differently at this + * one point. + */ +export const emitOpenAIImages = defineStage< + Record, + Record, + Fields<'response.openaiImages.canonical' | 'response.http.headers'>, + Fields<'response.openaiImages.rendered' | 'response.http.status' | 'response.http.headers'> +>({ + name: 'emitOpenAIImages', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { + needs: ['response.openaiImages.canonical', 'response.http.headers'], + consumes: ['response.openaiImages.canonical', 'response.http.headers'], + provides: ['response.openaiImages.rendered', 'response.http.status', 'response.http.headers'], + }, + }, + execute: async (facts, next) => { + const back = await next(facts); + const { 'response.openaiImages.canonical': answer, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + return { + ...rest, + 'response.http.headers': forwardable.length === headers.length ? headers : move(forwardable), + 'response.http.status': isFailure(answer) ? answer.status : 200, + 'response.openaiImages.rendered': move(rendered(answer)), + }; + }, +}); + +const rendered = (answer: OpenAIImagesFacts['response.openaiImages.canonical']): OpenAIImagesFacts['response.openaiImages.rendered'] => + isFailure(answer) ? renderErrorEnvelope(answer.message, answer.body) + : isFrames(answer) ? renderSSE(answer) + : renderOpenAIImagesResponse(answer); + +const renderSSE = (frames: OpenAIImagesFrames): AsyncIterable => ({ + // The frames the client reads are a reframing of the ones the record holds, so this key + // points at that same stream rather than at nothing. + ...streamReferenceOf(frames), + [Symbol.asyncIterator]: () => (async function* () { + for await (const event of frames) yield openaiImagesStreamEventToSSEFrame(event); + })(), +}); diff --git a/packages/gateway/src/data-plane/openai-images/facts.ts b/packages/gateway/src/data-plane/openai-images/facts.ts new file mode 100644 index 0000000000..573a870002 --- /dev/null +++ b/packages/gateway/src/data-plane/openai-images/facts.ts @@ -0,0 +1,47 @@ +import type { GatewayFacts, Failure } from '../pipeline/facts.ts'; +import type { StreamOutcome } from '../pipeline/serve.ts'; +import type { Deferred } from '@floway-dev/pipeline'; +import type { SseFrame } from '@floway-dev/protocols/common'; +import type { OpenAIImagesStreamEvent, CanonicalOpenAIImagesRequest, CanonicalOpenAIImagesResponse } from '@floway-dev/protocols/openai-images'; + +/** The answer while it is still the upstream's, one event at a time. There is no transport + * sentinel below the events — this protocol ends at its completed event — so what travels is + * the events themselves rather than frames with a terminal arm the protocol has not got. */ +export type OpenAIImagesFrames = AsyncIterable; + +/** OpenAI Images' own keys. They extend the shared space and never merge into it, so a stage + * written against the gateway alone cannot name one. */ +export interface OpenAIImagesFacts extends GatewayFacts { + /** Whether the client asked for the answer as a stream. It stays put, as every `ingress.*` + * key does: the same flag travels to the upstream inside the parameters, and the answer is + * written back in the shape it asked for. */ + 'ingress.openaiImages.wantsStream': boolean; + 'request.openaiImages.canonical': CanonicalOpenAIImagesRequest; + /** The answer, whichever kind it turned out to be. A stream, a value and a failure sit at + * one key: telling them apart is reading a value, and each stage does that where it needs + * to. */ + 'response.openaiImages.canonical': CanonicalOpenAIImagesResponse | OpenAIImagesFrames | Failure; + /** What the upstream will have reported by the time the events run out, and `null` on every + * path that does not stream. A streamed image states its usage in the completed event, + * which is after this run has answered — so the numbers cannot be in + * `response.usage.billable`, which says what had been reported when the ending stage handed + * up: the entity, and no quantities. Settling from this is the epilogue's job, after the + * drain. */ + 'response.openaiImages.streamedUsage': Deferred | null; + /** What the client is actually sent, in the OpenAI Images protocol — a JSON body, or the SSE + * frames of a stream. The edge provides it, so a dump shows the body the client received + * rather than the gateway's canonical form. */ + 'response.openaiImages.rendered': Record | AsyncIterable; +} + +export type Fields = { [P in K]: OpenAIImagesFacts[P] }; + +/** What a caller must bring. `ingress.http.headers`, `ingress.openaiImages.wantsStream` and + * `request.openaiImages.canonical` are in it although `compose` cannot derive them — the ending + * stage reads all three and a return-only stage declares no request side — so this type is the + * whole statement and `entryNeeds` is part of it. */ +export type OpenAIImagesServeEntry = Fields<'ingress.http.headers' | 'ingress.openaiImages.wantsStream' | 'request.openaiImages.canonical' | 'serve.model'>; + +export type OpenAIImagesServeExit = Fields< + 'response.openaiImages.rendered' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable' +>; diff --git a/packages/gateway/src/data-plane/openai-images/pipeline.ts b/packages/gateway/src/data-plane/openai-images/pipeline.ts index 4151a93971..5bce6d5af8 100644 --- a/packages/gateway/src/data-plane/openai-images/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-images/pipeline.ts @@ -1,417 +1,13 @@ -// OpenAI Images as a pipeline. Two endpoints and one family: `generations` sends JSON and -// `edits` sends either JSON or a multipart form, so what the operation decides is the request -// fact and the provider method the ending calls, and the four stages are the same either way. -// Both endpoints accept `stream: true`, so the answer is a stream whenever the client asked for -// one — the same request field decides both what the upstream is asked for and what shape comes -// back. -// -// emitOpenAIImages the edge: serializes the answer into the OpenAI Images protocol, -// SSE framing included -// resolveCandidates narrows to the upstreams that expose this operation's endpoint -// failover runs what follows once per candidate -// callOpenAIImagesUpstream the ending: dials, and provides what came back - -import { recordStream, streamReferenceOf } from '../../dump/turn-dump.ts'; -import type { UsageQuantities } from '../../repo/types.ts'; -import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; +import { callOpenAIImagesUpstream } from './call-upstream.ts'; +import { emitOpenAIImages } from './emit.ts'; +import type { OpenAIImagesServeEntry, OpenAIImagesServeExit } from './facts.ts'; +import { narrowing } from './target.ts'; import { isFailure } from '../pipeline/facts.ts'; -import type { StreamOutcome } from '../pipeline/serve.ts'; -import type { GatewayServices } from '../pipeline/services.ts'; +import { failover } from '../pipeline/failover.ts'; +import { resolveCandidates } from '../pipeline/resolve-candidates.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; -import { failover, resolveCandidates } from '../pipeline/stages.ts'; -import { dialFailure, readUpstreamBody } from '../pipeline/upstream-body.ts'; -import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; -import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; -import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; -import { compose, defer, defineStage, move, own, type Deferred, type Owned, type Pipeline } from '@floway-dev/pipeline'; -import { - eventFrame, - isEventStreamMediaType, - mediaTypeEssence, - parseDecimalString, - renderErrorEnvelope, - upstreamErrorMessage, - type ModelEndpointKey, - type SseFrame, -} from '@floway-dev/protocols/common'; -import { - OPENAI_IMAGES_MISSING_TERMINAL_MESSAGE, - openaiImagesStreamEventToSSEFrame, - isOpenAIImagesTerminalEvent, - parseOpenAIImagesResponse, - parseOpenAIImagesStream, - parseOpenAIImagesUsage, - renderOpenAIImagesResponse, - type CanonicalOpenAIImagesEditsRequest, - type CanonicalOpenAIImagesRequest, - type CanonicalOpenAIImagesResponse, - type CanonicalOpenAIImagesUsage, - type OpenAIImageEditReference, - type OpenAIImagesEditImage, - type OpenAIImagesOperation, - type OpenAIImagesStreamEvent, -} from '@floway-dev/protocols/openai-images'; -import { isBase64ImageDataUrl, providerModelOf } from '@floway-dev/provider'; -import type { OpenAIImagesEditsRequest, OpenAIImagesEditsSource, ModelCandidate, PerformanceOperation, TelemetryModelIdentity } from '@floway-dev/provider'; - -/** The answer while it is still the upstream's, one event at a time. There is no transport - * sentinel below the events — this protocol ends at its completed event — so what travels is - * the events themselves rather than frames with a terminal arm the protocol has not got. */ -export type OpenAIImagesFrames = AsyncIterable; - -/** A stream as a value the record can hold, as a wrapper around the generator rather than the - * generator itself. What the wrapper says is where the resource is: the one resource in an - * OpenAI Images run is the upstream's body at `response.http.body`, claimed with `own()`, and - * this keeps a frame view from reading as another. */ -const view = (frames: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => frames }); - -/** OpenAI Images' own keys. They extend the shared space and never merge into it, so a stage - * written against the gateway alone cannot name one. */ -export interface OpenAIImagesFacts extends GatewayFacts { - /** Whether the client asked for the answer as a stream. It stays put, as every `ingress.*` - * key does: the same flag travels to the upstream inside the parameters, and the answer is - * written back in the shape it asked for. */ - 'ingress.openaiImages.wantsStream': boolean; - 'request.openaiImages.canonical': CanonicalOpenAIImagesRequest; - /** The answer, whichever kind it turned out to be. A stream, a value and a failure sit at - * one key: telling them apart is reading a value, and each stage does that where it needs - * to. */ - 'response.openaiImages.canonical': CanonicalOpenAIImagesResponse | OpenAIImagesFrames | Failure; - /** What the upstream will have reported by the time the events run out, and `null` on every - * path that does not stream. A streamed image states its usage in the completed event, - * which is after this run has answered — so the numbers cannot be in - * `response.usage.billable`, which says what had been reported when the ending stage handed - * up: the entity, and no quantities. Settling from this is the epilogue's job, after the - * drain. */ - 'response.openaiImages.streamedUsage': Deferred | null; - /** What the client is actually sent, in the OpenAI Images protocol — a JSON body, or the SSE - * frames of a stream. The edge provides it, so a dump shows the body the client received - * rather than the gateway's canonical form. */ - 'response.openaiImages.rendered': Record | AsyncIterable; -} - -type I = { [P in K]: OpenAIImagesFacts[P] }; - -const ENDPOINT = { - generations: 'openaiImagesGenerations', - edits: 'openaiImagesEdits', -} as const satisfies Record; - -const PERFORMANCE_OPERATION = { - generations: 'image_generation', - edits: 'image_edit', -} as const satisfies Record; - -const isFrames = (answer: CanonicalOpenAIImagesResponse | OpenAIImagesFrames): answer is OpenAIImagesFrames => - Symbol.asyncIterator in answer; - -/** - * The outermost edge. It names nothing on the way down — the family has one protocol, so there - * is no request to read to know how to answer — and coming back it renders the answer, keeps - * the upstream headers a client may see, and decides the status. The status is decided here - * rather than carried up because a refusal that never reached an upstream has none to carry. - * - * SSE framing is produced here and nowhere else: below this stage a stream carries protocol - * events, so the same assembly would serve another transport by rendering differently at this - * one point. - */ -const emitOpenAIImages = defineStage< - Record, - Record, - I<'response.openaiImages.canonical' | 'response.http.headers'>, - I<'response.openaiImages.rendered' | 'response.http.status' | 'response.http.headers'> ->({ - name: 'emitOpenAIImages', - through: { - request: { needs: [], consumes: [], provides: [] }, - response: { - needs: ['response.openaiImages.canonical', 'response.http.headers'], - consumes: ['response.openaiImages.canonical', 'response.http.headers'], - provides: ['response.openaiImages.rendered', 'response.http.status', 'response.http.headers'], - }, - }, - execute: async (facts, next) => { - const back = await next(facts); - const { 'response.openaiImages.canonical': answer, 'response.http.headers': headers, ...rest } = back; - // Vendor traces and quota state stay visible; what an intermediary must strip, and what - // would misdescribe a body this gateway serialized itself, does not. A filter that removed - // nothing hands the same array on, so the record shows no change where none happened. - const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); - return { - ...rest, - 'response.http.headers': forwardable.length === headers.length ? headers : move(forwardable), - 'response.http.status': isFailure(answer) ? answer.status : 200, - 'response.openaiImages.rendered': move(rendered(answer)), - }; - }, -}); - -const rendered = (answer: OpenAIImagesFacts['response.openaiImages.canonical']): OpenAIImagesFacts['response.openaiImages.rendered'] => - isFailure(answer) ? renderErrorEnvelope(answer.message, answer.body) - : isFrames(answer) ? renderSSE(answer) - : renderOpenAIImagesResponse(answer); - -const renderSSE = (frames: OpenAIImagesFrames): AsyncIterable => ({ - // The frames the client reads are a reframing of the ones the record holds, so this key - // points at that same stream rather than at nothing. - ...streamReferenceOf(frames), - [Symbol.asyncIterator]: () => (async function* () { - for await (const event of frames) yield openaiImagesStreamEventToSSEFrame(event); - })(), -}); - -/** - * The ending. It dials, reads what came back on the shape the request asked for, and provides - * the canonical answer, the headers that came with it, the raw HTTP body beneath it, and what - * the call is billable for. A failure is a value: an upstream that refused, and one that - * answered with something this protocol cannot read, are both outcomes the fork above can take - * to the next candidate rather than faults that end the run. - */ -const callOpenAIImagesUpstream = defineStage< - I<'ingress.openaiImages.wantsStream' | 'request.openaiImages.canonical' | 'route.attempt' | 'ingress.http.headers'>, - I<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.headers' - | 'response.http.body' | 'response.usage.billable'>, - GatewayServices ->({ - name: 'callOpenAIImagesUpstream', - return: { - provides: [ - 'response.openaiImages.canonical', - 'response.openaiImages.streamedUsage', - 'response.http.headers', - 'response.http.body', - 'response.usage.billable', - ], - }, - execute: async (facts, use) => { - const candidate = use.resolveAttempt(facts['route.attempt']); - const request = facts['request.openaiImages.canonical']; - const options = buildUpstreamCallOptions( - candidate, - use.gateway, - // The client's own headers reach the upstream from the record, not from a live request - // object: what a provider is allowed to forward is filtered per provider, and the dump - // shows what was there to filter. - new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value])), - ); - const model = providerModelOf(candidate); - // Attribution is set before the dial, so an attempt that never completes still names the - // candidate it was made against rather than the one tried before it. - use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, PERFORMANCE_OPERATION[request.operation]); - - let result; - try { - // No abort signal, as on this family's endpoints from the beginning: an image the upstream - // has already begun is charged for whether or not the client waited for it, so dropping the - // call would lose the usage reading and save nothing. Reading a stream is the other half of - // that and does take the signal — what it drops there is a connection, not an image. - result = request.operation === 'generations' - ? await candidate.provider.instance.callOpenAIImagesGenerations(model, request.parameters, undefined, options) - : await candidate.provider.instance.callOpenAIImagesEdits(model, providerEditsRequest(request), undefined, options); - } catch (error) { - use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); - // A dial that never completed reached no upstream, so nothing was billed and there are - // no headers to carry. What it leaves behind is the performance row settlement writes. - return move({ - ...facts, - 'response.openaiImages.canonical': dialFailure(error), - 'response.openaiImages.streamedUsage': null, - 'response.http.headers': [], - 'response.http.body': spentBody(null), - 'response.usage.billable': [], - }); - } - - const identity = telemetryModelIdentity(candidate, result.modelKey); - // What came back, unfiltered: the edge is where a client's view of it is decided. - const headers = [...result.response.headers]; - // An entity with no quantities is how "the upstream was called and reported nothing" is - // said, which is a different situation from reporting zero. - const called: readonly BillableEntity[] = [{ identity, quantities: {} }]; - const read = (canonical: CanonicalOpenAIImagesResponse | Failure, billable: readonly BillableEntity[]) => move({ - ...facts, - 'response.openaiImages.canonical': canonical, - 'response.openaiImages.streamedUsage': null, - 'response.http.headers': headers, - 'response.http.body': spentBody(result.response.body), - 'response.usage.billable': billable, - }); - - if (!result.response.ok) { - use.log.warn('upstream refused', { status: result.response.status }); - // An upstream error body is JSON like any other body. Reading it here is also what - // leaves a losing attempt with nothing open behind it. - const body = await readUpstreamBody(result.response); - return read({ - status: result.response.status, - message: upstreamErrorMessage(body.json) ?? body.text, - ...('json' in body ? { body: body.json } : {}), - }, called); - } - - const mediaType = result.response.headers.get('content-type'); - // Both halves have to hold. `stream` is what the client asked for, and an upstream that - // ignores it answers the single JSON body the arm below reads — so what settles which - // shape arrived is the media type, and what settles which shape the client is owed is the - // request. - if (facts['ingress.openaiImages.wantsStream'] && isEventStreamMediaType(mediaType)) { - if (result.response.body === null) { - return read({ status: 502, message: 'Upstream returned a streaming response with no body.' }, called); - } - // Usage is observed here, closest to the upstream and on the protocol it spoke, by - // folding the events as they pass — so the reading costs one pass and the client's own - // stream is what drives it. What it finds arrives with the completed event, long after - // this stage has handed up, which is why the entity above carries no quantities. - const metered = meterFrames(result.response.body, identity, use.gateway.abortSignal); - return move({ - ...facts, - // This protocol's stream is bare events rather than protocol frames, so the record is - // told how one becomes a frame instead of being left to assume. - 'response.openaiImages.canonical': recordStream(metered.frames, use.gateway.dump, eventFrame), - 'response.openaiImages.streamedUsage': metered.outcome, - 'response.http.headers': headers, - // Releasing this body is reading those events to the end: they are one reader over one - // connection, and a second reader is not something a `ReadableStream` allows. - 'response.http.body': own(result.response.body, async (): Promise => { for await (const _event of metered.frames) { /* to end of stream */ } }), - 'response.usage.billable': called, - }); - } - - const body = await readUpstreamBody(result.response); - if (!('json' in body)) { - // Every protocol the gateway carries is one it fully understands, so a body it cannot - // read is not handed on unread. - const essence = mediaTypeEssence(mediaType) ?? 'no media type'; - use.log.warn('upstream answered with a body that is not JSON', { status: result.response.status, mediaType: essence }); - return read({ - status: 502, - message: `The upstream answered ${result.response.status} with ${essence}, and the OpenAI Images protocol is JSON.`, - body: body.text, - }, called); - } - - let canonical: CanonicalOpenAIImagesResponse; - try { - canonical = parseOpenAIImagesResponse(body.json); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - use.log.warn('upstream answered with a body the OpenAI Images protocol cannot read', { message }); - return read({ status: 502, message, body: body.json }, called); - } - return read(canonical, [{ identity, quantities: billed(canonical.usage) }]); - }, -}); - -/** A body this stage has already read to the end, or one the upstream never sent. The record - * holds a body as a stream and `failover` releases the losing attempts', so every path hands - * one up; what says an answer was unusable is the failure at the canonical key, not this. */ -const spentBody = (body: ReadableStream | null): ReadableStream & Owned => - own(body ?? new ReadableStream({ start: controller => controller.close() }), (): Promise => Promise.resolve()); - -interface MeteredFrames { - readonly frames: OpenAIImagesFrames; - readonly outcome: Deferred; -} - -const meterFrames = ( - body: ReadableStream, - identity: TelemetryModelIdentity, - signal: AbortSignal | undefined, -): MeteredFrames => { - let settle!: (outcome: StreamOutcome) => void; - // Declared as this run's own unfinished work, so the runner waits for it at teardown where - // it can see it rather than the reading being started and forgotten. - const outcome = defer(new Promise(resolve => { settle = resolve; })); - // Running out without the completed event is what "it did not finish" means, and it is known - // at the same moment the usage is. - let sawTerminal = false; - const frames = view((async function* () { - let usage: CanonicalOpenAIImagesUsage | undefined; - try { - for await (const event of parseOpenAIImagesStream(body, { signal })) { - if (isOpenAIImagesTerminalEvent(event)) { - usage = parseOpenAIImagesUsage(event); - sawTerminal = true; - yield event; - // The image is complete, so there is nothing further to read. An upstream that holds - // the connection open past this point would otherwise keep the client's own stream - // open with it; returning here closes the read, which cancels the upstream. - return; - } - yield event; - } - } finally { - // Reached however the events ended — the completed one, a client that stopped reading, - // or a broken upstream — because what the upstream already metered is billable whatever - // happened to the downstream half. - settle({ billable: [{ identity, quantities: billed(usage) }], failed: !sawTerminal }); - } - // Only an upstream that ended its body without ever completing the image reaches here: the - // arm above returns on the terminal event. A client has been sent partial images and no - // image, which is a failed answer however far it got. - throw new Error(OPENAI_IMAGES_MISSING_TERMINAL_MESSAGE); - })()); - return { frames, outcome }; -}; - -/** An image is billed by the tokens its upstream reports: `BILLING_METRICS` names no per-image - * or per-size unit, and per-size pricing is a selector coordinate rather than a metric, so - * there is nothing else here to record a count against. A reported zero is kept — it says the - * upstream reported, which an absent metric would not. */ -const billed = (usage: CanonicalOpenAIImagesUsage | undefined): UsageQuantities => { - const quantities: UsageQuantities = {}; - if (usage?.inputTokens !== undefined) quantities.input_tokens = parseDecimalString(String(usage.inputTokens)); - if (usage?.inputImageTokens !== undefined) quantities.input_image_tokens = parseDecimalString(String(usage.inputImageTokens)); - if (usage?.outputTokens !== undefined) quantities.output_tokens = parseDecimalString(String(usage.outputTokens)); - if (usage?.outputImageTokens !== undefined) quantities.output_image_tokens = parseDecimalString(String(usage.outputImageTokens)); - return quantities; -}; - -/** The provider's own shape, built where the dial happens. What it needs from a reference is - * whether the data URL inside it can become a file, because that decides whether the edit can - * ride as a multipart form; the canonical fact holds the reference the client wrote and leaves - * that question to the serializer that has it. */ -const providerEditsRequest = (request: CanonicalOpenAIImagesEditsRequest): OpenAIImagesEditsRequest => ({ - images: request.images.map(providerEditsSource), - ...(request.mask === undefined ? {} : { mask: providerEditsSource(request.mask) }), - parameters: request.parameters, -}); - -const providerEditsSource = (image: OpenAIImagesEditImage): OpenAIImagesEditsSource => { - if (image.kind === 'file') { - return { type: 'upload', file: new File([image.file.bytes], image.file.fileName, { type: image.file.mediaType }) }; - } - const { reference } = image; - return typeof reference.image_url === 'string' && isBase64ImageDataUrl(reference.image_url) - ? { type: 'inline', reference: reference as OpenAIImageEditReference & { image_url: string } } - : { type: 'reference', reference }; -}; - -/** A candidate that cannot serve *this* request is not a candidate. One family covers two - * endpoints and an upstream may expose either without the other, so which one is asked for is - * what narrows the list. */ -const narrowing = (request: CanonicalOpenAIImagesRequest) => ({ - kind: 'image' as const, - reject: (candidate: ModelCandidate) => candidate.model.endpoints[ENDPOINT[request.operation]] === undefined - ? `the upstream does not expose the OpenAI Images ${request.operation} endpoint` - : null, - unsupported: (model: string) => `Model ${model} does not support the /images/${request.operation} endpoint.`, - refuse: (status: number, message: string) => ({ - 'response.openaiImages.canonical': { status, message } as Failure, - 'response.openaiImages.streamedUsage': null, - }), - refuses: ['response.openaiImages.canonical', 'response.openaiImages.streamedUsage'] as const, -}); - -/** What a caller must bring. `ingress.http.headers`, `ingress.openaiImages.wantsStream` and - * `request.openaiImages.canonical` are in it although `compose` cannot derive them — the ending - * stage reads all three and a return-only stage declares no request side — so this type is the - * whole statement and `entryNeeds` is part of it. */ -export type OpenAIImagesServeEntry = I<'ingress.http.headers' | 'ingress.openaiImages.wantsStream' | 'request.openaiImages.canonical' | 'serve.model'>; - -export type OpenAIImagesServeExit = I< - 'response.openaiImages.rendered' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable' ->; +import { compose, type Pipeline } from '@floway-dev/pipeline'; +import type { CanonicalOpenAIImagesRequest } from '@floway-dev/protocols/openai-images'; export const openaiImagesServePipeline = (request: CanonicalOpenAIImagesRequest): Pipeline => compose('openaiImagesServe', [ diff --git a/packages/gateway/src/data-plane/openai-images/target.ts b/packages/gateway/src/data-plane/openai-images/target.ts new file mode 100644 index 0000000000..bc83782a13 --- /dev/null +++ b/packages/gateway/src/data-plane/openai-images/target.ts @@ -0,0 +1,25 @@ +import type { Failure } from '../pipeline/facts.ts'; +import type { ModelEndpointKey } from '@floway-dev/protocols/common'; +import type { OpenAIImagesOperation, CanonicalOpenAIImagesRequest } from '@floway-dev/protocols/openai-images'; +import type { ModelCandidate } from '@floway-dev/provider'; + +const ENDPOINT = { + generations: 'openaiImagesGenerations', + edits: 'openaiImagesEdits', +} as const satisfies Record; + +/** A candidate that cannot serve *this* request is not a candidate. One family covers two + * endpoints and an upstream may expose either without the other, so which one is asked for is + * what narrows the list. */ +export const narrowing = (request: CanonicalOpenAIImagesRequest) => ({ + kind: 'image' as const, + reject: (candidate: ModelCandidate) => candidate.model.endpoints[ENDPOINT[request.operation]] === undefined + ? `the upstream does not expose the OpenAI Images ${request.operation} endpoint` + : null, + unsupported: (model: string) => `Model ${model} does not support the /images/${request.operation} endpoint.`, + refuse: (status: number, message: string) => ({ + 'response.openaiImages.canonical': { status, message } as Failure, + 'response.openaiImages.streamedUsage': null, + }), + refuses: ['response.openaiImages.canonical', 'response.openaiImages.streamedUsage'] as const, +}); diff --git a/packages/gateway/src/data-plane/pipeline/failover.ts b/packages/gateway/src/data-plane/pipeline/failover.ts new file mode 100644 index 0000000000..cc7193f1c5 --- /dev/null +++ b/packages/gateway/src/data-plane/pipeline/failover.ts @@ -0,0 +1,62 @@ +import type { GatewayServices } from './services.ts'; +import type { Slice } from './stage-contracts.ts'; +import { defineStage, move, type Facts } from '@floway-dev/pipeline'; + +/** + * Runs what follows it once per candidate and returns the first that did not fail. + * + * The losing attempts' bodies are its own — that is what `consumes` declares on the way up + * — and the winner's rides onward, which is what `provides` declares. Failover never + * breaks a stream: once one has opened there is nothing to fail over to, and the family's + * edge is what settles that by sitting above this stage rather than below it. + */ +export interface Forking { + /** How this family reads an attempt's outcome. Branching is the framework's; what + * something branches on is the domain's. */ + readonly failed: (handedUp: Facts) => boolean; + /** The keys at which this family's attempts hand up something the run owns — an upstream + * body still open, most often. A family whose ending reads its answer to the end owns + * nothing and names nothing here, and one that streams names the key it streams at. + * + * It cannot be a fixed key. Declaring `provides` for a key a family never produces makes + * the runner throw on the first real request, and declaring `consumes` for one it does + * produce and hands up makes it throw the other way. Which keys carry a resource is a + * statement only the family can make. */ + readonly owns: readonly string[]; +} + +export const failover = ({ failed, owns }: Forking) => defineStage< + Slice<'serve.candidates'>, + Slice<'serve.candidates' | 'route.attempt'>, + Slice<'response.usage.billable'>, + Slice<'response.usage.billable'>, + GatewayServices +>({ + name: 'failover', + through: { + request: { needs: ['serve.candidates'], consumes: [], provides: ['route.attempt'] }, + response: { + needs: ['response.usage.billable'], + // Owned on the way up and handed onward: every attempt's is this stage's to release, + // and the one it adopts rides up with ownership going with it. + consumes: owns as never, + provides: owns as never, + }, + }, + execute: async (facts, next, use) => { + let last: Slice<'response.usage.billable'> | undefined; + for (const candidate of facts['serve.candidates']) { + // Per-attempt telemetry state, cleared before control leaves, so a mid-attempt throw + // still attributes its performance row to the candidate that was being tried. + use.gateway.attempt.timing.upstreamCallStartedAt = null; + use.gateway.attempt.timing.firstOutputTokenAt = null; + last = await next({ ...facts, 'route.attempt': move(candidate) }); + if (!failed(last as Facts)) return last; + use.log.info('candidate failed, trying the next', { upstream: candidate.upstreamId }); + } + if (last === undefined) throw new Error('failover: assembly handed it an empty candidate list'); + // Every candidate failed, and the last failure is the base — so the client sees real + // upstream telemetry rather than a synthesized gateway envelope. + return last; + }, +}); diff --git a/packages/gateway/src/data-plane/pipeline/stages.ts b/packages/gateway/src/data-plane/pipeline/resolve-candidates.ts similarity index 56% rename from packages/gateway/src/data-plane/pipeline/stages.ts rename to packages/gateway/src/data-plane/pipeline/resolve-candidates.ts index f4c434ea09..8f94456c9a 100644 --- a/packages/gateway/src/data-plane/pipeline/stages.ts +++ b/packages/gateway/src/data-plane/pipeline/resolve-candidates.ts @@ -1,26 +1,11 @@ -// Resolving a model to the upstreams that can serve it, and running the suffix once per -// candidate until one answers. Both are ordinary stages: nothing in the framework knows -// what a retry is, and array position is the whole of the statement that what follows -// `failover` is the per-attempt segment. -// -// Neither names a family's own key. Branching is a capability of the framework and what -// something branches on is a concept in the domain, so the family hands in the two -// domain-shaped things — how to narrow a candidate, and how to read an attempt's outcome — -// and these stages stay written against the shared space alone. That is also what lets -// them compose into a pipeline over any family's larger space with no variance question to -// lose: assembly reasons over declarations, which are strings. - -import type { GatewayFacts } from './facts.ts'; import type { GatewayServices } from './services.ts'; +import type { Slice } from './stage-contracts.ts'; import { enumerateModelCandidates } from '../providers/resolution.ts'; import { appendFailedUpstreams } from '../shared/failed-upstreams.ts'; import { defineStage, move } from '@floway-dev/pipeline'; -import type { Facts } from '@floway-dev/pipeline'; import type { ModelKind } from '@floway-dev/protocols/common'; import type { ModelCandidate } from '@floway-dev/provider'; -type Slice = { [P in K]: GatewayFacts[P] }; - /** What a family narrows its candidates by, and what it says when nothing is left. A * candidate that resolves but cannot serve this request — no endpoint for the kind, or a * payload the target protocol cannot express — is not a candidate, and saying why is what @@ -108,62 +93,3 @@ export const resolveCandidates = (narrowing: Narrowing boolean; - /** The keys at which this family's attempts hand up something the run owns — an upstream - * body still open, most often. A family whose ending reads its answer to the end owns - * nothing and names nothing here, and one that streams names the key it streams at. - * - * It cannot be a fixed key. Declaring `provides` for a key a family never produces makes - * the runner throw on the first real request, and declaring `consumes` for one it does - * produce and hands up makes it throw the other way. Which keys carry a resource is a - * statement only the family can make. */ - readonly owns: readonly string[]; -} - -export const failover = ({ failed, owns }: Forking) => defineStage< - Slice<'serve.candidates'>, - Slice<'serve.candidates' | 'route.attempt'>, - Slice<'response.usage.billable'>, - Slice<'response.usage.billable'>, - GatewayServices ->({ - name: 'failover', - through: { - request: { needs: ['serve.candidates'], consumes: [], provides: ['route.attempt'] }, - response: { - needs: ['response.usage.billable'], - // Owned on the way up and handed onward: every attempt's is this stage's to release, - // and the one it adopts rides up with ownership going with it. - consumes: owns as never, - provides: owns as never, - }, - }, - execute: async (facts, next, use) => { - let last: Slice<'response.usage.billable'> | undefined; - for (const candidate of facts['serve.candidates']) { - // Per-attempt telemetry state, cleared before control leaves, so a mid-attempt throw - // still attributes its performance row to the candidate that was being tried. - use.gateway.attempt.timing.upstreamCallStartedAt = null; - use.gateway.attempt.timing.firstOutputTokenAt = null; - last = await next({ ...facts, 'route.attempt': move(candidate) }); - if (!failed(last as Facts)) return last; - use.log.info('candidate failed, trying the next', { upstream: candidate.upstreamId }); - } - if (last === undefined) throw new Error('failover: assembly handed it an empty candidate list'); - // Every candidate failed, and the last failure is the base — so the client sees real - // upstream telemetry rather than a synthesized gateway envelope. - return last; - }, -}); diff --git a/packages/gateway/src/data-plane/pipeline/stage-contracts.ts b/packages/gateway/src/data-plane/pipeline/stage-contracts.ts new file mode 100644 index 0000000000..74e6a1923c --- /dev/null +++ b/packages/gateway/src/data-plane/pipeline/stage-contracts.ts @@ -0,0 +1,3 @@ +import type { GatewayFacts } from './facts.ts'; + +export type Slice = { [P in K]: GatewayFacts[P] }; diff --git a/packages/gateway/src/data-plane/rerank/call-upstream.ts b/packages/gateway/src/data-plane/rerank/call-upstream.ts new file mode 100644 index 0000000000..e2d7c1f585 --- /dev/null +++ b/packages/gateway/src/data-plane/rerank/call-upstream.ts @@ -0,0 +1,155 @@ +import type { Fields } from './facts.ts'; +import type { UsageQuantities } from '../../repo/types.ts'; +import type { BillableEntity } from '../pipeline/facts.ts'; +import type { GatewayServices } from '../pipeline/services.ts'; +import { dialFailure, readUpstreamBody, unreadableBody } from '../pipeline/upstream-body.ts'; +import { upstreamPerformanceContext, telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; +import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { upstreamErrorMessage, parseDecimalString } from '@floway-dev/protocols/common'; +import { parseRerankUsage, parseRerankResponse, type CanonicalRerankResponse } from '@floway-dev/protocols/rerank'; +import { providerModelOf } from '@floway-dev/provider'; + +/** + * The ending. It dials, reads the upstream's body, and provides the canonical answer and + * what the call is billable for. A failure is a value: a 429 here is what an earlier stage + * fails over, and even a 400 can be, because the next candidate's path and flags may differ. + */ +export const callRerankUpstream = defineStage< + Fields<'request.rerank.canonical' | 'route.attempt' | 'ingress.http.headers' | 'ingress.rerank.sourceProtocol'>, + Fields<'response.rerank.canonical' | 'response.rerank.targetProtocol' | 'response.http.headers' | 'response.usage.billable'>, + GatewayServices +>({ + name: 'callRerankUpstream', + return: { + provides: ['response.rerank.canonical', 'response.rerank.targetProtocol', 'response.http.headers', 'response.usage.billable'], + }, + execute: async (facts, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + const request = facts['request.rerank.canonical']; + const model = providerModelOf(candidate); + // Configuration refuses a rerank model without a target, so this holds for every candidate + // the narrowing kept; saying so here is what lets a dial that never answered still name + // the protocol it spoke. + const configuredTarget = model.rerankTarget; + if (configuredTarget === undefined) { + throw new Error(`${candidate.provider.upstreamId} serves rerank for ${candidate.model.id} without a target protocol`); + } + // Attribution is set before the dial, so an attempt that never completes still names the + // candidate it was made against rather than the one tried before it. + use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'rerank'); + + let result; + try { + result = await candidate.provider.instance.callRerank( + model, + request, + use.gateway.abortSignal, + // The client's own headers reach the upstream from the record, not from a live + // request object: what a provider is allowed to forward is filtered per provider, + // and the dump shows what was there to filter. + buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), + ); + } catch (error) { + use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); + // A dial that never completed reached no upstream, so nothing was billed and there are + // no headers to carry. What it leaves behind is the performance row settlement writes. + return move({ + ...facts, + 'response.rerank.canonical': dialFailure(error), + 'response.rerank.targetProtocol': configuredTarget.protocol, + 'response.http.headers': [], + 'response.usage.billable': [], + }); + } + + const identity = telemetryModelIdentity(candidate, result.modelKey); + // What came back, unfiltered: the edge is where a client's view of it is decided. + const headers = [...result.response.headers]; + const body = await readUpstreamBody(result.response); + + if (!result.response.ok) { + use.log.warn('upstream refused', { status: result.response.status }); + return move({ + ...facts, + 'response.rerank.canonical': { + status: result.response.status, + message: upstreamErrorMessage(body.json) ?? body.text, + ...('json' in body ? { body: body.json } : {}), + }, + 'response.rerank.targetProtocol': result.target.protocol, + 'response.http.headers': headers, + // The upstream was called and reported nothing, which is a different situation + // from reporting zero — so the entity is present with no quantities. + 'response.usage.billable': [{ identity, quantities: {} }], + }); + } + + if (!('json' in body)) { + return move({ + ...facts, + 'response.rerank.canonical': unreadableBody(result.response, body, 'the rerank protocol'), + 'response.rerank.targetProtocol': result.target.protocol, + 'response.http.headers': headers, + 'response.usage.billable': [{ identity, quantities: {} }], + }); + } + + // A usage block the reader cannot make sense of is a report we cannot parse, which from + // here is no report. It is read before the results, so an answer this gateway could not + // model still bills for what the upstream did meter — the two readings are independent + // and one of them failing is not a reason to discard the other. + let usage: Pick; + try { + usage = parseRerankUsage(result.target.protocol, body.json); + } catch (error) { + use.log.warn('upstream reported usage the rerank protocol cannot read', { error: String(error) }); + usage = {}; + } + const metered: readonly BillableEntity[] = [{ + identity, + quantities: billed(usage), + // A rerank rate can depend on how large the input was and not only on how much of it + // there was, so the token total is a pricing input as well as a quantity. + ...(usage.totalTokens === undefined ? {} : { pricingFacts: { inputTokens: usage.totalTokens } }), + }]; + + // An answer the client's own protocol produced is what that client already reads, so the + // edge renders it back out unchanged and nothing here has to model it. Reading the results + // is what a *translation* needs, and only a cross-protocol run does one — so a result item + // this gateway cannot model is a failure there and a field it simply carries here. + const translating = facts['ingress.rerank.sourceProtocol'] !== result.target.protocol; + let canonical: CanonicalRerankResponse; + try { + canonical = parseRerankResponse(result.target.protocol, body.json); + } catch (error) { + if (translating) { + use.log.warn('upstream answered with results the rerank protocol cannot read', { error: String(error) }); + return move({ + ...facts, + 'response.rerank.canonical': unreadableBody(result.response, body, 'the rerank protocol'), + 'response.rerank.targetProtocol': result.target.protocol, + 'response.http.headers': headers, + 'response.usage.billable': metered, + }); + } + use.log.debug('same-protocol answer carries results this gateway does not model', { error: String(error) }); + canonical = { raw: body.json as Record, results: [] }; + } + + return move({ + ...facts, + 'response.rerank.canonical': canonical, + 'response.rerank.targetProtocol': result.target.protocol, + 'response.http.headers': headers, + 'response.usage.billable': metered, + }); + }, +}); + +const billed = (usage: Pick | undefined): UsageQuantities => { + const quantities: UsageQuantities = {}; + if (usage?.searchUnits !== undefined) quantities.rerank_searches = parseDecimalString(String(usage.searchUnits)); + if (usage?.totalTokens !== undefined) quantities.input_tokens = parseDecimalString(String(usage.totalTokens)); + return quantities; +}; diff --git a/packages/gateway/src/data-plane/rerank/emit.ts b/packages/gateway/src/data-plane/rerank/emit.ts new file mode 100644 index 0000000000..060fc6c6be --- /dev/null +++ b/packages/gateway/src/data-plane/rerank/emit.ts @@ -0,0 +1,78 @@ +import type { Fields } from './facts.ts'; +import { isFailure } from '../pipeline/facts.ts'; +import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { renderErrorEnvelope } from '@floway-dev/protocols/common'; +import { renderRerankResponse } from '@floway-dev/protocols/rerank'; + +/** + * The outermost edge. Renders the canonical answer into the protocol the client spoke — + * which is why `ingress.rerank.sourceProtocol` is an ingress fact and not a request one: + * it survives the switch to whatever protocol the upstream turned out to speak. + */ +export const emitRerank = defineStage< + Fields<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical'>, + Fields<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical'>, + Fields<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical' | 'response.rerank.canonical' | 'response.rerank.targetProtocol' | 'response.http.headers'>, + Fields<'response.rerank.rendered' | 'response.http.status' | 'response.http.headers'> +>({ + name: 'emitRerank', + through: { + request: { + needs: ['ingress.rerank.sourceProtocol', 'request.rerank.canonical'], + consumes: [], + provides: [], + }, + response: { + needs: ['response.rerank.canonical', 'response.http.headers'], + consumes: ['response.rerank.canonical', 'response.rerank.targetProtocol', 'response.http.headers'], + provides: ['response.rerank.rendered', 'response.http.status', 'response.http.headers'], + }, + }, + execute: async (facts, next) => { + const back = await next(facts); + const { 'response.rerank.canonical': answer, 'response.rerank.targetProtocol': target, 'response.http.headers': headers, ...rest } = back; + // Vendor traces and quota state stay visible; what an intermediary must strip, and what + // would misdescribe a body this gateway serialized itself, does not. A filter that removed + // nothing hands the same array on, so the record shows no change where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); + if (isFailure(answer)) { + return { + ...rest, + 'response.http.headers': forClient, + 'response.rerank.rendered': move(renderErrorEnvelope(answer.message, answer.body)), + // The upstream's own status, or the gateway's own when it refused before dialling. + // A client is not owed the upstream's exact bytes; it is owed the truth about what + // happened, and a 429 arriving as a 200 is not that. + 'response.http.status': answer.status, + }; + } + // Translating can fail on an answer that parsed: a result may index a document the + // request never sent. The upstream answered and the gateway cannot put that answer in the + // protocol the client speaks, which is the gateway failing to serve rather than anything + // the client can fix — so it is 502, and a value like every other failure here. + let rendered: Record; + try { + rendered = renderRerankResponse( + back['ingress.rerank.sourceProtocol'], + target, + answer, + back['request.rerank.canonical'], + ); + } catch (error) { + return { + ...rest, + 'response.http.headers': forClient, + 'response.rerank.rendered': move(renderErrorEnvelope(error instanceof Error ? error.message : String(error))), + 'response.http.status': 502, + }; + } + return { + ...rest, + 'response.http.headers': forClient, + 'response.http.status': 200, + 'response.rerank.rendered': move(rendered), + }; + }, +}); diff --git a/packages/gateway/src/data-plane/rerank/facts.ts b/packages/gateway/src/data-plane/rerank/facts.ts new file mode 100644 index 0000000000..ae73bab404 --- /dev/null +++ b/packages/gateway/src/data-plane/rerank/facts.ts @@ -0,0 +1,22 @@ +import type { GatewayFacts, Failure } from '../pipeline/facts.ts'; +import type { RerankSourceProtocol, RerankProtocol } from '@floway-dev/protocols/common'; +import type { CanonicalRerankRequest, CanonicalRerankResponse } from '@floway-dev/protocols/rerank'; + +/** Rerank's own keys. They extend the shared space and never merge into it, so a stage + * written against the gateway alone cannot name one. */ +export interface RerankFacts extends GatewayFacts { + /** Which of the four rerank protocols the client spoke. It belongs to the ingress and + * stays put: the answer is rendered back into it whatever the upstream spoke. */ + 'ingress.rerank.sourceProtocol': RerankSourceProtocol; + 'request.rerank.canonical': CanonicalRerankRequest; + 'response.rerank.canonical': CanonicalRerankResponse | Failure; + /** Which protocol the attempt spoke. A response fact, because an upstream picks the target + * it answers in and the edge needs it to render back into the client's — a run that never + * reached one carries the target its request was serialized for. */ + 'response.rerank.targetProtocol': RerankProtocol; + /** What the client is actually sent, in its own protocol. The edge provides it, so a + * dump shows the bytes the client received rather than the gateway's canonical form. */ + 'response.rerank.rendered': Record; +} + +export type Fields = { [P in K]: RerankFacts[P] }; diff --git a/packages/gateway/src/data-plane/rerank/pipeline.ts b/packages/gateway/src/data-plane/rerank/pipeline.ts index 17e62b8cf3..79bd32fd50 100644 --- a/packages/gateway/src/data-plane/rerank/pipeline.ts +++ b/packages/gateway/src/data-plane/rerank/pipeline.ts @@ -1,294 +1,17 @@ -// Rerank as a pipeline. The family with a canonical contract already — parse, render, -// serialize and a usage reading — so it is the one that proves the shared stages on real -// business before the families that need their contracts written first. -// -// The shape every family repeats: -// -// emitRerank the edge: serializes the answer into the client's protocol -// writeSettlement above the fork, so a run bills once however many attempts it made -// resolveCandidates narrows to the upstreams that can serve this request -// failover runs what follows once per candidate -// callRerankUpstream the ending: dials, and provides what came back - -import type { UsageQuantities } from '../../repo/types.ts'; -import type { BillableEntity, Failure, GatewayFacts } from '../pipeline/facts.ts'; +import { callRerankUpstream } from './call-upstream.ts'; +import { emitRerank } from './emit.ts'; +import type { Fields } from './facts.ts'; +import { narrowing } from './target.ts'; import { isFailure } from '../pipeline/facts.ts'; -import type { GatewayServices } from '../pipeline/services.ts'; +import { failover } from '../pipeline/failover.ts'; +import { resolveCandidates } from '../pipeline/resolve-candidates.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; -import { failover, resolveCandidates } from '../pipeline/stages.ts'; -import { dialFailure, readUpstreamBody, unreadableBody } from '../pipeline/upstream-body.ts'; -import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; -import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; -import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; -import type { Pipeline } from '@floway-dev/pipeline'; -import { defineStage, move, compose } from '@floway-dev/pipeline'; -import { parseDecimalString, renderErrorEnvelope, upstreamErrorMessage, type RerankProtocol, type RerankSourceProtocol } from '@floway-dev/protocols/common'; -import { - parseRerankResponse, - parseRerankUsage, - renderRerankResponse, - rerankRequestIncompatibility, - type CanonicalRerankRequest, - type CanonicalRerankResponse, -} from '@floway-dev/protocols/rerank'; -import { providerModelOf } from '@floway-dev/provider'; - -/** Rerank's own keys. They extend the shared space and never merge into it, so a stage - * written against the gateway alone cannot name one. */ -export interface RerankFacts extends GatewayFacts { - /** Which of the four rerank protocols the client spoke. It belongs to the ingress and - * stays put: the answer is rendered back into it whatever the upstream spoke. */ - 'ingress.rerank.sourceProtocol': RerankSourceProtocol; - 'request.rerank.canonical': CanonicalRerankRequest; - 'response.rerank.canonical': CanonicalRerankResponse | Failure; - /** Which protocol the attempt spoke. A response fact, because an upstream picks the target - * it answers in and the edge needs it to render back into the client's — a run that never - * reached one carries the target its request was serialized for. */ - 'response.rerank.targetProtocol': RerankProtocol; - /** What the client is actually sent, in its own protocol. The edge provides it, so a - * dump shows the bytes the client received rather than the gateway's canonical form. */ - 'response.rerank.rendered': Record; -} - -type R = { [P in K]: RerankFacts[P] }; - -/** - * The outermost edge. Renders the canonical answer into the protocol the client spoke — - * which is why `ingress.rerank.sourceProtocol` is an ingress fact and not a request one: - * it survives the switch to whatever protocol the upstream turned out to speak. - */ -const emitRerank = defineStage< - R<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical'>, - R<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical'>, - R<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical' | 'response.rerank.canonical' | 'response.rerank.targetProtocol' | 'response.http.headers'>, - R<'response.rerank.rendered' | 'response.http.status' | 'response.http.headers'> ->({ - name: 'emitRerank', - through: { - request: { - needs: ['ingress.rerank.sourceProtocol', 'request.rerank.canonical'], - consumes: [], - provides: [], - }, - response: { - needs: ['response.rerank.canonical', 'response.http.headers'], - consumes: ['response.rerank.canonical', 'response.rerank.targetProtocol', 'response.http.headers'], - provides: ['response.rerank.rendered', 'response.http.status', 'response.http.headers'], - }, - }, - execute: async (facts, next) => { - const back = await next(facts); - const { 'response.rerank.canonical': answer, 'response.rerank.targetProtocol': target, 'response.http.headers': headers, ...rest } = back; - // Vendor traces and quota state stay visible; what an intermediary must strip, and what - // would misdescribe a body this gateway serialized itself, does not. A filter that removed - // nothing hands the same array on, so the record shows no change where none happened. - const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); - const forClient = forwardable.length === headers.length ? headers : move(forwardable); - if (isFailure(answer)) { - return { - ...rest, - 'response.http.headers': forClient, - 'response.rerank.rendered': move(renderErrorEnvelope(answer.message, answer.body)), - // The upstream's own status, or the gateway's own when it refused before dialling. - // A client is not owed the upstream's exact bytes; it is owed the truth about what - // happened, and a 429 arriving as a 200 is not that. - 'response.http.status': answer.status, - }; - } - // Translating can fail on an answer that parsed: a result may index a document the - // request never sent. The upstream answered and the gateway cannot put that answer in the - // protocol the client speaks, which is the gateway failing to serve rather than anything - // the client can fix — so it is 502, and a value like every other failure here. - let rendered: Record; - try { - rendered = renderRerankResponse( - back['ingress.rerank.sourceProtocol'], - target, - answer, - back['request.rerank.canonical'], - ); - } catch (error) { - return { - ...rest, - 'response.http.headers': forClient, - 'response.rerank.rendered': move(renderErrorEnvelope(error instanceof Error ? error.message : String(error))), - 'response.http.status': 502, - }; - } - return { - ...rest, - 'response.http.headers': forClient, - 'response.http.status': 200, - 'response.rerank.rendered': move(rendered), - }; - }, -}); - -/** - * The ending. It dials, reads the upstream's body, and provides the canonical answer and - * what the call is billable for. A failure is a value: a 429 here is what an earlier stage - * fails over, and even a 400 can be, because the next candidate's path and flags may differ. - */ -const callRerankUpstream = defineStage< - R<'request.rerank.canonical' | 'route.attempt' | 'ingress.http.headers' | 'ingress.rerank.sourceProtocol'>, - R<'response.rerank.canonical' | 'response.rerank.targetProtocol' | 'response.http.headers' | 'response.usage.billable'>, - GatewayServices ->({ - name: 'callRerankUpstream', - return: { - provides: ['response.rerank.canonical', 'response.rerank.targetProtocol', 'response.http.headers', 'response.usage.billable'], - }, - execute: async (facts, use) => { - const candidate = use.resolveAttempt(facts['route.attempt']); - const request = facts['request.rerank.canonical']; - const model = providerModelOf(candidate); - // Configuration refuses a rerank model without a target, so this holds for every candidate - // the narrowing kept; saying so here is what lets a dial that never answered still name - // the protocol it spoke. - const configuredTarget = model.rerankTarget; - if (configuredTarget === undefined) { - throw new Error(`${candidate.provider.upstreamId} serves rerank for ${candidate.model.id} without a target protocol`); - } - // Attribution is set before the dial, so an attempt that never completes still names the - // candidate it was made against rather than the one tried before it. - use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'rerank'); - - let result; - try { - result = await candidate.provider.instance.callRerank( - model, - request, - use.gateway.abortSignal, - // The client's own headers reach the upstream from the record, not from a live - // request object: what a provider is allowed to forward is filtered per provider, - // and the dump shows what was there to filter. - buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), - ); - } catch (error) { - use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); - // A dial that never completed reached no upstream, so nothing was billed and there are - // no headers to carry. What it leaves behind is the performance row settlement writes. - return move({ - ...facts, - 'response.rerank.canonical': dialFailure(error), - 'response.rerank.targetProtocol': configuredTarget.protocol, - 'response.http.headers': [], - 'response.usage.billable': [], - }); - } - - const identity = telemetryModelIdentity(candidate, result.modelKey); - // What came back, unfiltered: the edge is where a client's view of it is decided. - const headers = [...result.response.headers]; - const body = await readUpstreamBody(result.response); - - if (!result.response.ok) { - use.log.warn('upstream refused', { status: result.response.status }); - return move({ - ...facts, - 'response.rerank.canonical': { - status: result.response.status, - message: upstreamErrorMessage(body.json) ?? body.text, - ...('json' in body ? { body: body.json } : {}), - }, - 'response.rerank.targetProtocol': result.target.protocol, - 'response.http.headers': headers, - // The upstream was called and reported nothing, which is a different situation - // from reporting zero — so the entity is present with no quantities. - 'response.usage.billable': [{ identity, quantities: {} }], - }); - } - - if (!('json' in body)) { - return move({ - ...facts, - 'response.rerank.canonical': unreadableBody(result.response, body, 'the rerank protocol'), - 'response.rerank.targetProtocol': result.target.protocol, - 'response.http.headers': headers, - 'response.usage.billable': [{ identity, quantities: {} }], - }); - } - - // A usage block the reader cannot make sense of is a report we cannot parse, which from - // here is no report. It is read before the results, so an answer this gateway could not - // model still bills for what the upstream did meter — the two readings are independent - // and one of them failing is not a reason to discard the other. - let usage: Pick; - try { - usage = parseRerankUsage(result.target.protocol, body.json); - } catch (error) { - use.log.warn('upstream reported usage the rerank protocol cannot read', { error: String(error) }); - usage = {}; - } - const metered: readonly BillableEntity[] = [{ - identity, - quantities: billed(usage), - // A rerank rate can depend on how large the input was and not only on how much of it - // there was, so the token total is a pricing input as well as a quantity. - ...(usage.totalTokens === undefined ? {} : { pricingFacts: { inputTokens: usage.totalTokens } }), - }]; - - // An answer the client's own protocol produced is what that client already reads, so the - // edge renders it back out unchanged and nothing here has to model it. Reading the results - // is what a *translation* needs, and only a cross-protocol run does one — so a result item - // this gateway cannot model is a failure there and a field it simply carries here. - const translating = facts['ingress.rerank.sourceProtocol'] !== result.target.protocol; - let canonical: CanonicalRerankResponse; - try { - canonical = parseRerankResponse(result.target.protocol, body.json); - } catch (error) { - if (translating) { - use.log.warn('upstream answered with results the rerank protocol cannot read', { error: String(error) }); - return move({ - ...facts, - 'response.rerank.canonical': unreadableBody(result.response, body, 'the rerank protocol'), - 'response.rerank.targetProtocol': result.target.protocol, - 'response.http.headers': headers, - 'response.usage.billable': metered, - }); - } - use.log.debug('same-protocol answer carries results this gateway does not model', { error: String(error) }); - canonical = { raw: body.json as Record, results: [] }; - } - - return move({ - ...facts, - 'response.rerank.canonical': canonical, - 'response.rerank.targetProtocol': result.target.protocol, - 'response.http.headers': headers, - 'response.usage.billable': metered, - }); - }, -}); - -const billed = (usage: Pick | undefined): UsageQuantities => { - const quantities: UsageQuantities = {}; - if (usage?.searchUnits !== undefined) quantities.rerank_searches = parseDecimalString(String(usage.searchUnits)); - if (usage?.totalTokens !== undefined) quantities.input_tokens = parseDecimalString(String(usage.totalTokens)); - return quantities; -}; - -/** A candidate that cannot serve *this* request is not a candidate. Saying why is what - * turns an empty list into a 400 a client can act on. */ -const narrowing = (request: CanonicalRerankRequest) => ({ - kind: 'rerank' as const, - reject: (candidate: Parameters[0]) => { - const model = providerModelOf(candidate); - if (candidate.model.endpoints.rerank === undefined || model.rerankTarget === undefined) { - return 'the upstream does not expose a rerank endpoint'; - } - return rerankRequestIncompatibility(model.rerankTarget.protocol, request); - }, - unsupported: (model: string, reasons: readonly string[]) => reasons.length === 0 - ? `Model ${model} does not support rerank.` - : `Model ${model} does not support this rerank request: ${reasons.join('; ')}.`, - refuse: (status: number, message: string) => ({ 'response.rerank.canonical': { status, message } }), - refuses: ['response.rerank.canonical'] as const, -}); +import { compose, type Pipeline } from '@floway-dev/pipeline'; +import type { CanonicalRerankRequest } from '@floway-dev/protocols/rerank'; export const rerankServePipeline = (request: CanonicalRerankRequest): Pipeline< - R<'ingress.http.headers' | 'ingress.rerank.sourceProtocol' | 'request.rerank.canonical' | 'serve.model'>, - R<'response.rerank.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> + Fields<'ingress.http.headers' | 'ingress.rerank.sourceProtocol' | 'request.rerank.canonical' | 'serve.model'>, + Fields<'response.rerank.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> > => compose('rerankServe', [ emitRerank, writeSettlement(handedUp => isFailure((handedUp as { 'response.rerank.canonical'?: unknown })['response.rerank.canonical'])), @@ -299,5 +22,3 @@ export const rerankServePipeline = (request: CanonicalRerankRequest): Pipeline< }), callRerankUpstream, ]); - -export type { BillableEntity }; diff --git a/packages/gateway/src/data-plane/rerank/target.ts b/packages/gateway/src/data-plane/rerank/target.ts new file mode 100644 index 0000000000..3246ad0d99 --- /dev/null +++ b/packages/gateway/src/data-plane/rerank/target.ts @@ -0,0 +1,20 @@ +import { rerankRequestIncompatibility, type CanonicalRerankRequest } from '@floway-dev/protocols/rerank'; +import { providerModelOf } from '@floway-dev/provider'; + +/** A candidate that cannot serve *this* request is not a candidate. Saying why is what + * turns an empty list into a 400 a client can act on. */ +export const narrowing = (request: CanonicalRerankRequest) => ({ + kind: 'rerank' as const, + reject: (candidate: Parameters[0]) => { + const model = providerModelOf(candidate); + if (candidate.model.endpoints.rerank === undefined || model.rerankTarget === undefined) { + return 'the upstream does not expose a rerank endpoint'; + } + return rerankRequestIncompatibility(model.rerankTarget.protocol, request); + }, + unsupported: (model: string, reasons: readonly string[]) => reasons.length === 0 + ? `Model ${model} does not support rerank.` + : `Model ${model} does not support this rerank request: ${reasons.join('; ')}.`, + refuse: (status: number, message: string) => ({ 'response.rerank.canonical': { status, message } }), + refuses: ['response.rerank.canonical'] as const, +}); From 46759dabba80c1b6ecbb8b796ff80408e4c346c6 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 30 Sep 2026 21:41:41 +0800 Subject: [PATCH 59/79] fix(pipeline): preserve audio transcription SSE frames Observe completion and usage without replacing upstream event labels, reserializing data, or rejecting valid JSON extension frames. Cover labeled frames, original spelling and unknown extensions through the assembled pipeline. Document stage ownership and ordered audio uploads. --- README.md | 13 ++++++--- .../data-plane/openai-audio-pipeline_test.ts | 27 +++++++++++++++++++ .../data-plane/openai-audio/call-upstream.ts | 11 ++++---- .../src/data-plane/openai-audio/emit.ts | 21 +++------------ .../src/data-plane/openai-audio/facts.ts | 4 +-- packages/protocols/src/openai-audio/index.ts | 1 - packages/protocols/src/openai-audio/stream.ts | 9 ------- 7 files changed, 48 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 7d1105257e..a3fbf2eb1b 100644 --- a/README.md +++ b/README.md @@ -90,9 +90,10 @@ Rerank models are manual Custom models. Each model selects its outbound Cohere, Jina, Voyage, DashScope-compatible, or DashScope-native protocol and may override that protocol's canonical path; there is no upstream-wide rerank path. -Audio transcription is a buffered multipart passthrough for Custom, Azure, and -Ollama-compatible upstreams. JSON, text, subtitle, and transcription SSE -responses retain their upstream wire shape. +Audio transcription parses multipart uploads into ordered entries and rebuilds +them for each Custom, Azure, or Ollama-compatible upstream attempt. JSON, text, +and subtitle responses retain their upstream document; transcription SSE +responses preserve event labels and data while usage is observed separately. ### Upstreams @@ -184,6 +185,12 @@ also available as a root script. Route type generation runs first because the web app's generated types are not checked in and its lint configuration is type-aware. The web build includes assertions on the emitted bundle. +Pipeline endpoints are assembled from individual stage modules. Each stage owns +its request and response contract; a large stage keeps local helpers in the same +file and can place independent parts in a sibling directory of the same name. +Protocol-family folders own their stages, while pipeline and wire modules +compose the stages in traversal order. + The protocol tests cover v1 and v2 opaque-blob carrier compatibility, lossless UTF-16 code-unit recovery, and retained-memory growth during history replay. The memory regression runs a bounded fixture in a separate Node.js process with diff --git a/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts index 8d4344ace9..61c2888e01 100644 --- a/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts @@ -160,6 +160,33 @@ describe('the OpenAI Audio Transcriptions pipeline', () => { .toEqual(['transcript.text.delta', 'transcript.text.done']); }); + it('preserves upstream SSE event labels and data spelling', async () => { + const delta = '{ "type": "transcript.text.delta", "delta": "hi", "vendor": true }'; + const extension = '{ "vendor_extension": "no type discriminator" }'; + const done = '{ "type": "transcript.text.done", "text": "hi", "usage": { "type": "tokens", "input_tokens": 3, "output_tokens": 1 } }'; + resolves([candidate(async () => ({ + response: new Response(`event: transcript.delta\ndata: ${delta}\n\nevent: vendor.extension\ndata: ${extension}\n\nevent: transcript.completed\ndata: ${done}\n\n`, { + headers: { 'content-type': 'text/event-stream' }, + }), + modelKey: 'whisper-key', + } as ProviderCallResult))]); + + const { facts, drain } = await serve(); + const frames: SseFrame[] = []; + for await (const frame of facts['response.openaiAudioTranscription.rendered'] as AsyncIterable) frames.push(frame); + await drain(); + + expect(frames).toEqual([ + { type: 'sse', event: 'transcript.delta', data: delta }, + { type: 'sse', event: 'vendor.extension', data: extension }, + { type: 'sse', event: 'transcript.completed', data: done }, + ]); + expect(await facts['response.openaiAudioTranscription.streamedOutcome']).toMatchObject({ + failed: false, + billable: [{ quantities: { input_tokens: '3', output_tokens: '1' } }], + }); + }); + it('keeps the events view clear of what answers to release', () => { // Neither the generator nor the view around it is a resource. Whether the host marks a // generator disposable varies — Node 24 does, Node 22 does not — and the run's answer diff --git a/packages/gateway/src/data-plane/openai-audio/call-upstream.ts b/packages/gateway/src/data-plane/openai-audio/call-upstream.ts index 0f10a1a72a..a8898f3dce 100644 --- a/packages/gateway/src/data-plane/openai-audio/call-upstream.ts +++ b/packages/gateway/src/data-plane/openai-audio/call-upstream.ts @@ -8,7 +8,7 @@ import { upstreamPerformanceContext, telemetryModelIdentity } from '../shared/te import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; import { defineStage, move, own, defer, type Owned, type Logger, type Deferred } from '@floway-dev/pipeline'; import { isEventStreamMediaType, eventFrame, parseSSEStream, parseDecimalString } from '@floway-dev/protocols/common'; -import { parseOpenAIAudioTranscription, parseOpenAIAudioTranscriptionUsage, parseOpenAIAudioTranscriptionStreamEvent, isOpenAIAudioTranscriptionDoneEvent, parseOpenAIAudioTranscriptionStreamUsage, type OpenAIAudioTranscriptionResponseFormat, type CanonicalOpenAIAudioTranscription, type OpenAIAudioTranscriptionUsage } from '@floway-dev/protocols/openai-audio'; +import { parseOpenAIAudioTranscription, parseOpenAIAudioTranscriptionUsage, isOpenAIAudioTranscriptionDoneEvent, parseOpenAIAudioTranscriptionStreamUsage, type OpenAIAudioTranscriptionResponseFormat, type CanonicalOpenAIAudioTranscription, type OpenAIAudioTranscriptionUsage } from '@floway-dev/protocols/openai-audio'; import { providerModelOf, type TelemetryModelIdentity } from '@floway-dev/provider'; const viewOf = (events: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => events }); @@ -121,8 +121,7 @@ export const callOpenAIAudioTranscriptionUpstream = defineStage< const metered = meterEvents(result.response.body, identity, use.gateway.abortSignal, use.log); return move({ ...facts, - // This protocol's stream is bare events rather than protocol frames, so the record is - // told how one becomes a frame instead of being left to assume. + // SSE labels and data remain upstream-owned; parsing below only observes billing and completion. 'response.openaiAudioTranscription.canonical': recordStream(metered.events, use.gateway.dump, eventFrame), 'response.openaiAudioTranscription.mediaType': mediaType, 'response.openaiAudioTranscription.streamedOutcome': metered.outcome, @@ -230,17 +229,17 @@ const meterEvents = ( let completed = false; try { for await (const frame of parseSSEStream(body, { signal })) { - const event = parseOpenAIAudioTranscriptionStreamEvent(JSON.parse(frame.data) as unknown); + const event: unknown = JSON.parse(frame.data); if (isOpenAIAudioTranscriptionDoneEvent(event)) { usage = readUsage(() => parseOpenAIAudioTranscriptionStreamUsage(event), log); completed = true; - yield event; + yield frame; // The transcript is complete, so there is nothing further to read. An upstream that // holds the connection open past this point would otherwise keep the client's own // stream open with it; returning here closes the read, which cancels the upstream. return; } - yield event; + yield frame; } } finally { // Reached however the events ended — the terminal one, a client that stopped reading, diff --git a/packages/gateway/src/data-plane/openai-audio/emit.ts b/packages/gateway/src/data-plane/openai-audio/emit.ts index a51093eea6..b240f942f6 100644 --- a/packages/gateway/src/data-plane/openai-audio/emit.ts +++ b/packages/gateway/src/data-plane/openai-audio/emit.ts @@ -1,9 +1,8 @@ import type { OpenAIAudioTranscriptionEvents, Fields } from './facts.ts'; -import { streamReferenceOf } from '../../dump/turn-dump.ts'; import { isFailure } from '../pipeline/facts.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import { defineStage, move } from '@floway-dev/pipeline'; -import { renderErrorEnvelope, sseFrame, type SseFrame } from '@floway-dev/protocols/common'; +import { renderErrorEnvelope } from '@floway-dev/protocols/common'; import { renderOpenAIAudioTranscription, type CanonicalOpenAIAudioTranscription } from '@floway-dev/protocols/openai-audio'; const isEvents = (answer: CanonicalOpenAIAudioTranscription | OpenAIAudioTranscriptionEvents): answer is OpenAIAudioTranscriptionEvents => @@ -16,11 +15,8 @@ const isEvents = (answer: CanonicalOpenAIAudioTranscription | OpenAIAudioTranscr * for a document that is carried rather than rewritten the upstream's label is the only true * description there is. * - * SSE framing is produced here and nowhere else — below this stage the answer is parsed - * events, so the same assembly would serve another transport by rendering differently at - * this one point. The upstream's own `event:` label is not carried: it is transport, and - * OpenAI's clients read this endpoint's frames by their payload's `type` rather than by the - * label. https://github.com/openai/openai-python/blob/10ee3f0da2ac6f93345c1204bd7bb1a2faa79ff2/src/openai/_streaming.py#L61-L107 + * Streamed answers retain the upstream's parsed SSE frames. Reading their event payloads + * for usage must not replace labels or reserialize data seen by the client. */ export const emitOpenAIAudioTranscription = defineStage< Fields<'ingress.openaiAudioTranscription.responseFormat'>, @@ -69,17 +65,8 @@ export const emitOpenAIAudioTranscription = defineStage< 'response.http.headers': forClient, 'response.http.status': 200, 'response.openaiAudioTranscription.rendered': move(isEvents(answer) - ? renderSSE(answer) + ? answer : renderOpenAIAudioTranscription(back['ingress.openaiAudioTranscription.responseFormat'], answer)), }; }, }); - -const renderSSE = (events: OpenAIAudioTranscriptionEvents): AsyncIterable => ({ - // The frames the client reads are a reframing of the events the record holds, so this key - // points at that same stream rather than at nothing. - ...streamReferenceOf(events), - [Symbol.asyncIterator]: () => (async function* () { - for await (const event of events) yield sseFrame(JSON.stringify(event)); - })(), -}); diff --git a/packages/gateway/src/data-plane/openai-audio/facts.ts b/packages/gateway/src/data-plane/openai-audio/facts.ts index 61bcf473f5..b1f63a5901 100644 --- a/packages/gateway/src/data-plane/openai-audio/facts.ts +++ b/packages/gateway/src/data-plane/openai-audio/facts.ts @@ -1,7 +1,7 @@ import type { BillableEntity, GatewayFacts, Failure } from '../pipeline/facts.ts'; import type { Deferred } from '@floway-dev/pipeline'; import type { SseFrame } from '@floway-dev/protocols/common'; -import type { OpenAIAudioTranscriptionStreamEvent, OpenAIAudioTranscriptionResponseFormat, CanonicalOpenAIAudioTranscription } from '@floway-dev/protocols/openai-audio'; +import type { OpenAIAudioTranscriptionResponseFormat, CanonicalOpenAIAudioTranscription } from '@floway-dev/protocols/openai-audio'; import type { OpenAIAudioTranscriptionFormEntry } from '@floway-dev/provider'; /** The answer while it is still the upstream's, one event at a time. It is a view and not a @@ -11,7 +11,7 @@ import type { OpenAIAudioTranscriptionFormEntry } from '@floway-dev/provider'; * A view is a wrapper around the generator rather than the generator itself, which is what * says where the resource is: the upstream's body at `response.http.body`, claimed with * `own()`, and nothing else here. */ -export type OpenAIAudioTranscriptionEvents = AsyncIterable; +export type OpenAIAudioTranscriptionEvents = AsyncIterable; /** What settling this run will be told once the events run out: what the upstream metered, * and whether the transcript ever finished. */ diff --git a/packages/protocols/src/openai-audio/index.ts b/packages/protocols/src/openai-audio/index.ts index d712dd51ab..ee0dc444c5 100644 --- a/packages/protocols/src/openai-audio/index.ts +++ b/packages/protocols/src/openai-audio/index.ts @@ -24,6 +24,5 @@ export { export type { OpenAIAudioTranscriptionDoneEvent, OpenAIAudioTranscriptionStreamEvent } from './stream.ts'; export { isOpenAIAudioTranscriptionDoneEvent, - parseOpenAIAudioTranscriptionStreamEvent, parseOpenAIAudioTranscriptionStreamUsage, } from './stream.ts'; diff --git a/packages/protocols/src/openai-audio/stream.ts b/packages/protocols/src/openai-audio/stream.ts index 94e90ff10f..55d73047d6 100644 --- a/packages/protocols/src/openai-audio/stream.ts +++ b/packages/protocols/src/openai-audio/stream.ts @@ -23,15 +23,6 @@ export const isOpenAIAudioTranscriptionDoneEvent = (event: unknown): event is Op && (event as { type?: unknown }).type === 'transcript.text.done' && typeof (event as { text?: unknown }).text === 'string'; -/** The reading every frame goes through, so the value at the canonical key is a parsed - * event and the edge re-serializes it rather than relaying the bytes it arrived in. */ -export const parseOpenAIAudioTranscriptionStreamEvent = (value: unknown): OpenAIAudioTranscriptionStreamEvent => { - if (typeof value !== 'object' || value === null || typeof (value as { type?: unknown }).type !== 'string') { - throw new Error('OpenAI Audio Transcriptions stream event must be an object carrying a string type'); - } - return value as OpenAIAudioTranscriptionStreamEvent; -}; - /** A streamed transcription states its usage once, in the terminal event, in the same shape * a JSON body states it. */ export const parseOpenAIAudioTranscriptionStreamUsage = ( From d67dc84565acec4941871fd3b341c918811f9b2a Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 03:05:16 +0800 Subject: [PATCH 60/79] fix(pipeline): preserve observed response outcomes Keep successful upstream HTTP statuses through protocol emission. Record each billable observation while summing request diagnostics, and derive output performance from the observed quantities even on partial failures. Normalize audio uploads into portable content facts before recording. --- README.md | 16 ++++-- .../data-plane/openai-embeddings/http_test.ts | 23 ++++++++ .../data-plane/pipeline/settlement_test.ts | 53 +++++++++++++++++++ .../data-plane/openai-audio/call-upstream.ts | 6 ++- .../src/data-plane/openai-audio/emit.ts | 9 ++-- .../src/data-plane/openai-audio/facts.ts | 18 +++++-- .../src/data-plane/openai-audio/http.ts | 15 ++++-- .../src/data-plane/openai-completions/emit.ts | 9 ++-- .../openai-embeddings/call-upstream.ts | 6 ++- .../src/data-plane/openai-embeddings/emit.ts | 6 +-- .../data-plane/openai-images/call-upstream.ts | 6 ++- .../src/data-plane/openai-images/emit.ts | 6 +-- .../data-plane/pipeline/resolve-candidates.ts | 9 ++-- .../src/data-plane/pipeline/settlement.ts | 7 ++- .../src/data-plane/rerank/call-upstream.ts | 9 +++- .../gateway/src/data-plane/rerank/emit.ts | 6 +-- packages/gateway/src/dump/attribution.ts | 5 +- 17 files changed, 160 insertions(+), 49 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/pipeline/settlement_test.ts diff --git a/README.md b/README.md index a3fbf2eb1b..b491c8af1f 100644 --- a/README.md +++ b/README.md @@ -90,10 +90,18 @@ Rerank models are manual Custom models. Each model selects its outbound Cohere, Jina, Voyage, DashScope-compatible, or DashScope-native protocol and may override that protocol's canonical path; there is no upstream-wide rerank path. -Audio transcription parses multipart uploads into ordered entries and rebuilds -them for each Custom, Azure, or Ollama-compatible upstream attempt. JSON, text, -and subtitle responses retain their upstream document; transcription SSE -responses preserve event labels and data while usage is observed separately. +Audio transcription parses multipart uploads into ordered content entries, +retaining each file's name, media type, timestamp and bytes. Each upstream +attempt builds its own multipart body. JSON, text and subtitle responses retain +their upstream document; transcription SSE responses preserve event labels and +data while usage is observed separately. Successful pipeline responses retain +the upstream HTTP status. + +Billing records each actual upstream usage observation independently, including +multiple model calls within one hosted-tool turn. Pricing uses each observation's +model identity and pricing facts. Request diagnostics sum the observed token +quantities; performance uses the observed output count for TPOT and partial +output failures. ### Upstreams diff --git a/packages/gateway/__tests__/data-plane/openai-embeddings/http_test.ts b/packages/gateway/__tests__/data-plane/openai-embeddings/http_test.ts index 0595fed5b8..a3c6410441 100644 --- a/packages/gateway/__tests__/data-plane/openai-embeddings/http_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-embeddings/http_test.ts @@ -6,6 +6,29 @@ import { buildCustomUpstreamRecord, copilotModels, flushAsyncWork, requestAppWit import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { jsonResponse, withMockedFetch, assertEquals, assertExists } from '@floway-dev/test-utils'; +test('/v1/embeddings preserves an upstream success status', async () => { + const { apiKey, repo } = await setupAppTest(); + await repo.upstreams.deleteAll(); + await saveUpstreamForTest(repo.upstreams, buildCustomUpstreamRecord({ + config: { + baseUrl: 'https://embeddings.test', authStyle: 'bearer', apiKey: 'sk-test', ingressHeadersRules: [], endpoints: {}, + modelsFetch: { enabled: false }, + models: [{ upstreamModelId: 'embedding', kind: 'embedding', endpoints: { openaiEmbeddings: {} } }], + }, + })); + await withMockedFetch(async () => jsonResponse({ + object: 'list', model: 'embedding', data: [{ object: 'embedding', index: 0, embedding: [0.5] }], + usage: { prompt_tokens: 1, total_tokens: 1 }, + }, 201), async () => { + const response = await requestAppWithWarmModels('/v1/embeddings', { + method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + body: JSON.stringify({ model: 'embedding', input: 'hello', encoding_format: 'float' }), + }); + assertEquals(response.status, 201); + assertEquals((await response.json() as { data: unknown[] }).data.length, 1); + }); +}); + test('/v1/embeddings keeps distinct alias candidates on the same upstream through failover', async () => { const { apiKey, repo } = await setupAppTest(); await repo.upstreams.deleteAll(); diff --git a/packages/gateway/__tests__/data-plane/pipeline/settlement_test.ts b/packages/gateway/__tests__/data-plane/pipeline/settlement_test.ts new file mode 100644 index 0000000000..547cecc6af --- /dev/null +++ b/packages/gateway/__tests__/data-plane/pipeline/settlement_test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { settleBillable } from '../../../src/data-plane/pipeline/settlement.ts'; +import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts'; +import { openRunDump } from '../../../src/dump/run-sink.ts'; +import { initRepo } from '../../../src/repo/index.ts'; +import { installDumpStubs, runRecordOf } from '../../dump/test-fixtures.ts'; +import { InMemoryRepo } from '../../repo/memory.ts'; +import { mockGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { mockPerfTelemetryContext } from '@floway-dev/test-utils'; + +describe('pipeline settlement', () => { + it.each([false, true])('keeps call rows, totals and output performance on failed=%s', async failed => { + const repo = new InMemoryRepo(); + initRepo(repo); + const stubs = installDumpStubs(initDumpStore, initDumpBroker); + const pending: Promise[] = []; + const background = (work: Promise) => { pending.push(work); }; + const key = { + id: 'key_test', userId: 1, name: 'Test key', key: 'raw-key', serverSecret: '11'.repeat(32), + createdAt: '2026-01-01T00:00:00.000Z', upstreamIds: null, deletedAt: null, + dumpRetentionSeconds: 3600, openaiResponsesRetentionSeconds: 0, + }; + const recordUsage = vi.spyOn(repo.usage, 'record'); + const timing = { upstreamCallStartedAt: 50, firstOutputTokenAt: 100 }; + const dump = openRunDump(key, { + method: 'POST', path: '/v1/responses', body: { bytes: new Uint8Array(), streamError: null }, + }, background, true, timing); + const gateway = mockGatewayCtx({ + dump, backgroundScheduler: background, + attempt: { timing, telemetry: mockPerfTelemetryContext({ keyId: key.id }) }, + }); + const identity = { model: 'm', upstream: 'u', modelKey: 'm', pricing: null }; + settleBillable({ gateway, background, log: { debug: () => {}, info: () => {}, warn: () => {}, error: () => {} } }, [ + { identity, quantities: { input_tokens: '10', output_tokens: '2' } }, + { identity, quantities: { input_tokens: '20', output_tokens: '3' } }, + ], failed, 500); + await Promise.all(pending); + const usage = await repo.usage.listAll(); + expect(recordUsage.mock.calls.map(([row]) => row.metrics.map(metric => metric.quantity))).toEqual([['10', '2'], ['20', '3']]); + expect(usage).toMatchObject([{ requests: 2 }]); + expect(await repo.performance.listAll()).toMatchObject([{ + requests: 1, tpotSamples: 1, tpotUsSum: 100_000, + errorsWithOutput: failed ? 1 : 0, errorsNoOutput: 0, + }]); + dump!.finalize(200, 0); + await Promise.all(pending); + expect(runRecordOf(stubs.stored[0]?.record).meta).toMatchObject({ + inputTokens: 30, outputTokens: 5, model: 'm', + error: failed ? { kind: 'failed' } : null, + }); + }); +}); diff --git a/packages/gateway/src/data-plane/openai-audio/call-upstream.ts b/packages/gateway/src/data-plane/openai-audio/call-upstream.ts index a8898f3dce..3008961818 100644 --- a/packages/gateway/src/data-plane/openai-audio/call-upstream.ts +++ b/packages/gateway/src/data-plane/openai-audio/call-upstream.ts @@ -51,7 +51,11 @@ export const callOpenAIAudioTranscriptionUpstream = defineStage< try { result = await candidate.provider.instance.callOpenAIAudioTranscriptions( providerModelOf(candidate), - { entries: facts['request.openaiAudioTranscription.form'] }, + { + entries: facts['request.openaiAudioTranscription.form'].map(({ name, value }) => ({ + name, value: typeof value === 'string' ? value : new File([value.bytes], value.name, { type: value.type, lastModified: value.lastModified }), + })), + }, use.gateway.abortSignal, // The client's own headers reach the upstream from the record, not from a live request // object: what a provider is allowed to forward is filtered per provider, and the dump diff --git a/packages/gateway/src/data-plane/openai-audio/emit.ts b/packages/gateway/src/data-plane/openai-audio/emit.ts index b240f942f6..6f7f3b1fe4 100644 --- a/packages/gateway/src/data-plane/openai-audio/emit.ts +++ b/packages/gateway/src/data-plane/openai-audio/emit.ts @@ -22,7 +22,7 @@ export const emitOpenAIAudioTranscription = defineStage< Fields<'ingress.openaiAudioTranscription.responseFormat'>, Fields<'ingress.openaiAudioTranscription.responseFormat'>, Fields<'ingress.openaiAudioTranscription.responseFormat' | 'response.openaiAudioTranscription.canonical' | 'response.openaiAudioTranscription.mediaType'> - & { 'response.http.headers': readonly (readonly [string, string])[] }, + & { 'response.http.status': number; 'response.http.headers': readonly (readonly [string, string])[] }, Fields<'response.openaiAudioTranscription.rendered' | 'response.openaiAudioTranscription.mediaType'> & { 'response.http.status': number; 'response.http.headers': readonly (readonly [string, string])[] } >({ @@ -34,7 +34,7 @@ export const emitOpenAIAudioTranscription = defineStage< provides: [], }, response: { - needs: ['response.openaiAudioTranscription.canonical', 'response.openaiAudioTranscription.mediaType', 'response.http.headers'], + needs: ['response.openaiAudioTranscription.canonical', 'response.openaiAudioTranscription.mediaType', 'response.http.headers', 'response.http.status'], consumes: ['response.openaiAudioTranscription.canonical', 'response.http.headers'], provides: ['response.openaiAudioTranscription.rendered', 'response.openaiAudioTranscription.mediaType', 'response.http.status', 'response.http.headers'], }, @@ -57,13 +57,10 @@ export const emitOpenAIAudioTranscription = defineStage< 'response.openaiAudioTranscription.rendered': move(renderErrorEnvelope(answer.message, answer.body)), }; } - // Everything that reaches here answered. The same key carried the upstream's own status - // further down; re-providing it is what makes the top of the record the response the - // client gets rather than the one the upstream gave. return { ...rest, 'response.http.headers': forClient, - 'response.http.status': 200, + 'response.http.status': back['response.http.status'], 'response.openaiAudioTranscription.rendered': move(isEvents(answer) ? answer : renderOpenAIAudioTranscription(back['ingress.openaiAudioTranscription.responseFormat'], answer)), diff --git a/packages/gateway/src/data-plane/openai-audio/facts.ts b/packages/gateway/src/data-plane/openai-audio/facts.ts index b1f63a5901..543ed864a3 100644 --- a/packages/gateway/src/data-plane/openai-audio/facts.ts +++ b/packages/gateway/src/data-plane/openai-audio/facts.ts @@ -2,7 +2,18 @@ import type { BillableEntity, GatewayFacts, Failure } from '../pipeline/facts.ts import type { Deferred } from '@floway-dev/pipeline'; import type { SseFrame } from '@floway-dev/protocols/common'; import type { OpenAIAudioTranscriptionResponseFormat, CanonicalOpenAIAudioTranscription } from '@floway-dev/protocols/openai-audio'; -import type { OpenAIAudioTranscriptionFormEntry } from '@floway-dev/provider'; + +export interface AudioUpload { + readonly name: string; + readonly type: string; + readonly lastModified: number; + readonly bytes: Uint8Array; +} + +export interface AudioFormEntry { + readonly name: string; + readonly value: string | AudioUpload; +} /** The answer while it is still the upstream's, one event at a time. It is a view and not a * resource: what owns the connection is `response.http.body`, which is where release and @@ -32,9 +43,8 @@ export interface OpenAIAudioTranscriptionFacts extends GatewayFacts { 'ingress.openaiAudioTranscription.responseFormat': OpenAIAudioTranscriptionResponseFormat; /** The multipart form as ordered semantic entries. The body is parsed before routing * because field order is unconstrained, and every candidate builds a fresh body from - * these, so a retry never reuses a consumed one. The bytes the client sent are recorded - * at `ingress.http.body`, which is where a dump reads the upload itself. */ - 'request.openaiAudioTranscription.form': readonly OpenAIAudioTranscriptionFormEntry[]; + * these, so a retry never reuses a consumed one. Uploaded files carry their metadata and bytes as portable content values. */ + 'request.openaiAudioTranscription.form': readonly AudioFormEntry[]; /** The one transcription, whichever rendering carried it — or the events it is arriving * as, or the failure that came instead. A stream, a value and a failure sit at one key: * telling them apart is reading a value, and each stage does that where it needs to. */ diff --git a/packages/gateway/src/data-plane/openai-audio/http.ts b/packages/gateway/src/data-plane/openai-audio/http.ts index ee5b286054..2a7b2effa3 100644 --- a/packages/gateway/src/data-plane/openai-audio/http.ts +++ b/packages/gateway/src/data-plane/openai-audio/http.ts @@ -8,13 +8,13 @@ import type { Context } from 'hono'; +import type { AudioFormEntry } from './facts.ts'; import { openaiAudioTranscriptionServePipeline } from './pipeline.ts'; import { isFrames, openPrologue, readIngress, serveThrough } from '../pipeline/serve.ts'; import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; import { move } from '@floway-dev/pipeline'; import { isMultipartFormDataMediaType } from '@floway-dev/protocols/common'; import { parseOpenAIAudioTranscriptionResponseFormat, type OpenAIAudioTranscriptionResponseFormat } from '@floway-dev/protocols/openai-audio'; -import type { OpenAIAudioTranscriptionFormEntry } from '@floway-dev/provider'; type PreparedOpenAIAudioTranscription = | { @@ -22,7 +22,7 @@ type PreparedOpenAIAudioTranscription = readonly model: string; readonly responseFormat: OpenAIAudioTranscriptionResponseFormat; readonly wantsStream: boolean; - readonly entries: readonly OpenAIAudioTranscriptionFormEntry[]; + readonly entries: readonly AudioFormEntry[]; } | { readonly type: 'invalid'; readonly message: string }; @@ -57,9 +57,16 @@ const prepareOpenAIAudioTranscription = async (bytes: Uint8Array, contentType: s return { type: 'invalid', message: error instanceof Error ? error.message : String(error) }; } - const entries: OpenAIAudioTranscriptionFormEntry[] = []; + const entries: AudioFormEntry[] = []; for (const [name, value] of form.entries()) { - entries.push({ name, value }); + entries.push({ + name, value: typeof value === 'string' ? value : { + name: value.name, + type: value.type, + lastModified: value.lastModified, + bytes: new Uint8Array(await value.arrayBuffer()), + }, + }); } return { type: 'ok', diff --git a/packages/gateway/src/data-plane/openai-completions/emit.ts b/packages/gateway/src/data-plane/openai-completions/emit.ts index 481290b227..05b0080399 100644 --- a/packages/gateway/src/data-plane/openai-completions/emit.ts +++ b/packages/gateway/src/data-plane/openai-completions/emit.ts @@ -21,7 +21,7 @@ const isFrames = (answer: OpenAICompletionsFacts['response.openaiCompletions.pay export const emitOpenAICompletions = defineStage< Fields<'ingress.openaiCompletions.wantsStream' | 'ingress.openaiCompletions.wantsUsageChunk' | 'request.openaiCompletions.payload'>, Fields<'ingress.openaiCompletions.wantsStream' | 'ingress.openaiCompletions.wantsUsageChunk' | 'request.openaiCompletions.payload'>, - Fields<'ingress.openaiCompletions.wantsUsageChunk' | 'response.openaiCompletions.payload' | 'response.http.headers'>, + Fields<'ingress.openaiCompletions.wantsUsageChunk' | 'response.openaiCompletions.payload' | 'response.http.status' | 'response.http.headers'>, Fields<'response.openaiCompletions.rendered' | 'response.http.status' | 'response.http.headers'> >({ name: 'emitOpenAICompletions', @@ -32,7 +32,7 @@ export const emitOpenAICompletions = defineStage< provides: ['request.openaiCompletions.payload'], }, response: { - needs: ['response.openaiCompletions.payload', 'response.http.headers'], + needs: ['response.openaiCompletions.payload', 'response.http.headers', 'response.http.status'], consumes: ['response.openaiCompletions.payload', 'response.http.headers'], provides: ['response.openaiCompletions.rendered', 'response.http.status', 'response.http.headers'], }, @@ -51,13 +51,10 @@ export const emitOpenAICompletions = defineStage< // would misdescribe a body this gateway serialized itself, does not. A filter that removed // nothing hands the same array on, so the record shows no change where none happened. const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); - // A refusal keeps the status the upstream gave it. Anything that answered is a 200 the - // gateway says itself, because what the client receives is serialized here rather than - // relayed — the upstream's own status is a fact further down for whoever wants it. return { ...rest, 'response.http.headers': forwardable.length === headers.length ? headers : move(forwardable), - 'response.http.status': isFailure(answer) ? answer.status : 200, + 'response.http.status': isFailure(answer) ? answer.status : back['response.http.status'], 'response.openaiCompletions.rendered': move(rendered(answer, back['ingress.openaiCompletions.wantsUsageChunk'])), }; }, diff --git a/packages/gateway/src/data-plane/openai-embeddings/call-upstream.ts b/packages/gateway/src/data-plane/openai-embeddings/call-upstream.ts index a23f438eec..0f89029e24 100644 --- a/packages/gateway/src/data-plane/openai-embeddings/call-upstream.ts +++ b/packages/gateway/src/data-plane/openai-embeddings/call-upstream.ts @@ -17,12 +17,12 @@ import { providerModelOf } from '@floway-dev/provider'; */ export const callOpenAIEmbeddingsUpstream = defineStage< Fields<'request.openaiEmbeddings.canonical' | 'route.attempt' | 'ingress.http.headers' | 'serve.model'>, - Fields<'response.openaiEmbeddings.canonical' | 'response.http.headers' | 'response.usage.billable'>, + Fields<'response.openaiEmbeddings.canonical' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'>, GatewayServices >({ name: 'callOpenAIEmbeddingsUpstream', return: { - provides: ['response.openaiEmbeddings.canonical', 'response.http.headers', 'response.usage.billable'], + provides: ['response.openaiEmbeddings.canonical', 'response.http.status', 'response.http.headers', 'response.usage.billable'], }, execute: async (facts, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); @@ -48,6 +48,7 @@ export const callOpenAIEmbeddingsUpstream = defineStage< return move({ ...facts, 'response.openaiEmbeddings.canonical': dialFailure(error), + 'response.http.status': 502, 'response.http.headers': [], 'response.usage.billable': [], }); @@ -62,6 +63,7 @@ export const callOpenAIEmbeddingsUpstream = defineStage< const answered = (canonical: CanonicalOpenAIEmbeddingsResponse | Failure, quantities: UsageQuantities) => move({ ...facts, 'response.openaiEmbeddings.canonical': canonical, + 'response.http.status': result.response.status, 'response.http.headers': headers, 'response.usage.billable': [{ identity, quantities }], }); diff --git a/packages/gateway/src/data-plane/openai-embeddings/emit.ts b/packages/gateway/src/data-plane/openai-embeddings/emit.ts index 0671161d47..1b0ce7a199 100644 --- a/packages/gateway/src/data-plane/openai-embeddings/emit.ts +++ b/packages/gateway/src/data-plane/openai-embeddings/emit.ts @@ -14,7 +14,7 @@ import { renderOpenAIEmbeddingsResponse } from '@floway-dev/protocols/openai-emb export const emitOpenAIEmbeddings = defineStage< Fields<'ingress.openaiEmbeddings.encodingFormat'>, Fields<'ingress.openaiEmbeddings.encodingFormat'>, - Fields<'ingress.openaiEmbeddings.encodingFormat' | 'response.openaiEmbeddings.canonical' | 'response.http.headers'>, + Fields<'ingress.openaiEmbeddings.encodingFormat' | 'response.openaiEmbeddings.canonical' | 'response.http.status' | 'response.http.headers'>, Fields<'response.openaiEmbeddings.rendered' | 'response.http.status' | 'response.http.headers'> >({ name: 'emitOpenAIEmbeddings', @@ -25,7 +25,7 @@ export const emitOpenAIEmbeddings = defineStage< provides: [], }, response: { - needs: ['response.openaiEmbeddings.canonical', 'response.http.headers'], + needs: ['response.openaiEmbeddings.canonical', 'response.http.headers', 'response.http.status'], consumes: ['response.openaiEmbeddings.canonical', 'response.http.headers'], provides: ['response.openaiEmbeddings.rendered', 'response.http.status', 'response.http.headers'], }, @@ -52,7 +52,7 @@ export const emitOpenAIEmbeddings = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.http.status': 200, + 'response.http.status': back['response.http.status'], 'response.openaiEmbeddings.rendered': move(renderOpenAIEmbeddingsResponse(back['ingress.openaiEmbeddings.encodingFormat'], answer)), }; }, diff --git a/packages/gateway/src/data-plane/openai-images/call-upstream.ts b/packages/gateway/src/data-plane/openai-images/call-upstream.ts index 2c7f3df8de..b434f13c6c 100644 --- a/packages/gateway/src/data-plane/openai-images/call-upstream.ts +++ b/packages/gateway/src/data-plane/openai-images/call-upstream.ts @@ -32,7 +32,7 @@ const PERFORMANCE_OPERATION = { */ export const callOpenAIImagesUpstream = defineStage< Fields<'ingress.openaiImages.wantsStream' | 'request.openaiImages.canonical' | 'route.attempt' | 'ingress.http.headers'>, - Fields<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.headers' + Fields<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.http.body' | 'response.usage.billable'>, GatewayServices >({ @@ -41,6 +41,7 @@ export const callOpenAIImagesUpstream = defineStage< provides: [ 'response.openaiImages.canonical', 'response.openaiImages.streamedUsage', + 'response.http.status', 'response.http.headers', 'response.http.body', 'response.usage.billable', @@ -79,6 +80,7 @@ export const callOpenAIImagesUpstream = defineStage< ...facts, 'response.openaiImages.canonical': dialFailure(error), 'response.openaiImages.streamedUsage': null, + 'response.http.status': 502, 'response.http.headers': [], 'response.http.body': spentBody(null), 'response.usage.billable': [], @@ -95,6 +97,7 @@ export const callOpenAIImagesUpstream = defineStage< ...facts, 'response.openaiImages.canonical': canonical, 'response.openaiImages.streamedUsage': null, + 'response.http.status': result.response.status, 'response.http.headers': headers, 'response.http.body': spentBody(result.response.body), 'response.usage.billable': billable, @@ -132,6 +135,7 @@ export const callOpenAIImagesUpstream = defineStage< // told how one becomes a frame instead of being left to assume. 'response.openaiImages.canonical': recordStream(metered.frames, use.gateway.dump, eventFrame), 'response.openaiImages.streamedUsage': metered.outcome, + 'response.http.status': result.response.status, 'response.http.headers': headers, // Releasing this body is reading those events to the end: they are one reader over one // connection, and a second reader is not something a `ReadableStream` allows. diff --git a/packages/gateway/src/data-plane/openai-images/emit.ts b/packages/gateway/src/data-plane/openai-images/emit.ts index 9cee0e6d4d..35154cd14a 100644 --- a/packages/gateway/src/data-plane/openai-images/emit.ts +++ b/packages/gateway/src/data-plane/openai-images/emit.ts @@ -22,14 +22,14 @@ const isFrames = (answer: CanonicalOpenAIImagesResponse | OpenAIImagesFrames): a export const emitOpenAIImages = defineStage< Record, Record, - Fields<'response.openaiImages.canonical' | 'response.http.headers'>, + Fields<'response.openaiImages.canonical' | 'response.http.status' | 'response.http.headers'>, Fields<'response.openaiImages.rendered' | 'response.http.status' | 'response.http.headers'> >({ name: 'emitOpenAIImages', through: { request: { needs: [], consumes: [], provides: [] }, response: { - needs: ['response.openaiImages.canonical', 'response.http.headers'], + needs: ['response.openaiImages.canonical', 'response.http.headers', 'response.http.status'], consumes: ['response.openaiImages.canonical', 'response.http.headers'], provides: ['response.openaiImages.rendered', 'response.http.status', 'response.http.headers'], }, @@ -44,7 +44,7 @@ export const emitOpenAIImages = defineStage< return { ...rest, 'response.http.headers': forwardable.length === headers.length ? headers : move(forwardable), - 'response.http.status': isFailure(answer) ? answer.status : 200, + 'response.http.status': isFailure(answer) ? answer.status : back['response.http.status'], 'response.openaiImages.rendered': move(rendered(answer)), }; }, diff --git a/packages/gateway/src/data-plane/pipeline/resolve-candidates.ts b/packages/gateway/src/data-plane/pipeline/resolve-candidates.ts index 8f94456c9a..b6021b00dd 100644 --- a/packages/gateway/src/data-plane/pipeline/resolve-candidates.ts +++ b/packages/gateway/src/data-plane/pipeline/resolve-candidates.ts @@ -38,9 +38,9 @@ export interface Narrowing { export const resolveCandidates = (narrowing: Narrowing) => defineStage< Slice<'serve.model'>, // what arrives Slice<'serve.model' | 'serve.candidates'>, // what it hands down - Slice<'response.usage.billable' | 'response.http.headers'>, // what comes back - Slice<'response.usage.billable' | 'response.http.headers'>, // what it hands up, having descended - Slice<'response.usage.billable' | 'response.http.headers'> & Refusal, // and what it answers with instead + Slice<'response.usage.billable' | 'response.http.headers' | 'response.http.status'>, // what comes back + Slice<'response.usage.billable' | 'response.http.headers' | 'response.http.status'>, // what it hands up, having descended + Slice<'response.usage.billable' | 'response.http.headers' | 'response.http.status'> & Refusal, // and what it answers with instead GatewayServices >({ name: 'resolveCandidates', @@ -48,7 +48,7 @@ export const resolveCandidates = (narrowing: Narrowing { const model = facts['serve.model']; const { candidates, sawModel, failedUpstreams } = await enumerateModelCandidates({ @@ -67,6 +67,7 @@ export const resolveCandidates = (narrowing: Narrowing & { readonly log: Logger }, billable: readonly BillableEntity[], failed: boolean, - outputTokens = 0, finishedAt: number = performance.now(), ): void => { for (const entity of billable) { - // The dump names the upstream that answered and what it metered, which is the same - // reading the row is written from rather than a second one taken separately. - if (!failed) services.gateway.dump?.success(entity.identity, dumpUsage(entity.quantities)); + services.gateway.dump?.success(entity.identity, dumpUsage(entity.quantities)); services.background(recordUsage( services.gateway.apiKeyId, entity.identity, @@ -68,6 +65,8 @@ export const settleBillable = ( services.log.error('failed to record usage', { error: String(error) }); })); } + if (failed) services.gateway.dump?.failed('The upstream response failed.'); + const outputTokens = billable.reduce((total, entity) => total + Number(entity.quantities.output_tokens ?? 0), 0); recordPerformance(services.gateway, services.gateway.attempt.telemetry, failed, outputTokens, finishedAt); }; diff --git a/packages/gateway/src/data-plane/rerank/call-upstream.ts b/packages/gateway/src/data-plane/rerank/call-upstream.ts index e2d7c1f585..59ee71306d 100644 --- a/packages/gateway/src/data-plane/rerank/call-upstream.ts +++ b/packages/gateway/src/data-plane/rerank/call-upstream.ts @@ -17,12 +17,12 @@ import { providerModelOf } from '@floway-dev/provider'; */ export const callRerankUpstream = defineStage< Fields<'request.rerank.canonical' | 'route.attempt' | 'ingress.http.headers' | 'ingress.rerank.sourceProtocol'>, - Fields<'response.rerank.canonical' | 'response.rerank.targetProtocol' | 'response.http.headers' | 'response.usage.billable'>, + Fields<'response.rerank.canonical' | 'response.rerank.targetProtocol' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'>, GatewayServices >({ name: 'callRerankUpstream', return: { - provides: ['response.rerank.canonical', 'response.rerank.targetProtocol', 'response.http.headers', 'response.usage.billable'], + provides: ['response.rerank.canonical', 'response.rerank.targetProtocol', 'response.http.status', 'response.http.headers', 'response.usage.billable'], }, execute: async (facts, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); @@ -58,6 +58,7 @@ export const callRerankUpstream = defineStage< ...facts, 'response.rerank.canonical': dialFailure(error), 'response.rerank.targetProtocol': configuredTarget.protocol, + 'response.http.status': 502, 'response.http.headers': [], 'response.usage.billable': [], }); @@ -78,6 +79,7 @@ export const callRerankUpstream = defineStage< ...('json' in body ? { body: body.json } : {}), }, 'response.rerank.targetProtocol': result.target.protocol, + 'response.http.status': result.response.status, 'response.http.headers': headers, // The upstream was called and reported nothing, which is a different situation // from reporting zero — so the entity is present with no quantities. @@ -90,6 +92,7 @@ export const callRerankUpstream = defineStage< ...facts, 'response.rerank.canonical': unreadableBody(result.response, body, 'the rerank protocol'), 'response.rerank.targetProtocol': result.target.protocol, + 'response.http.status': result.response.status, 'response.http.headers': headers, 'response.usage.billable': [{ identity, quantities: {} }], }); @@ -129,6 +132,7 @@ export const callRerankUpstream = defineStage< ...facts, 'response.rerank.canonical': unreadableBody(result.response, body, 'the rerank protocol'), 'response.rerank.targetProtocol': result.target.protocol, + 'response.http.status': result.response.status, 'response.http.headers': headers, 'response.usage.billable': metered, }); @@ -141,6 +145,7 @@ export const callRerankUpstream = defineStage< ...facts, 'response.rerank.canonical': canonical, 'response.rerank.targetProtocol': result.target.protocol, + 'response.http.status': result.response.status, 'response.http.headers': headers, 'response.usage.billable': metered, }); diff --git a/packages/gateway/src/data-plane/rerank/emit.ts b/packages/gateway/src/data-plane/rerank/emit.ts index 060fc6c6be..0a1a60029b 100644 --- a/packages/gateway/src/data-plane/rerank/emit.ts +++ b/packages/gateway/src/data-plane/rerank/emit.ts @@ -13,7 +13,7 @@ import { renderRerankResponse } from '@floway-dev/protocols/rerank'; export const emitRerank = defineStage< Fields<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical'>, Fields<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical'>, - Fields<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical' | 'response.rerank.canonical' | 'response.rerank.targetProtocol' | 'response.http.headers'>, + Fields<'ingress.rerank.sourceProtocol' | 'request.rerank.canonical' | 'response.rerank.canonical' | 'response.rerank.targetProtocol' | 'response.http.status' | 'response.http.headers'>, Fields<'response.rerank.rendered' | 'response.http.status' | 'response.http.headers'> >({ name: 'emitRerank', @@ -24,7 +24,7 @@ export const emitRerank = defineStage< provides: [], }, response: { - needs: ['response.rerank.canonical', 'response.http.headers'], + needs: ['response.rerank.canonical', 'response.http.headers', 'response.http.status'], consumes: ['response.rerank.canonical', 'response.rerank.targetProtocol', 'response.http.headers'], provides: ['response.rerank.rendered', 'response.http.status', 'response.http.headers'], }, @@ -71,7 +71,7 @@ export const emitRerank = defineStage< return { ...rest, 'response.http.headers': forClient, - 'response.http.status': 200, + 'response.http.status': back['response.http.status'], 'response.rerank.rendered': move(rendered), }; }, diff --git a/packages/gateway/src/dump/attribution.ts b/packages/gateway/src/dump/attribution.ts index c240c82f4e..0a32bc255e 100644 --- a/packages/gateway/src/dump/attribution.ts +++ b/packages/gateway/src/dump/attribution.ts @@ -107,8 +107,9 @@ export class DumpAttribution { success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void { this.model = identity.model; this.upstreamId = identity.upstream; - this.inputTokens = tokenUsageInput(usage); - this.outputTokens = usage?.output ?? null; + const input = tokenUsageInput(usage); + if (input !== null) this.inputTokens = (this.inputTokens ?? 0) + input; + if (usage?.output !== undefined) this.outputTokens = (this.outputTokens ?? 0) + usage.output; } async metadata(outcome: DumpTurnOutcome): Promise { From 83d6ecae4624a87f964b2a428ba317107aaa06b5 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 04:31:13 +0800 Subject: [PATCH 61/79] feat: expose non-chat provider operations as pipelines Enter provider-owned fact spaces through operation-specific chains and finish each supported request with the shared HTTP stage. Keep URL, authentication headers and immutable request content visible as facts, and track each response body through one ownership identity. Use portable model flags and file snapshots, stream replayable multipart content without native file facts, preserve transport error chains, and retain replaced Codex endpoint observations and full 401 diagnostics across credential retries. Validation: HTTP/provider/test-utils typechecks passed; 124 HTTP and provider test files passed with 1658 tests; targeted ESLint and staged diff checks passed. --- packages/http/__tests__/pipeline_test.ts | 122 ++++++++++++++++++ packages/http/package.json | 10 ++ packages/http/src/http.ts | 34 +++++ packages/http/src/pipeline-types.ts | 45 +++++++ packages/http/src/pipeline.ts | 3 + packages/http/src/request-content.ts | 92 +++++++++++++ packages/http/src/response.ts | 21 +++ .../provider-azure/__tests__/provider_test.ts | 18 +-- packages/provider-azure/package.json | 2 + packages/provider-azure/src/index.ts | 1 + packages/provider-azure/src/pipelines.ts | 13 ++ packages/provider-azure/src/provider.ts | 2 + .../src/stages/prepare-request.ts | 73 +++++++++++ packages/provider-claude-code/src/provider.ts | 1 + .../provider-codex/__tests__/provider_test.ts | 78 ++++++++++- packages/provider-codex/package.json | 2 + packages/provider-codex/src/fetch.ts | 74 ++++++----- packages/provider-codex/src/index.ts | 1 + packages/provider-codex/src/models.ts | 2 +- packages/provider-codex/src/pipeline-facts.ts | 65 ++++++++++ packages/provider-codex/src/pipelines.ts | 16 +++ packages/provider-codex/src/provider.ts | 2 + .../provider-codex/src/stages/authenticate.ts | 31 +++++ .../src/stages/authorize-headers.ts | 22 ++++ .../src/stages/observe-response.ts | 35 +++++ .../src/stages/prepare-request.ts | 61 +++++++++ .../provider-codex/src/stages/read-account.ts | 12 ++ .../provider-codex/src/stages/retry-access.ts | 32 +++++ .../__tests__/provider_test.ts | 4 +- packages/provider-copilot/package.json | 2 + packages/provider-copilot/src/auth.ts | 34 +++-- packages/provider-copilot/src/index.ts | 1 + .../provider-copilot/src/operation-model.ts | 71 ++++++++++ .../provider-copilot/src/pipeline-facts.ts | 17 +++ packages/provider-copilot/src/pipelines.ts | 12 ++ packages/provider-copilot/src/provider.ts | 72 +---------- .../src/stages/authenticate.ts | 41 ++++++ .../src/stages/observe-quota.ts | 28 ++++ .../src/stages/prepare-request.ts | 38 ++++++ .../provider-custom/__tests__/fetch_test.ts | 10 +- .../__tests__/ingress-header-rules_test.ts | 23 ++-- .../__tests__/provider_test.ts | 27 ++-- packages/provider-custom/package.json | 2 + packages/provider-custom/src/index.ts | 1 + packages/provider-custom/src/pipelines.ts | 15 +++ packages/provider-custom/src/provider.ts | 2 + .../src/stages/prepare-request.ts | 114 ++++++++++++++++ .../__tests__/provider_test.ts | 6 +- packages/provider-ollama/package.json | 2 + packages/provider-ollama/src/index.ts | 1 + packages/provider-ollama/src/pipelines.ts | 16 +++ packages/provider-ollama/src/provider.ts | 2 + .../src/stages/observe-account.ts | 31 +++++ .../src/stages/prepare-request.ts | 53 ++++++++ packages/provider/__tests__/audio_test.ts | 7 +- packages/provider/__tests__/images_test.ts | 14 +- .../provider/__tests__/invocation_test.ts | 1 + packages/provider/__tests__/pipeline_test.ts | 21 +++ packages/provider/package.json | 4 +- packages/provider/src/audio.ts | 14 +- packages/provider/src/http-headers.ts | 19 +++ packages/provider/src/images.ts | 45 ++++--- packages/provider/src/index.ts | 5 +- packages/provider/src/pipeline.ts | 60 +++++++++ packages/provider/src/provider.ts | 2 + packages/test-utils/package.json | 2 + packages/test-utils/src/index.ts | 1 + packages/test-utils/src/provider-pipeline.ts | 50 +++++++ packages/test-utils/src/stubs.ts | 1 + pnpm-lock.yaml | 45 +++++++ 70 files changed, 1581 insertions(+), 205 deletions(-) create mode 100644 packages/http/__tests__/pipeline_test.ts create mode 100644 packages/http/src/http.ts create mode 100644 packages/http/src/pipeline-types.ts create mode 100644 packages/http/src/pipeline.ts create mode 100644 packages/http/src/request-content.ts create mode 100644 packages/http/src/response.ts create mode 100644 packages/provider-azure/src/pipelines.ts create mode 100644 packages/provider-azure/src/stages/prepare-request.ts create mode 100644 packages/provider-codex/src/pipeline-facts.ts create mode 100644 packages/provider-codex/src/pipelines.ts create mode 100644 packages/provider-codex/src/stages/authenticate.ts create mode 100644 packages/provider-codex/src/stages/authorize-headers.ts create mode 100644 packages/provider-codex/src/stages/observe-response.ts create mode 100644 packages/provider-codex/src/stages/prepare-request.ts create mode 100644 packages/provider-codex/src/stages/read-account.ts create mode 100644 packages/provider-codex/src/stages/retry-access.ts create mode 100644 packages/provider-copilot/src/operation-model.ts create mode 100644 packages/provider-copilot/src/pipeline-facts.ts create mode 100644 packages/provider-copilot/src/pipelines.ts create mode 100644 packages/provider-copilot/src/stages/authenticate.ts create mode 100644 packages/provider-copilot/src/stages/observe-quota.ts create mode 100644 packages/provider-copilot/src/stages/prepare-request.ts create mode 100644 packages/provider-custom/src/pipelines.ts create mode 100644 packages/provider-custom/src/stages/prepare-request.ts create mode 100644 packages/provider-ollama/src/pipelines.ts create mode 100644 packages/provider-ollama/src/stages/observe-account.ts create mode 100644 packages/provider-ollama/src/stages/prepare-request.ts create mode 100644 packages/provider/__tests__/pipeline_test.ts create mode 100644 packages/provider/src/http-headers.ts create mode 100644 packages/provider/src/pipeline.ts create mode 100644 packages/test-utils/src/provider-pipeline.ts diff --git a/packages/http/__tests__/pipeline_test.ts b/packages/http/__tests__/pipeline_test.ts new file mode 100644 index 0000000000..1321288b7b --- /dev/null +++ b/packages/http/__tests__/pipeline_test.ts @@ -0,0 +1,122 @@ +import { describe, expect, test, vi } from 'vitest'; + +import { isReplayableBody, type Fetcher } from '../src/index.ts'; +import { exchangeResponse, http, type HttpRequestFacts, type HttpResponseFacts, type HttpServices } from '../src/pipeline.ts'; +import { multipartBody, serializeHttpBody, type HttpFile } from '../src/request-content.ts'; +import { compose, move, run, secret, setRelease } from '@floway-dev/pipeline'; + +const request = (overrides: Partial = {}): HttpRequestFacts => move({ + 'request.http.callId': 17, + 'request.http.url': 'https://upstream.test/route', + 'request.http.method': 'POST', + 'request.http.headers': [['Content-Type', 'application/json']], + 'request.http.body': { message: 'hello' }, + 'request.http.encoding': 'json', + ...overrides, +}); + +const execute = (facts: HttpRequestFacts, fetcher: Fetcher, wrapUpstreamCall = async (dispatch: () => Promise): Promise => await dispatch()) => { + const services: HttpServices = { + httpCall: callId => { + expect(callId).toBe(17); + return { fetcher, wrapUpstreamCall, signal: undefined, waitUntil: () => {} }; + }, + }; + return run(compose('httpTest', [http]), facts, services); +}; + +describe('shared HTTP stage', () => { + test('multipart quoting and newline encoding match native Fetch for arbitrary field names and empty filenames', async () => { + const bytes = new Uint8Array([1, 2, 255]); + const entries = [{ name: 'line\nname', value: 'line\nvalue' }, { name: 'upload', value: { name: '', type: '', bytes } }]; + const form = new FormData(); + form.append(entries[0]!.name, entries[0]!.value as string); + form.append('upload', new File([bytes], '', { type: '' })); + const native = new Request('https://upstream.test', { method: 'POST', body: form }); + const boundary = native.headers.get('content-type')!.split('boundary=')[1]!; + const encoded = serializeHttpBody({ boundary, entries }, 'multipart'); + if (!isReplayableBody(encoded)) throw new Error('expected replayable multipart content'); + const actual = await new Response(encoded.open()).arrayBuffer(); + expect(new Uint8Array(actual)).toEqual(new Uint8Array(await native.arrayBuffer())); + expect(actual.byteLength).toBe(encoded.contentLength); + }); + + test('forwards secret values, repeated field lines and immutable JSON content, preserving upstream status and body', async () => { + const item = { content: 'shared nested object' }; + const payload = { input: [item], token: secret('body-secret') }; + const fetcher: Fetcher = async (url, init) => { + expect(url).toBe('https://upstream.test/route'); + expect(init.headers).toEqual([['authorization', 'Bearer header-secret'], ['x-value', 'one'], ['x-value', 'two']]); + if (!isReplayableBody(init.body)) throw new Error('expected a replayable JSON request'); + const bytes = await new Response(init.body.open()).arrayBuffer(); + expect(bytes.byteLength).toBe(init.body.contentLength); + expect(JSON.parse(new TextDecoder().decode(bytes))).toEqual({ input: [item], token: 'body-secret' }); + expect(payload.input[0]).toBe(item); + return new Response('created', { status: 201, statusText: 'Created', headers: { 'x-upstream': 'retained' } }); + }; + const result = await execute(request({ + 'request.http.headers': [['authorization', secret('Bearer header-secret')], ['x-value', 'one'], ['x-value', 'two']], + 'request.http.body': payload, + }), fetcher); + const exchange = result.facts['response.http.exchange']; + if (exchange.type !== 'response') throw new Error('expected upstream response'); + expect(result.facts['response.http.body']).toBe(exchange.body); + expect(exchange.status).toBe(201); + expect(exchange.headers).toContainEqual(['x-upstream', 'retained']); + expect(await exchangeResponse(exchange).text()).toBe('created'); + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + await result.drain(); + }); + + test('multipart request content retains file bytes, names, media type and repeated fields without native file facts', async () => { + const file: HttpFile = { name: 'voice.wav', type: 'audio/wav', bytes: new Uint8Array([0, 1, 2, 255]), lastModified: 42 }; + const body = multipartBody([{ name: 'file', value: file }, { name: 'timestamp[]', value: 'word' }, { name: 'timestamp[]', value: 'segment' }]); + const result = await execute(request({ + 'request.http.headers': [['Content-Type', `multipart/form-data; boundary=${body.boundary}`]], + 'request.http.body': body, + 'request.http.encoding': 'multipart', + }), async (_url, init) => { + if (!isReplayableBody(init.body)) throw new Error('expected replayable multipart content'); + const response = new Response(init.body.open(), { headers: init.headers }); + const form = await response.formData(); + const uploaded = form.get('file') as File; + expect(uploaded.name).toBe('voice.wav'); + expect(uploaded.type).toBe('audio/wav'); + expect(new Uint8Array(await uploaded.arrayBuffer())).toEqual(file.bytes); + expect(form.getAll('timestamp[]')).toEqual(['word', 'segment']); + return new Response(null, { status: 204 }); + }); + expect(result.facts['response.http.body']).toBeNull(); + await result.drain(); + }); + + test('represents actual transport rejection as a value retaining the original error chain', async () => { + const cause = new Error('socket closed'); + const error = new TypeError('fetch failed', { cause }); + const result = await execute(request(), async () => { throw error; }); + expect(result.facts['response.http.exchange']).toEqual({ type: 'transportFailure', error }); + expect(result.facts['response.http.body']).toBeNull(); + await result.drain(); + }); + + test('serialization and timing-wrapper faults propagate as programming errors', async () => { + const fetcher = vi.fn(); + await expect(execute(request({ 'request.http.body': { count: 1n } }), fetcher)).rejects.toThrow(); + const error = new Error('timing wrapper fault'); + await expect(execute(request(), fetcher, async () => { throw error; })).rejects.toBe(error); + expect(fetcher).not.toHaveBeenCalled(); + }); + + test('owns an unread response body immediately and permits the decoder to replace its release action once', async () => { + const cancel = vi.fn(); + const body = new ReadableStream({ cancel }); + const result = await execute(request(), async () => new Response(body)); + const owned = result.facts['response.http.body']; + if (owned === null) throw new Error('expected owned body'); + const finish = vi.fn(async () => { await owned.cancel(); }); + setRelease(owned, finish); + await result.drain(); + expect(finish).toHaveBeenCalledTimes(1); + expect(cancel).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/http/package.json b/packages/http/package.json index ec9bbb9b7a..41a06717c2 100644 --- a/packages/http/package.json +++ b/packages/http/package.json @@ -4,6 +4,14 @@ "version": "0.0.0", "type": "module", "exports": { + "./pipeline": { + "import": "./src/pipeline.ts", + "types": "./src/pipeline.ts" + }, + "./request-content": { + "import": "./src/request-content.ts", + "types": "./src/request-content.ts" + }, ".": { "import": "./src/index.ts", "types": "./src/index.ts" @@ -13,6 +21,8 @@ "typecheck": "tsc --noEmit" }, "dependencies": { + "@discoveryjs/json-ext": "1.1.0", + "@floway-dev/pipeline": "workspace:*", "@noble/hashes": "^2.2.0", "@reclaimprotocol/tls": "0.1.2", "@scure/base": "^2.2.0" diff --git a/packages/http/src/http.ts b/packages/http/src/http.ts new file mode 100644 index 0000000000..1da37bed01 --- /dev/null +++ b/packages/http/src/http.ts @@ -0,0 +1,34 @@ +import type { HttpExchange, HttpRequestFacts, HttpResponseFacts, HttpServices } from './pipeline-types.ts'; +import { serializeHttpBody } from './request-content.ts'; +import { takeHttpResponse } from './response.ts'; +import { defineStage, isSecret, move } from '@floway-dev/pipeline'; + +export const http = defineStage({ + name: 'http', + return: { provides: ['response.http.exchange', 'response.http.body'] }, + execute: async (facts, use) => { + const call = use.httpCall(facts['request.http.callId']); + const headers = facts['request.http.headers'].map(([name, value]): [string, string] => + [name, isSecret(value) ? value.reveal() : value]); + const body = serializeHttpBody(facts['request.http.body'], facts['request.http.encoding']); + const init = { + method: facts['request.http.method'], + headers, + body, + signal: call.signal, + }; + new URL(facts['request.http.url']); + new Headers(headers); + const exchange: HttpExchange = await call.wrapUpstreamCall(async () => { + let response: Response; + try { + response = await call.fetcher(facts['request.http.url'], init); + } catch (error) { + use.log.warn('upstream transport failed', { error }); + return { type: 'transportFailure' as const, error }; + } + return takeHttpResponse(response); + }); + return move({ ...facts, 'response.http.exchange': exchange, 'response.http.body': exchange.type === 'response' ? exchange.body : null }); + }, +}); diff --git a/packages/http/src/pipeline-types.ts b/packages/http/src/pipeline-types.ts new file mode 100644 index 0000000000..2aeda93b06 --- /dev/null +++ b/packages/http/src/pipeline-types.ts @@ -0,0 +1,45 @@ +import type { HttpBody, HttpBodyEncoding } from './request-content.ts'; +import type { Fetcher } from './types.ts'; +import type { Owned, Secret } from '@floway-dev/pipeline'; + +export interface HttpCall { + readonly fetcher: Fetcher; + readonly signal: AbortSignal | undefined; + readonly waitUntil: (work: Promise) => void; + readonly wrapUpstreamCall: (dispatch: () => Promise) => Promise; +} + +export type HttpServices = { + readonly httpCall: (callId: number) => HttpCall; +}; + +export type HttpHeaders = readonly (readonly [string, string | Secret])[]; + +export interface HttpRequestFacts { + 'request.http.callId': number; + 'request.http.url': string; + 'request.http.method': string; + 'request.http.headers': HttpHeaders; + 'request.http.body': HttpBody; + 'request.http.encoding': HttpBodyEncoding; +} + +export interface HttpResponseExchange { + readonly type: 'response'; + readonly status: number; + readonly statusText: string; + readonly headers: readonly (readonly [string, string])[]; + readonly body: (ReadableStream & Owned) | null; +} + +export interface HttpTransportFailure { + readonly type: 'transportFailure'; + readonly error: unknown; +} + +export type HttpExchange = HttpResponseExchange | HttpTransportFailure; + +export interface HttpResponseFacts { + 'response.http.exchange': HttpExchange; + 'response.http.body': (ReadableStream & Owned) | null; +} diff --git a/packages/http/src/pipeline.ts b/packages/http/src/pipeline.ts new file mode 100644 index 0000000000..7dab3cb1be --- /dev/null +++ b/packages/http/src/pipeline.ts @@ -0,0 +1,3 @@ +export { http } from './http.ts'; +export { exchangeResponse, takeHttpResponse } from './response.ts'; +export type { HttpCall, HttpExchange, HttpHeaders, HttpRequestFacts, HttpResponseExchange, HttpResponseFacts, HttpServices, HttpTransportFailure } from './pipeline-types.ts'; diff --git a/packages/http/src/request-content.ts b/packages/http/src/request-content.ts new file mode 100644 index 0000000000..b324f7ca55 --- /dev/null +++ b/packages/http/src/request-content.ts @@ -0,0 +1,92 @@ +import { stringifyChunked } from '@discoveryjs/json-ext'; + +import type { ReplayableBody } from './types.ts'; +import { isSecret } from '@floway-dev/pipeline'; + +export interface HttpFile { + readonly name: string; + readonly type: string; + readonly lastModified?: number; + readonly bytes: Uint8Array; +} + +export interface HttpFormEntry { + readonly name: string; + readonly value: string | HttpFile; +} + +export interface HttpMultipartBody { + readonly boundary: string; + readonly entries: readonly HttpFormEntry[]; +} + +export type HttpBody = object | string | Uint8Array | null; +export type HttpBodyEncoding = 'json' | 'multipart' | 'direct'; + +const encode = (value: string): Uint8Array => new TextEncoder().encode(value); +const quoted = (value: string): string => value.replaceAll('\r', '%0D').replaceAll('\n', '%0A').replaceAll('"', '%22'); +const normalizeNewlines = (value: string): string => value.replace(/\r\n|\r|\n/g, '\r\n'); + +// Preserve Fetch's multipart wire encoding while retaining shared upload bytes. +// https://github.com/nodejs/undici/blob/01a912e49a50c48009ed2639d2a457a6ec26752a/lib/web/fetch/body.js +const multipartChunks = function* (body: HttpMultipartBody): Generator { + for (const { name, value } of body.entries) { + const disposition = `--${body.boundary}\r\nContent-Disposition: form-data; name="${quoted(normalizeNewlines(name))}"`; + if (typeof value === 'string') { + yield encode(`${disposition}\r\n\r\n${normalizeNewlines(value)}\r\n`); + } else { + const filename = value.name.length === 0 ? '' : `; filename="${quoted(value.name)}"`; + yield encode(`${disposition}${filename}\r\nContent-Type: ${value.type || 'application/octet-stream'}\r\n\r\n`); + yield value.bytes; + yield encode('\r\n'); + } + } + yield encode(`--${body.boundary}--\r\n`); +}; + +export const jsonByteChunks = function* (value: object): Generator { + for (const chunk of stringifyChunked(value, (_key: string, field: unknown) => isSecret(field) ? field.reveal() : field)) yield encode(chunk); +}; + +const replayable = (chunks: () => Generator): ReplayableBody => { + let contentLength = 0; + for (const chunk of chunks()) { + contentLength += chunk.byteLength; + if (!Number.isSafeInteger(contentLength)) throw new RangeError('HTTP request content exceeds the supported content length'); + } + return { + contentLength, + open: () => { + const iterator = chunks(); + return new ReadableStream({ + pull(controller) { + const next = iterator.next(); + if (next.done) controller.close(); + else controller.enqueue(next.value); + }, + cancel() { + iterator.return(undefined); + }, + }); + }, + }; +}; + +export const serializeHttpBody = (body: HttpBody, encoding: HttpBodyEncoding): string | ReplayableBody | null => { + if (encoding === 'json') { + if (body === null || typeof body !== 'object' || body instanceof Uint8Array) throw new TypeError('JSON HTTP content must be a parsed object'); + return replayable(() => jsonByteChunks(body)); + } + if (encoding === 'multipart') { + const multipart = body as HttpMultipartBody; + return replayable(() => multipartChunks(multipart)); + } + if (body instanceof Uint8Array) return replayable(function* () { yield body; }); + if (body === null || typeof body === 'string') return body; + throw new TypeError('Direct HTTP content must be text or bytes'); +}; + +export const multipartBody = (entries: readonly HttpFormEntry[]): HttpMultipartBody => ({ + boundary: `floway-${crypto.randomUUID()}`, + entries, +}); diff --git a/packages/http/src/response.ts b/packages/http/src/response.ts new file mode 100644 index 0000000000..8b0a1a3dbd --- /dev/null +++ b/packages/http/src/response.ts @@ -0,0 +1,21 @@ +import type { HttpResponseExchange } from './pipeline-types.ts'; +import { own } from '@floway-dev/pipeline'; + +export const takeHttpResponse = (response: Response): HttpResponseExchange => { + const rawBody = response.body; + const body = rawBody === null ? null : own(rawBody, async () => { await rawBody.cancel(); }); + return { + type: 'response' as const, + status: response.status, + statusText: response.statusText, + headers: [...response.headers], + body, + }; +}; + +export const exchangeResponse = (exchange: HttpResponseExchange): Response => + new Response(exchange.body, { + status: exchange.status, + statusText: exchange.statusText, + headers: exchange.headers.map(([name, value]): [string, string] => [name, value]), + }); diff --git a/packages/provider-azure/__tests__/provider_test.ts b/packages/provider-azure/__tests__/provider_test.ts index 672def8545..a6bc549012 100644 --- a/packages/provider-azure/__tests__/provider_test.ts +++ b/packages/provider-azure/__tests__/provider_test.ts @@ -3,7 +3,7 @@ import { test } from 'vitest'; import { createAzureProvider } from '../src/provider.ts'; import type { UpstreamRecord } from '@floway-dev/provider'; import { directFetcher } from '@floway-dev/provider'; -import { assertEquals, noopUpstreamCallOptions, sseResponse, withMockedFetch } from '@floway-dev/test-utils'; +import { callProviderPipeline, assertEquals, noopUpstreamCallOptions, sseResponse, withMockedFetch } from '@floway-dev/test-utils'; const azureRecord = (overrides: Partial = {}): UpstreamRecord => { const config = { @@ -133,7 +133,7 @@ test('createAzureProvider sends upstream model ids in OpenAI-shaped request bodi }], }], }, 'generate', undefined, noopUpstreamCallOptions()); - const embeddings = await instance.instance.callOpenAIEmbeddings(providerModel, { input: 'hello' }, undefined, noopUpstreamCallOptions()); + const embeddings = await callProviderPipeline(instance, 'openaiEmbeddings', providerModel, { input: 'hello' }, undefined, noopUpstreamCallOptions()); assertEquals(chat.modelKey, 'gpt-prod'); assertEquals(responses.modelKey, 'gpt-prod'); @@ -474,7 +474,7 @@ test('createAzureProvider exposes image models and routes generations with api-v const models = await provider.instance.getProvidedModels(directFetcher); assertEquals(models[0].kind, 'image'); assertEquals(models[0].endpoints, { openaiImagesGenerations: {}, openaiImagesEdits: {} }); - const result = await provider.instance.callOpenAIImagesGenerations(models[0], { prompt: 'hello' }, undefined, noopUpstreamCallOptions()); + const result = await callProviderPipeline(provider, 'openaiImagesGenerations', models[0], { prompt: 'hello' }, undefined, noopUpstreamCallOptions()); assertEquals(result.modelKey, 'gpt-image-2'); assertEquals(result.response.status, 200); }, @@ -521,11 +521,11 @@ test('createAzureProvider callOpenAIImagesEdits posts multipart with model repla async () => { const provider = createAzureProvider(record); const models = await provider.instance.getProvidedModels(directFetcher); - const result = await provider.instance.callOpenAIImagesEdits(models[0], { + const result = await callProviderPipeline(provider, 'openaiImagesEdits', models[0], { parameters: { prompt: 'replace sky' }, images: [{ type: 'upload', - file: new File([new Uint8Array([1, 2, 3])], 'photo.png', { type: 'image/png' }), + file: { bytes: new Uint8Array([1, 2, 3]), name: 'photo.png', type: 'image/png' }, }], }, undefined, noopUpstreamCallOptions()); assertEquals(result.modelKey, 'gpt-image-2'); @@ -556,9 +556,9 @@ test('createAzureProvider callOpenAIAudioTranscriptions selects the deployment i async () => { const provider = createAzureProvider(record); const [model] = await provider.instance.getProvidedModels(directFetcher); - const result = await provider.instance.callOpenAIAudioTranscriptions(model, { + const result = await callProviderPipeline(provider, 'openaiAudioTranscriptions', model, { entries: [ - { name: 'file', value: new File(['audio'], 'clip.mp3', { type: 'audio/mpeg' }) }, + { name: 'file', value: { bytes: new TextEncoder().encode('audio'), name: 'clip.mp3', type: 'audio/mpeg' } }, { name: 'model', value: 'public-model' }, { name: 'response_format', value: 'vtt' }, ], @@ -590,10 +590,10 @@ test('createAzureProvider callOpenAIAudioTranscriptions reduces a Foundry endpoi async () => { const provider = createAzureProvider(record); const [model] = await provider.instance.getProvidedModels(directFetcher); - await provider.instance.callOpenAIAudioTranscriptions(model, { + await callProviderPipeline(provider, 'openaiAudioTranscriptions', model, { entries: [ { name: 'model', value: 'public-model' }, - { name: 'file', value: new File(['audio'], 'clip.wav', { type: 'audio/wav' }) }, + { name: 'file', value: { bytes: new TextEncoder().encode('audio'), name: 'clip.wav', type: 'audio/wav' } }, ], }, undefined, noopUpstreamCallOptions()); }, diff --git a/packages/provider-azure/package.json b/packages/provider-azure/package.json index 2944619f6a..432f72a585 100644 --- a/packages/provider-azure/package.json +++ b/packages/provider-azure/package.json @@ -10,6 +10,8 @@ "typecheck": "tsc --noEmit" }, "dependencies": { + "@floway-dev/http": "workspace:*", + "@floway-dev/pipeline": "workspace:*", "@floway-dev/interceptor": "workspace:*", "@floway-dev/protocols": "workspace:*", "@floway-dev/provider": "workspace:*" diff --git a/packages/provider-azure/src/index.ts b/packages/provider-azure/src/index.ts index 744129dbdc..1b26b37a02 100644 --- a/packages/provider-azure/src/index.ts +++ b/packages/provider-azure/src/index.ts @@ -1,6 +1,7 @@ import { AZURE_DEFAULT_FLAGS } from './defaults.ts'; import { createAzureProvider } from './provider.ts'; import type { ProviderModule } from '@floway-dev/provider'; +export { createAzurePipelines } from './pipelines.ts'; export const azureProviderModule: ProviderModule = { create: createAzureProvider, diff --git a/packages/provider-azure/src/pipelines.ts b/packages/provider-azure/src/pipelines.ts new file mode 100644 index 0000000000..61803e73d2 --- /dev/null +++ b/packages/provider-azure/src/pipelines.ts @@ -0,0 +1,13 @@ +import type { AzureUpstreamConfig } from './config.ts'; +import { prepareAzureRequest } from './stages/prepare-request.ts'; +import { http } from '@floway-dev/http/pipeline'; +import { compose } from '@floway-dev/pipeline'; +import type { ProviderOperationPayloads, ProviderOperationResponse, ProviderPipelines, ProviderRequest } from '@floway-dev/provider'; + +export const createAzurePipelines = (config: AzureUpstreamConfig): ProviderPipelines => ({ + openaiCompletions: compose, ProviderOperationResponse<'openaiCompletions'>>('azure.openaiCompletions', [prepareAzureRequest(config, 'openaiCompletions'), http]), + openaiEmbeddings: compose, ProviderOperationResponse<'openaiEmbeddings'>>('azure.openaiEmbeddings', [prepareAzureRequest(config, 'openaiEmbeddings'), http]), + openaiImagesGenerations: compose, ProviderOperationResponse<'openaiImagesGenerations'>>('azure.openaiImagesGenerations', [prepareAzureRequest(config, 'openaiImagesGenerations'), http]), + openaiImagesEdits: compose, ProviderOperationResponse<'openaiImagesEdits'>>('azure.openaiImagesEdits', [prepareAzureRequest(config, 'openaiImagesEdits'), http]), + openaiAudioTranscriptions: compose, ProviderOperationResponse<'openaiAudioTranscriptions'>>('azure.openaiAudioTranscriptions', [prepareAzureRequest(config, 'openaiAudioTranscriptions'), http]), +}); diff --git a/packages/provider-azure/src/provider.ts b/packages/provider-azure/src/provider.ts index cc8e8796cd..047a00c23f 100644 --- a/packages/provider-azure/src/provider.ts +++ b/packages/provider-azure/src/provider.ts @@ -3,6 +3,7 @@ import { AZURE_DEFAULT_FLAGS } from './defaults.ts'; import { azureFetchOpenAIAudioTranscriptions, azureFetchOpenAIChatCompletions, azureFetchOpenAICompletions, azureFetchOpenAIEmbeddings, azureFetchOpenAIImagesEdits, azureFetchOpenAIImagesGenerations, azureFetchAnthropicMessages, azureFetchAnthropicMessagesCountTokens, azureFetchOpenAIResponses, azureFetchOpenAIResponsesCompact } from './fetch.ts'; import { AZURE_OPENAI_RESPONSES_BOUNDARY } from './interceptors/openai-responses/index.ts'; import type { OpenAIResponsesBoundaryCtx } from './interceptors/openai-responses/types.ts'; +import { createAzurePipelines } from './pipelines.ts'; import { runInterceptors } from '@floway-dev/interceptor'; import { parseAnthropicMessagesStream } from '@floway-dev/protocols/anthropic-messages'; import { kindForEndpoints } from '@floway-dev/protocols/common'; @@ -131,6 +132,7 @@ export const createAzureProvider = (record: UpstreamRecord): Provider => { disabledPublicModelIds: azure.disabledPublicModelIds, modelPrefix: azure.modelPrefix, modelsCache: azure.modelsCache, + pipelines: createAzurePipelines(azure.config), instance, }; }; diff --git a/packages/provider-azure/src/stages/prepare-request.ts b/packages/provider-azure/src/stages/prepare-request.ts new file mode 100644 index 0000000000..b0b1ed9ae4 --- /dev/null +++ b/packages/provider-azure/src/stages/prepare-request.ts @@ -0,0 +1,73 @@ +import type { AzureUpstreamConfig } from '../config.ts'; +import { azureOpenAiV1BaseUrl } from '../endpoint.ts'; +import { type HttpRequestFacts, type HttpResponseFacts } from '@floway-dev/http/pipeline'; +import { multipartBody, type HttpBody, type HttpBodyEncoding } from '@floway-dev/http/request-content'; +import { defineStage, move, secret } from '@floway-dev/pipeline'; +import { joinBaseAndPath, mergeHttpHeaders, prepareOpenAIImagesEditsBody, withHttpContentType, type ProviderOperationPayloads, type ProviderRequest, type ProviderResponse, type ProviderServices } from '@floway-dev/provider'; + +type AzureOperation = 'openaiCompletions' | 'openaiEmbeddings' | 'openaiImagesGenerations' | 'openaiImagesEdits' | 'openaiAudioTranscriptions'; + +interface AzureHttpFacts extends HttpRequestFacts { + 'request.azure.deployment': string; +} + +const PATHS: Record, string> = { + openaiCompletions: '/completions', + openaiEmbeddings: '/embeddings', + openaiImagesGenerations: '/images/generations', + openaiImagesEdits: '/images/edits', +}; + +export const prepareAzureRequest = (config: AzureUpstreamConfig, operation: O) => { + const prepare = defineStage, AzureHttpFacts, HttpResponseFacts, ProviderResponse, ProviderServices>({ + name: `prepareAzure${operation.replace(/^openai/, 'OpenAI').replace(/^alpha/, 'Alpha').replace(/^rerank/, 'Rerank')}`, + through: { + request: { + needs: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'], + consumes: ['request.provider.model', 'request.provider.payload', 'request.http.headers'], + provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.azure.deployment'], + }, + response: { needs: ['response.http.exchange'], consumes: [], provides: ['response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'] }, + }, + execute: async (facts, next) => { + const { 'request.provider.model': model, 'request.provider.payload': payload, ...rest } = facts; + const deployment = (model.providerData as { upstreamModelId: string }).upstreamModelId; + let body: HttpBody; + let encoding: HttpBodyEncoding = 'json'; + let url: URL; + if (operation === 'openaiAudioTranscriptions') { + body = multipartBody((payload as ProviderOperationPayloads['openaiAudioTranscriptions']).entries.filter(entry => entry.name !== 'model')); + encoding = 'multipart'; + url = new URL(config.endpoint); + url.pathname = `/openai/deployments/${encodeURIComponent(deployment)}/audio/transcriptions`; + url.search = ''; + url.hash = ''; + // https://github.com/Azure/azure-rest-api-specs/blob/928047803788f7377fa003a26ba2bdc2e0fcccc0/specification/cognitiveservices/OpenAI.Inference/routes/audio_transcription.tsp#L19-L49 + url.searchParams.set('api-version', '2025-04-01-preview'); + } else { + url = new URL(joinBaseAndPath(azureOpenAiV1BaseUrl(config.endpoint), PATHS[operation as Exclude])); + if (operation === 'openaiImagesEdits') { + const prepared = await prepareOpenAIImagesEditsBody(payload as ProviderOperationPayloads['openaiImagesEdits'], deployment); + body = prepared.body; + encoding = prepared.encoding; + } else { + body = { ...payload, model: deployment }; + } + if (operation === 'openaiImagesGenerations' || operation === 'openaiImagesEdits') url.searchParams.append('api-version', 'preview'); + } + const base = withHttpContentType([['api-key', secret(config.apiKey)]], body, encoding); + const headers = mergeHttpHeaders(base, facts['request.http.headers']); + const back = await next(move({ + ...rest, + 'request.azure.deployment': deployment, + 'request.http.url': url.href, + 'request.http.method': 'POST', + 'request.http.headers': headers, + 'request.http.body': body, + 'request.http.encoding': encoding, + })); + return move({ ...back, 'response.provider.modelKey': deployment, 'response.provider.called': back['response.http.exchange'].type === 'response', 'response.provider.previousCalls': [] }); + }, + }); + return prepare; +}; diff --git a/packages/provider-claude-code/src/provider.ts b/packages/provider-claude-code/src/provider.ts index 3ba23d1ed4..e98b443753 100644 --- a/packages/provider-claude-code/src/provider.ts +++ b/packages/provider-claude-code/src/provider.ts @@ -126,6 +126,7 @@ export const createClaudeCodeProvider = (record: UpstreamRecord): Provider => { disabledPublicModelIds: record.disabledPublicModelIds, modelPrefix: record.modelPrefix, modelsCache: record.modelsCache, + pipelines: {}, instance, }; }; diff --git a/packages/provider-codex/__tests__/provider_test.ts b/packages/provider-codex/__tests__/provider_test.ts index 5f1f773868..5d88d8f638 100644 --- a/packages/provider-codex/__tests__/provider_test.ts +++ b/packages/provider-codex/__tests__/provider_test.ts @@ -4,8 +4,10 @@ import { createUpstreamStateRepoStub, type UpstreamStateRepoStub } from './upstr import { CODEX_RESPONSES_LITE_CLIENT_METADATA_KEY, CODEX_RESPONSES_LITE_HEADER } from '../src/constants.ts'; import { createCodexProvider } from '../src/provider.ts'; import type { CodexAccessTokenEntry, CodexUpstreamState } from '../src/state.ts'; -import { directFetcher, initProviderRepo, type UpstreamRecord } from '@floway-dev/provider'; -import { noopUpstreamCallOptions, readJsonRequest, stubProviderModel } from '@floway-dev/test-utils'; +import { exchangeResponse } from '@floway-dev/http/pipeline'; +import { move, run, setRelease, type Event } from '@floway-dev/pipeline'; +import { directFetcher, initProviderRepo, providerModelFacts, type UpstreamRecord } from '@floway-dev/provider'; +import { callProviderPipeline, noopUpstreamCallOptions, readJsonRequest, stubProviderModel } from '@floway-dev/test-utils'; const farFutureMs = Date.now() + 24 * 60 * 60 * 1000; @@ -91,6 +93,68 @@ const oauthTokenResponse = (overrides: Partial<{ access_token: string; refresh_t }), { status: 200, headers: new Headers({ 'content-type': 'application/json' }) }); describe('createCodexProvider', () => { + test('Codex Images records the complete parsed 401 body and preserves its preceding endpoint observation', async () => { + const refusal = { error: { code: 'invalid_token', message: 'expired bearer' }, diagnostic: 'x'.repeat(600) }; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => { + if (String(input) === 'https://auth.openai.com/oauth/token') return oauthTokenResponse(); + return new Headers(init?.headers).get('authorization') === 'Bearer at' + ? Response.json(refusal, { status: 401 }) + : Response.json({ created: 1, data: [] }); + }); + const provider = createCodexProvider(current!); + const model = stubProviderModel({ id: 'gpt-image-2', kind: 'image', endpoints: { openaiImagesGenerations: {}, openaiImagesEdits: {} } }); + const events: Event[] = []; + const options = noopUpstreamCallOptions(); + const executed = await run(provider.pipelines.openaiImagesGenerations!, move({ + 'request.provider.model': providerModelFacts(model), + 'request.provider.payload': { prompt: 'circle' }, + 'request.http.callId': 0, + 'request.http.headers': [], + }), { httpCall: () => ({ ...options, signal: undefined }), dump: (event: Event) => { events.push(event); } }); + expect(events.some(event => event.type === 'stage.leaved' && JSON.stringify(event.facts['response.codex.failureBody']) === JSON.stringify(refusal))).toBe(true); + expect(executed.facts['response.provider.previousCalls']).toEqual([{ modelKey: 'gpt-image-2' }]); + const exchange = executed.facts['response.http.exchange']; + if (exchange.type !== 'response') throw new Error('expected final model reply'); + await exchangeResponse(exchange).arrayBuffer(); + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + await executed.drain(); + }); + + test('Codex Images pipeline preserves the image turn and payload across a renewable 401 retry', async () => { + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => { + if (String(input) === 'https://auth.openai.com/oauth/token') return oauthTokenResponse(); + const headers = new Headers(init?.headers); + return headers.get('authorization') === 'Bearer at' + ? new Response(JSON.stringify({ error: { code: 'invalid_token', message: 'expired bearer' } }), { status: 401 }) + : new Response(JSON.stringify({ created: 1, data: [{ b64_json: 'image' }] }), { status: 201, headers: { 'x-upstream': 'retained' } }); + }); + const provider = createCodexProvider(current!); + const model = stubProviderModel({ id: 'gpt-image-2', kind: 'image', endpoints: { openaiImagesGenerations: {}, openaiImagesEdits: {} } }); + const options = noopUpstreamCallOptions(); + options.headers.set('x-codex-image-turn-id', 'stable-image-turn'); + const result = await callProviderPipeline(provider, 'openaiImagesGenerations', model, { prompt: 'blue circle' }, undefined, options); + expect(result.called).toBe(true); + expect(result.previousCalls).toEqual([{ modelKey: 'gpt-image-2' }]); + expect(result.response.status).toBe(201); + expect(result.response.headers.get('x-upstream')).toBe('retained'); + const imageCalls = fetchSpy.mock.calls.filter(([input]) => String(input).endsWith('/codex/images/generations')); + expect(imageCalls).toHaveLength(2); + expect(imageCalls.map(([, init]) => new Headers(init?.headers).get('x-codex-image-turn-id'))).toEqual(['stable-image-turn', 'stable-image-turn']); + expect(await Promise.all(imageCalls.map(([, init]) => readJsonRequest(init!)))).toEqual([{ prompt: 'blue circle', model: 'gpt-image-2' }, { prompt: 'blue circle', model: 'gpt-image-2' }]); + }); + + test('Codex Images pipeline marks a preflight Free-plan refusal as an uncalled model endpoint', async () => { + current = { ...current!, state: { accounts: [{ ...(current!.state as CodexUpstreamState).accounts[0], accessToken: { ...freshAccessToken, planType: 'free' } }] } }; + const fetchSpy = vi.spyOn(globalThis, 'fetch'); + const provider = createCodexProvider(current); + const model = stubProviderModel({ id: 'gpt-image-2', kind: 'image', endpoints: { openaiImagesGenerations: {}, openaiImagesEdits: {} } }); + const result = await callProviderPipeline(provider, 'openaiImagesGenerations', model, { prompt: 'blue circle' }); + expect(result.response.status).toBe(403); + expect(result.called).toBe(false); + expect(result.previousCalls).toEqual([]); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + test('owns request identity and turn metadata headers on the instance', () => { const provider = createCodexProvider(baseRecord); @@ -338,7 +402,7 @@ describe('createCodexProvider', () => { const model = stubProviderModel({ id: 'gpt-image-2', display_name: 'GPT-Image-2', kind: 'image', endpoints: { openaiImagesGenerations: {}, openaiImagesEdits: {} } }); const options = noopUpstreamCallOptions(); options.headers.set('x-codex-image-turn-id', 'turn-image'); - const result = await instance.instance.callOpenAIImagesGenerations(model, { prompt: 'an orange circle', quality: 'low' }, undefined, options); + const result = await callProviderPipeline(instance, 'openaiImagesGenerations', model, { prompt: 'an orange circle', quality: 'low' }, undefined, options); expect(result.response.status).toBe(200); expect(result.modelKey).toBe('gpt-image-2'); const [url, init] = fetchSpy.mock.calls[0]; @@ -358,7 +422,7 @@ describe('createCodexProvider', () => { }; const instance = createCodexProvider(freeRecord); const model = stubProviderModel({ id: 'gpt-image-2', display_name: 'GPT-Image-2', kind: 'image', endpoints: { openaiImagesGenerations: {}, openaiImagesEdits: {} } }); - const result = await instance.instance.callOpenAIImagesGenerations(model, { prompt: 'an orange circle' }, undefined, noopUpstreamCallOptions()); + const result = await callProviderPipeline(instance, 'openaiImagesGenerations', model, { prompt: 'an orange circle' }, undefined, noopUpstreamCallOptions()); expect(result.response.status).toBe(403); expect(fetchSpy).not.toHaveBeenCalled(); }); @@ -370,7 +434,7 @@ describe('createCodexProvider', () => { }; const instance = createCodexProvider(freeRecord); const model = stubProviderModel({ id: 'gpt-image-2', display_name: 'GPT-Image-2', kind: 'image', endpoints: { openaiImagesGenerations: {}, openaiImagesEdits: {} } }); - const result = await instance.instance.callOpenAIImagesEdits(model, { + const result = await callProviderPipeline(instance, 'openaiImagesEdits', model, { images: [{ type: 'reference', reference: { image_url: 'https://example.test/image.png' } }], parameters: { prompt: 'edit' }, }, undefined, noopUpstreamCallOptions()); @@ -389,8 +453,8 @@ describe('createCodexProvider', () => { const model = stubProviderModel({ id: 'gpt-image-2', display_name: 'GPT-Image-2', kind: 'image', endpoints: { openaiImagesGenerations: {}, openaiImagesEdits: {} } }); const options = noopUpstreamCallOptions(); options.headers.set('originator', 'chatgpt_cca'); - const result = await instance.instance.callOpenAIImagesEdits(model, { - images: [{ type: 'upload', file: new File(['image'], 'image.png', { type: 'image/png' }) }], + const result = await callProviderPipeline(instance, 'openaiImagesEdits', model, { + images: [{ type: 'upload', file: { bytes: new TextEncoder().encode('image'), name: 'image.png', type: 'image/png' } }], parameters: { prompt: 'make it blue' }, }, undefined, options); expect(result.response.status).toBe(200); diff --git a/packages/provider-codex/package.json b/packages/provider-codex/package.json index feb0f68892..3e70f76ebf 100644 --- a/packages/provider-codex/package.json +++ b/packages/provider-codex/package.json @@ -10,6 +10,8 @@ "typecheck": "tsc --noEmit" }, "dependencies": { + "@floway-dev/http": "workspace:*", + "@floway-dev/pipeline": "workspace:*", "@floway-dev/interceptor": "workspace:*", "@floway-dev/protocols": "workspace:*", "@floway-dev/provider": "workspace:*", diff --git a/packages/provider-codex/src/fetch.ts b/packages/provider-codex/src/fetch.ts index 05af18bb00..cd87a49328 100644 --- a/packages/provider-codex/src/fetch.ts +++ b/packages/provider-codex/src/fetch.ts @@ -52,10 +52,10 @@ export interface CodexCallEffects { // Account selection shared by Codex backend calls. Every surface uses the same // OAuth credential, quota state, terminal-session classification, and refresh // retry contract; each operation owns its wire body and response decoding. -interface CodexBackendCallBase { +export interface CodexBackendCallBase { upstreamId: string; account: CodexAccountCredential; - model: ProviderModel; + model: Pick; headers: Headers; signal?: AbortSignal; effects: CodexCallEffects; @@ -142,7 +142,7 @@ export const callCodexOpenAIImagesEdits = async (opts: CallCodexOpenAIImagesEdit return await performImageCall(opts, request, prepared.accessToken, prepared.effectivePlan, false); }; -const prepareCodexCall = async (opts: CodexBackendCallBase): Promise<{ ok: true; accessToken: CodexAccessTokenEntry } | { ok: false; response: Response }> => { +export const prepareCodexCall = async (opts: CodexBackendCallBase): Promise<{ ok: true; accessToken: CodexAccessTokenEntry } | { ok: false; response: Response }> => { if (opts.account.state !== 'active') { return { ok: false, response: synthetic503(`Codex upstream is ${opts.account.state}`) }; } @@ -548,7 +548,7 @@ const dispatchCodexHttpCall = async ( return await classifyCodexHttpResponse(opts, response); }; -const classifyCodexHttpResponse = async ( +export const classifyCodexHttpResponse = async ( opts: CodexBackendCallBase, response: Response, quotaPolicy: 'always' | 'when-present' = 'always', @@ -564,25 +564,7 @@ const classifyCodexHttpResponse = async ( } if (response.status === 401) { - const bodyText = await response.text(); - const { code, message } = parseUpstreamError(bodyText); - if (opts.account.refresh_token === null) { - // An access-only credential cannot recover from a rejected bearer, so - // the row is marked best-effort — but a storage failure must never - // replace the upstream status, headers, or body the caller needs to - // diagnose what happened. - try { - await opts.effects.persistTerminalState('session_terminated', message); - } catch { - // The upstream response remains authoritative. - } - return new Response(bodyText, { status: 401, statusText: response.statusText, headers: response.headers }); - } - if (code === 'token_invalidated') { - await opts.effects.persistTerminalState('session_terminated', message); - return synthetic503(`Codex session terminated: ${message}`); - } - return new Response(bodyText, { status: 401, statusText: response.statusText, headers: response.headers }); + return await classifyCodexUnauthorizedResponse(opts, response, decodeCodexUpstreamError(await response.text())); } return response; @@ -624,7 +606,7 @@ const dispatchCodexImageCall = async ( // credential has nothing to re-mint from, its 401 was already classified as // terminal in `classifyCodexHttpResponse`, and the verbatim upstream response // is what reaches the client. -const refreshAccessTokenForRetry = async ( +export const refreshAccessTokenForRetry = async ( opts: CodexBackendCallBase, failedEntry: CodexAccessTokenEntry, fallbackPlan?: CodexPlanObservation, @@ -862,17 +844,45 @@ const performImageCall = async ( return { modelKey: opts.model.id, response }; }; -const parseUpstreamError = (rawText: string): { code: string | null; message: string } => { +export interface CodexUpstreamError { + readonly rawText: string; + readonly body: unknown; + readonly code: string | null; + readonly message: string; +} + +export const decodeCodexUpstreamError = (rawText: string): CodexUpstreamError => { + let body: unknown; try { - const obj = JSON.parse(rawText) as { error?: { code?: unknown; message?: unknown }; detail?: unknown }; - const code = obj.error && typeof obj.error === 'object' && typeof obj.error.code === 'string' ? obj.error.code : null; - const message = obj.error && typeof obj.error === 'object' && typeof obj.error.message === 'string' - ? obj.error.message - : typeof obj.detail === 'string' ? obj.detail : rawText.slice(0, 256); - return { code, message }; + body = JSON.parse(rawText); } catch { - return { code: null, message: rawText.slice(0, 256) }; + return { rawText, body: rawText, code: null, message: rawText.slice(0, 256) }; + } + const obj = typeof body === 'object' && body !== null ? body as { error?: { code?: unknown; message?: unknown }; detail?: unknown } : undefined; + const code = obj?.error && typeof obj.error === 'object' && typeof obj.error.code === 'string' ? obj.error.code : null; + const message = obj?.error && typeof obj.error === 'object' && typeof obj.error.message === 'string' + ? obj.error.message + : typeof obj?.detail === 'string' ? obj.detail : rawText.slice(0, 256); + return { rawText, body, code, message }; +}; + +export const classifyCodexUnauthorizedResponse = async (opts: CodexBackendCallBase, response: Response, parsed: CodexUpstreamError): Promise => { + const { rawText, code, message } = parsed; + if (opts.account.refresh_token === null) { + // Access-only credentials retain the authoritative upstream reply even when + // the terminal-state write fails. + try { + await opts.effects.persistTerminalState('session_terminated', message); + } catch { + // The upstream response remains authoritative. + } + return new Response(rawText, { status: 401, statusText: response.statusText, headers: response.headers }); + } + if (code === 'token_invalidated') { + await opts.effects.persistTerminalState('session_terminated', message); + return synthetic503(`Codex session terminated: ${message}`); } + return new Response(rawText, { status: 401, statusText: response.statusText, headers: response.headers }); }; const imageUnavailableResult = (modelKey: string): ProviderCallResult => ({ diff --git a/packages/provider-codex/src/index.ts b/packages/provider-codex/src/index.ts index 6f84a51d6b..b25ee2e346 100644 --- a/packages/provider-codex/src/index.ts +++ b/packages/provider-codex/src/index.ts @@ -1,6 +1,7 @@ import { CODEX_DEFAULT_FLAGS } from './defaults.ts'; import { createCodexProvider } from './provider.ts'; import type { ProviderModule } from '@floway-dev/provider'; +export { createCodexPipelines } from './pipelines.ts'; export const codexProviderModule: ProviderModule = { create: createCodexProvider, diff --git a/packages/provider-codex/src/models.ts b/packages/provider-codex/src/models.ts index dfbf9ae1ba..d844249fe8 100644 --- a/packages/provider-codex/src/models.ts +++ b/packages/provider-codex/src/models.ts @@ -141,7 +141,7 @@ export const codexModelContextWindow = (model: ProviderModel): CodexContextWindo }; }; -export const codexModelUsesResponsesLite = (model: ProviderModel): boolean => { +export const codexModelUsesResponsesLite = (model: Pick): boolean => { const providerData = model.providerData; if (providerData === undefined) return false; if (!isPlainRecord(providerData)) { diff --git a/packages/provider-codex/src/pipeline-facts.ts b/packages/provider-codex/src/pipeline-facts.ts new file mode 100644 index 0000000000..7edaa35088 --- /dev/null +++ b/packages/provider-codex/src/pipeline-facts.ts @@ -0,0 +1,65 @@ +import type { CodexPlanObservation } from './access-token.ts'; +import type { CodexCallEffects, CodexBackendCallBase } from './fetch.ts'; +import type { CodexAccountCredential, CodexAccessTokenEntry } from './state.ts'; +import type { HttpRequestFacts } from '@floway-dev/http/pipeline'; +import { secret, type Secret } from '@floway-dev/pipeline'; +import type { ProviderModelFacts, ProviderOperationPayloads, ProviderRequest, ProviderServices } from '@floway-dev/provider'; + +export type CodexOperation = 'alphaSearch' | 'openaiImagesGenerations' | 'openaiImagesEdits'; +export type CodexRequest = ProviderRequest; +export type CodexToken = Omit & { readonly token: Secret }; +export type CodexAccount = Omit & { + readonly refresh_token: Secret | null; + readonly accessToken: CodexToken | null; +}; +export type CodexAccountFacts = CodexRequest & { + 'request.codex.account': CodexAccount; +}; +export type CodexAuthenticatedFacts = CodexAccountFacts & { + 'request.codex.accessToken': CodexToken; + 'request.codex.plan': CodexPlanObservation | null; +}; +export interface CodexHttpFacts extends HttpRequestFacts { + 'request.codex.model': ProviderModelFacts; + 'request.codex.account': CodexAccount; + 'request.codex.accessToken': CodexToken; + 'request.codex.plan': CodexPlanObservation | null; + 'request.codex.modelKey': string; +} + +export interface CodexPipelineConfig { + readonly upstreamId: string; + readonly fallbackPlanType: string | undefined; + readonly readAccount: () => Promise; + readonly effects: CodexCallEffects; +} + +export const tokenFacts = (entry: CodexAccessTokenEntry): CodexToken => ({ ...entry, token: secret(entry.token) }); +export const accountFacts = (account: CodexAccountCredential): CodexAccount => ({ + ...account, + refresh_token: account.refresh_token === null ? null : secret(account.refresh_token), + accessToken: account.accessToken === null ? null : tokenFacts(account.accessToken), +}); +export const accountValue = (account: CodexAccount): CodexAccountCredential => ({ + ...account, + refresh_token: account.refresh_token === null ? null : account.refresh_token.reveal(), + accessToken: account.accessToken === null ? null : { ...account.accessToken, token: account.accessToken.token.reveal() }, +}); + +export const codexCall = ( + config: CodexPipelineConfig, + facts: { 'request.codex.account': CodexAccount; 'request.http.callId': number }, + use: ProviderServices, + model: CodexBackendCallBase['model'], +): CodexBackendCallBase => { + const call = use.httpCall(facts['request.http.callId']); + return { + upstreamId: config.upstreamId, + account: accountValue(facts['request.codex.account']), + model, + headers: new Headers(), + signal: call.signal, + effects: config.effects, + call: { ...call, headers: new Headers() }, + }; +}; diff --git a/packages/provider-codex/src/pipelines.ts b/packages/provider-codex/src/pipelines.ts new file mode 100644 index 0000000000..330a78d19f --- /dev/null +++ b/packages/provider-codex/src/pipelines.ts @@ -0,0 +1,16 @@ +import type { CodexPipelineConfig, CodexRequest } from './pipeline-facts.ts'; +import { authenticateCodex } from './stages/authenticate.ts'; +import { authorizeCodexHeaders } from './stages/authorize-headers.ts'; +import { observeCodexResponse } from './stages/observe-response.ts'; +import { prepareCodexRequest } from './stages/prepare-request.ts'; +import { readCodexAccount } from './stages/read-account.ts'; +import { retryCodexAccess } from './stages/retry-access.ts'; +import { http } from '@floway-dev/http/pipeline'; +import { compose } from '@floway-dev/pipeline'; +import type { ProviderPipelines, ProviderResponse } from '@floway-dev/provider'; + +export const createCodexPipelines = (config: CodexPipelineConfig): ProviderPipelines => ({ + alphaSearch: compose, ProviderResponse>('codex.alphaSearch', [readCodexAccount<'alphaSearch'>(config), authenticateCodex(config, 'alphaSearch'), prepareCodexRequest('alphaSearch'), retryCodexAccess(config, 'alphaSearch'), authorizeCodexHeaders, observeCodexResponse(config, 'alphaSearch'), http]), + openaiImagesGenerations: compose, ProviderResponse>('codex.openaiImagesGenerations', [readCodexAccount<'openaiImagesGenerations'>(config), authenticateCodex(config, 'openaiImagesGenerations'), prepareCodexRequest('openaiImagesGenerations'), retryCodexAccess(config, 'openaiImagesGenerations'), authorizeCodexHeaders, observeCodexResponse(config, 'openaiImagesGenerations'), http]), + openaiImagesEdits: compose, ProviderResponse>('codex.openaiImagesEdits', [readCodexAccount<'openaiImagesEdits'>(config), authenticateCodex(config, 'openaiImagesEdits'), prepareCodexRequest('openaiImagesEdits'), retryCodexAccess(config, 'openaiImagesEdits'), authorizeCodexHeaders, observeCodexResponse(config, 'openaiImagesEdits'), http]), +}); diff --git a/packages/provider-codex/src/provider.ts b/packages/provider-codex/src/provider.ts index e5b2c1d337..a725dce359 100644 --- a/packages/provider-codex/src/provider.ts +++ b/packages/provider-codex/src/provider.ts @@ -4,6 +4,7 @@ import { assertCodexUpstreamRecord, type CodexUpstreamConfig } from './config.ts import { CODEX_DEFAULT_FLAGS } from './defaults.ts'; import { callCodexAlphaSearch, callCodexOpenAIImagesEdits, callCodexOpenAIImagesGenerations, callCodexOpenAIResponses, callCodexOpenAIResponsesCompact, type CodexCallEffects } from './fetch.ts'; import { CODEX_OPENAI_RESPONSES_BOUNDARY } from './interceptors/openai-responses/index.ts'; +import { createCodexPipelines } from './pipelines.ts'; import type { OpenAIResponsesBoundaryCtx } from './interceptors/openai-responses/types.ts'; import { codexImageProviderModel, codexPlanSupportsImages, codexRawToProviderModel, fetchCodexCatalog } from './models.ts'; import { assertCodexUpstreamState, findCodexAccountIndex, persistCodexRefreshTokenRotation, persistCodexTerminalState } from './state.ts'; @@ -169,6 +170,7 @@ export const createCodexProvider = (record: UpstreamRecord): Provider => { disabledPublicModelIds: record.disabledPublicModelIds, modelPrefix: record.modelPrefix, modelsCache: record.modelsCache, + pipelines: createCodexPipelines({ upstreamId: record.id, fallbackPlanType: accountIdentity.planType ?? undefined, readAccount: async () => (await readActiveAccount()).account, effects }), instance, }; }; diff --git a/packages/provider-codex/src/stages/authenticate.ts b/packages/provider-codex/src/stages/authenticate.ts new file mode 100644 index 0000000000..0adb1dbe20 --- /dev/null +++ b/packages/provider-codex/src/stages/authenticate.ts @@ -0,0 +1,31 @@ +import { codexPlanObservation } from '../access-token.ts'; +import { prepareCodexCall } from '../fetch.ts'; +import { codexPlanSupportsImages } from '../models.ts'; +import { codexCall, tokenFacts, type CodexAccountFacts, type CodexAuthenticatedFacts, type CodexOperation, type CodexPipelineConfig } from '../pipeline-facts.ts'; +import { takeHttpResponse } from '@floway-dev/http/pipeline'; +import { defineStage, move, type Handed } from '@floway-dev/pipeline'; +import type { ProviderResponse, ProviderServices } from '@floway-dev/provider'; + +export const authenticateCodex = (config: CodexPipelineConfig, operation: O) => defineStage, CodexAuthenticatedFacts, ProviderResponse, ProviderResponse, ProviderResponse, ProviderServices>({ + name: 'authenticateCodex', + through: { + request: { needs: ['request.codex.account', 'request.provider.model', 'request.http.callId'], consumes: [], provides: ['request.codex.accessToken', 'request.codex.plan'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + return: { provides: ['response.http.exchange', 'response.http.body', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'] }, + execute: async (facts, next, use) => { + const model = facts['request.provider.model']; + const ready = await prepareCodexCall(codexCall(config, facts, use, model)); + const failed = (response: Response) => { + const exchange = takeHttpResponse(response); + return move({ ...facts, 'response.http.exchange': exchange, 'response.http.body': exchange.body, 'response.provider.modelKey': model.id, 'response.provider.called': false, 'response.provider.previousCalls': [] }); + }; + if (!ready.ok) return failed(ready.response); + const observedPlan = codexPlanObservation(ready.accessToken); + const plan = observedPlan ?? (config.fallbackPlanType === undefined ? null : { planType: config.fallbackPlanType }); + if (operation !== 'alphaSearch' && plan !== null && !codexPlanSupportsImages(plan.planType)) { + return failed(Response.json({ error: { type: 'image_tools_unavailable', message: 'ChatGPT Free accounts do not provide Codex image tools.' } }, { status: 403 })); + } + return await next(move({ ...facts, 'request.codex.accessToken': tokenFacts(ready.accessToken), 'request.codex.plan': plan })) as Handed; + }, +}); diff --git a/packages/provider-codex/src/stages/authorize-headers.ts b/packages/provider-codex/src/stages/authorize-headers.ts new file mode 100644 index 0000000000..5ae01de3a9 --- /dev/null +++ b/packages/provider-codex/src/stages/authorize-headers.ts @@ -0,0 +1,22 @@ +import type { CodexHttpFacts } from '../pipeline-facts.ts'; +import type { HttpResponseFacts } from '@floway-dev/http/pipeline'; +import { defineStage, move, secret, type Handed } from '@floway-dev/pipeline'; +import { replaceHttpHeader, type ProviderServices } from '@floway-dev/provider'; + +export const authorizeCodexHeaders = defineStage({ + name: 'authorizeCodexHeaders', + through: { + request: { + needs: ['request.codex.account', 'request.codex.accessToken', 'request.http.headers'], + consumes: ['request.http.headers'], + provides: ['request.http.headers'], + }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => { + const accountId = facts['request.codex.account'].chatgptAccountId; + let headers = replaceHttpHeader(facts['request.http.headers'], 'authorization', secret(`Bearer ${facts['request.codex.accessToken'].token.reveal()}`)); + if (accountId !== null) headers = replaceHttpHeader(headers, 'chatgpt-account-id', accountId); + return await next(move({ ...facts, 'request.http.headers': headers })) as Handed; + }, +}); diff --git a/packages/provider-codex/src/stages/observe-response.ts b/packages/provider-codex/src/stages/observe-response.ts new file mode 100644 index 0000000000..72078ff26a --- /dev/null +++ b/packages/provider-codex/src/stages/observe-response.ts @@ -0,0 +1,35 @@ +import { classifyCodexHttpResponse, classifyCodexUnauthorizedResponse, decodeCodexUpstreamError } from '../fetch.ts'; +import { codexCall, type CodexHttpFacts, type CodexOperation, type CodexPipelineConfig } from '../pipeline-facts.ts'; +import { exchangeResponse, takeHttpResponse, type HttpResponseFacts } from '@floway-dev/http/pipeline'; +import { defer, defineStage, move, setRelease, type Deferred } from '@floway-dev/pipeline'; +import type { ProviderServices } from '@floway-dev/provider'; + +type Observed = HttpResponseFacts & { 'response.codex.background': Deferred; 'response.codex.failureBody': unknown }; + +export const observeCodexResponse = (config: CodexPipelineConfig, operation: CodexOperation) => defineStage({ + name: 'observeCodexResponse', + through: { + request: { needs: ['request.codex.account', 'request.codex.model', 'request.http.callId'], consumes: [], provides: [] }, + response: { needs: ['response.http.exchange', 'response.http.body'], consumes: ['response.http.exchange', 'response.http.body'], provides: ['response.http.exchange', 'response.http.body', 'response.codex.background', 'response.codex.failureBody'] }, + }, + execute: async (facts, next, use) => { + const back = await next(move({ ...facts })); + const exchange = back['response.http.exchange']; + const pending: Promise[] = []; + if (exchange.type === 'transportFailure') return move({ ...back, 'response.codex.background': defer(Promise.all(pending)), 'response.codex.failureBody': null }); + const call = codexCall(config, facts, use, facts['request.codex.model']); + const response = exchangeResponse(exchange); + let classified: Response; + let failureBody: unknown = null; + if (exchange.status === 401) { + const parsed = decodeCodexUpstreamError(await response.text()); + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + failureBody = parsed.body; + classified = await classifyCodexUnauthorizedResponse(call, response, parsed); + } else { + classified = await classifyCodexHttpResponse({ ...call, call: { ...call.call, waitUntil: work => { pending.push(work); } } }, response, operation === 'alphaSearch' ? 'always' : 'when-present'); + } + const retained = classified === response ? exchange : takeHttpResponse(classified); + return move({ ...back, 'response.http.exchange': retained, 'response.http.body': retained.body, 'response.codex.background': defer(Promise.all(pending)), 'response.codex.failureBody': failureBody }); + }, +}); diff --git a/packages/provider-codex/src/stages/prepare-request.ts b/packages/provider-codex/src/stages/prepare-request.ts new file mode 100644 index 0000000000..993fb4c0b6 --- /dev/null +++ b/packages/provider-codex/src/stages/prepare-request.ts @@ -0,0 +1,61 @@ +import { CODEX_ALPHA_SEARCH_PATH, CODEX_BACKEND_BASE, CODEX_CLI_VERSION, CODEX_OPENAI_IMAGES_EDITS_PATH, CODEX_OPENAI_IMAGES_GENERATIONS_PATH, CODEX_ORIGINATOR, CODEX_USER_AGENT } from '../constants.ts'; +import { uuidV7 } from '../ids.ts'; +import type { CodexAuthenticatedFacts, CodexHttpFacts, CodexOperation } from '../pipeline-facts.ts'; +import { defineStage, isSecret, move } from '@floway-dev/pipeline'; +import { serializeOpenAIImagesEditsJsonPayload, type ProviderCallResponse, type ProviderOperationPayloads, type ProviderResponse, type ProviderServices } from '@floway-dev/provider'; + +const header = (facts: CodexAuthenticatedFacts, name: string): string | null => { + const value = facts['request.http.headers'].find(([key]) => key.toLowerCase() === name)?.[1]; + const text = value === undefined ? '' : isSecret(value) ? value.reveal() : value; + return text.trim() || null; +}; + +export const prepareCodexRequest = (operation: O) => defineStage, CodexHttpFacts, ProviderCallResponse, ProviderResponse, ProviderServices>({ + name: `prepareCodex${operation.replace(/^openai/, 'OpenAI').replace(/^alpha/, 'Alpha')}`, + through: { + request: { + needs: ['request.provider.model', 'request.provider.payload', 'request.http.headers', 'request.http.callId', 'request.codex.account', 'request.codex.accessToken', 'request.codex.plan'], + consumes: ['request.provider.model', 'request.provider.payload', 'request.http.headers'], + provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.codex.modelKey', 'request.codex.model'], + }, + response: { needs: ['response.http.exchange', 'response.provider.called', 'response.provider.previousCalls'], consumes: [], provides: ['response.provider.modelKey'] }, + }, + execute: async (facts, next) => { + const { 'request.provider.model': model, 'request.provider.payload': payload, ...rest } = facts; + let body: Record; + let path: string; + const headers: [string, string][] = [ + ['originator', operation === 'alphaSearch' ? CODEX_ORIGINATOR : header(facts, 'originator') ?? CODEX_ORIGINATOR], + ['user-agent', CODEX_USER_AGENT], + ['version', CODEX_CLI_VERSION], + ['accept', 'application/json'], + ['content-type', 'application/json'], + ]; + if (operation === 'alphaSearch') { + const source = payload as ProviderOperationPayloads['alphaSearch']; + const requestId = typeof source.id === 'string' && source.id.trim().length > 0 ? source.id.trim() : uuidV7(); + body = { ...source, id: requestId, model: model.id }; + path = CODEX_ALPHA_SEARCH_PATH; + headers.push(['session-id', requestId], ['thread-id', requestId], ['x-client-request-id', requestId], ['x-codex-window-id', `${requestId}:0`]); + const metadata = header(facts, 'x-codex-turn-metadata'); + if (metadata !== null) headers.push(['x-codex-turn-metadata', metadata]); + } else { + body = operation === 'openaiImagesEdits' + ? await serializeOpenAIImagesEditsJsonPayload(payload as ProviderOperationPayloads['openaiImagesEdits'], model.id) + : { ...payload, model: model.id }; + path = operation === 'openaiImagesEdits' ? CODEX_OPENAI_IMAGES_EDITS_PATH : CODEX_OPENAI_IMAGES_GENERATIONS_PATH; + headers.push(['x-codex-image-turn-id', header(facts, 'x-codex-image-turn-id') ?? uuidV7()]); + } + const back = await next(move({ + ...rest, + 'request.codex.model': model, + 'request.codex.modelKey': model.id, + 'request.http.url': `${CODEX_BACKEND_BASE}${path}`, + 'request.http.method': 'POST', + 'request.http.headers': headers, + 'request.http.body': body, + 'request.http.encoding': 'json' as const, + })); + return move({ ...back, 'response.provider.modelKey': model.id }); + }, +}); diff --git a/packages/provider-codex/src/stages/read-account.ts b/packages/provider-codex/src/stages/read-account.ts new file mode 100644 index 0000000000..5900f933a3 --- /dev/null +++ b/packages/provider-codex/src/stages/read-account.ts @@ -0,0 +1,12 @@ +import { accountFacts, type CodexAccountFacts, type CodexOperation, type CodexPipelineConfig, type CodexRequest } from '../pipeline-facts.ts'; +import { defineStage, move, type Handed } from '@floway-dev/pipeline'; +import type { ProviderResponse, ProviderServices } from '@floway-dev/provider'; + +export const readCodexAccount = (config: CodexPipelineConfig) => defineStage, CodexAccountFacts, ProviderResponse, ProviderResponse, ProviderServices>({ + name: 'readCodexAccount', + through: { + request: { needs: [], consumes: [], provides: ['request.codex.account'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => await next(move({ ...facts, 'request.codex.account': accountFacts(await config.readAccount()) })) as Handed, +}); diff --git a/packages/provider-codex/src/stages/retry-access.ts b/packages/provider-codex/src/stages/retry-access.ts new file mode 100644 index 0000000000..da73e6a0fc --- /dev/null +++ b/packages/provider-codex/src/stages/retry-access.ts @@ -0,0 +1,32 @@ +import { codexPlanObservation } from '../access-token.ts'; +import { refreshAccessTokenForRetry } from '../fetch.ts'; +import { codexPlanSupportsImages } from '../models.ts'; +import { codexCall, tokenFacts, type CodexHttpFacts, type CodexOperation, type CodexPipelineConfig } from '../pipeline-facts.ts'; +import { takeHttpResponse, type HttpResponseFacts } from '@floway-dev/http/pipeline'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { ProviderCallResponse, ProviderServices } from '@floway-dev/provider'; + +export const retryCodexAccess = (config: CodexPipelineConfig, operation: CodexOperation) => defineStage({ + name: 'retryCodexAccess', + through: { + request: { needs: ['request.codex.account', 'request.codex.accessToken', 'request.codex.plan', 'request.codex.model', 'request.codex.modelKey', 'request.http.callId'], consumes: ['request.codex.accessToken', 'request.codex.plan'], provides: ['request.codex.accessToken', 'request.codex.plan'] }, + response: { needs: ['response.http.exchange', 'response.http.body'], consumes: ['response.http.exchange', 'response.http.body'], provides: ['response.http.exchange', 'response.http.body', 'response.provider.called', 'response.provider.previousCalls'] }, + }, + execute: async (facts, next, use) => { + const first = await next(move({ ...facts })); + const exchange = first['response.http.exchange']; + if (exchange.type !== 'response' || exchange.status !== 401 || facts['request.codex.account'].refresh_token === null) return move({ ...first, 'response.provider.called': exchange.type === 'response', 'response.provider.previousCalls': [] }); + if (exchange.body !== null) await exchange.body[Symbol.asyncDispose](); + const failedToken = facts['request.codex.accessToken']; + const fresh = await refreshAccessTokenForRetry(codexCall(config, facts, use, facts['request.codex.model']), { ...failedToken, token: failedToken.token.reveal() }, facts['request.codex.plan'] ?? undefined); + const declined = (response: Response) => { + const nextExchange = takeHttpResponse(response); + return move({ ...first, 'response.http.exchange': nextExchange, 'response.http.body': nextExchange.body, 'response.provider.called': false, 'response.provider.previousCalls': [{ modelKey: facts['request.codex.modelKey'] }] }); + }; + if (!fresh.ok) return declined(fresh.response); + const plan = codexPlanObservation(fresh.accessToken) ?? facts['request.codex.plan']; + if (operation !== 'alphaSearch' && plan !== null && !codexPlanSupportsImages(plan.planType)) return declined(Response.json({ error: { type: 'image_tools_unavailable', message: 'ChatGPT Free accounts do not provide Codex image tools.' } }, { status: 403 })); + const retried = await next(move({ ...facts, 'request.codex.accessToken': tokenFacts(fresh.accessToken), 'request.codex.plan': plan })); + return move({ ...retried, 'response.provider.called': retried['response.http.exchange'].type === 'response', 'response.provider.previousCalls': [{ modelKey: facts['request.codex.modelKey'] }] }); + }, +}); diff --git a/packages/provider-copilot/__tests__/provider_test.ts b/packages/provider-copilot/__tests__/provider_test.ts index 6d6439a420..616a9a6000 100644 --- a/packages/provider-copilot/__tests__/provider_test.ts +++ b/packages/provider-copilot/__tests__/provider_test.ts @@ -10,7 +10,7 @@ import { createInMemoryImageProcessor, initImageProcessor } from '@floway-dev/pl import type { AnthropicMessagesPayload } from '@floway-dev/protocols/anthropic-messages'; import type { UpstreamRecord } from '@floway-dev/provider'; import { directFetcher, initProviderRepo } from '@floway-dev/provider'; -import { assertEquals, assertRejects, assertThrows, jsonResponse, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, sseResponse, withMockedFetch } from '@floway-dev/test-utils'; +import { callProviderPipeline, assertEquals, assertRejects, assertThrows, jsonResponse, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, sseResponse, withMockedFetch } from '@floway-dev/test-utils'; const mergeVariantsControl = vi.hoisted<{ override: ((merged: CopilotRawModel[]) => CopilotRawModel[]) | null; @@ -594,7 +594,7 @@ test('Copilot provider forces stream=true for streaming endpoints and leaves cou await provider.callOpenAIResponses(byId.get('gpt-resp')!, { input: [] }, 'generate', undefined, opts); await provider.callAnthropicMessages(byId.get('claude-msg')!, { max_tokens: 10, messages: [{ role: 'user', content: 'hi' }] }, undefined, anthropicMessagesOpts); await provider.callAnthropicMessagesCountTokens(byId.get('claude-msg')!, { max_tokens: 10, messages: [{ role: 'user', content: 'hi' }] }, undefined, anthropicMessagesOpts); - await provider.callOpenAIEmbeddings(byId.get('emb-mini')!, { input: 'hi' }, undefined, opts); + await callProviderPipeline(instance, 'openaiEmbeddings', byId.get('emb-mini')!, { input: 'hi' }, undefined, opts); }, ); diff --git a/packages/provider-copilot/package.json b/packages/provider-copilot/package.json index 378cbb240b..d813df5f67 100644 --- a/packages/provider-copilot/package.json +++ b/packages/provider-copilot/package.json @@ -10,6 +10,8 @@ "typecheck": "tsc --noEmit" }, "dependencies": { + "@floway-dev/http": "workspace:*", + "@floway-dev/pipeline": "workspace:*", "@floway-dev/interceptor": "workspace:*", "@floway-dev/platform": "workspace:*", "@floway-dev/protocols": "workspace:*", diff --git a/packages/provider-copilot/src/auth.ts b/packages/provider-copilot/src/auth.ts index 75fe443235..98a1e4b06c 100644 --- a/packages/provider-copilot/src/auth.ts +++ b/packages/provider-copilot/src/auth.ts @@ -127,7 +127,7 @@ function isTokenValid(token: string | null, expiresAt: number): boolean { return expiresAt > now + 60; } -async function getCopilotToken(upstreamId: string, githubHost: string, githubToken: string, fetcher: Fetcher, signal: AbortSignal | undefined): Promise { +export async function getCopilotToken(upstreamId: string, githubHost: string, githubToken: string, fetcher: Fetcher, signal: AbortSignal | undefined): Promise { const now = Date.now(); const cached = inProcessTokenCache.get(upstreamId); if (cached && isTokenValid(cached.entry.token, cached.entry.expiresAt) && now - cached.cachedAt < IN_PROCESS_TTL_MS) { @@ -225,21 +225,12 @@ export interface CopilotAuth { githubToken: string; } -export async function copilotAuthedFetch(path: string, init: FetchInit, auth: CopilotAuth, options: CopilotFetchOptions): Promise { - const signal = init.signal ?? undefined; - let ownedInit: FetchInit | undefined = init; - // The token exchange is the only await before the data-plane dispatch. Keep - // the body in an explicit owner and replace the generator parameter so the - // final network wait cannot retain both copies after ownership transfers. - init = { signal }; - const entry = await getCopilotToken(auth.id, auth.githubHost, auth.githubToken, options.fetcher, signal); - +export const copilotRequestHeaders = (entry: CopilotTokenEntry, initHeaders: HeadersInit | undefined, attached: HttpHeaderLines | undefined): Headers => { // x-request-id and x-agent-task-id share a single per-call UUID, mirroring // VSCode Copilot Chat's "one id ties the request to its background task" pattern. const requestId = crypto.randomUUID(); - if (ownedInit === undefined) throw new Error('Copilot request ownership missing before dispatch'); - const headers = new Headers(ownedInit.headers); + const headers = new Headers(initHeaders); headers.set('Authorization', `Bearer ${entry.token}`); headers.set('Content-Type', 'application/json'); headers.set('editor-version', EDITOR_VERSION); @@ -265,13 +256,28 @@ export async function copilotAuthedFetch(path: string, init: FetchInit, auth: Co // otherwise pin. An interceptor that wants to clear an arbitrary downstream // header value must do so by name through this sentinel; the layer does not // otherwise expose a per-header delete API. - if (options.headers) { - for (const [name, value] of options.headers) { + if (attached) { + for (const [name, value] of attached) { if (value === '') headers.delete(name); else headers.set(name, value); } } + return headers; +}; + +export async function copilotAuthedFetch(path: string, init: FetchInit, auth: CopilotAuth, options: CopilotFetchOptions): Promise { + const signal = init.signal ?? undefined; + let ownedInit: FetchInit | undefined = init; + // The token exchange is the only await before the data-plane dispatch. Keep + // the body in an explicit owner and replace the generator parameter so the + // final network wait cannot retain both copies after ownership transfers. + init = { signal }; + const entry = await getCopilotToken(auth.id, auth.githubHost, auth.githubToken, options.fetcher, signal); + + if (ownedInit === undefined) throw new Error('Copilot request ownership missing before dispatch'); + const headers = copilotRequestHeaders(entry, ownedInit.headers, options.headers); + const request = { ...ownedInit, headers }; ownedInit = undefined; // Do not await here: the dispatch owner clears its body synchronously, then diff --git a/packages/provider-copilot/src/index.ts b/packages/provider-copilot/src/index.ts index daeb678809..abe5ffd267 100644 --- a/packages/provider-copilot/src/index.ts +++ b/packages/provider-copilot/src/index.ts @@ -1,6 +1,7 @@ import { COPILOT_DEFAULT_FLAGS } from './defaults.ts'; import { createCopilotProvider } from './provider.ts'; import type { ProviderModule } from '@floway-dev/provider'; +export { createCopilotPipelines } from './pipelines.ts'; export const copilotProviderModule: ProviderModule = { create: createCopilotProvider, diff --git a/packages/provider-copilot/src/operation-model.ts b/packages/provider-copilot/src/operation-model.ts new file mode 100644 index 0000000000..9f47751214 --- /dev/null +++ b/packages/provider-copilot/src/operation-model.ts @@ -0,0 +1,71 @@ +import { resolveCopilotRawModel, type ModelSelectionHints } from './model-selection.ts'; +import type { CopilotRawModel } from './types.ts'; +import type { ModelEndpointKey } from '@floway-dev/protocols/common'; +import type { ProviderModel } from '@floway-dev/provider'; + +interface CopilotProviderData { readonly rawModels: readonly CopilotRawModel[] } + +// Copilot's `/models` reports each model's served endpoints as public paths; map +// one onto our structured endpoint key. Both `/x` and `/v1/x` spellings appear. +// Copilot is the only upstream whose catalog speaks paths — operator config and +// our own constants are structured — so this lives here, not in a shared helper. +const copilotPathToModelEndpoint = (path: string): ModelEndpointKey | undefined => { + switch (path) { + case '/chat/completions': + case '/v1/chat/completions': + return 'openaiChatCompletions'; + case '/responses': + case '/v1/responses': + return 'openaiResponses'; + case '/v1/messages': + case '/messages': + return 'anthropicMessages'; + case '/embeddings': + case '/v1/embeddings': + return 'openaiEmbeddings'; + case '/images/generations': + case '/v1/images/generations': + return 'openaiImagesGenerations'; + case '/images/edits': + case '/v1/images/edits': + return 'openaiImagesEdits'; + default: + return undefined; + } +}; + +export const rawModelSupportsEndpoint = (model: CopilotRawModel, endpoint: ModelEndpointKey): boolean => { + if ((model.supported_endpoints ?? []).some(path => copilotPathToModelEndpoint(path) === endpoint)) return true; + // Copilot's Anthropic-family entries have historically under-reported their + // native Anthropic Messages path, so treat claude-* as Anthropic-Messages-capable. + if (endpoint === 'anthropicMessages' && model.id.startsWith('claude-')) return true; + if (endpoint === 'openaiChatCompletions') { + return model.supported_endpoints === undefined && model.capabilities?.type === 'chat'; + } + if (endpoint === 'openaiEmbeddings') return model.supported_endpoints === undefined && model.capabilities?.type === 'embeddings'; + return false; +}; + +export const rawModelFor = (model: Pick, endpoint: ModelEndpointKey, hints: ModelSelectionHints = {}): CopilotRawModel => { + // Copilot exposes one canonical public Claude model id per family. Raw + // variant selection is derived from request body fields and parsed Anthropic Messages + // beta intent, not from the client's original model alias string. + const rawModels = (model.providerData as CopilotProviderData).rawModels.filter(rawModel => rawModelSupportsEndpoint(rawModel, endpoint)); + if (rawModels.length === 0) { + throw new Error(`Copilot provider exposed ${endpoint} for ${model.id}, but no raw variant supports that endpoint`); + } + return resolveCopilotRawModel({ object: 'list', data: rawModels }, model.id, hints) ?? rawModels[0]; +}; + +export const copilotOpenAIEmbeddingsBody = (body: Record): Record => { + if (typeof body.input !== 'string') return body; + + // OpenAI-compatible clients may send scalar string input, but Copilot's + // upstream /embeddings endpoint currently returns 400 unless text input is + // wrapped as an array. + // References: + // https://platform.openai.com/docs/api-reference/embeddings/create + // https://github.com/ericc-ch/copilot-api/blob/0ea08febdd7e3e055b03dd298bf57e669500b5c1/src/services/copilot/create-embeddings.ts#L19-L21 + // https://github.com/BerriAI/litellm/blob/c8fb77f119ad69a80f5fde088efd3a1aa77f458b/litellm/proxy/proxy_server.py#L7826-L7839 + return { ...body, input: [body.input] }; +}; diff --git a/packages/provider-copilot/src/pipeline-facts.ts b/packages/provider-copilot/src/pipeline-facts.ts new file mode 100644 index 0000000000..cf269f3767 --- /dev/null +++ b/packages/provider-copilot/src/pipeline-facts.ts @@ -0,0 +1,17 @@ +import type { HttpRequestFacts, HttpResponseFacts } from '@floway-dev/http/pipeline'; +import type { Deferred, Secret } from '@floway-dev/pipeline'; +import type { ProviderOperationPayloads, ProviderRequest } from '@floway-dev/provider'; + +export type EmbeddingsRequest = ProviderRequest; +export interface CopilotSession { + readonly token: Secret; + readonly expiresAt: number; + readonly baseUrl: string; +} +export interface CopilotRequest extends EmbeddingsRequest { + 'request.copilot.session': CopilotSession; +} +export interface CopilotHttpFacts extends HttpRequestFacts { + 'request.copilot.modelKey': string; +} +export type QuotaResponse = HttpResponseFacts & { 'response.copilot.quota': Deferred }; diff --git a/packages/provider-copilot/src/pipelines.ts b/packages/provider-copilot/src/pipelines.ts new file mode 100644 index 0000000000..73636dc2d8 --- /dev/null +++ b/packages/provider-copilot/src/pipelines.ts @@ -0,0 +1,12 @@ +import type { CopilotAuth } from './auth.ts'; +import type { EmbeddingsRequest } from './pipeline-facts.ts'; +import { authenticateCopilot } from './stages/authenticate.ts'; +import { observeCopilotQuota } from './stages/observe-quota.ts'; +import { prepareCopilotEmbeddings } from './stages/prepare-request.ts'; +import { http } from '@floway-dev/http/pipeline'; +import { compose } from '@floway-dev/pipeline'; +import type { ProviderPipelines, ProviderResponse } from '@floway-dev/provider'; + +export const createCopilotPipelines = (auth: CopilotAuth): ProviderPipelines => ({ + openaiEmbeddings: compose('copilot.openaiEmbeddings', [authenticateCopilot(auth), prepareCopilotEmbeddings(), observeCopilotQuota(auth.id), http]), +}); diff --git a/packages/provider-copilot/src/provider.ts b/packages/provider-copilot/src/provider.ts index 35fbabdf55..1fdd720c0a 100644 --- a/packages/provider-copilot/src/provider.ts +++ b/packages/provider-copilot/src/provider.ts @@ -12,14 +12,16 @@ import { COPILOT_OPENAI_RESPONSES_BOUNDARY } from './interceptors/openai-respons import type { OpenAIResponsesBoundaryCtx } from './interceptors/openai-responses/types.ts'; import { emptyKnownModels, mergeKnownModels, projectKnownModels } from './known-models.ts'; import { mergeCopilotVariants } from './merge-variants.ts'; -import { CONTEXT_1M_BETA, copilotModelSupportsFastVariant, type ModelSelectionHints, resolveCopilotRawModel } from './model-selection.ts'; +import { CONTEXT_1M_BETA, copilotModelSupportsFastVariant } from './model-selection.ts'; import { copilotVariantIndex } from './model-variants.ts'; +import { copilotOpenAIEmbeddingsBody, rawModelFor, rawModelSupportsEndpoint } from './operation-model.ts'; +import { createCopilotPipelines } from './pipelines.ts'; import { pricingForCopilotPublicModelId } from './pricing.ts'; import { readCopilotUpstreamState, type CopilotUpstreamState } from './state.ts'; import type { CopilotRawModel } from './types.ts'; import { runInterceptors } from '@floway-dev/interceptor'; import { parseAnthropicMessagesStream, type AnthropicMessagesPayload, type AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; -import { type ModelEndpointKey, type ModelEndpoints, type ProtocolFrame, isFastServiceTier, kindForEndpoints } from '@floway-dev/protocols/common'; +import { type ModelEndpoints, type ProtocolFrame, isFastServiceTier, kindForEndpoints } from '@floway-dev/protocols/common'; import { parseOpenAIChatCompletionsStream, type OpenAIChatCompletionsPayload, type OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import { parseOpenAIResponsesStream, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesResult } from '@floway-dev/protocols/openai-responses'; import { eventResult, getProviderRepo, headersForAnthropicMessagesCall, jsonRequestBody, readUpstreamApiError, streamingProviderCall, apiErrorToResponse, resolveEffectiveFlags, type ExecuteResult, type FetchInit, type FlagOverrides, type HttpHeaderLines, type ProviderInstance, type Provider, type ProviderCallResult, type ProviderModel, type ProviderOpenAIResponsesResult, type ProviderStreamResult, type TelemetryModelIdentity, type UpstreamCallOptions, type UpstreamRecord } from '@floway-dev/provider'; @@ -53,47 +55,6 @@ const copilotRawToProviderModel = (model: CopilotRawModel): Omit { - switch (path) { - case '/chat/completions': - case '/v1/chat/completions': - return 'openaiChatCompletions'; - case '/responses': - case '/v1/responses': - return 'openaiResponses'; - case '/v1/messages': - case '/messages': - return 'anthropicMessages'; - case '/embeddings': - case '/v1/embeddings': - return 'openaiEmbeddings'; - case '/images/generations': - case '/v1/images/generations': - return 'openaiImagesGenerations'; - case '/images/edits': - case '/v1/images/edits': - return 'openaiImagesEdits'; - default: - return undefined; - } -}; - -const rawModelSupportsEndpoint = (model: CopilotRawModel, endpoint: ModelEndpointKey): boolean => { - if ((model.supported_endpoints ?? []).some(path => copilotPathToModelEndpoint(path) === endpoint)) return true; - // Copilot's Anthropic-family entries have historically under-reported their - // native Anthropic Messages path, so treat claude-* as Anthropic-Messages-capable. - if (endpoint === 'anthropicMessages' && model.id.startsWith('claude-')) return true; - if (endpoint === 'openaiChatCompletions') { - return model.supported_endpoints === undefined && model.capabilities?.type === 'chat'; - } - if (endpoint === 'openaiEmbeddings') return model.supported_endpoints === undefined && model.capabilities?.type === 'embeddings'; - return false; -}; - const copilotModelEndpoints = (rawModels: readonly CopilotRawModel[]): ModelEndpoints => { if (rawModels.some(model => rawModelSupportsEndpoint(model, 'openaiResponses'))) { return { openaiResponses: {} }; @@ -133,30 +94,6 @@ const anthropicMessagesBoundaryContext = ( const rejectUnsupported = (capability: string) => (): Promise => Promise.reject(new Error(`Copilot provider does not implement ${capability}`)); -const rawModelFor = (model: ProviderModel, endpoint: ModelEndpointKey, hints: ModelSelectionHints = {}): CopilotRawModel => { - // Copilot exposes one canonical public Claude model id per family. Raw - // variant selection is derived from request body fields and parsed Anthropic Messages - // beta intent, not from the client's original model alias string. - const rawModels = (model.providerData as CopilotProviderData).rawModels.filter(rawModel => rawModelSupportsEndpoint(rawModel, endpoint)); - if (rawModels.length === 0) { - throw new Error(`Copilot provider exposed ${endpoint} for ${model.id}, but no raw variant supports that endpoint`); - } - return resolveCopilotRawModel({ object: 'list', data: rawModels }, model.id, hints) ?? rawModels[0]; -}; - -const copilotOpenAIEmbeddingsBody = (body: Record): Record => { - if (typeof body.input !== 'string') return body; - - // OpenAI-compatible clients may send scalar string input, but Copilot's - // upstream /embeddings endpoint currently returns 400 unless text input is - // wrapped as an array. - // References: - // https://platform.openai.com/docs/api-reference/embeddings/create - // https://github.com/ericc-ch/copilot-api/blob/0ea08febdd7e3e055b03dd298bf57e669500b5c1/src/services/copilot/create-embeddings.ts#L19-L21 - // https://github.com/BerriAI/litellm/blob/c8fb77f119ad69a80f5fde088efd3a1aa77f458b/litellm/proxy/proxy_server.py#L7826-L7839 - return { ...body, input: [body.input] }; -}; - const finalizeCopilotModels = ( rawModels: CopilotRawModel[], upstreamOverrides: FlagOverrides, @@ -500,6 +437,7 @@ export const createCopilotProvider = (record: UpstreamRecord): Provider => { disabledPublicModelIds: copilot.disabledPublicModelIds, modelPrefix: copilot.modelPrefix, modelsCache: copilot.modelsCache, + pipelines: createCopilotPipelines(upstreamConfig), instance, }; }; diff --git a/packages/provider-copilot/src/stages/authenticate.ts b/packages/provider-copilot/src/stages/authenticate.ts new file mode 100644 index 0000000000..179f117e21 --- /dev/null +++ b/packages/provider-copilot/src/stages/authenticate.ts @@ -0,0 +1,41 @@ +import { getCopilotToken, isCopilotTokenFetchError, type CopilotAuth } from '../auth.ts'; +import { rawModelFor } from '../operation-model.ts'; +import type { EmbeddingsRequest, CopilotRequest } from '../pipeline-facts.ts'; +import { takeHttpResponse } from '@floway-dev/http/pipeline'; +import { defineStage, move, secret, type Handed } from '@floway-dev/pipeline'; +import type { ProviderResponse, ProviderServices } from '@floway-dev/provider'; + +export const authenticateCopilot = (auth: CopilotAuth) => { + const authenticate = defineStage({ + name: 'authenticateCopilot', + through: { + request: { + needs: ['request.provider.model', 'request.http.callId'], + consumes: [], + provides: ['request.copilot.session'], + }, + response: { needs: ['response.http.exchange', 'response.provider.modelKey', 'response.provider.called'], consumes: [], provides: [] }, + }, + return: { provides: ['response.http.exchange', 'response.http.body', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'] }, + execute: async (facts, next, use) => { + const call = use.httpCall(facts['request.http.callId']); + let entry; + try { + entry = await getCopilotToken(auth.id, auth.githubHost, auth.githubToken, call.fetcher, call.signal); + } catch (error) { + if (!isCopilotTokenFetchError(error)) throw error; + const exchange = takeHttpResponse(new Response(error.body, { status: error.status, headers: error.headers })); + return move({ + ...facts, + 'response.http.exchange': exchange, + 'response.http.body': exchange.body, + 'response.provider.modelKey': rawModelFor(facts['request.provider.model'], 'openaiEmbeddings').id, + 'response.provider.called': false, + 'response.provider.previousCalls': [], + }); + } + return await next(move({ ...facts, 'request.copilot.session': { ...entry, token: secret(entry.token) } })) as Handed; + }, + }); + return authenticate; +}; diff --git a/packages/provider-copilot/src/stages/observe-quota.ts b/packages/provider-copilot/src/stages/observe-quota.ts new file mode 100644 index 0000000000..22cdb1d138 --- /dev/null +++ b/packages/provider-copilot/src/stages/observe-quota.ts @@ -0,0 +1,28 @@ +import type { QuotaResponse } from '../pipeline-facts.ts'; +import { parseCopilotQuotaHeaders, putCopilotQuota } from '../quota.ts'; +import type { HttpRequestFacts, HttpResponseFacts } from '@floway-dev/http/pipeline'; +import { defer, defineStage, move } from '@floway-dev/pipeline'; +import type { ProviderServices } from '@floway-dev/provider'; + +export const observeCopilotQuota = (upstreamId: string) => { + const observeQuota = defineStage, Pick, HttpResponseFacts, QuotaResponse, ProviderServices>({ + name: 'observeCopilotQuota', + through: { + request: { needs: ['request.http.callId'], consumes: [], provides: [] }, + response: { needs: ['response.http.exchange'], consumes: [], provides: ['response.copilot.quota'] }, + }, + execute: async (facts, next, use) => { + const back = await next(move({ ...facts })); + const exchange = back['response.http.exchange']; + const pending: Promise[] = []; + if (exchange.type === 'response') { + const snapshot = parseCopilotQuotaHeaders(new Headers(exchange.headers.map(([name, value]): [string, string] => [name, value])), new Date()); + if (snapshot !== null) pending.push(putCopilotQuota(upstreamId, snapshot).catch(error => { + use.log.warn('Copilot quota snapshot persistence failed', { upstreamId, error }); + })); + } + return move({ ...back, 'response.copilot.quota': defer(Promise.all(pending)) }); + }, + }); + return observeQuota; +}; diff --git a/packages/provider-copilot/src/stages/prepare-request.ts b/packages/provider-copilot/src/stages/prepare-request.ts new file mode 100644 index 0000000000..5d33ca5bfc --- /dev/null +++ b/packages/provider-copilot/src/stages/prepare-request.ts @@ -0,0 +1,38 @@ +import { copilotRequestHeaders } from '../auth.ts'; +import { copilotOpenAIEmbeddingsBody, rawModelFor } from '../operation-model.ts'; +import type { CopilotRequest, CopilotHttpFacts } from '../pipeline-facts.ts'; +import type { HttpResponseFacts } from '@floway-dev/http/pipeline'; +import { defineStage, isSecret, move, secret, type Secret } from '@floway-dev/pipeline'; +import type { ProviderResponse, ProviderServices } from '@floway-dev/provider'; + +export const prepareCopilotEmbeddings = () => { + const prepare = defineStage({ + name: 'prepareCopilotOpenAIEmbeddings', + through: { + request: { + needs: ['request.provider.model', 'request.provider.payload', 'request.http.headers', 'request.http.callId', 'request.copilot.session'], + consumes: ['request.provider.model', 'request.provider.payload', 'request.copilot.session', 'request.http.headers'], + provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.copilot.modelKey'], + }, + response: { needs: ['response.http.exchange'], consumes: [], provides: ['response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'] }, + }, + execute: async (facts, next) => { + const { 'request.provider.model': model, 'request.provider.payload': payload, 'request.copilot.session': session, ...rest } = facts; + const modelKey = rawModelFor(model, 'openaiEmbeddings').id; + const admitted = facts['request.http.headers'].map(([name, value]): [string, string] => [name, isSecret(value) ? value.reveal() : value]); + const headers = [...copilotRequestHeaders({ ...session, token: session.token.reveal() }, undefined, admitted)] + .map(([name, value]): [string, string | Secret] => [name, name === 'authorization' ? secret(value) : value]); + const back = await next(move({ + ...rest, + 'request.copilot.modelKey': modelKey, + 'request.http.url': `${session.baseUrl}/embeddings`, + 'request.http.method': 'POST', + 'request.http.headers': headers, + 'request.http.body': { ...copilotOpenAIEmbeddingsBody(payload), model: modelKey }, + 'request.http.encoding': 'json' as const, + })); + return move({ ...back, 'response.provider.modelKey': modelKey, 'response.provider.called': back['response.http.exchange'].type === 'response', 'response.provider.previousCalls': [] }); + }, + }); + return prepare; +}; diff --git a/packages/provider-custom/__tests__/fetch_test.ts b/packages/provider-custom/__tests__/fetch_test.ts index 3f4669cad4..62a9b52bc8 100644 --- a/packages/provider-custom/__tests__/fetch_test.ts +++ b/packages/provider-custom/__tests__/fetch_test.ts @@ -15,7 +15,7 @@ import { import { createCustomProvider } from '../src/provider.ts'; import type { UpstreamRecord } from '@floway-dev/provider'; import { directFetcher, identityWrapUpstreamCall } from '@floway-dev/provider'; -import { assertEquals, assertExists, jsonResponse, noopUpstreamCallOptions, withMockedFetch } from '@floway-dev/test-utils'; +import { callProviderPipeline, assertEquals, assertExists, jsonResponse, noopUpstreamCallOptions, withMockedFetch } from '@floway-dev/test-utils'; const baseRecord: UpstreamRecord = { id: 'up_test', @@ -286,11 +286,11 @@ test('Custom provider callOpenAIImagesEdits forwards multipart body with model f async () => { const provider = createCustomProvider(record); const [model] = await provider.instance.getProvidedModels(directFetcher); - const result = await provider.instance.callOpenAIImagesEdits(model, { + const result = await callProviderPipeline(provider, 'openaiImagesEdits', model, { parameters: { prompt: 'add a kite' }, images: [{ type: 'upload', - file: new File([new Uint8Array([1, 2, 3])], 'photo.png', { type: 'image/png' }), + file: { bytes: new Uint8Array([1, 2, 3]), name: 'photo.png', type: 'image/png' }, }], }, undefined, noopUpstreamCallOptions()); assertEquals(result.modelKey, 'gpt-image-2'); @@ -326,9 +326,9 @@ test('Custom provider callOpenAIAudioTranscriptions preserves multipart entries async () => { const provider = createCustomProvider(record); const [model] = await provider.instance.getProvidedModels(directFetcher); - const result = await provider.instance.callOpenAIAudioTranscriptions(model, { + const result = await callProviderPipeline(provider, 'openaiAudioTranscriptions', model, { entries: [ - { name: 'file', value: new File([new Uint8Array([7, 8])], 'voice.ogg', { type: 'audio/ogg' }) }, + { name: 'file', value: { bytes: new Uint8Array([7, 8]), name: 'voice.ogg', type: 'audio/ogg' } }, { name: 'model', value: 'public-model' }, { name: 'language', value: 'en' }, ], diff --git a/packages/provider-custom/__tests__/ingress-header-rules_test.ts b/packages/provider-custom/__tests__/ingress-header-rules_test.ts index 3ac1139f2c..293c754954 100644 --- a/packages/provider-custom/__tests__/ingress-header-rules_test.ts +++ b/packages/provider-custom/__tests__/ingress-header-rules_test.ts @@ -4,7 +4,7 @@ import type { CustomIngressHeaderRule } from '../src/config.ts'; import { createCustomProvider } from '../src/provider.ts'; import { parseRerankRequest } from '@floway-dev/protocols/rerank'; import { directFetcher, type Fetcher, type UpstreamModelConfig, type UpstreamRecord } from '@floway-dev/provider'; -import { assertEquals, assertExists, jsonResponse, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, sseResponse, withMockedFetch } from '@floway-dev/test-utils'; +import { callProviderPipeline, assertEquals, assertExists, jsonResponse, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, sseResponse, withMockedFetch } from '@floway-dev/test-utils'; const HEADER = 'x-route'; @@ -195,28 +195,27 @@ test('every endpoint resolves the same rules', async () => { const messagesOpts = () => noopAnthropicMessagesUpstreamCallOptions({ headers: headers() }); const messagesBody = { max_tokens: 10, messages: [{ role: 'user' as const, content: 'hi' }] }; - await provider.instance.callAlphaSearch(model, { query: 'hi' }, undefined, opts()); + await callProviderPipeline(provider, 'alphaSearch', model, { query: 'hi' }, undefined, opts()); await provider.instance.callOpenAIChatCompletions(model, { messages: [] }, undefined, opts()); - await provider.instance.callOpenAICompletions(model, { prompt: 'hi' }, undefined, opts()); + await callProviderPipeline(provider, 'openaiCompletions', model, { prompt: 'hi' }, undefined, opts()); await provider.instance.callOpenAIResponses(model, { input: [] }, 'generate', undefined, opts()); await provider.instance.callOpenAIResponses(model, { input: [] }, 'compact', undefined, opts()); await provider.instance.callAnthropicMessages(model, messagesBody, undefined, messagesOpts()); await provider.instance.callAnthropicMessagesCountTokens(model, messagesBody, undefined, messagesOpts()); - await provider.instance.callOpenAIEmbeddings(model, { input: 'hi' }, undefined, opts()); - await provider.instance.callOpenAIImagesGenerations(model, { prompt: 'hi' }, undefined, opts()); - await provider.instance.callOpenAIImagesEdits(model, { + await callProviderPipeline(provider, 'openaiEmbeddings', model, { input: 'hi' }, undefined, opts()); + await callProviderPipeline(provider, 'openaiImagesGenerations', model, { prompt: 'hi' }, undefined, opts()); + await callProviderPipeline(provider, 'openaiImagesEdits', model, { parameters: { prompt: 'hi' }, - images: [{ type: 'upload', file: new File([new Uint8Array([1])], 'photo.png', { type: 'image/png' }) }], + images: [{ type: 'upload', file: { bytes: new Uint8Array([1]), name: 'photo.png', type: 'image/png' } }], }, undefined, opts()); - await provider.instance.callOpenAIAudioTranscriptions(model, { - entries: [{ name: 'file', value: new File([new Uint8Array([1])], 'voice.ogg', { type: 'audio/ogg' }) }], + await callProviderPipeline(provider, 'openaiAudioTranscriptions', model, { + entries: [{ name: 'file', value: { bytes: new Uint8Array([1]), name: 'voice.ogg', type: 'audio/ogg' } }], }, undefined, opts()); - await provider.instance.callRerank( + await callProviderPipeline(provider, 'rerank', rerankModel, parseRerankRequest('cohere-v1', { model: 'reranker', query: 'query', documents: ['one'] }).request, undefined, - opts(), - ); + opts()); }, ); diff --git a/packages/provider-custom/__tests__/provider_test.ts b/packages/provider-custom/__tests__/provider_test.ts index 1d8267b046..31092d1c19 100644 --- a/packages/provider-custom/__tests__/provider_test.ts +++ b/packages/provider-custom/__tests__/provider_test.ts @@ -5,7 +5,7 @@ import type { ModelPricing } from '@floway-dev/protocols/common'; import { parseRerankRequest } from '@floway-dev/protocols/rerank'; import type { UpstreamModelConfig, UpstreamRecord } from '@floway-dev/provider'; import { directFetcher } from '@floway-dev/provider'; -import { assertEquals, assertExists, assertRejects, jsonResponse, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, sseResponse, withMockedFetch } from '@floway-dev/test-utils'; +import { callProviderPipeline, assertEquals, assertExists, assertRejects, jsonResponse, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, sseResponse, withMockedFetch } from '@floway-dev/test-utils'; interface BuildOptions { ingressHeadersRules?: { key: string; value: string | null }[]; @@ -340,12 +340,11 @@ test('callRerank uses the model target protocol, raw model id, and canonical pat return jsonResponse({ results: [] }); }, async () => { - const result = await instance.instance.callRerank( + const result = await callProviderPipeline(instance, 'rerank', model, parseRerankRequest('cohere-v1', { model: 'public-reranker', query: 'query', documents: ['one'], top_n: 1 }).request, undefined, - noopUpstreamCallOptions(), - ); + noopUpstreamCallOptions()); assertEquals(result.target, { protocol: 'cohere-v2' }); assertEquals(result.modelKey, 'raw-reranker'); }, @@ -373,19 +372,19 @@ test('callRerank honors the per-model path without adding an upstream path overr return jsonResponse({ output: { results: [] } }); }, async () => { - await instance.instance.callRerank( + await callProviderPipeline(instance, 'rerank', model, parseRerankRequest('jina-v1', { model: 'raw-reranker', query: 'query', documents: ['one'] }).request, undefined, - noopUpstreamCallOptions(), - ); + noopUpstreamCallOptions()); }, ); assertEquals(requestUrl, 'https://custom.example.com/workspace/rerank'); }); test('Custom provider forces stream=true for streaming endpoints and leaves count-tokens/embeddings alone', async () => { - const provider = createCustomProvider(buildCustomUpstream()).instance; + const instance = createCustomProvider(buildCustomUpstream()); + const provider = instance.instance; const bodies: Record> = {}; const betas: Record = {}; @@ -414,7 +413,7 @@ test('Custom provider forces stream=true for streaming endpoints and leaves coun await provider.callOpenAIResponses(model, { input: [] }, 'generate', undefined, opts); await provider.callAnthropicMessages(model, { max_tokens: 10, messages: [{ role: 'user', content: 'hi' }] }, undefined, anthropicMessagesOpts); await provider.callAnthropicMessagesCountTokens(model, { max_tokens: 10, messages: [{ role: 'user', content: 'hi' }] }, undefined, anthropicMessagesOpts); - await provider.callOpenAIEmbeddings(model, { input: 'hi' }, undefined, opts); + await callProviderPipeline(instance, 'openaiEmbeddings', model, { input: 'hi' }, undefined, opts); }, ); @@ -434,7 +433,8 @@ test('Custom provider uses configured endpoints regardless of per-model hints in await withMockedFetch( () => jsonResponse({ object: 'list', data: [{ id: 'm-1', supported_endpoints: ['/some/random/path'] }] }), async () => { - const provider = createCustomProvider(buildCustomUpstream()).instance; + const instance = createCustomProvider(buildCustomUpstream()); + const provider = instance.instance; const [model] = await provider.getProvidedModels(directFetcher); assertEquals(model.endpoints, { openaiChatCompletions: {} }); assertEquals(model.kind, 'chat'); @@ -493,7 +493,7 @@ test('Custom provider callOpenAIImagesGenerations posts JSON with model re-injec async () => { const provider = createCustomProvider(buildCustomUpstream()); const [model] = await provider.instance.getProvidedModels(directFetcher); - const result = await provider.instance.callOpenAIImagesGenerations(model, { prompt: 'hi' }, undefined, noopUpstreamCallOptions()); + const result = await callProviderPipeline(provider, 'openaiImagesGenerations', model, { prompt: 'hi' }, undefined, noopUpstreamCallOptions()); assertEquals(result.modelKey, 'gpt-image-2'); assertEquals(result.response.status, 200); }, @@ -518,12 +518,11 @@ test('Custom provider callAlphaSearch posts JSON to /v1/alpha/search with the up async () => { const provider = createCustomProvider(buildCustomUpstream()); const [model] = await provider.instance.getProvidedModels(directFetcher); - const result = await provider.instance.callAlphaSearch( + const result = await callProviderPipeline(provider, 'alphaSearch', model, { id: 'search-session', commands: { search_query: [{ q: 'Floway' }] } }, undefined, - noopUpstreamCallOptions(), - ); + noopUpstreamCallOptions()); assertEquals(result.response.status, 200); assertEquals(result.modelKey, 'gpt-search'); }, diff --git a/packages/provider-custom/package.json b/packages/provider-custom/package.json index 34987f358a..4558c177a4 100644 --- a/packages/provider-custom/package.json +++ b/packages/provider-custom/package.json @@ -11,6 +11,8 @@ "typecheck": "tsc --noEmit" }, "dependencies": { + "@floway-dev/http": "workspace:*", + "@floway-dev/pipeline": "workspace:*", "@floway-dev/protocols": "workspace:*", "@floway-dev/provider": "workspace:*" }, diff --git a/packages/provider-custom/src/index.ts b/packages/provider-custom/src/index.ts index 19f4a9c616..ddc575039f 100644 --- a/packages/provider-custom/src/index.ts +++ b/packages/provider-custom/src/index.ts @@ -10,3 +10,4 @@ export const customProviderModule: ProviderModule = { export { assertCustomUpstreamRecord, type CustomIngressHeaderRule, type CustomModelsFetch, type CustomUpstreamConfig } from './config.ts'; export { fetchCustomModels, type CustomModelsResponse, type CustomRawModel } from './fetch-models.ts'; export { projectCustomModels, projectCustomDiscoveredModels } from './provider.ts'; +export { createCustomPipelines } from './pipelines.ts'; diff --git a/packages/provider-custom/src/pipelines.ts b/packages/provider-custom/src/pipelines.ts new file mode 100644 index 0000000000..ce3438c8d0 --- /dev/null +++ b/packages/provider-custom/src/pipelines.ts @@ -0,0 +1,15 @@ +import type { CustomUpstreamConfig } from './config.ts'; +import { prepareCustomRequest } from './stages/prepare-request.ts'; +import { http } from '@floway-dev/http/pipeline'; +import { compose } from '@floway-dev/pipeline'; +import type { ProviderOperationPayloads, ProviderOperationResponse, ProviderPipelines, ProviderRequest } from '@floway-dev/provider'; + +export const createCustomPipelines = (config: CustomUpstreamConfig): ProviderPipelines => ({ + alphaSearch: compose, ProviderOperationResponse<'alphaSearch'>>('custom.alphaSearch', [prepareCustomRequest(config, 'alphaSearch'), http]), + openaiCompletions: compose, ProviderOperationResponse<'openaiCompletions'>>('custom.openaiCompletions', [prepareCustomRequest(config, 'openaiCompletions'), http]), + openaiEmbeddings: compose, ProviderOperationResponse<'openaiEmbeddings'>>('custom.openaiEmbeddings', [prepareCustomRequest(config, 'openaiEmbeddings'), http]), + openaiImagesGenerations: compose, ProviderOperationResponse<'openaiImagesGenerations'>>('custom.openaiImagesGenerations', [prepareCustomRequest(config, 'openaiImagesGenerations'), http]), + openaiImagesEdits: compose, ProviderOperationResponse<'openaiImagesEdits'>>('custom.openaiImagesEdits', [prepareCustomRequest(config, 'openaiImagesEdits'), http]), + openaiAudioTranscriptions: compose, ProviderOperationResponse<'openaiAudioTranscriptions'>>('custom.openaiAudioTranscriptions', [prepareCustomRequest(config, 'openaiAudioTranscriptions'), http]), + rerank: compose, ProviderOperationResponse<'rerank'>>('custom.rerank', [prepareCustomRequest(config, 'rerank'), http]), +}); diff --git a/packages/provider-custom/src/provider.ts b/packages/provider-custom/src/provider.ts index bff2f9dcfc..7742d6a6dd 100644 --- a/packages/provider-custom/src/provider.ts +++ b/packages/provider-custom/src/provider.ts @@ -3,6 +3,7 @@ import { CUSTOM_DEFAULT_FLAGS } from './defaults.ts'; import { fetchCustomModels, type CustomModelsResponse, type CustomRawModel } from './fetch-models.ts'; import { customFetchAlphaSearch, customFetchOpenAIAudioTranscriptions, customFetchOpenAIChatCompletions, customFetchOpenAICompletions, customFetchOpenAIEmbeddings, customFetchOpenAIImagesEdits, customFetchOpenAIImagesGenerations, customFetchAnthropicMessages, customFetchAnthropicMessagesCountTokens, customFetchRerank, customFetchOpenAIResponses, customFetchOpenAIResponsesCompact } from './fetch.ts'; import { inferEndpointsFromModelId } from './infer-endpoints.ts'; +import { createCustomPipelines } from './pipelines.ts'; import { parseAnthropicMessagesStream } from '@floway-dev/protocols/anthropic-messages'; import { type ModelEndpoints, kindForEndpoints } from '@floway-dev/protocols/common'; import { parseOpenAIChatCompletionsStream } from '@floway-dev/protocols/openai-chat-completions'; @@ -288,6 +289,7 @@ export const createCustomProvider = (record: UpstreamRecord): Provider => { disabledPublicModelIds: record.disabledPublicModelIds, modelPrefix: record.modelPrefix, modelsCache: record.modelsCache, + pipelines: createCustomPipelines(config), instance, }; }; diff --git a/packages/provider-custom/src/stages/prepare-request.ts b/packages/provider-custom/src/stages/prepare-request.ts new file mode 100644 index 0000000000..b8c7f5dfc9 --- /dev/null +++ b/packages/provider-custom/src/stages/prepare-request.ts @@ -0,0 +1,114 @@ +import type { CustomPathOverrideKey, CustomUpstreamConfig } from '../config.ts'; +import { type HttpHeaders, type HttpRequestFacts, type HttpResponseFacts } from '@floway-dev/http/pipeline'; +import { multipartBody, type HttpBody, type HttpBodyEncoding } from '@floway-dev/http/request-content'; +import { defineStage, isSecret, move, secret, type Secret } from '@floway-dev/pipeline'; +import type { RerankTarget } from '@floway-dev/protocols/common'; +import { DEFAULT_RERANK_PATHS, serializeRerankRequest } from '@floway-dev/protocols/rerank'; +import { joinBaseAndPath, mergeHttpHeaders, prepareOpenAIImagesEditsBody, replaceHttpHeader, withHttpContentType, type ProviderOperation, type ProviderOperationPayloads, type ProviderRequest, type ProviderResponse, type ProviderServices } from '@floway-dev/provider'; + +interface CustomHttpFacts extends HttpRequestFacts { + 'request.custom.modelKey': string; + 'request.custom.path': string; +} + +type CustomResponse = ProviderResponse & { 'response.provider.rerankTarget'?: RerankTarget }; + +const PATHS: Partial> = { + alphaSearch: '/alpha/search', + openaiCompletions: '/completions', + openaiEmbeddings: '/embeddings', + openaiImagesGenerations: '/images/generations', + openaiImagesEdits: '/images/edits', + openaiAudioTranscriptions: '/audio/transcriptions', +}; + +const resolvedHeaders = (config: CustomUpstreamConfig, incoming: HttpHeaders): HttpHeaders => { + const names = new Set(config.ingressHeadersRules.map(rule => rule.key.toLowerCase())); + const output: (readonly [string, string | Secret])[] = incoming.filter(([name]) => !names.has(name.toLowerCase())); + for (const rule of config.ingressHeadersRules) { + if (rule.value !== null) output.push([rule.key, rule.value]); + else { + const values = incoming.filter(([name]) => name.toLowerCase() === rule.key.toLowerCase()).map(([, value]) => value); + if (values.length === 1) output.push([rule.key, values[0]!]); + else if (values.length > 1) { + const value = values.map(value => isSecret(value) ? value.reveal() : value).join(', '); + output.push([rule.key, values.some(isSecret) ? secret(value) : value]); + } + } + } + return output; +}; + +export const prepareCustomRequest = (config: CustomUpstreamConfig, operation: O) => { + const prepare = defineStage, CustomHttpFacts, HttpResponseFacts, CustomResponse, ProviderServices>({ + name: `prepareCustom${operation.replace(/^openai/, 'OpenAI').replace(/^alpha/, 'Alpha').replace(/^rerank/, 'Rerank')}`, + through: { + request: { + needs: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'], + consumes: ['request.provider.model', 'request.provider.payload', 'request.http.headers'], + provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.custom.modelKey', 'request.custom.path'], + }, + response: { + needs: ['response.http.exchange'], + consumes: [], + provides: ['response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', ...(operation === 'rerank' ? ['response.provider.rerankTarget' as const] : [])], + }, + }, + execute: async (facts, next) => { + const { 'request.provider.model': model, 'request.provider.payload': payload, ...rest } = facts; + const modelKey = model.providerData as string; + let body: HttpBody; + let encoding: HttpBodyEncoding = 'json'; + let target: RerankTarget | undefined; + let path: string; + switch (operation) { + case 'rerank': + target = model.rerankTarget; + if (target === undefined) throw new Error(`Rerank model ${model.id} has no outbound target`); + path = target.path ?? DEFAULT_RERANK_PATHS[target.protocol]; + body = serializeRerankRequest(target.protocol, modelKey, payload as ProviderOperationPayloads['rerank']); + break; + case 'openaiImagesEdits': { + const prepared = await prepareOpenAIImagesEditsBody(payload as ProviderOperationPayloads['openaiImagesEdits'], modelKey); + body = prepared.body; + encoding = prepared.encoding; + path = config.pathOverrides?.['/images/edits'] ?? '/v1/images/edits'; + break; + } + case 'openaiAudioTranscriptions': { + const request = payload as ProviderOperationPayloads['openaiAudioTranscriptions']; + body = multipartBody(request.entries.map(entry => entry.name === 'model' ? { ...entry, value: modelKey } : entry)); + encoding = 'multipart'; + path = config.pathOverrides?.['/audio/transcriptions'] ?? '/v1/audio/transcriptions'; + break; + } + default: { + const key = PATHS[operation]!; + path = config.pathOverrides?.[key] ?? `/v1${key}`; + body = { ...payload, model: modelKey }; + } + } + let base: HttpHeaders = []; + if (config.authStyle === 'anthropic') { + // https://docs.anthropic.com/en/api/versioning + base = [['x-api-key', secret(config.apiKey)], ['anthropic-version', '2023-06-01']]; + } else if (config.authStyle === 'bearer') { + base = replaceHttpHeader(base, 'Authorization', secret(`Bearer ${config.apiKey}`)); + } + base = withHttpContentType(base, body, encoding); + const headers = mergeHttpHeaders(base, resolvedHeaders(config, facts['request.http.headers'])); + const back = await next(move({ + ...rest, + 'request.custom.modelKey': modelKey, + 'request.custom.path': path, + 'request.http.url': joinBaseAndPath(config.baseUrl, path), + 'request.http.method': 'POST', + 'request.http.headers': headers, + 'request.http.body': body, + 'request.http.encoding': encoding, + })); + return move({ ...back, 'response.provider.modelKey': modelKey, 'response.provider.called': back['response.http.exchange'].type === 'response', 'response.provider.previousCalls': [], ...(target === undefined ? {} : { 'response.provider.rerankTarget': target }) }); + }, + }); + return prepare; +}; diff --git a/packages/provider-ollama/__tests__/provider_test.ts b/packages/provider-ollama/__tests__/provider_test.ts index 8d7b96bc3e..9fd4aead69 100644 --- a/packages/provider-ollama/__tests__/provider_test.ts +++ b/packages/provider-ollama/__tests__/provider_test.ts @@ -2,7 +2,7 @@ import { test } from 'vitest'; import { createOllamaProvider } from '../src/provider.ts'; import { initProviderRepo, type UpstreamRecord } from '@floway-dev/provider'; -import { assertEquals, assertExists, jsonResponse, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, testFetcher, withMockedFetch } from '@floway-dev/test-utils'; +import { callProviderPipeline, assertEquals, assertExists, jsonResponse, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, testFetcher, withMockedFetch } from '@floway-dev/test-utils'; // A cloud upstream writes its usage snapshot after the calls it serves, so the // provider needs a repo to write into wherever those calls are exercised. @@ -161,9 +161,9 @@ test('manual transcription models call Ollama without auto-advertising the endpo const models = await instance.instance.getProvidedModels(testFetcher); assertEquals(models.map(model => model.kind), ['transcription']); assertEquals(models[0]?.chat, undefined); - await instance.instance.callOpenAIAudioTranscriptions(models[0], { + await callProviderPipeline(instance, 'openaiAudioTranscriptions', models[0], { entries: [ - { name: 'file', value: new File(['audio'], 'clip.wav', { type: 'audio/wav' }) }, + { name: 'file', value: { bytes: new TextEncoder().encode('audio'), name: 'clip.wav', type: 'audio/wav' } }, { name: 'model', value: 'public-model' }, ], }, undefined, noopUpstreamCallOptions()); diff --git a/packages/provider-ollama/package.json b/packages/provider-ollama/package.json index dc5a39ae59..f99c1eee9f 100644 --- a/packages/provider-ollama/package.json +++ b/packages/provider-ollama/package.json @@ -10,6 +10,8 @@ "typecheck": "tsc --noEmit" }, "dependencies": { + "@floway-dev/http": "workspace:*", + "@floway-dev/pipeline": "workspace:*", "@floway-dev/protocols": "workspace:*", "@floway-dev/provider": "workspace:*" }, diff --git a/packages/provider-ollama/src/index.ts b/packages/provider-ollama/src/index.ts index 4d522ba5b0..3eb8eea4be 100644 --- a/packages/provider-ollama/src/index.ts +++ b/packages/provider-ollama/src/index.ts @@ -1,6 +1,7 @@ import { OLLAMA_DEFAULT_FLAGS } from './defaults.ts'; import { createOllamaProvider } from './provider.ts'; import type { ProviderModule } from '@floway-dev/provider'; +export { createOllamaPipelines } from './pipelines.ts'; export const ollamaProviderModule: ProviderModule = { create: createOllamaProvider, diff --git a/packages/provider-ollama/src/pipelines.ts b/packages/provider-ollama/src/pipelines.ts new file mode 100644 index 0000000000..495674a2ab --- /dev/null +++ b/packages/provider-ollama/src/pipelines.ts @@ -0,0 +1,16 @@ +import type { OllamaUpstreamConfig } from './config.ts'; +import { observeOllamaAccount } from './stages/observe-account.ts'; +import { prepareOllamaRequest } from './stages/prepare-request.ts'; +import type { OllamaUpstreamState } from './state.ts'; +import { http } from '@floway-dev/http/pipeline'; +import { compose } from '@floway-dev/pipeline'; +import type { ProviderOperationPayloads, ProviderOperationResponse, ProviderPipelines, ProviderRequest } from '@floway-dev/provider'; + +export const createOllamaPipelines = (upstreamId: string, config: OllamaUpstreamConfig, state: OllamaUpstreamState): ProviderPipelines => { + const observe = observeOllamaAccount(upstreamId, config, state); + return { + openaiCompletions: compose, ProviderOperationResponse<'openaiCompletions'>>('ollama.openaiCompletions', [observe, prepareOllamaRequest(config, 'openaiCompletions'), http]), + openaiEmbeddings: compose, ProviderOperationResponse<'openaiEmbeddings'>>('ollama.openaiEmbeddings', [observe, prepareOllamaRequest(config, 'openaiEmbeddings'), http]), + openaiAudioTranscriptions: compose, ProviderOperationResponse<'openaiAudioTranscriptions'>>('ollama.openaiAudioTranscriptions', [observe, prepareOllamaRequest(config, 'openaiAudioTranscriptions'), http]), + }; +}; diff --git a/packages/provider-ollama/src/provider.ts b/packages/provider-ollama/src/provider.ts index 0754a0e900..c7fb5f9a54 100644 --- a/packages/provider-ollama/src/provider.ts +++ b/packages/provider-ollama/src/provider.ts @@ -31,6 +31,7 @@ import { OLLAMA_DEFAULT_FLAGS } from './defaults.ts'; import { fetchOllamaCatalog, type OllamaCatalog } from './fetch-models.ts'; import { ollamaFetchOpenAIAudioTranscriptions, ollamaFetchOpenAIChatCompletions, ollamaFetchOpenAICompletions, ollamaFetchOpenAIEmbeddings, ollamaFetchAnthropicMessages, ollamaFetchAnthropicMessagesCountTokens, ollamaFetchOpenAIResponses, ollamaFetchOpenAIResponsesCompact } from './fetch.ts'; import { pricingForOllamaModelKey } from './pricing.ts'; +import { createOllamaPipelines } from './pipelines.ts'; import { readOllamaUpstreamState } from './state.ts'; import { scheduleOllamaUsageProbe } from './usage-probe.ts'; import { parseAnthropicMessagesStream } from '@floway-dev/protocols/anthropic-messages'; @@ -228,6 +229,7 @@ export const createOllamaProvider = (record: UpstreamRecord): Provider => { disabledPublicModelIds: record.disabledPublicModelIds, modelPrefix: record.modelPrefix, modelsCache: record.modelsCache, + pipelines: createOllamaPipelines(record.id, config, state), instance, }; }; diff --git a/packages/provider-ollama/src/stages/observe-account.ts b/packages/provider-ollama/src/stages/observe-account.ts new file mode 100644 index 0000000000..937891269d --- /dev/null +++ b/packages/provider-ollama/src/stages/observe-account.ts @@ -0,0 +1,31 @@ +import { scheduleOllamaAccountProbe } from '../account-probe.ts'; +import type { OllamaUpstreamConfig } from '../config.ts'; +import type { OllamaUpstreamState } from '../state.ts'; +import { scheduleOllamaUsageProbe } from '../usage-probe.ts'; +import type { HttpRequestFacts } from '@floway-dev/http/pipeline'; +import { defer, defineStage, move, type Deferred } from '@floway-dev/pipeline'; +import type { ProviderResponse, ProviderServices } from '@floway-dev/provider'; + +type ProbeResponse = ProviderResponse & { 'response.ollama.probes': Deferred }; + +export const observeOllamaAccount = (upstreamId: string, config: OllamaUpstreamConfig, state: OllamaUpstreamState) => { + const observeAccount = defineStage, Pick, ProviderResponse, ProbeResponse, ProviderServices>({ + name: 'observeOllamaAccount', + through: { + request: { needs: ['request.http.callId'], consumes: [], provides: [] }, + response: { needs: ['response.http.exchange'], consumes: [], provides: ['response.ollama.probes'] }, + }, + execute: async (facts, next, use) => { + const back = await next(move({ ...facts })); + const pending: Promise[] = []; + if (back['response.http.exchange'].type === 'response') { + const call = use.httpCall(facts['request.http.callId']); + const background = (work: Promise): void => { pending.push(work); }; + scheduleOllamaUsageProbe(upstreamId, config, state, call.fetcher, background); + scheduleOllamaAccountProbe(upstreamId, config, state, call.fetcher, background); + } + return move({ ...back, 'response.ollama.probes': defer(Promise.all(pending)) }); + }, + }); + return observeAccount; +}; diff --git a/packages/provider-ollama/src/stages/prepare-request.ts b/packages/provider-ollama/src/stages/prepare-request.ts new file mode 100644 index 0000000000..075563c12d --- /dev/null +++ b/packages/provider-ollama/src/stages/prepare-request.ts @@ -0,0 +1,53 @@ +import type { OllamaUpstreamConfig } from '../config.ts'; +import type { HttpRequestFacts, HttpResponseFacts } from '@floway-dev/http/pipeline'; +import { multipartBody } from '@floway-dev/http/request-content'; +import { defineStage, move, secret } from '@floway-dev/pipeline'; +import { joinBaseAndPath, mergeHttpHeaders, withHttpContentType, type ProviderOperationPayloads, type ProviderRequest, type ProviderResponse, type ProviderServices } from '@floway-dev/provider'; + +type OllamaOperation = 'openaiCompletions' | 'openaiEmbeddings' | 'openaiAudioTranscriptions'; + +interface OllamaHttpFacts extends HttpRequestFacts { + 'request.ollama.modelKey': string; +} + +const PATHS: Record = { + openaiCompletions: '/v1/completions', + openaiEmbeddings: '/v1/embeddings', + // https://github.com/ollama/ollama/blob/573386c35eac76124ffce571f4b0fefa0a7fe13c/server/routes.go#L1916-L1922 + openaiAudioTranscriptions: '/v1/audio/transcriptions', +}; + +export const prepareOllamaRequest = (config: OllamaUpstreamConfig, operation: O) => { + const prepare = defineStage, OllamaHttpFacts, HttpResponseFacts, ProviderResponse, ProviderServices>({ + name: `prepareOllama${operation.replace(/^openai/, 'OpenAI').replace(/^alpha/, 'Alpha').replace(/^rerank/, 'Rerank')}`, + through: { + request: { + needs: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'], + consumes: ['request.provider.model', 'request.provider.payload', 'request.http.headers'], + provides: ['request.http.url', 'request.http.method', 'request.http.headers', 'request.http.body', 'request.http.encoding', 'request.ollama.modelKey'], + }, + response: { needs: ['response.http.exchange'], consumes: [], provides: ['response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'] }, + }, + execute: async (facts, next) => { + const { 'request.provider.model': model, 'request.provider.payload': payload, ...rest } = facts; + const modelKey = model.providerData as string; + const encoding = operation === 'openaiAudioTranscriptions' ? 'multipart' as const : 'json' as const; + const body = operation === 'openaiAudioTranscriptions' + ? multipartBody((payload as ProviderOperationPayloads['openaiAudioTranscriptions']).entries.map(entry => entry.name === 'model' ? { ...entry, value: modelKey } : entry)) + : { ...payload, model: modelKey }; + const base = config.apiKey ? [['Authorization', secret(`Bearer ${config.apiKey}`)] as const] : []; + const headers = mergeHttpHeaders(withHttpContentType(base, body, encoding), facts['request.http.headers']); + const back = await next(move({ + ...rest, + 'request.ollama.modelKey': modelKey, + 'request.http.url': joinBaseAndPath(config.baseUrl, PATHS[operation]), + 'request.http.method': 'POST', + 'request.http.headers': headers, + 'request.http.body': body, + 'request.http.encoding': encoding, + })); + return move({ ...back, 'response.provider.modelKey': modelKey, 'response.provider.called': back['response.http.exchange'].type === 'response', 'response.provider.previousCalls': [] }); + }, + }); + return prepare; +}; diff --git a/packages/provider/__tests__/audio_test.ts b/packages/provider/__tests__/audio_test.ts index 62fb7541c6..25aba03ac8 100644 --- a/packages/provider/__tests__/audio_test.ts +++ b/packages/provider/__tests__/audio_test.ts @@ -4,10 +4,7 @@ import { serializeModelPathOpenAIAudioTranscriptionRequest, serializeModelFieldO import { assertEquals, assertExists } from '@floway-dev/test-utils'; test('serializeModelFieldOpenAIAudioTranscriptionRequest preserves ordered fields and file metadata while replacing model', async () => { - const file = new File([new Uint8Array([1, 2, 3, 4])], 'meeting.wav', { - type: 'audio/wav', - lastModified: 1_700_000_000_000, - }); + const file = { bytes: new Uint8Array([1, 2, 3, 4]), name: 'meeting.wav', type: 'audio/wav', lastModified: 1_700_000_000_000 }; const form = serializeModelFieldOpenAIAudioTranscriptionRequest({ entries: [ { name: 'file', value: file }, @@ -34,7 +31,7 @@ test('serializeModelPathOpenAIAudioTranscriptionRequest omits model fields selec const form = serializeModelPathOpenAIAudioTranscriptionRequest({ entries: [ { name: 'model', value: 'public-model' }, - { name: 'file', value: new File(['audio'], 'meeting.wav', { type: 'audio/wav' }) }, + { name: 'file', value: { bytes: new TextEncoder().encode('audio'), name: 'meeting.wav', type: 'audio/wav' } }, { name: 'response_format', value: 'json' }, ], }); diff --git a/packages/provider/__tests__/images_test.ts b/packages/provider/__tests__/images_test.ts index fea11462ab..e8e38848f9 100644 --- a/packages/provider/__tests__/images_test.ts +++ b/packages/provider/__tests__/images_test.ts @@ -12,7 +12,7 @@ test('serializeOpenAIImagesEditsRequest preserves reference fields and encodes m const serialized = await serializeOpenAIImagesEditsRequest({ images: [ { type: 'reference', reference: { image_url: 'https://example.test/image.png', detail: 'future-field' } }, - { type: 'upload', file: new File(['inline'], 'inline.png', { type: 'image/png' }) }, + { type: 'upload', file: { bytes: new TextEncoder().encode('inline'), name: 'inline.png', type: 'image/png' } }, ], mask: { type: 'reference', reference: { file_id: 'file-mask' } }, parameters: { prompt: 'edit', background: null }, @@ -32,7 +32,7 @@ test('serializeOpenAIImagesEditsRequest preserves reference fields and encodes m test('serializeOpenAIImagesEditsJsonPayload forces upload sources into data URLs', async () => { const body = await serializeOpenAIImagesEditsJsonPayload({ - images: [{ type: 'upload', file: new File(['image'], 'image.png', { type: 'image/png' }) }], + images: [{ type: 'upload', file: { bytes: new TextEncoder().encode('image'), name: 'image.png', type: 'image/png' } }], parameters: { prompt: 'edit' }, }, 'gpt-image-2'); assertEquals(body, { @@ -43,15 +43,16 @@ test('serializeOpenAIImagesEditsJsonPayload forces upload sources into data URLs }); test('serializeOpenAIImagesEditsRequest uses the singular field for one upload and the array field for many', async () => { - const first = new File(['first'], 'first.png', { type: 'image/png' }); - const second = new File(['second'], 'second.png', { type: 'image/png' }); + const first = { bytes: new TextEncoder().encode('first'), name: 'first.png', type: 'image/png' }; + const second = { bytes: new TextEncoder().encode('second'), name: 'second.png', type: 'image/png' }; const single = await serializeOpenAIImagesEditsRequest({ images: [{ type: 'upload', file: first }], parameters: { prompt: 'single' }, }, 'gpt-image'); assertEquals(single instanceof FormData, true); const singleForm = single as FormData; - assertEquals(singleForm.get('image'), first); + assertEquals((singleForm.get('image') as File).name, first.name); + assertEquals(new Uint8Array(await (singleForm.get('image') as File).arrayBuffer()), first.bytes); assertEquals(singleForm.getAll('image[]'), []); const multiple = await serializeOpenAIImagesEditsRequest({ @@ -60,7 +61,8 @@ test('serializeOpenAIImagesEditsRequest uses the singular field for one upload a }, 'gpt-image'); assertEquals(multiple instanceof FormData, true); const multipleForm = multiple as FormData; - assertEquals(multipleForm.getAll('image[]'), [first, second]); + assertEquals(multipleForm.getAll('image[]').map(value => (value as File).name), [first.name, second.name]); + assertEquals(await Promise.all(multipleForm.getAll('image[]').map(async value => new Uint8Array(await (value as File).arrayBuffer()))), [first.bytes, second.bytes]); assertEquals(multipleForm.get('image'), null); assertEquals(multipleForm.get('model'), 'gpt-image'); }); diff --git a/packages/provider/__tests__/invocation_test.ts b/packages/provider/__tests__/invocation_test.ts index ed390a615e..20d486eed7 100644 --- a/packages/provider/__tests__/invocation_test.ts +++ b/packages/provider/__tests__/invocation_test.ts @@ -65,6 +65,7 @@ test('providerModelOf throws the alias-row diagnostic when the candidate names a disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + pipelines: {}, instance: stubProvider(), }, model: { diff --git a/packages/provider/__tests__/pipeline_test.ts b/packages/provider/__tests__/pipeline_test.ts new file mode 100644 index 0000000000..f2512387c6 --- /dev/null +++ b/packages/provider/__tests__/pipeline_test.ts @@ -0,0 +1,21 @@ +import { expect, test } from 'vitest'; + +import type { FlagId } from '../src/flags.ts'; +import { providerModelFacts } from '../src/pipeline.ts'; +import { encodeRun, move } from '@floway-dev/pipeline'; +import { stubProviderModel } from '@floway-dev/test-utils'; + +test('provider model facts retain flags as immutable dump content while sharing unchanged metadata', () => { + const enabledFlags = new Set(['vendor-kimi']); + const model = stubProviderModel({ enabledFlags, providerData: { id: 'upstream-model' } }); + const snapshot = move(providerModelFacts(model)); + expect(snapshot.limits).toBe(model.limits); + expect(snapshot.providerData).toBe(model.providerData); + expect(snapshot.enabledFlags).toEqual(['vendor-kimi']); + expect(() => (snapshot.enabledFlags as FlagId[]).push('vendor-deepseek')).toThrow(); + enabledFlags.add('vendor-deepseek'); + expect(snapshot.enabledFlags).toEqual(['vendor-kimi']); + const encoded = encodeRun([{ type: 'stage.entered', stageId: 0, name: 'provider', parentStageId: null, facts: { 'request.provider.model': snapshot } }]); + expect(JSON.stringify(encoded)).toContain('vendor-kimi'); + expect(JSON.stringify(encoded)).not.toContain('vendor-deepseek'); +}); diff --git a/packages/provider/package.json b/packages/provider/package.json index 13cba0bdc7..729c91450a 100644 --- a/packages/provider/package.json +++ b/packages/provider/package.json @@ -9,7 +9,8 @@ "./join": { "import": "./src/join.ts", "types": "./src/join.ts" }, "./model": { "import": "./src/model.ts", "types": "./src/model.ts" }, "./model-config": { "import": "./src/model-config.ts", "types": "./src/model-config.ts" }, - "./model-prefix": { "import": "./src/model-prefix.ts", "types": "./src/model-prefix.ts" } + "./model-prefix": { "import": "./src/model-prefix.ts", "types": "./src/model-prefix.ts" }, + "./pipeline": { "import": "./src/pipeline.ts", "types": "./src/pipeline.ts" } }, "scripts": { "typecheck": "tsc --noEmit" @@ -17,6 +18,7 @@ "dependencies": { "@discoveryjs/json-ext": "1.1.0", "@floway-dev/http": "workspace:*", + "@floway-dev/pipeline": "workspace:*", "@floway-dev/protocols": "workspace:*", "@noble/hashes": "^2.2.0", "klona": "2.0.6" diff --git a/packages/provider/src/audio.ts b/packages/provider/src/audio.ts index 3fd5b68bc4..0fa30e54d9 100644 --- a/packages/provider/src/audio.ts +++ b/packages/provider/src/audio.ts @@ -1,13 +1,6 @@ -// Buffered multipart carrier for OpenAI-compatible audio transcription. -// Entries stay ordered and may repeat, matching FormData semantics. File -// objects retain their bytes, filename, media type, and lastModified metadata; -// providers rebuild a fresh FormData for every candidate so retries never -// reuse a consumed request body. +import type { HttpFormEntry } from '@floway-dev/http/request-content'; -export interface OpenAIAudioTranscriptionFormEntry { - readonly name: string; - readonly value: string | File; -} +export type OpenAIAudioTranscriptionFormEntry = HttpFormEntry; export interface OpenAIAudioTranscriptionRequest { readonly entries: readonly OpenAIAudioTranscriptionFormEntry[]; @@ -28,7 +21,8 @@ const serializeOpenAIAudioTranscriptionRequest = ( } else if (typeof entry.value === 'string') { form.append(entry.name, entry.value); } else { - form.append(entry.name, entry.value, entry.value.name); + const file = entry.value; + form.append(entry.name, new File([file.bytes as Uint8Array], file.name, { type: file.type, lastModified: file.lastModified })); } } return form; diff --git a/packages/provider/src/http-headers.ts b/packages/provider/src/http-headers.ts new file mode 100644 index 0000000000..a242d27faf --- /dev/null +++ b/packages/provider/src/http-headers.ts @@ -0,0 +1,19 @@ +import type { HttpHeaders } from '@floway-dev/http/pipeline'; +import type { HttpBody, HttpBodyEncoding, HttpMultipartBody } from '@floway-dev/http/request-content'; +import type { Secret } from '@floway-dev/pipeline'; + +export const replaceHttpHeader = (headers: HttpHeaders, name: string, value: string | Secret): HttpHeaders => + [...headers.filter(([key]) => key.toLowerCase() !== name.toLowerCase()), [name, value]]; + +export const mergeHttpHeaders = (base: HttpHeaders, incoming: HttpHeaders): HttpHeaders => { + const supplied = new Set(incoming.map(([name]) => name.toLowerCase())); + return [...base.filter(([name]) => !supplied.has(name.toLowerCase())), ...incoming]; +}; + +export const withHttpContentType = (headers: HttpHeaders, body: HttpBody, encoding: HttpBodyEncoding): HttpHeaders => { + if (body === null || headers.some(([name]) => name.toLowerCase() === 'content-type')) return headers; + if (encoding === 'multipart') { + return [...headers, ['Content-Type', `multipart/form-data; boundary=${(body as HttpMultipartBody).boundary}`]]; + } + return encoding === 'json' ? [...headers, ['Content-Type', 'application/json']] : headers; +}; diff --git a/packages/provider/src/images.ts b/packages/provider/src/images.ts index 458bd7b4d2..c5d100a074 100644 --- a/packages/provider/src/images.ts +++ b/packages/provider/src/images.ts @@ -1,14 +1,15 @@ import { base64ToBytes, bytesToBase64, parseBase64ImageDataUrl } from './image-helpers.ts'; import { jsonRequestBody } from './json-request.ts'; import type { ReplayableBody } from './options.ts'; +import { multipartBody as httpMultipartBody, type HttpFile, type HttpFormEntry, type HttpBody, type HttpBodyEncoding } from '@floway-dev/http/request-content'; import type { OpenAIImageEditReference } from '@floway-dev/protocols/openai-images'; // Each source stores one authoritative representation. Multipart requires // upload-like sources with no extra reference fields plus scalar parameters; -// every other request uses JSON and encodes File uploads as data URLs. +// every other request uses JSON and encodes upload bytes as data URLs. interface UploadedOpenAIImagesEditsSource { type: 'upload'; - file: File; + file: HttpFile; } interface InlineOpenAIImagesEditsSource { @@ -29,7 +30,7 @@ export interface OpenAIImagesEditsRequest { parameters: Record; } -const uploadedFile = (source: OpenAIImagesEditsSource, index: number): File | null => { +const uploadedFile = (source: OpenAIImagesEditsSource, index: number): HttpFile | null => { if (source.type === 'upload') return source.file; if (source.type === 'reference') return null; const parsed = parseBase64ImageDataUrl(source.reference.image_url); @@ -40,13 +41,12 @@ const uploadedFile = (source: OpenAIImagesEditsSource, index: number): File | nu } catch { return null; } - return new File([bytes], `image-${index}`, { type: parsed.mimeType }); + return { bytes, name: `image-${index}`, type: parsed.mimeType }; }; const jsonReference = async (source: OpenAIImagesEditsSource): Promise => { if (source.type === 'inline' || source.type === 'reference') return source.reference; - const bytes = new Uint8Array(await source.file.arrayBuffer()); - return { image_url: `data:${source.file.type};base64,${bytesToBase64(bytes)}` }; + return { image_url: `data:${source.file.type};base64,${bytesToBase64(source.file.bytes)}` }; }; const jsonBody = async (request: OpenAIImagesEditsRequest): Promise> => { @@ -64,7 +64,7 @@ export const serializeOpenAIImagesEditsJsonPayload = async ( model: string, ): Promise> => ({ ...await jsonBody(request), model }); -const multipartBody = (request: OpenAIImagesEditsRequest, model: string): FormData | null => { +const multipartEntries = (request: OpenAIImagesEditsRequest, model: string): readonly HttpFormEntry[] | null => { const sources = [...request.images, ...(request.mask === undefined ? [] : [request.mask])]; const compatibleSources = sources.every(source => source.type === 'upload' @@ -73,7 +73,7 @@ const multipartBody = (request: OpenAIImagesEditsRequest, model: string): FormDa typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'); if (!compatibleSources || !compatibleParameters) return null; - const images: File[] = []; + const images: HttpFile[] = []; for (const [index, source] of request.images.entries()) { const file = uploadedFile(source, index); if (file === null) return null; @@ -82,17 +82,30 @@ const multipartBody = (request: OpenAIImagesEditsRequest, model: string): FormDa const mask = request.mask === undefined ? undefined : uploadedFile(request.mask, images.length); if (mask === null) return null; - const form = new FormData(); - for (const [name, value] of Object.entries(request.parameters)) form.append(name, String(value)); + const entries: HttpFormEntry[] = Object.entries(request.parameters).map(([name, value]) => ({ name, value: String(value) })); const imageField = images.length === 1 ? 'image' : 'image[]'; - for (const image of images) form.append(imageField, image); - if (mask !== undefined) form.append('mask', mask); - form.append('model', model); - return form; + for (const image of images) entries.push({ name: imageField, value: image }); + if (mask !== undefined) entries.push({ name: 'mask', value: mask }); + entries.push({ name: 'model', value: model }); + return entries; +}; + +export const prepareOpenAIImagesEditsBody = async (request: OpenAIImagesEditsRequest, model: string): Promise<{ body: HttpBody; encoding: HttpBodyEncoding }> => { + const entries = multipartEntries(request, model); + return entries === null + ? { body: await serializeOpenAIImagesEditsJsonPayload(request, model), encoding: 'json' } + : { body: httpMultipartBody(entries), encoding: 'multipart' }; }; export const serializeOpenAIImagesEditsRequest = async (request: OpenAIImagesEditsRequest, model: string): Promise => { - const multipart = multipartBody(request, model); - if (multipart !== null) return multipart; + const entries = multipartEntries(request, model); + if (entries !== null) { + const form = new FormData(); + for (const { name, value } of entries) { + if (typeof value === 'string') form.append(name, value); + else form.append(name, new File([value.bytes as Uint8Array], value.name, { type: value.type, lastModified: value.lastModified })); + } + return form; + } return jsonRequestBody(await serializeOpenAIImagesEditsJsonPayload(request, model)); }; diff --git a/packages/provider/src/index.ts b/packages/provider/src/index.ts index e01253bc15..2ffdd520d8 100644 --- a/packages/provider/src/index.ts +++ b/packages/provider/src/index.ts @@ -7,6 +7,9 @@ export type { OpenAIResponsesInvocation, } from './invocation.ts'; export { providerModelOf } from './invocation.ts'; +export type { ProviderCallResponse, ProviderModelFacts, ProviderOperation, ProviderOperationPayloads, ProviderOperationResponse, ProviderPipeline, ProviderPipelines, ProviderRequest, ProviderResponse, ProviderRerankResponse, ProviderServices } from './pipeline.ts'; +export { providerModelFacts } from './pipeline.ts'; +export { mergeHttpHeaders, replaceHttpHeader, withHttpContentType } from './http-headers.ts'; export type { InternalDebugError } from './error.ts'; export { toInternalDebugError } from './error.ts'; @@ -60,7 +63,7 @@ export type { } from './provider.ts'; export { headersForAnthropicMessagesCall } from './anthropic-messages.ts'; export type { OpenAIImagesEditsRequest, OpenAIImagesEditsSource } from './images.ts'; -export { serializeOpenAIImagesEditsJsonPayload, serializeOpenAIImagesEditsRequest } from './images.ts'; +export { prepareOpenAIImagesEditsBody, serializeOpenAIImagesEditsJsonPayload, serializeOpenAIImagesEditsRequest } from './images.ts'; export type { OpenAIAudioTranscriptionFormEntry, OpenAIAudioTranscriptionRequest } from './audio.ts'; export { serializeModelFieldOpenAIAudioTranscriptionRequest, serializeModelPathOpenAIAudioTranscriptionRequest } from './audio.ts'; export type { ProviderStreamParser } from './streaming.ts'; diff --git a/packages/provider/src/pipeline.ts b/packages/provider/src/pipeline.ts new file mode 100644 index 0000000000..2dfaaedee9 --- /dev/null +++ b/packages/provider/src/pipeline.ts @@ -0,0 +1,60 @@ +import type { OpenAIAudioTranscriptionRequest } from './audio.ts'; +import type { FlagId } from './flags.ts'; +import type { OpenAIImagesEditsRequest } from './images.ts'; +import type { ProviderModel } from './model.ts'; +import type { HttpResponseFacts, HttpHeaders, HttpServices } from '@floway-dev/http/pipeline'; +import type { Pipeline } from '@floway-dev/pipeline'; +import type { RerankTarget } from '@floway-dev/protocols/common'; +import type { OpenAICompletionsPayload } from '@floway-dev/protocols/openai-completions'; +import type { OpenAIEmbeddingsPayload } from '@floway-dev/protocols/openai-embeddings'; +import type { OpenAIImagesGenerationsPayload } from '@floway-dev/protocols/openai-images'; +import type { CanonicalRerankRequest } from '@floway-dev/protocols/rerank'; + +export interface ProviderOperationPayloads { + alphaSearch: Record; + openaiCompletions: Omit; + openaiEmbeddings: Omit; + openaiImagesGenerations: Omit; + openaiImagesEdits: OpenAIImagesEditsRequest; + openaiAudioTranscriptions: OpenAIAudioTranscriptionRequest; + rerank: CanonicalRerankRequest; +} + +export type ProviderOperation = keyof ProviderOperationPayloads; + +export type ProviderModelFacts = Omit & { readonly enabledFlags: readonly FlagId[] }; + +export const providerModelFacts = (model: ProviderModel): ProviderModelFacts => ({ ...model, enabledFlags: [...model.enabledFlags] }); + +export interface ProviderRequest

{ + 'request.provider.model': ProviderModelFacts; + 'request.provider.payload': P; + 'request.http.callId': number; + 'request.http.headers': HttpHeaders; +} + +export interface ProviderCallResponse extends HttpResponseFacts { + /** A preflight reply does not represent a call to the model endpoint. */ + 'response.provider.called': boolean; + /** Credential retries retain replaced endpoint replies so settlement counts each call. */ + 'response.provider.previousCalls': readonly { readonly modelKey: string }[]; +} + +export interface ProviderResponse extends ProviderCallResponse { + 'response.provider.modelKey': string; +} + +export interface ProviderRerankResponse extends ProviderResponse { + 'response.provider.rerankTarget': RerankTarget; +} + +export type ProviderOperationResponse = + O extends 'rerank' ? ProviderRerankResponse : ProviderResponse; + +export type ProviderPipeline = Pipeline< + ProviderRequest, + ProviderOperationResponse +>; + +export type ProviderPipelines = { readonly [O in ProviderOperation]?: ProviderPipeline }; +export type ProviderServices = HttpServices; diff --git a/packages/provider/src/provider.ts b/packages/provider/src/provider.ts index dd604ef02d..fbce3b1561 100644 --- a/packages/provider/src/provider.ts +++ b/packages/provider/src/provider.ts @@ -4,6 +4,7 @@ import type { OpenAIImagesEditsRequest } from './images.ts'; import type { ModelPrefixConfig } from './model-prefix.ts'; import type { ProviderModel, UpstreamModelsCache, UpstreamProviderKind, UpstreamRecord } from './model.ts'; import type { Fetcher } from './options.ts'; +import type { ProviderPipelines } from './pipeline.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEvent } from '@floway-dev/protocols/anthropic-messages'; import type { ProtocolFrame, RerankTarget } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; @@ -41,6 +42,7 @@ export interface Provider { // The row's persisted catalog snapshot, mirrored so resolution does not pay // a second round trip after the row has already been loaded. modelsCache: UpstreamModelsCache | null; + pipelines: ProviderPipelines; instance: ProviderInstance; } diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index be7bac2563..f2c0b82448 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -10,6 +10,8 @@ "typecheck": "tsc --noEmit" }, "dependencies": { + "@floway-dev/http": "workspace:*", + "@floway-dev/pipeline": "workspace:*", "@floway-dev/provider": "workspace:*" } } diff --git a/packages/test-utils/src/index.ts b/packages/test-utils/src/index.ts index e911f1fbd0..a88fcf7e28 100644 --- a/packages/test-utils/src/index.ts +++ b/packages/test-utils/src/index.ts @@ -8,4 +8,5 @@ export { assertThrows, } from './assert.ts'; export { jsonResponse, readJsonRequest, sseResponse, testFetcher, withMockedFetch } from './mock-fetch.ts'; +export { callProviderPipeline } from './provider-pipeline.ts'; export { mockPerfTelemetryContext, noopAnthropicMessagesUpstreamCallOptions, noopUpstreamCallOptions, stubInternalModel, stubProvider, stubProviderModel, stubModelCandidate, testTelemetryModelIdentity } from './stubs.ts'; diff --git a/packages/test-utils/src/provider-pipeline.ts b/packages/test-utils/src/provider-pipeline.ts new file mode 100644 index 0000000000..b6dae79a9f --- /dev/null +++ b/packages/test-utils/src/provider-pipeline.ts @@ -0,0 +1,50 @@ +import { noopUpstreamCallOptions } from './stubs.ts'; +import { exchangeResponse, type HttpServices } from '@floway-dev/http/pipeline'; +import { move, run, setRelease } from '@floway-dev/pipeline'; +import type { Provider, ProviderCallResult, ProviderModel, ProviderOperation, ProviderOperationPayloads, ProviderOperationResponse, ProviderPipeline, ProviderRerankCallResult, ProviderRequest, UpstreamCallOptions } from '@floway-dev/provider'; +import { providerModelFacts } from '@floway-dev/provider'; + +type CallResult = + (O extends 'rerank' ? ProviderRerankCallResult : ProviderCallResult) & { readonly called: boolean; readonly previousCalls: readonly { readonly modelKey: string }[] }; + +export const callProviderPipeline = async ( + provider: Provider, + operation: O, + model: ProviderModel, + payload: ProviderOperationPayloads[O], + signal: AbortSignal | undefined = undefined, + options: UpstreamCallOptions = noopUpstreamCallOptions(), +): Promise> => { + const pipeline = provider.pipelines[operation]; + if (pipeline === undefined) throw new Error(`${provider.kind} has no ${operation} pipeline`); + const entry: ProviderRequest = move({ + 'request.provider.model': providerModelFacts(model), + 'request.provider.payload': payload, + 'request.http.callId': 0, + 'request.http.headers': [...options.headers], + }); + const services: HttpServices = { httpCall: () => ({ ...options, signal }) }; + const executed = await run, ProviderOperationResponse, HttpServices>(pipeline as ProviderPipeline, entry, services); + const facts = executed.facts; + const exchange = facts['response.http.exchange']; + if (exchange.type === 'transportFailure') { + await executed.drain(); + throw exchange.error; + } + const response = exchangeResponse(exchange); + let bytes: ArrayBuffer; + try { + bytes = await response.arrayBuffer(); + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + } finally { + await executed.drain(); + } + const result = { + response: new Response(exchange.body === null ? null : bytes, { status: exchange.status, statusText: exchange.statusText, headers: response.headers }), + modelKey: facts['response.provider.modelKey'], + called: facts['response.provider.called'], + previousCalls: facts['response.provider.previousCalls'], + ...(operation === 'rerank' ? { target: (facts as ProviderOperationResponse<'rerank'>)['response.provider.rerankTarget'] } : {}), + }; + return result as CallResult; +}; diff --git a/packages/test-utils/src/stubs.ts b/packages/test-utils/src/stubs.ts index 2a92d3ee78..7b2fd727a3 100644 --- a/packages/test-utils/src/stubs.ts +++ b/packages/test-utils/src/stubs.ts @@ -113,6 +113,7 @@ export const stubModelCandidate = (overrides: { disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, + pipelines: {}, instance: stubProvider(), }; const modelOverrides = overrides.model ?? {}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b2a6984fc4..eef958a243 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -409,6 +409,12 @@ importers: packages/http: dependencies: + '@discoveryjs/json-ext': + specifier: 1.1.0 + version: 1.1.0 + '@floway-dev/pipeline': + specifier: workspace:* + version: link:../pipeline '@noble/hashes': specifier: ^2.2.0 version: 2.2.0 @@ -480,6 +486,9 @@ importers: '@floway-dev/http': specifier: workspace:* version: link:../http + '@floway-dev/pipeline': + specifier: workspace:* + version: link:../pipeline '@floway-dev/protocols': specifier: workspace:* version: link:../protocols @@ -496,9 +505,15 @@ importers: packages/provider-azure: dependencies: + '@floway-dev/http': + specifier: workspace:* + version: link:../http '@floway-dev/interceptor': specifier: workspace:* version: link:../interceptor + '@floway-dev/pipeline': + specifier: workspace:* + version: link:../pipeline '@floway-dev/protocols': specifier: workspace:* version: link:../protocols @@ -534,9 +549,15 @@ importers: packages/provider-codex: dependencies: + '@floway-dev/http': + specifier: workspace:* + version: link:../http '@floway-dev/interceptor': specifier: workspace:* version: link:../interceptor + '@floway-dev/pipeline': + specifier: workspace:* + version: link:../pipeline '@floway-dev/protocols': specifier: workspace:* version: link:../protocols @@ -553,9 +574,15 @@ importers: packages/provider-copilot: dependencies: + '@floway-dev/http': + specifier: workspace:* + version: link:../http '@floway-dev/interceptor': specifier: workspace:* version: link:../interceptor + '@floway-dev/pipeline': + specifier: workspace:* + version: link:../pipeline '@floway-dev/platform': specifier: workspace:* version: link:../platform @@ -581,6 +608,12 @@ importers: packages/provider-custom: dependencies: + '@floway-dev/http': + specifier: workspace:* + version: link:../http + '@floway-dev/pipeline': + specifier: workspace:* + version: link:../pipeline '@floway-dev/protocols': specifier: workspace:* version: link:../protocols @@ -594,6 +627,12 @@ importers: packages/provider-ollama: dependencies: + '@floway-dev/http': + specifier: workspace:* + version: link:../http + '@floway-dev/pipeline': + specifier: workspace:* + version: link:../pipeline '@floway-dev/protocols': specifier: workspace:* version: link:../protocols @@ -628,6 +667,12 @@ importers: packages/test-utils: dependencies: + '@floway-dev/http': + specifier: workspace:* + version: link:../http + '@floway-dev/pipeline': + specifier: workspace:* + version: link:../pipeline '@floway-dev/provider': specifier: workspace:* version: link:../provider From 2c6ef6b65223c4be6403b86dd67dbc8ea7e9fd4e Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 05:01:39 +0800 Subject: [PATCH 62/79] feat: enter provider pipelines from non-chat runs Use portable model and upload facts for typed provider handoffs and preserve a single owned HTTP body. Collect provider retries and candidate attempts as separate billing observations, retain pricing facts and measured zeros, and settle each logical run at its exit. Preserve original diagnostics when deferred settlement marks a failure and recover completed calls on exceptional exits. --- README.md | 8 ++ .../data-plane/alpha-search/routes_test.ts | 2 +- .../chat/anthropic-messages/attempt_test.ts | 2 +- .../chat/anthropic-messages/http_test.ts | 2 +- .../chat/anthropic-messages/serve_test.ts | 2 +- .../gemini-generate-content/attempt_test.ts | 2 +- .../chat/gemini-generate-content/http_test.ts | 2 +- .../gemini-generate-content/serve_test.ts | 2 +- .../openai-chat-completions/attempt_test.ts | 2 +- .../chat/openai-chat-completions/http_test.ts | 2 +- .../openai-chat-completions/serve_test.ts | 2 +- .../affinity/copilot-roundtrip_test.ts | 2 +- .../chat/openai-responses/attempt_test.ts | 6 +- .../chat/openai-responses/http_test.ts | 2 +- .../image-generation-integration_test.ts | 12 +- .../chat/openai-responses/serve_test.ts | 2 +- .../data-plane/openai-audio-pipeline_test.ts | 20 +--- .../openai-completions-pipeline_test.ts | 91 +++++++++++---- .../openai-completions/http_test.ts | 8 +- .../openai-completions/usage_test.ts | 24 ++-- .../openai-embeddings-pipeline_test.ts | 21 +--- .../openai-embeddings/failures_test.ts | 5 +- .../data-plane/openai-images-pipeline_test.ts | 23 +--- .../data-plane/pipeline/settlement_test.ts | 3 +- .../data-plane/rerank-pipeline_test.ts | 19 +-- .../data-plane/rerank-status_test.ts | 3 + .../data-plane/search-pipeline_test.ts | 15 +-- .../shared/iterate-candidates_test.ts | 2 +- .../__tests__/test-utils/provider-pipeline.ts | 35 ++++++ .../data-plane/alpha-search/call-upstream.ts | 61 +++++----- .../src/data-plane/alpha-search/emit.ts | 6 +- .../alpha-search/execute-operations.ts | 5 +- .../alpha-search/parse-operations.ts | 9 +- .../src/data-plane/alpha-search/pipeline.ts | 5 +- .../server-tools/image-generation.ts | 4 +- .../data-plane/openai-audio/call-upstream.ts | 96 +++++++-------- .../src/data-plane/openai-audio/http.ts | 3 +- .../src/data-plane/openai-audio/pipeline.ts | 6 +- .../openai-completions/call-upstream.ts | 101 +++++++--------- .../src/data-plane/openai-completions/http.ts | 1 - .../data-plane/openai-completions/pipeline.ts | 6 +- .../data-plane/openai-completions/usage.ts | 6 +- .../openai-embeddings/call-upstream.ts | 54 +++++---- .../data-plane/openai-embeddings/pipeline.ts | 4 +- .../data-plane/openai-images/call-upstream.ts | 89 ++++++-------- .../src/data-plane/openai-images/http.ts | 1 - .../src/data-plane/openai-images/pipeline.ts | 6 +- .../src/data-plane/pipeline/candidates.ts | 10 +- .../gateway/src/data-plane/pipeline/facts.ts | 5 +- .../src/data-plane/pipeline/failover.ts | 37 ++++-- .../src/data-plane/pipeline/provider-entry.ts | 20 ++++ .../src/data-plane/pipeline/provider-usage.ts | 25 ++++ .../gateway/src/data-plane/pipeline/serve.ts | 49 ++++---- .../src/data-plane/pipeline/services.ts | 3 +- .../src/data-plane/pipeline/settlement.ts | 109 +++++++++--------- .../src/data-plane/pipeline/upstream-body.ts | 7 ++ .../src/data-plane/rerank/call-upstream.ts | 62 +++++----- .../gateway/src/data-plane/rerank/pipeline.ts | 4 +- packages/gateway/src/dump/attribution.ts | 23 ++-- packages/gateway/src/dump/run-sink.ts | 4 +- packages/gateway/src/dump/turn-dump.ts | 2 +- 61 files changed, 595 insertions(+), 549 deletions(-) create mode 100644 packages/gateway/__tests__/test-utils/provider-pipeline.ts create mode 100644 packages/gateway/src/data-plane/pipeline/provider-entry.ts create mode 100644 packages/gateway/src/data-plane/pipeline/provider-usage.ts diff --git a/README.md b/README.md index b491c8af1f..538ece627b 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,14 @@ also available as a root script. Route type generation runs first because the web app's generated types are not checked in and its lint configuration is type-aware. The web build includes assertions on the emitted bundle. +Provider operations use typed pipeline handoffs. Portable model facts snapshot flags as arrays; +upload facts contain bytes and file metadata. The shared HTTP stage adopts each native body +once, and decoders replace its release action as ownership moves through the run. Provider +retry observations and failed-over attempts join the same billable collection. Settlement +wraps rendering and writes each observed call once, including calls completed before a later +stage throws. Streaming quantities settle at the deferred exit. Explicit protocol and +transport errors take precedence over the settlement's generic failure marker. + Pipeline endpoints are assembled from individual stage modules. Each stage owns its request and response contract; a large stage keeps local helpers in the same file and can place independent parts in a sibling directory of the same name. diff --git a/packages/gateway/__tests__/data-plane/alpha-search/routes_test.ts b/packages/gateway/__tests__/data-plane/alpha-search/routes_test.ts index 1ff9267365..7ebebbc66f 100644 --- a/packages/gateway/__tests__/data-plane/alpha-search/routes_test.ts +++ b/packages/gateway/__tests__/data-plane/alpha-search/routes_test.ts @@ -362,7 +362,7 @@ describe('/alpha/search data plane', () => { const entered = eventsOf(record) .filter(event => event.type === 'stage.entered') .map(event => event.name); - expect(entered).toEqual(['emitAlphaSearch', 'writeSettlement', 'parseSearchOperations', 'executeSearchOperations']); + expect(entered).toEqual(['writeSettlement', 'emitAlphaSearch', 'parseSearchOperations', 'executeSearchOperations']); }); }); diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/attempt_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/attempt_test.ts index ba995f2afe..f809fa0d70 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/attempt_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/attempt_test.ts @@ -69,7 +69,7 @@ const makeCandidate = (overrides: { return { provider: { upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, instance: provider, + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: provider, }, model: stubInternalModel({ endpoints, diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts index 5a4e96443e..4fb932c576 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/http_test.ts @@ -115,7 +115,7 @@ const makeCandidate = (overrides: { return { provider: { upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, instance: provider, + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: provider, }, model: stubInternalModel(overrides.endpoints ? { endpoints: overrides.endpoints } : {}, upstream), fetcher: directFetcher, diff --git a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/serve_test.ts b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/serve_test.ts index ac38b981c5..d57fc9c9c2 100644 --- a/packages/gateway/__tests__/data-plane/chat/anthropic-messages/serve_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/anthropic-messages/serve_test.ts @@ -143,7 +143,7 @@ const makeCandidate = (overrides: { return { provider: { upstreamId: upstream, kind, name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, instance: provider, + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: provider, }, model: stubInternalModel({ id: modelId, diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/attempt_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/attempt_test.ts index 6d7e3fcb54..5547742f6b 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/attempt_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/attempt_test.ts @@ -89,7 +89,7 @@ const makeCandidate = (overrides: { return { provider: { upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, instance: provider, + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: provider, }, model: stubInternalModel(overrides.endpoints ? { endpoints: overrides.endpoints } : {}, upstream), fetcher: directFetcher, diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts index 19c15dcee0..887168f737 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/http_test.ts @@ -118,7 +118,7 @@ const makeCandidate = (overrides: { return { provider: { upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, instance: provider, + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: provider, }, model: stubInternalModel({ endpoints }, upstream), fetcher: directFetcher, diff --git a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/serve_test.ts b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/serve_test.ts index 958a1fc71f..b428789a8b 100644 --- a/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/serve_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/gemini-generate-content/serve_test.ts @@ -139,7 +139,7 @@ const makeCandidate = (overrides: { return { provider: { upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, instance: provider, + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: provider, }, model: stubInternalModel({ endpoints }, upstream), fetcher: directFetcher, diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/attempt_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/attempt_test.ts index 6672b3bbab..77a22b00d2 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/attempt_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/attempt_test.ts @@ -77,7 +77,7 @@ const makeCandidate = (overrides: { return { provider: { upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, instance: provider, + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: provider, }, model: stubInternalModel({ endpoints, diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts index e3877ca1c8..4fb01b74b7 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/http_test.ts @@ -112,7 +112,7 @@ const makeCandidate = (overrides: { return { provider: { upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, instance: provider, + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: provider, }, model: stubInternalModel(overrides.endpoints ? { endpoints: overrides.endpoints } : {}, upstream), fetcher: directFetcher, diff --git a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/serve_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/serve_test.ts index df553b4326..3c3a9609e5 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/serve_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-chat-completions/serve_test.ts @@ -95,7 +95,7 @@ const makeCandidate = (overrides: { return { provider: { upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, instance: provider, + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: provider, }, model: stubInternalModel({ id: overrides.modelId ?? 'test-model', diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/affinity/copilot-roundtrip_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/affinity/copilot-roundtrip_test.ts index 0dc945f6ba..5470fb6c1c 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/affinity/copilot-roundtrip_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/affinity/copilot-roundtrip_test.ts @@ -85,7 +85,7 @@ test('Copilot item-id and generic affinity trailers compose and unwrap in bounda inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, - modelsCache: null, + modelsCache: null, pipelines: {}, instance: stubProvider(), }, model: { id: 'gpt-test', endpoints: { openaiResponses: {} } }, diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/attempt_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/attempt_test.ts index 617494d650..8fd1b0bb01 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/attempt_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/attempt_test.ts @@ -71,7 +71,7 @@ const makeCandidate = ( inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, - modelsCache: null, + modelsCache: null, pipelines: {}, instance: provider, }, model: stubInternalModel({ @@ -328,7 +328,7 @@ test('generate defers the role rewrite until after translation to OpenAI Chat Co inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, - modelsCache: null, + modelsCache: null, pipelines: {}, instance: stubProvider({ callOpenAIChatCompletions }), }, model: stubInternalModel({ @@ -479,7 +479,7 @@ test('generate strips disallowed headers and injects external image loading acro const candidate: ModelCandidate = { provider: { upstreamId: 'up_test', kind: 'custom', name: 'up_test', inboundHeaderAllowlist: [], - disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, instance: anthropicMessagesProvider, + disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, pipelines: {}, instance: anthropicMessagesProvider, }, model: upstreamModel, fetcher: directFetcher, diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts index 095e034697..c3ba4dbd04 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/http_test.ts @@ -140,7 +140,7 @@ const makeCandidate = (overrides: { inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, - modelsCache: null, + modelsCache: null, pipelines: {}, instance: provider, }, model: stubInternalModel({ diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tools/image-generation-integration_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tools/image-generation-integration_test.ts index ad5593ab84..f4f8a71e80 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tools/image-generation-integration_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/interceptors/server-tools/image-generation-integration_test.ts @@ -367,7 +367,7 @@ test('an image generated in turn 1 is re-collected as an edit source in turn 2', assertEquals(request.images.length, 1); const image = request.images[0]; assert(image.type === 'upload'); - const bytes = await image.file.text(); + const bytes = new TextDecoder().decode(image.file.bytes); assertEquals(bytes, 'AAAA'); }); @@ -407,7 +407,7 @@ test('a prefetched remote edit source remains visible to orchestration and is re const request = stub.editsRequests[0]; const image = request.images[0]; assert(image.type === 'upload'); - assertEquals(new Uint8Array(await image.file.arrayBuffer()), Uint8Array.from(atob(REMOTE_PNG_B64), c => c.charCodeAt(0))); + assertEquals(new Uint8Array(image.file.bytes), Uint8Array.from(atob(REMOTE_PNG_B64), c => c.charCodeAt(0))); }); test('mask-only GIF edit transcodes one shared image and mask to WebP', async () => { @@ -437,8 +437,8 @@ test('mask-only GIF edit transcodes one shared image and mask to WebP', async () assert(mask?.type === 'upload'); assertEquals(image.file.type, 'image/webp'); assertEquals(mask.file.type, 'image/webp'); - assertEquals(await image.file.text(), 'WEBP'); - assertEquals(await mask.file.text(), 'WEBP'); + assertEquals(new TextDecoder().decode(image.file.bytes), 'WEBP'); + assertEquals(new TextDecoder().decode(mask.file.bytes), 'WEBP'); }); test('identical GIF source and mask share one transcode', async () => { @@ -467,8 +467,8 @@ test('identical GIF source and mask share one transcode', async () => { const image = request.images[0]; assert(image.type === 'upload'); assert(request.mask?.type === 'upload'); - assertEquals(await image.file.text(), 'WEBP'); - assertEquals(await request.mask.file.text(), 'WEBP'); + assertEquals(new TextDecoder().decode(image.file.bytes), 'WEBP'); + assertEquals(new TextDecoder().decode(request.mask.file.bytes), 'WEBP'); }); test('image transcoding failure becomes a terminal image tool failure', async () => { diff --git a/packages/gateway/__tests__/data-plane/chat/openai-responses/serve_test.ts b/packages/gateway/__tests__/data-plane/chat/openai-responses/serve_test.ts index 5a6a61d117..7b3dc9c1f9 100644 --- a/packages/gateway/__tests__/data-plane/chat/openai-responses/serve_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/openai-responses/serve_test.ts @@ -134,7 +134,7 @@ const makeCandidate = (overrides: { inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, - modelsCache: null, + modelsCache: null, pipelines: {}, instance: provider, }, // Default keeps stubInternalModel's three-endpoint map intact; tests that diff --git a/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts index 61c2888e01..5cface65f9 100644 --- a/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts @@ -11,6 +11,7 @@ import { createCandidateRegistry } from '../../src/data-plane/pipeline/candidate import { enumerateModelCandidates } from '../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../src/repo/index.ts'; import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts'; +import { stubProviderPipeline } from '../test-utils/provider-pipeline.ts'; import { isOwned, move, run } from '@floway-dev/pipeline'; import type { SseFrame } from '@floway-dev/protocols/common'; import { directFetcher, type ModelCandidate, type ProviderCallResult } from '@floway-dev/provider'; @@ -30,6 +31,7 @@ const candidate = (callOpenAIAudioTranscriptions: () => Promise await run( beforeEach(() => { vi.mocked(enumerateModelCandidates).mockReset(); initRepo({ + apiKeys: { update: async () => {} }, usage: { record: async () => {} }, performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, } as never); @@ -78,26 +81,13 @@ beforeEach(() => { describe('the OpenAI Audio Transcriptions pipeline', () => { it('assembles, and asks its caller for what the descending stages need', () => { expect([...openaiAudioTranscriptionServePipeline.entryNeeds].sort()).toEqual([ + 'ingress.http.headers', 'ingress.openaiAudioTranscription.responseFormat', + 'request.openaiAudioTranscription.form', 'serve.model', ]); }); - // The ending stage reads `request.openaiAudioTranscription.form` and `ingress.http.headers`, - // and the entry contract mentions neither. That is not this family's defect: a stage whose - // only trait is `return` declares no request side at all, by ruling — "when it - // short-circuits, only `provides`" — so assembly cannot see what an ending stage reads. - // - // It bites harder here than it does for a family whose edge needs the request payload to - // render: this family's edge does not, so the payload the whole endpoint exists to send is - // among the keys assembly cannot ask for. A caller who omits it reaches the deepest stage - // before failing. The type layer still catches it at the definition site, which is why this - // is a gap and not a break. - it('cannot see the request payload, because only a return-only stage reads it', () => { - expect(openaiAudioTranscriptionServePipeline.entryNeeds).not.toContain('request.openaiAudioTranscription.form'); - expect(openaiAudioTranscriptionServePipeline.entryNeeds).not.toContain('ingress.http.headers'); - }); - it('names the entry key a caller did not bring, before any stage runs', async () => { await expect(run(openaiAudioTranscriptionServePipeline, move({ 'serve.model': 'whisper-1' }) as never, {})) .rejects.toThrow('run(openaiAudioTranscriptionServe): openaiAudioTranscriptionServe needs'); diff --git a/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts index 9577ef34f8..c39a4e0562 100644 --- a/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-completions-pipeline_test.ts @@ -11,9 +11,10 @@ import { createCandidateRegistry } from '../../src/data-plane/pipeline/candidate import { enumerateModelCandidates } from '../../src/data-plane/providers/resolution.ts'; import { initRepo } from '../../src/repo/index.ts'; import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts'; -import { isDeferred, move, run } from '@floway-dev/pipeline'; +import { stubProviderPipeline } from '../test-utils/provider-pipeline.ts'; +import { compose, defineStage, isDeferred, move, run } from '@floway-dev/pipeline'; import type { SseFrame } from '@floway-dev/protocols/common'; -import { directFetcher, type ModelCandidate, type ProviderCallResult, type UpstreamCallOptions } from '@floway-dev/provider'; +import { directFetcher, type ModelCandidate, type ProviderCallResult, type UpstreamCallOptions, type ProviderRequest, type ProviderResponse, type ProviderOperationPayloads } from '@floway-dev/provider'; import { stubInternalModel, stubProvider, stubProviderModel } from '@floway-dev/test-utils'; vi.mock('../../src/data-plane/providers/resolution.ts', async importOriginal => ({ @@ -47,6 +48,7 @@ const candidate = ( upstreamId: upstream, kind: 'custom', name: upstream, inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, modelsCache: null, instance: stubProvider({ callOpenAICompletions }), + pipelines: { openaiCompletions: stubProviderPipeline('openaiCompletions', callOpenAICompletions) }, }, model: stubInternalModel({ id: 'text-model', endpoints, providerModels: { [upstream]: stubProviderModel({ id: 'text-model', endpoints }) } }, upstream), fetcher: directFetcher, @@ -74,6 +76,7 @@ beforeEach(() => { recorded.usage = []; recorded.performance = []; initRepo({ + apiKeys: { update: async () => {} }, usage: { record: async (row: unknown) => { recorded.usage.push(row); } }, performance: { recordNeutral: async (dims: unknown) => { recorded.performance.push(dims); }, @@ -113,6 +116,7 @@ beforeEach(() => { vi.mocked(enumerateModelCandidates).mockReset(); }); describe('the OpenAI Completions pipeline', () => { it('assembles, and asks its caller for what the descending stages need', () => { expect([...openaiCompletionsServePipeline.entryNeeds].sort()).toEqual([ + 'ingress.http.headers', 'ingress.openaiCompletions.wantsStream', 'ingress.openaiCompletions.wantsUsageChunk', 'request.openaiCompletions.payload', @@ -120,16 +124,6 @@ describe('the OpenAI Completions pipeline', () => { ]); }); - // `callOpenAICompletionsUpstream` reads `ingress.http.headers`, and the entry contract does - // not mention it. That is not this family's defect: a stage whose only trait is `return` - // declares no request side at all, by ruling — "when it short-circuits, only `provides`" — - // so assembly cannot see what an ending stage reads, and every family's ending stage reads - // something. A caller who omits that key gets a runtime failure at the deepest stage - // instead of the assembly error the entry contract exists to give it. - it('cannot see what an ending stage reads, because a return-only stage declares no needs', () => { - expect(openaiCompletionsServePipeline.entryNeeds).not.toContain('ingress.http.headers'); - }); - it('renders the upstream frames as SSE, hiding the usage chunk the client did not ask for', async () => { let sent: Record | undefined; resolves([candidate('up_a', async (_model, body) => { @@ -203,7 +197,8 @@ describe('the OpenAI Completions pipeline', () => { expect(facts['response.usage.billable']).toEqual([ { identity: { model: 'text-model', upstream: 'up_a', modelKey: 'text-model-key', pricing: null }, - quantities: { input_tokens: '5', output_tokens: '7' }, + quantities: { input_tokens: '5', input_cache_read_tokens: '0', input_cache_write_tokens: '0', output_tokens: '7' }, + pricingFacts: { inputTokens: 5, serviceTier: null }, }, ]); await drain(); @@ -245,11 +240,65 @@ describe('the OpenAI Completions pipeline', () => { expect(drained).toContain('winner'); }); - // Settlement is above the fork, so a run bills once however many candidates it tried — - // and it is unconditional, so a run that reached no upstream still writes a row that names - // no billed entity. Nothing asserted either until the review found the stage was composed - // into no pipeline at all. - it('writes exactly one usage row per run, however many candidates it tried', async () => { + it.each([false, true])('includes the provider retry observation for stream=%s', async streaming => { + const upstream = candidate('up_a', async () => ({ + response: streaming ? sse(chunk('hi'), usageChunk, '[DONE]') : Response.json({ + id: 'cmpl_1', choices: [], service_tier: 'flex', usage: { prompt_tokens: 0, completion_tokens: 0 }, + }), + modelKey: 'final-model', + })); + const terminal = upstream.provider.pipelines.openaiCompletions!; + upstream.provider.pipelines = { + openaiCompletions: compose('retryingProvider', [ + defineStage, ProviderRequest, ProviderResponse, ProviderResponse>({ + name: 'retryObservation', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { needs: [], consumes: [], provides: ['response.provider.previousCalls'] }, + }, + execute: async (facts, next) => ({ ...await next({ ...facts }), 'response.provider.previousCalls': move([{ modelKey: 'initial-model' }]) }), + }), + defineStage, ProviderRequest, ProviderResponse, ProviderResponse>({ + name: 'providerEntry', + into: { + request: { needs: [], consumes: [], provides: [] }, + response: { needs: [], consumes: [], provides: [] }, + }, + execute: async (facts, next) => ({ ...await next({ ...facts }, terminal) }), + }), + ]), + }; + resolves([upstream]); + const { facts, drain } = await serve(entryFacts({ 'ingress.openaiCompletions.wantsStream': streaming })); + if (streaming) await collect(facts['response.openaiCompletions.rendered']); + await drain(); + expect(recorded.usage).toMatchObject([ + { modelKey: 'initial-model', requests: 1, metrics: [] }, + { modelKey: 'final-model', requests: 1 }, + ]); + if (!streaming) { + expect(facts['response.usage.billable'][1]).toMatchObject({ + quantities: { input_tokens: '0', output_tokens: '0' }, + pricingFacts: { inputTokens: 0, serviceTier: 'flex' }, + }); + } + }); + + it('settles completed attempts when a later refusal body throws, preserving the original exception', async () => { + const original = new Error('response socket reset', { cause: new Error('connection closed') }); + resolves([ + candidate('up_a', async () => ({ response: Response.json({ error: 'retry' }, { status: 429 }), modelKey: 'first' })), + candidate('up_b', async () => ({ + response: new Response(new ReadableStream({ start: controller => { controller.error(original); } }), { status: 429 }), + modelKey: 'second', + })), + ]); + await expect(serve(entryFacts({ 'ingress.openaiCompletions.wantsStream': false }))).rejects.toBe(original); + expect(recorded.usage).toMatchObject([{ modelKey: 'first', requests: 1 }, { modelKey: 'second', requests: 1 }]); + expect(recorded.performance).toHaveLength(1); + }); + + it('settles each upstream observation across failed-over attempts', async () => { resolves([ candidate('up_a', async () => ({ response: Response.json({ error: 'nope' }, { status: 429 }), modelKey: 'k' })), candidate('up_b', async () => ({ @@ -262,7 +311,7 @@ describe('the OpenAI Completions pipeline', () => { 'request.openaiCompletions.payload': { model: 'text-model', prompt: 'hello' }, })); await drain(); - expect(recorded.usage).toHaveLength(1); + expect(recorded.usage).toHaveLength(2); expect(recorded.performance).toHaveLength(1); }); @@ -274,9 +323,9 @@ describe('the OpenAI Completions pipeline', () => { resolves([candidate('up_b', async () => ({ response: sse(chunk('hi'), usageChunk, '[DONE]'), modelKey: 'k' }))]); const { facts, drain } = await serve(entryFacts()); - await drain(); - expect(recorded.usage).toHaveLength(0); + await drain(); + expect(recorded.usage).toHaveLength(1); const streamed = facts['response.openaiCompletions.streamedUsage']; expect(streamed).not.toBeNull(); const outcome = await streamed!; diff --git a/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts b/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts index 6aec63a1bd..08466a8f8c 100644 --- a/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts @@ -130,7 +130,7 @@ test('/v1/completions non-streaming forwards body to upstream /v1/completions an await flushAsyncWork(); const usageRows = await repo.usage.listAll(); assertEquals(usageRows.length, 1); - assertEquals(tokenCountsFromUsage(usageRows[0]!), { input: 5, output: 1 }); + assertEquals(tokenCountsFromUsage(usageRows[0]!), { input: 5, input_cache_read: 0, input_cache_write: 0, output: 1 }); }); test('/v1/completions streaming forces stream_options.include_usage upstream', async () => { @@ -195,7 +195,7 @@ test('/v1/completions streaming strips usage chunk when client did not request i await flushAsyncWork(); const usageRows = await repo.usage.listAll(); assertEquals(usageRows.length, 1); - assertEquals(tokenCountsFromUsage(usageRows[0]!), { input: 4, output: 2 }); + assertEquals(tokenCountsFromUsage(usageRows[0]!), { input: 4, input_cache_read: 0, input_cache_write: 0, output: 2 }); }); test('/v1/completions streaming forwards usage chunk when the client opted in', async () => { @@ -343,7 +343,7 @@ test('/v1/completions non-streaming records usage row, performance neutral row ( const usage = await repo.usage.listAll(); assertEquals(usage.length, 1); assertEquals(usage[0]?.model, 'davinci-002'); - assertEquals(tokenCountsFromUsage(usage[0]!), { input: 7, output: 2 }); + assertEquals(tokenCountsFromUsage(usage[0]!), { input: 7, input_cache_read: 0, input_cache_write: 0, output: 2 }); const performance = await repo.performance.listAll(); assertEquals(performance.length, 1); @@ -416,7 +416,7 @@ test('/v1/completions streaming records usage row, performance neutral row (text const usage = await repo.usage.listAll(); assertEquals(usage.length, 1); - assertEquals(tokenCountsFromUsage(usage[0]!), { input: 4, output: 2 }); + assertEquals(tokenCountsFromUsage(usage[0]!), { input: 4, input_cache_read: 0, input_cache_write: 0, output: 2 }); const performance = await repo.performance.listAll(); assertEquals(performance.length, 1); diff --git a/packages/gateway/__tests__/data-plane/openai-completions/usage_test.ts b/packages/gateway/__tests__/data-plane/openai-completions/usage_test.ts index daf3ef7075..5d2eac8481 100644 --- a/packages/gateway/__tests__/data-plane/openai-completions/usage_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-completions/usage_test.ts @@ -6,7 +6,7 @@ import { assertEquals } from '@floway-dev/test-utils'; test('tokenUsageFromOpenAICompletionsUsage maps the OpenAI bare shape to bare input + output', () => { assertEquals( tokenUsageFromOpenAICompletionsUsage({ prompt_tokens: 12, completion_tokens: 3, total_tokens: 15 }, undefined, false, 'up/model'), - { input: 12, output: 3 }, + { input: 12, input_cache_read: 0, input_cache_write: 0, output: 3, tier: null }, ); }); @@ -19,7 +19,7 @@ test('tokenUsageFromOpenAICompletionsUsage splits prompt_tokens into cache_read { prompt_tokens: 100, completion_tokens: 7, total_tokens: 107, prompt_tokens_details: { cached_tokens: 80 } }, undefined, false, 'up/model', ), - { input: 20, input_cache_read: 80, output: 7 }, + { input: 20, input_cache_read: 80, input_cache_write: 0, output: 7, tier: null }, ); }); @@ -32,7 +32,7 @@ test('tokenUsageFromOpenAICompletionsUsage reads DeepSeek prompt_cache_hit_token { prompt_tokens: 200, completion_tokens: 5, total_tokens: 205, prompt_cache_hit_tokens: 128, prompt_cache_miss_tokens: 72 }, undefined, false, 'up/model', ), - { input: 72, input_cache_read: 128, output: 5 }, + { input: 72, input_cache_read: 128, input_cache_write: 0, output: 5, tier: null }, ); }); @@ -42,7 +42,7 @@ test('tokenUsageFromOpenAICompletionsUsage reads the flat top-level cached_token { prompt_tokens: 50, completion_tokens: 3, total_tokens: 53, cached_tokens: 32 }, undefined, false, 'up/model', ), - { input: 18, input_cache_read: 32, output: 3 }, + { input: 18, input_cache_read: 32, input_cache_write: 0, output: 3, tier: null }, ); }); @@ -73,15 +73,15 @@ test('tokenUsageFromOpenAICompletionsUsage runs serviceTier through billableServ // aggregate with rows that have no tier; null/undefined stays null. assertEquals( tokenUsageFromOpenAICompletionsUsage({ prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 }, 'priority', false, 'up/model'), - { input: 5, output: 2, tier: 'priority' }, + { input: 5, input_cache_read: 0, input_cache_write: 0, output: 2, tier: 'priority' }, ); assertEquals( tokenUsageFromOpenAICompletionsUsage({ prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 }, 'default', false, 'up/model'), - { input: 5, output: 2 }, + { input: 5, input_cache_read: 0, input_cache_write: 0, output: 2, tier: null }, ); assertEquals( tokenUsageFromOpenAICompletionsUsage({ prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 }, null, false, 'up/model'), - { input: 5, output: 2 }, + { input: 5, input_cache_read: 0, input_cache_write: 0, output: 2, tier: null }, ); }); @@ -93,7 +93,7 @@ test('tokenUsageFromOpenAICompletionsUsage folds the cache counts back in on the { prompt_tokens: 479, completion_tokens: 373, total_tokens: 14164, prompt_tokens_details: { cached_tokens: 13312 } }, undefined, false, 'up/model', ), - { input: 479, input_cache_read: 13312, output: 373 }, + { input: 479, input_cache_read: 13312, input_cache_write: 0, output: 373, tier: null }, ); }); @@ -103,7 +103,7 @@ test('tokenUsageFromOpenAICompletionsUsage folds on the flag when the totals wit { prompt_tokens: 479, completion_tokens: 373, prompt_tokens_details: { cached_tokens: 13312 } }, undefined, true, 'up/model', ), - { input: 479, input_cache_read: 13312, output: 373 }, + { input: 479, input_cache_read: 13312, input_cache_write: 0, output: 373, tier: null }, ); }); @@ -112,3 +112,9 @@ test('tokenUsageFromOpenAICompletionsUsage returns null on malformed input', () assertEquals(tokenUsageFromOpenAICompletionsUsage({}, undefined, false, 'up/model'), null); assertEquals(tokenUsageFromOpenAICompletionsUsage({ prompt_tokens: 'no' }, undefined, false, 'up/model'), null); }); + +test('tokenUsageFromOpenAICompletionsUsage retains a measured zero', () => { + expect(tokenUsageFromOpenAICompletionsUsage({ prompt_tokens: 0, completion_tokens: 0 }, null, false, 'up/model')).toEqual({ + input: 0, input_cache_read: 0, input_cache_write: 0, output: 0, tier: null, + }); +}); diff --git a/packages/gateway/__tests__/data-plane/openai-embeddings-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-embeddings-pipeline_test.ts index d9e19c3a28..b29a7ac194 100644 --- a/packages/gateway/__tests__/data-plane/openai-embeddings-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-embeddings-pipeline_test.ts @@ -11,30 +11,13 @@ import { move, run } from '@floway-dev/pipeline'; describe('the OpenAI Embeddings pipeline', () => { it('assembles, and asks its caller for what the descending stages need', () => { expect([...openaiEmbeddingsServePipeline.entryNeeds].sort()).toEqual([ + 'ingress.http.headers', 'ingress.openaiEmbeddings.encodingFormat', + 'request.openaiEmbeddings.canonical', 'serve.model', ]); }); - // Rerank records the same hole against `ingress.http.headers`; OpenAI Embeddings shows it - // twice over, and the second one is the sharper case. A stage whose only trait is `return` - // declares no request side at all, by ruling — "when it short-circuits, only `provides`" - // — so assembly cannot see what an ending stage reads. `callOpenAIEmbeddingsUpstream` reads - // both the headers and the request payload, and neither reaches the entry contract. - // - // Rerank's edge happened to need `request.rerank.canonical` for rendering, which put it - // in the contract for an unrelated reason. This family's edge needs only the encoding, - // so nothing above the ending stage names the payload — and the pipeline that cannot run - // without it does not ask for it. Written as a test rather than a comment because the - // hole has a consequence: a caller who omits either key gets a runtime failure at the - // deepest stage instead of an assembly error, and the entry contract exists to stop - // exactly that. The type layer still catches it at the definition site, which is why - // this is a gap and not a break. - it('cannot see what an ending stage reads, because a return-only stage declares no needs', () => { - expect(openaiEmbeddingsServePipeline.entryNeeds).not.toContain('request.openaiEmbeddings.canonical'); - expect(openaiEmbeddingsServePipeline.entryNeeds).not.toContain('ingress.http.headers'); - }); - it('names the entry key a caller did not bring, before any stage runs', async () => { await expect(run(openaiEmbeddingsServePipeline, move({ 'serve.model': 'text-embedding-3-small' }) as never, {})) .rejects.toThrow('run(openaiEmbeddingsServe): openaiEmbeddingsServe needs'); diff --git a/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts b/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts index 4a6156d31b..405cec3acb 100644 --- a/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-embeddings/failures_test.ts @@ -43,7 +43,8 @@ test('a usage write that fails leaves the answer alone and still reports', async const { apiKey, repo } = await setupAppTest(); await registerOpenAIEmbeddingsUpstream(repo); - repo.usage.record = () => Promise.reject(new Error('simulated SQL write failure')); + const failure = new Error('simulated SQL write failure'); + repo.usage.record = () => Promise.reject(failure); const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); try { @@ -71,7 +72,7 @@ test('a usage write that fails leaves the answer alone and still reports', async }, ); - assertEquals(errorSpy.mock.calls.some(call => String(call[0]).includes('usage')), true); + assertEquals(errorSpy.mock.calls.some(call => call[1] === failure), true); } finally { errorSpy.mockRestore(); } diff --git a/packages/gateway/__tests__/data-plane/openai-images-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-images-pipeline_test.ts index a08cdadcc7..85998bcfcb 100644 --- a/packages/gateway/__tests__/data-plane/openai-images-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-images-pipeline_test.ts @@ -22,29 +22,10 @@ const edits: CanonicalOpenAIImagesRequest = { describe('the OpenAI Images pipeline', () => { it('assembles both endpoints as one array, and asks its caller for what the descending stages need', () => { - expect([...openaiImagesServePipeline(generations).entryNeeds].sort()).toEqual(['serve.model']); - expect([...openaiImagesServePipeline(edits).entryNeeds].sort()).toEqual(['serve.model']); + expect([...openaiImagesServePipeline(generations).entryNeeds].sort()).toEqual(['ingress.http.headers', 'ingress.openaiImages.wantsStream', 'request.openaiImages.canonical', 'serve.model']); + expect([...openaiImagesServePipeline(edits).entryNeeds].sort()).toEqual(['ingress.http.headers', 'ingress.openaiImages.wantsStream', 'request.openaiImages.canonical', 'serve.model']); }); - // `callOpenAIImagesUpstream` reads `request.openaiImages.canonical`, - // `ingress.openaiImages.wantsStream` and `ingress.http.headers`, and the derived contract - // mentions none of them. That is not this family's defect: a stage whose only trait is - // `return` declares no request side at all, by ruling — "when it short-circuits, only - // `provides`" — so assembly cannot see what an ending stage reads, and every family's ending - // stage reads something. - // - // Written as a test rather than a comment because the hole has a consequence: a caller who - // omits any of them gets a runtime failure at the deepest stage instead of the assembly error - // the entry contract exists to produce. - it('cannot see what its ending stage reads, because a return-only stage declares no needs', () => { - const derived = openaiImagesServePipeline(generations).entryNeeds; - expect(derived).not.toContain('request.openaiImages.canonical'); - expect(derived).not.toContain('ingress.openaiImages.wantsStream'); - expect(derived).not.toContain('ingress.http.headers'); - }); - - // What covers that hole, and the reason it is a gap rather than a break: the entry type names - // all four, so the caller `entryNeeds` would have let through does not compile. it('names every key a caller must bring in its entry type', () => { const entry: OpenAIImagesServeEntry = { 'ingress.http.headers': [['content-type', 'application/json']], diff --git a/packages/gateway/__tests__/data-plane/pipeline/settlement_test.ts b/packages/gateway/__tests__/data-plane/pipeline/settlement_test.ts index 547cecc6af..6f5af73df2 100644 --- a/packages/gateway/__tests__/data-plane/pipeline/settlement_test.ts +++ b/packages/gateway/__tests__/data-plane/pipeline/settlement_test.ts @@ -31,6 +31,7 @@ describe('pipeline settlement', () => { attempt: { timing, telemetry: mockPerfTelemetryContext({ keyId: key.id }) }, }); const identity = { model: 'm', upstream: 'u', modelKey: 'm', pricing: null }; + if (failed) dump!.failed(new Error('Original source failure', { cause: new Error('socket reset') })); settleBillable({ gateway, background, log: { debug: () => {}, info: () => {}, warn: () => {}, error: () => {} } }, [ { identity, quantities: { input_tokens: '10', output_tokens: '2' } }, { identity, quantities: { input_tokens: '20', output_tokens: '3' } }, @@ -47,7 +48,7 @@ describe('pipeline settlement', () => { await Promise.all(pending); expect(runRecordOf(stubs.stored[0]?.record).meta).toMatchObject({ inputTokens: 30, outputTokens: 5, model: 'm', - error: failed ? { kind: 'failed' } : null, + error: failed ? { kind: 'failed', reason: 'Original source failure' } : null, }); }); }); diff --git a/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts b/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts index 63ef01032f..77df0d4c5e 100644 --- a/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts @@ -20,26 +20,13 @@ const request: CanonicalRerankRequest = { describe('the rerank pipeline', () => { it('assembles, and asks its caller for what the descending stages need', () => { expect([...rerankServePipeline(request).entryNeeds].sort()).toEqual([ + 'ingress.http.headers', 'ingress.rerank.sourceProtocol', 'request.rerank.canonical', 'serve.model', ]); }); - // `callRerankUpstream` reads `ingress.http.headers`, and the entry contract does not - // mention it. That is not this family's defect: a stage whose only trait is `return` - // declares no request side at all, by ruling — "when it short-circuits, only `provides`" - // — so assembly cannot see what an ending stage reads, and every family's ending stage - // reads something. - // - // Written as a test rather than a comment because the hole has a consequence: a caller - // who omits that key gets a runtime failure at the deepest stage instead of an assembly - // error, and the entry contract exists to stop exactly that. The type layer still - // catches it at the definition site, which is why this is a gap and not a break. - it('cannot see what an ending stage reads, because a return-only stage declares no needs', () => { - expect(rerankServePipeline(request).entryNeeds).not.toContain('ingress.http.headers'); - }); - // `failover` used to declare that it provides `response.http.body` for every family. Three // of the six never produce one — they read their answer to the end — so those pipelines // composed cleanly and then threw on the first real request, at the deepest stage, with a @@ -51,13 +38,13 @@ describe('the rerank pipeline', () => { it('claims nothing on the way up when a family reads its answer to the end', () => { const reading = failover({ failed: () => false, owns: [] }); expect(reading.through?.response.consumes).toEqual([]); - expect(reading.through?.response.provides).toEqual([]); + expect(reading.through?.response.provides).toEqual(['response.usage.billable']); // And a family that streams claims the key it streams at, in both directions: every // attempt's is the fork's to release, and the one it adopts rides up with ownership. const streaming = failover({ failed: () => false, owns: ['response.http.body'] }); expect(streaming.through?.response.consumes).toEqual(['response.http.body']); - expect(streaming.through?.response.provides).toEqual(['response.http.body']); + expect(streaming.through?.response.provides).toEqual(['response.http.body', 'response.usage.billable']); }); // A live handle is never a fact, and the test for that is whether it can be rendered into diff --git a/packages/gateway/__tests__/data-plane/rerank-status_test.ts b/packages/gateway/__tests__/data-plane/rerank-status_test.ts index 80ca9acc89..9583da5a0d 100644 --- a/packages/gateway/__tests__/data-plane/rerank-status_test.ts +++ b/packages/gateway/__tests__/data-plane/rerank-status_test.ts @@ -11,6 +11,7 @@ import { enumerateModelCandidates } from '../../src/data-plane/providers/resolut import { rerankServePipeline } from '../../src/data-plane/rerank/pipeline.ts'; import { initRepo } from '../../src/repo/index.ts'; import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts'; +import { stubProviderPipeline } from '../test-utils/provider-pipeline.ts'; import { move, run } from '@floway-dev/pipeline'; import type { CanonicalRerankRequest } from '@floway-dev/protocols/rerank'; import { directFetcher, type ModelCandidate, type ProviderRerankCallResult } from '@floway-dev/provider'; @@ -29,6 +30,7 @@ const candidate = (upstream: string, callRerank: () => Promise await run( beforeEach(() => { initRepo({ + apiKeys: { update: async () => {} }, usage: { record: async () => {} }, performance: { recordNeutral: async () => {}, recordZeroOutputError: async () => {} }, } as never); diff --git a/packages/gateway/__tests__/data-plane/search-pipeline_test.ts b/packages/gateway/__tests__/data-plane/search-pipeline_test.ts index 924fc51495..79f6b1759a 100644 --- a/packages/gateway/__tests__/data-plane/search-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/search-pipeline_test.ts @@ -18,19 +18,8 @@ describe('the search pipeline', () => { ]); }); - // The pinned ending is one `return`-only stage, and such a stage declares no request side - // at all — by ruling, when it short-circuits there is only `provides`. So assembly sees - // nothing it reads and derives an empty contract, even though the stage cannot run without - // `request.search.alphaSearch` and `ingress.http.headers`. It is the same hole every - // family's ending stage has; here it swallows the family's whole entry contract, because - // the ending is the only stage below the edge. - // - // Written as a test rather than a comment because the hole has a consequence: a caller who - // omits either key gets a runtime failure at the deepest stage instead of an assembly - // error, and the entry contract exists to stop exactly that. The type layer still catches - // it at the definition site, which is why this is a gap and not a break. - it('cannot see what the pinned ending reads, because a return-only stage declares no needs', () => { - expect(searchServePipeline(pinned).entryNeeds).toEqual([]); + it('requires the pinned request payload and admitted headers before entering its provider', () => { + expect([...searchServePipeline(pinned).entryNeeds].sort()).toEqual(['ingress.http.headers', 'request.search.alphaSearch']); }); it('names the entry key a caller did not bring, before any stage runs', async () => { diff --git a/packages/gateway/__tests__/data-plane/shared/iterate-candidates_test.ts b/packages/gateway/__tests__/data-plane/shared/iterate-candidates_test.ts index 4c57155d14..0d05d21b96 100644 --- a/packages/gateway/__tests__/data-plane/shared/iterate-candidates_test.ts +++ b/packages/gateway/__tests__/data-plane/shared/iterate-candidates_test.ts @@ -16,7 +16,7 @@ const stubCandidate = (id: string, upstream = 'up'): ModelCandidate => inboundHeaderAllowlist: [], disabledPublicModelIds: [], modelPrefix: null, - modelsCache: null, + modelsCache: null, pipelines: {}, instance: stubProvider(), }, }); diff --git a/packages/gateway/__tests__/test-utils/provider-pipeline.ts b/packages/gateway/__tests__/test-utils/provider-pipeline.ts new file mode 100644 index 0000000000..ea11129195 --- /dev/null +++ b/packages/gateway/__tests__/test-utils/provider-pipeline.ts @@ -0,0 +1,35 @@ +import type { GatewayCtx } from '../../src/data-plane/shared/gateway-ctx.ts'; +import { takeHttpResponse } from '@floway-dev/http/pipeline'; +import { compose, defineStage, isSecret, move, type Pipeline } from '@floway-dev/pipeline'; +import { directFetcher, identityWrapUpstreamCall, type ProviderCallResult, type ProviderModelFacts, type ProviderOperation, type ProviderOperationPayloads, type ProviderResponse, type ProviderRerankResponse, type ProviderRequest, type UpstreamCallOptions } from '@floway-dev/provider'; + +export const stubProviderPipeline = ( + operation: O, + call: (model: ProviderModelFacts, payload: ProviderOperationPayloads[O], signal: AbortSignal | undefined, options: UpstreamCallOptions) => Promise, +): Pipeline, O extends 'rerank' ? ProviderRerankResponse : ProviderResponse> => compose, ProviderResponse>(`stub.${operation}`, [ + defineStage, ProviderResponse, { gateway: GatewayCtx }>({ + name: 'stubHttp', + return: { provides: ['response.http.exchange', 'response.http.body', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'] }, + execute: async (facts, use) => { + let result: ProviderCallResult; + // The fixture callback stands for the transport, so its rejection is a transport outcome. + try { + result = await call(facts['request.provider.model'], facts['request.provider.payload'], use.gateway.abortSignal, { + fetcher: directFetcher, waitUntil: use.gateway.backgroundScheduler, wrapUpstreamCall: identityWrapUpstreamCall, + headers: new Headers(facts['request.http.headers'].map(([name, value]): [string, string] => [name, isSecret(value) ? value.reveal() : value])), + }); + } catch (error) { + return move({ + ...facts, 'response.http.exchange': { type: 'transportFailure', error }, 'response.http.body': null, + 'response.provider.modelKey': facts['request.provider.model'].id, 'response.provider.called': false, 'response.provider.previousCalls': [], + }); + } + const exchange = takeHttpResponse(result.response); + return move({ + ...facts, 'response.http.exchange': exchange, 'response.http.body': exchange.body, + 'response.provider.modelKey': result.modelKey, 'response.provider.called': true, 'response.provider.previousCalls': [], + ...('target' in result ? { 'response.provider.rerankTarget': result.target } : {}), + }); + }, + }), +]) as Pipeline, O extends 'rerank' ? ProviderRerankResponse : ProviderResponse>; diff --git a/packages/gateway/src/data-plane/alpha-search/call-upstream.ts b/packages/gateway/src/data-plane/alpha-search/call-upstream.ts index 72238ed4ee..c7fa7ef1ef 100644 --- a/packages/gateway/src/data-plane/alpha-search/call-upstream.ts +++ b/packages/gateway/src/data-plane/alpha-search/call-upstream.ts @@ -1,12 +1,15 @@ import type { PinnedSearchUpstream, Fields } from './facts.ts'; import { parseAlphaSearchResponse } from './protocol.ts'; +import { providerEntry } from '../pipeline/provider-entry.ts'; +import { providerUsage } from '../pipeline/provider-usage.ts'; import type { GatewayServices } from '../pipeline/services.ts'; +import { dialFailure, spentBody } from '../pipeline/upstream-body.ts'; import { enumerateModelCandidates } from '../providers/resolution.ts'; import type { GatewayCtx } from '../shared/gateway-ctx.ts'; -import { filterInboundHeadersForProvider } from '../shared/inbound-headers.ts'; import { telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; +import { exchangeResponse } from '@floway-dev/http/pipeline'; import { defineStage, move } from '@floway-dev/pipeline'; -import { providerModelOf, identityWrapUpstreamCall, type ModelCandidate } from '@floway-dev/provider'; +import { type ModelCandidate, type ProviderRequest, type ProviderResponse, type ProviderOperationPayloads } from '@floway-dev/provider'; /** * A misconfigured pin is the operator's to see with its stack, the way the gateway surfaces @@ -62,41 +65,40 @@ const asJson = (raw: string): unknown => { */ export const callSearchUpstream = (pinned: PinnedSearchUpstream) => defineStage< Fields<'request.search.alphaSearch' | 'ingress.http.headers'>, - Fields<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>, + ProviderRequest, + ProviderResponse, + Fields<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status' | 'response.http.body'>, GatewayServices >({ name: 'callSearchUpstream', - return: { provides: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers'] }, - execute: async (facts, use) => { + into: { + request: { needs: ['request.search.alphaSearch', 'ingress.http.headers'], consumes: [], provides: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'] }, + response: { needs: ['response.http.exchange', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.http.body'], consumes: ['response.http.exchange', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'], provides: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers', 'response.http.status', 'response.http.body'] }, + }, + execute: async (facts, next, use) => { const candidate = await resolvePinnedUpstream(pinned, use.gateway); // The caller's model is dropped: this endpoint is pinned to the operator's model, and the // provider stamps that one on the way out. // TODO: pin SearchRequest.id to one provider account when Codex upstreams support account // pools. The current Codex provider has one active account. const { model: _named, ...request } = facts['request.search.alphaSearch']; - const result = await candidate.provider.instance.callAlphaSearch( - providerModelOf(candidate), - request, - use.gateway.abortSignal, - { - fetcher: candidate.fetcher, - waitUntil: use.gateway.backgroundScheduler, - // The client's own headers reach the upstream from the record, not from a live - // request object: what a provider is allowed to forward is filtered per provider, - // and the dump shows what was there to filter. - headers: filterInboundHeadersForProvider(turnMetadataHeaders(facts['ingress.http.headers']), candidate.provider), - // No `PerformanceOperation` names search, so there is no performance row for a - // stamping wrapper's interval to land on. - wrapUpstreamCall: identityWrapUpstreamCall, - }, - ); + const selector = use.rememberCandidates([candidate])[0]!; + const pipeline = candidate.provider.pipelines.alphaSearch; + if (pipeline === undefined) throw new Error(`Provider ${candidate.provider.kind} has no Alpha Search pipeline`); + const back = await next(providerEntry({ ...facts, 'route.attempt': selector }, candidate, request, [...turnMetadataHeaders(facts['ingress.http.headers'])]), pipeline); + const { 'response.http.exchange': exchange, 'response.provider.modelKey': modelKey, 'response.provider.called': wasCalled, 'response.provider.previousCalls': _previousCalls, ...rest } = back; + if (exchange.type === 'transportFailure') { + return move({ ...rest, 'response.search.alphaSearch': dialFailure(exchange.error), 'response.usage.billable': providerUsage(candidate, back, []), 'response.http.headers': [], 'response.http.status': 502 }); + } + const result = { response: exchangeResponse(exchange), modelKey }; // The alpha-search protocol reports no usage at all, so the entity is present with no // quantities — the upstream was called and reported nothing, which is a different // situation from reporting zero. - const billable = [{ identity: telemetryModelIdentity(candidate, result.modelKey), quantities: {} }]; + const billable = wasCalled ? [{ identity: telemetryModelIdentity(candidate, result.modelKey), quantities: {} }] : []; // Every protocol the gateway carries is one it fully understands: the body is read here // and serialized again at the edge, an error body included. + spentBody(exchange.body); const raw = await result.response.text(); if (!result.response.ok) { use.log.warn('upstream refused', { status: result.response.status }); @@ -105,14 +107,15 @@ export const callSearchUpstream = (pinned: PinnedSearchUpstream) => defineStage< // something the edge can serialize back out. const body = asJson(raw); return move({ - ...facts, + ...rest, 'response.search.alphaSearch': { status: result.response.status, message: raw, ...(body === undefined ? {} : { body }), }, - 'response.usage.billable': billable, + 'response.usage.billable': providerUsage(candidate, back, billable), 'response.http.headers': [...result.response.headers], + 'response.http.status': result.response.status, }); } @@ -121,21 +124,23 @@ export const callSearchUpstream = (pinned: PinnedSearchUpstream) => defineStage< // A protocol that requires JSON and receives something else synthesizes its own error, // which is also why the raw text rides along: a dump reader is owed what came back. return move({ - ...facts, + ...rest, 'response.search.alphaSearch': { status: 502, message: `The search upstream answered ${result.response.status} but not in the search protocol: ${verdict.reason}.`, body: raw, }, - 'response.usage.billable': billable, + 'response.usage.billable': providerUsage(candidate, back, billable), 'response.http.headers': [...result.response.headers], + 'response.http.status': result.response.status, }); } return move({ - ...facts, + ...rest, 'response.search.alphaSearch': verdict.response, - 'response.usage.billable': billable, + 'response.usage.billable': providerUsage(candidate, back, billable), 'response.http.headers': [...result.response.headers], + 'response.http.status': result.response.status, }); }, }); diff --git a/packages/gateway/src/data-plane/alpha-search/emit.ts b/packages/gateway/src/data-plane/alpha-search/emit.ts index addd33d040..316cdfc1c0 100644 --- a/packages/gateway/src/data-plane/alpha-search/emit.ts +++ b/packages/gateway/src/data-plane/alpha-search/emit.ts @@ -16,14 +16,14 @@ import { defineStage, move } from '@floway-dev/pipeline'; export const emitAlphaSearch = defineStage< Fields, Fields, - Fields<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>, + Fields<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status'>, Fields<'response.search.rendered' | 'response.http.status' | 'response.http.headers'> >({ name: 'emitAlphaSearch', through: { request: { needs: [], consumes: [], provides: [] }, response: { - needs: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers'], + needs: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers', 'response.http.status'], consumes: ['response.search.alphaSearch', 'response.http.headers'], provides: ['response.search.rendered', 'response.http.status', 'response.http.headers'], }, @@ -53,7 +53,7 @@ export const emitAlphaSearch = defineStage< ...rest, 'response.http.headers': forClient, 'response.search.rendered': move(renderAlphaSearchResponse(answer)), - 'response.http.status': 200, + 'response.http.status': back['response.http.status'], }; }, }); diff --git a/packages/gateway/src/data-plane/alpha-search/execute-operations.ts b/packages/gateway/src/data-plane/alpha-search/execute-operations.ts index 17439847c7..5df9f7964d 100644 --- a/packages/gateway/src/data-plane/alpha-search/execute-operations.ts +++ b/packages/gateway/src/data-plane/alpha-search/execute-operations.ts @@ -11,11 +11,11 @@ import { defineStage, move } from '@floway-dev/pipeline'; */ export const executeSearchOperations = defineStage< Fields<'request.search.operations' | 'request.search.filters'>, - Fields<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>, + Fields<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status'>, SearchServices >({ name: 'executeSearchOperations', - return: { provides: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers'] }, + return: { provides: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers', 'response.http.status'] }, execute: async (facts, use) => { const session: WebSearchExecutionSession = { getProvider: use.searchProvider, @@ -42,6 +42,7 @@ export const executeSearchOperations = defineStage< 'response.usage.billable': [], // Nothing was called, so there are no upstream headers to carry. 'response.http.headers': [], + 'response.http.status': 200, }); }, }); diff --git a/packages/gateway/src/data-plane/alpha-search/parse-operations.ts b/packages/gateway/src/data-plane/alpha-search/parse-operations.ts index 77da35444d..cdd857205d 100644 --- a/packages/gateway/src/data-plane/alpha-search/parse-operations.ts +++ b/packages/gateway/src/data-plane/alpha-search/parse-operations.ts @@ -11,6 +11,7 @@ const inBandOutput = (facts: Fields<'request.search.alphaSearch'>, output: strin 'response.usage.billable': [], // Nothing was called, so there are no upstream headers to carry. 'response.http.headers': [], + 'response.http.status': 200, }); /** @@ -21,9 +22,9 @@ const inBandOutput = (facts: Fields<'request.search.alphaSearch'>, output: strin export const parseSearchOperations = defineStage< Fields<'request.search.alphaSearch'>, Fields<'request.search.operations' | 'request.search.filters'>, - Fields<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>, - Fields<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'>, - Fields<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers'> + Fields<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status'>, + Fields<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status'>, + Fields<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status'> >({ name: 'parseSearchOperations', through: { @@ -34,7 +35,7 @@ export const parseSearchOperations = defineStage< }, response: { needs: [], consumes: [], provides: [] }, }, - return: { provides: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers'] }, + return: { provides: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers', 'response.http.status'] }, execute: async (facts, next) => { const { 'request.search.alphaSearch': request, ...rest } = facts; const commands = request.commands ?? {}; diff --git a/packages/gateway/src/data-plane/alpha-search/pipeline.ts b/packages/gateway/src/data-plane/alpha-search/pipeline.ts index 79254ed12e..f6e699fd6a 100644 --- a/packages/gateway/src/data-plane/alpha-search/pipeline.ts +++ b/packages/gateway/src/data-plane/alpha-search/pipeline.ts @@ -3,7 +3,6 @@ import { emitAlphaSearch } from './emit.ts'; import { executeSearchOperations } from './execute-operations.ts'; import type { SearchExecution, Fields } from './facts.ts'; import { parseSearchOperations } from './parse-operations.ts'; -import { isFailure } from '../pipeline/facts.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; import { compose, type Pipeline } from '@floway-dev/pipeline'; @@ -11,10 +10,10 @@ export const searchServePipeline = (execution: SearchExecution): Pipeline< Fields<'request.search.alphaSearch'>, Fields<'response.search.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> > => compose('searchServe', [ - emitAlphaSearch, // Unconditional, as everywhere: a run that searched locally reached no upstream and bills // for none, and saying so is a row that names no billed entity rather than no row. - writeSettlement(handedUp => isFailure((handedUp as { 'response.search.alphaSearch'?: unknown })['response.search.alphaSearch'])), + writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400), + emitAlphaSearch, ...(execution.kind === 'local' ? [parseSearchOperations, executeSearchOperations] : [callSearchUpstream(execution)]), diff --git a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tools/image-generation.ts b/packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tools/image-generation.ts index 72a9b75bb4..6e4e0a083c 100644 --- a/packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tools/image-generation.ts +++ b/packages/gateway/src/data-plane/chat/openai-responses/interceptors/server-tools/image-generation.ts @@ -962,11 +962,11 @@ const buildEditsRequest = ( }; const images = sources.map((source, index) => ({ type: 'upload' as const, - file: new File([source.bytes], `image_${index}.${editFileExt(source.mimeType)}`, { type: source.mimeType }), + file: { bytes: new Uint8Array(source.bytes), name: `image_${index}.${editFileExt(source.mimeType)}`, type: source.mimeType }, })); const maskFile = mask === undefined ? undefined - : new File([mask.bytes], `mask.${editFileExt(mask.mimeType)}`, { type: mask.mimeType }); + : { bytes: new Uint8Array(mask.bytes), name: `mask.${editFileExt(mask.mimeType)}`, type: mask.mimeType }; return { images, ...(maskFile === undefined ? {} : { mask: { type: 'upload' as const, file: maskFile } }), diff --git a/packages/gateway/src/data-plane/openai-audio/call-upstream.ts b/packages/gateway/src/data-plane/openai-audio/call-upstream.ts index 3008961818..922f2aa858 100644 --- a/packages/gateway/src/data-plane/openai-audio/call-upstream.ts +++ b/packages/gateway/src/data-plane/openai-audio/call-upstream.ts @@ -2,14 +2,16 @@ import type { Fields, OpenAIAudioTranscriptionEvents, OpenAIAudioTranscriptionSt import { recordStream } from '../../dump/turn-dump.ts'; import type { UsageQuantities } from '../../repo/types.ts'; import type { BillableEntity, Failure } from '../pipeline/facts.ts'; +import { providerEntry } from '../pipeline/provider-entry.ts'; +import { providerUsage } from '../pipeline/provider-usage.ts'; import type { GatewayServices } from '../pipeline/services.ts'; -import { dialFailure } from '../pipeline/upstream-body.ts'; +import { dialFailure, spentBody } from '../pipeline/upstream-body.ts'; import { upstreamPerformanceContext, telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; -import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; -import { defineStage, move, own, defer, type Owned, type Logger, type Deferred } from '@floway-dev/pipeline'; +import { exchangeResponse } from '@floway-dev/http/pipeline'; +import { defineStage, move, setRelease, defer, type Owned, type Logger, type Deferred } from '@floway-dev/pipeline'; import { isEventStreamMediaType, eventFrame, parseSSEStream, parseDecimalString } from '@floway-dev/protocols/common'; import { parseOpenAIAudioTranscription, parseOpenAIAudioTranscriptionUsage, isOpenAIAudioTranscriptionDoneEvent, parseOpenAIAudioTranscriptionStreamUsage, type OpenAIAudioTranscriptionResponseFormat, type CanonicalOpenAIAudioTranscription, type OpenAIAudioTranscriptionUsage } from '@floway-dev/protocols/openai-audio'; -import { providerModelOf, type TelemetryModelIdentity } from '@floway-dev/provider'; +import type { ProviderRequest, ProviderOperationPayloads, ProviderResponse, TelemetryModelIdentity } from '@floway-dev/provider'; const viewOf = (events: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => events }); @@ -23,60 +25,43 @@ const viewOf = (events: AsyncGenerator): AsyncIterable => ({ [Symbol.as */ export const callOpenAIAudioTranscriptionUpstream = defineStage< Fields<'ingress.openaiAudioTranscription.responseFormat' | 'request.openaiAudioTranscription.form' | 'route.attempt' | 'ingress.http.headers'>, + ProviderRequest, + ProviderResponse, Fields<'response.openaiAudioTranscription.canonical' | 'response.openaiAudioTranscription.mediaType' | 'response.openaiAudioTranscription.streamedOutcome'> & { 'response.usage.billable': readonly BillableEntity[]; 'response.http.status': number; 'response.http.headers': readonly (readonly [string, string])[]; - 'response.http.body': ReadableStream & Owned; }, + 'response.http.body': (ReadableStream & Owned) | null; }, GatewayServices >({ name: 'callOpenAIAudioTranscriptionUpstream', - return: { - provides: [ - 'response.openaiAudioTranscription.canonical', - 'response.openaiAudioTranscription.mediaType', - 'response.openaiAudioTranscription.streamedOutcome', - 'response.usage.billable', - 'response.http.status', - 'response.http.headers', - 'response.http.body', - ], + into: { + request: { needs: ['ingress.openaiAudioTranscription.responseFormat', 'request.openaiAudioTranscription.form', 'route.attempt', 'ingress.http.headers'], consumes: [], provides: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'] }, + response: { needs: ['response.http.exchange', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.http.body'], consumes: ['response.http.exchange', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'], provides: ['response.openaiAudioTranscription.canonical', 'response.openaiAudioTranscription.mediaType', 'response.openaiAudioTranscription.streamedOutcome', 'response.usage.billable', 'response.http.status', 'response.http.headers', 'response.http.body'] }, }, - execute: async (facts, use) => { + execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); // Attribution is set before the dial, so an attempt that never completes still names the // candidate it was made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'audio_transcription'); - let result; - try { - result = await candidate.provider.instance.callOpenAIAudioTranscriptions( - providerModelOf(candidate), - { - entries: facts['request.openaiAudioTranscription.form'].map(({ name, value }) => ({ - name, value: typeof value === 'string' ? value : new File([value.bytes], value.name, { type: value.type, lastModified: value.lastModified }), - })), - }, - use.gateway.abortSignal, - // The client's own headers reach the upstream from the record, not from a live request - // object: what a provider is allowed to forward is filtered per provider, and the dump - // shows what was there to filter. - buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), - ); - } catch (error) { - use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); - // A dial that never completed reached no upstream, so nothing was billed and there are - // no headers to carry. What it leaves behind is the performance row settlement writes. + const pipeline = candidate.provider.pipelines.openaiAudioTranscriptions; + if (pipeline === undefined) throw new Error(`Provider ${candidate.provider.kind} has no openaiAudioTranscriptions pipeline`); + const back = await next(providerEntry(facts, candidate, { entries: facts['request.openaiAudioTranscription.form'] }), pipeline); + const { 'response.http.exchange': exchange, 'response.provider.modelKey': modelKey, 'response.provider.called': wasCalled, 'response.provider.previousCalls': _previousCalls, ...rest } = back; + if (exchange.type === 'transportFailure') { return move({ - ...facts, - 'response.openaiAudioTranscription.canonical': dialFailure(error), + ...rest, + 'response.openaiAudioTranscription.canonical': dialFailure(exchange.error), 'response.openaiAudioTranscription.mediaType': null, 'response.openaiAudioTranscription.streamedOutcome': null, - 'response.usage.billable': [], + 'response.usage.billable': providerUsage(candidate, back, []), 'response.http.status': 502, 'response.http.headers': [], 'response.http.body': spentBody(null), }); } + const result = { response: exchangeResponse(exchange), modelKey }; + const identity = telemetryModelIdentity(candidate, result.modelKey); const format = facts['ingress.openaiAudioTranscription.responseFormat']; const status = result.response.status; @@ -84,21 +69,22 @@ export const callOpenAIAudioTranscriptionUpstream = defineStage< const headers = move([...result.response.headers] as readonly (readonly [string, string])[]); // An upstream that was called and reported nothing is a different situation from one // that reported zero, so the entity is present with no quantities. - const called: readonly BillableEntity[] = [{ identity, quantities: {} }]; + const called: readonly BillableEntity[] = wasCalled ? [{ identity, quantities: {} }] : []; if (!result.response.ok) { use.log.warn('upstream refused', { status }); + spentBody(exchange.body); // An upstream error body is JSON like any other body, and reading it here is also what // leaves a losing attempt with nothing open behind it. return move({ - ...facts, + ...rest, 'response.openaiAudioTranscription.canonical': await refusal(status, result.response), 'response.openaiAudioTranscription.mediaType': mediaType, 'response.openaiAudioTranscription.streamedOutcome': null, - 'response.usage.billable': called, + 'response.usage.billable': providerUsage(candidate, back, called), 'response.http.status': status, 'response.http.headers': headers, - 'response.http.body': spentBody(result.response.body), + 'response.http.body': spentBody(exchange.body), }); } @@ -109,11 +95,11 @@ export const callOpenAIAudioTranscriptionUpstream = defineStage< if (isEventStreamMediaType(mediaType)) { if (result.response.body === null) { return move({ - ...facts, + ...rest, 'response.openaiAudioTranscription.canonical': { status: 502, message: 'Upstream returned a streaming response with no body.' }, 'response.openaiAudioTranscription.mediaType': mediaType, 'response.openaiAudioTranscription.streamedOutcome': null, - 'response.usage.billable': called, + 'response.usage.billable': providerUsage(candidate, back, called), 'response.http.status': status, 'response.http.headers': headers, 'response.http.body': spentBody(null), @@ -123,18 +109,19 @@ export const callOpenAIAudioTranscriptionUpstream = defineStage< // here — closest to the upstream and on the protocol it spoke — by folding the events // as they pass, so the reading costs one pass and the client's own stream drives it. const metered = meterEvents(result.response.body, identity, use.gateway.abortSignal, use.log); + setRelease(exchange.body!, async () => { for await (const _event of metered.events) { /* drain */ } }); return move({ - ...facts, + ...rest, // SSE labels and data remain upstream-owned; parsing below only observes billing and completion. 'response.openaiAudioTranscription.canonical': recordStream(metered.events, use.gateway.dump, eventFrame), 'response.openaiAudioTranscription.mediaType': mediaType, - 'response.openaiAudioTranscription.streamedOutcome': metered.outcome, - 'response.usage.billable': called, + 'response.openaiAudioTranscription.streamedOutcome': defer(metered.outcome.then(outcome => ({ ...outcome, billable: providerUsage(candidate, back, outcome.billable) }))), + 'response.usage.billable': providerUsage(candidate, back, called), 'response.http.status': status, 'response.http.headers': headers, // Releasing this body is reading those events to the end: they are one reader over // one connection, and a second reader is not something a `ReadableStream` allows. - 'response.http.body': own(result.response.body, async (): Promise => { for await (const _event of metered.events) { /* to end of stream */ } }), + 'response.http.body': exchange.body, }); } @@ -142,26 +129,21 @@ export const callOpenAIAudioTranscriptionUpstream = defineStage< // client is sent is the document that arrived and not something serialized from a parse. // So there is nothing here to fail over from: the reading feeds the record and the usage // row, and the bytes travel either way. + spentBody(exchange.body); const read = readTranscription(format, new Uint8Array(await result.response.arrayBuffer()), use.log); return move({ - ...facts, + ...rest, 'response.openaiAudioTranscription.canonical': read.canonical, 'response.openaiAudioTranscription.mediaType': mediaType, 'response.openaiAudioTranscription.streamedOutcome': null, - 'response.usage.billable': [{ identity, quantities: billed(read.usage) }], + 'response.usage.billable': providerUsage(candidate, back, wasCalled ? [{ identity, quantities: billed(read.usage) }] : []), 'response.http.status': status, 'response.http.headers': headers, - 'response.http.body': spentBody(result.response.body), + 'response.http.body': spentBody(exchange.body), }); }, }); -/** A body this stage has already read to the end, or one the upstream never sent. The record - * holds a body as a stream and `failover` releases the losing attempts', so every path hands - * one up; what says an answer was unusable is the failure at the canonical key, not this. */ -const spentBody = (body: ReadableStream | null): ReadableStream & Owned => - own(body ?? new ReadableStream({ start: controller => controller.close() }), (): Promise => Promise.resolve()); - const refusal = async (status: number, response: Response): Promise => { const text = await response.text(); try { diff --git a/packages/gateway/src/data-plane/openai-audio/http.ts b/packages/gateway/src/data-plane/openai-audio/http.ts index 2a7b2effa3..5e1541baa5 100644 --- a/packages/gateway/src/data-plane/openai-audio/http.ts +++ b/packages/gateway/src/data-plane/openai-audio/http.ts @@ -91,7 +91,7 @@ const bodyOf = (rendered: Record | Uint8Array): Uint8Array => * that declared none — so `Rendered.contentType` carries a null and the seam answers without * the header rather than inventing one. And a transcription's stream states its own outcome in * its last event, which the family's meter reads because it is the one place that knows — so - * what `DeferredUsage` hands back is both what was billed and whether the stream finished, + * the deferred reading holds both what was billed and whether the stream finished, * settled from one promise while the request is still live. */ export const openaiAudioTranscriptions = async (c: Context): Promise => { @@ -127,6 +127,5 @@ export const openaiAudioTranscriptions = async (c: Context): Promise = // gateway carried rather than wrote is not one it can describe. return { body: bodyOf(rendered) as BodyInit, contentType: facts['response.openaiAudioTranscription.mediaType'] }; }, - facts => facts['response.openaiAudioTranscription.streamedOutcome'], ); }; diff --git a/packages/gateway/src/data-plane/openai-audio/pipeline.ts b/packages/gateway/src/data-plane/openai-audio/pipeline.ts index 1f3eac7caf..de18c50944 100644 --- a/packages/gateway/src/data-plane/openai-audio/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-audio/pipeline.ts @@ -14,15 +14,13 @@ export const openaiAudioTranscriptionServePipeline: Pipeline< & { 'response.http.status': number; 'response.usage.billable': readonly BillableEntity[]; 'response.http.headers': readonly (readonly [string, string])[]; } > = compose('openaiAudioTranscriptionServe', [ + writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, 'response.openaiAudioTranscription.streamedOutcome'), emitOpenAIAudioTranscription, - writeSettlement( - handedUp => isFailure((handedUp as { 'response.openaiAudioTranscription.canonical'?: unknown })['response.openaiAudioTranscription.canonical']), - handedUp => (handedUp as { 'response.openaiAudioTranscription.streamedOutcome'?: unknown })['response.openaiAudioTranscription.streamedOutcome'] !== null, - ), resolveCandidates(narrowing), failover({ failed: handedUp => isFailure((handedUp as { 'response.openaiAudioTranscription.canonical'?: unknown })['response.openaiAudioTranscription.canonical']), owns: ['response.http.body'], + pendingUsage: 'response.openaiAudioTranscription.streamedOutcome', }), callOpenAIAudioTranscriptionUpstream, ]); diff --git a/packages/gateway/src/data-plane/openai-completions/call-upstream.ts b/packages/gateway/src/data-plane/openai-completions/call-upstream.ts index 9b1d5018ed..8fb2b5cbf3 100644 --- a/packages/gateway/src/data-plane/openai-completions/call-upstream.ts +++ b/packages/gateway/src/data-plane/openai-completions/call-upstream.ts @@ -1,18 +1,19 @@ import type { Fields, OpenAICompletionsFacts, OpenAICompletionsFrames } from './facts.ts'; import { tokenUsageFromOpenAICompletionsUsage } from './usage.ts'; import { recordStream, type TurnDump } from '../../dump/turn-dump.ts'; -import type { UsageQuantities } from '../../repo/types.ts'; -import { tokenUsageQuantities } from '../../repo/usage-metrics.ts'; import { isFailure, type BillableEntity, type Failure } from '../pipeline/facts.ts'; +import { providerEntry } from '../pipeline/provider-entry.ts'; +import { providerUsage } from '../pipeline/provider-usage.ts'; import type { StreamOutcome } from '../pipeline/serve.ts'; import type { GatewayServices } from '../pipeline/services.ts'; -import { dialFailure } from '../pipeline/upstream-body.ts'; +import { dialFailure, spentBody } from '../pipeline/upstream-body.ts'; import { upstreamPerformanceContext, telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; -import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; -import { defineStage, move, own, defer, type Owned, type Deferred } from '@floway-dev/pipeline'; +import { tokenUsageMeasurement } from '../shared/telemetry/usage.ts'; +import { exchangeResponse } from '@floway-dev/http/pipeline'; +import { defineStage, move, setRelease, defer, type Owned, type Deferred } from '@floway-dev/pipeline'; import { isOpenAIUsageOnlyEventShape, type ProtocolFrame } from '@floway-dev/protocols/common'; import { parseOpenAICompletionsStream, parseOpenAICompletionsResult, type OpenAICompletionsResult, type OpenAICompletionsStreamEvent } from '@floway-dev/protocols/openai-completions'; -import { providerModelOf, type TelemetryModelIdentity, type ModelCandidate } from '@floway-dev/provider'; +import { providerModelOf, type ProviderRequest, type ProviderOperationPayloads, type ProviderResponse, type TelemetryModelIdentity, type ModelCandidate } from '@floway-dev/provider'; /** * A stream as a value the record can hold: a wrapper around the generator rather than the @@ -33,22 +34,18 @@ const view = (frames: AsyncGenerator): AsyncIterable => ({ [Symbol.asyn */ export const callOpenAICompletionsUpstream = defineStage< Fields<'ingress.openaiCompletions.wantsStream' | 'request.openaiCompletions.payload' | 'route.attempt' | 'ingress.http.headers'>, + ProviderRequest, + ProviderResponse, Fields<'response.openaiCompletions.payload' | 'response.openaiCompletions.streamedUsage' | 'response.usage.billable' | 'response.http.status' | 'response.http.headers' | 'response.http.body'>, GatewayServices >({ name: 'callOpenAICompletionsUpstream', - return: { - provides: [ - 'response.openaiCompletions.payload', - 'response.openaiCompletions.streamedUsage', - 'response.usage.billable', - 'response.http.status', - 'response.http.headers', - 'response.http.body', - ], + into: { + request: { needs: ['ingress.openaiCompletions.wantsStream', 'request.openaiCompletions.payload', 'route.attempt', 'ingress.http.headers'], consumes: [], provides: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'] }, + response: { needs: ['response.http.exchange', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.http.body'], consumes: ['response.http.exchange', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'], provides: ['response.openaiCompletions.payload', 'response.openaiCompletions.streamedUsage', 'response.usage.billable', 'response.http.status', 'response.http.headers', 'response.http.body'] }, }, - execute: async (facts, use) => { + execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); // The provider re-stamps whatever id it resolved upstream, so the id the client // addressed does not travel with the body. @@ -57,31 +54,23 @@ export const callOpenAICompletionsUpstream = defineStage< // candidate it was made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'text_completion'); - let result; - try { - result = await candidate.provider.instance.callOpenAICompletions( - providerModelOf(candidate), - body, - use.gateway.abortSignal, - // The client's own headers reach the upstream from the record, not from a live request - // object: what a provider is allowed to forward is filtered per provider, and the dump - // shows what was there to filter. - buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), - ); - } catch (error) { - use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); - // A dial that never completed reached no upstream, so nothing was billed and there are - // no headers to carry. What it leaves behind is the performance row settlement writes. + const pipeline = candidate.provider.pipelines.openaiCompletions; + if (pipeline === undefined) throw new Error(`Provider ${candidate.provider.kind} has no openaiCompletions pipeline`); + const back = await next(providerEntry(facts, candidate, body), pipeline); + const { 'response.http.exchange': exchange, 'response.provider.modelKey': modelKey, 'response.provider.called': wasCalled, 'response.provider.previousCalls': _previousCalls, ...rest } = back; + if (exchange.type === 'transportFailure') { return move({ - ...facts, - 'response.openaiCompletions.payload': dialFailure(error), + ...rest, + 'response.openaiCompletions.payload': dialFailure(exchange.error), 'response.openaiCompletions.streamedUsage': null, - 'response.usage.billable': [], + 'response.usage.billable': providerUsage(candidate, back, []), 'response.http.status': 502, 'response.http.headers': [], 'response.http.body': spentBody(null), }); } + const result = { response: exchangeResponse(exchange), modelKey }; + if (!result.response.ok) use.log.warn('upstream refused', { status: result.response.status }); const answer = await readUpstream( @@ -91,12 +80,14 @@ export const callOpenAICompletionsUpstream = defineStage< candidate, use.gateway.abortSignal, use.gateway.dump, + exchange.body, + wasCalled, ); return move({ - ...facts, + ...rest, 'response.openaiCompletions.payload': answer.payload, - 'response.openaiCompletions.streamedUsage': answer.streamedUsage, - 'response.usage.billable': answer.billable, + 'response.openaiCompletions.streamedUsage': answer.streamedUsage === null ? null : defer(answer.streamedUsage.then(outcome => ({ ...outcome, billable: providerUsage(candidate, back, outcome.billable) }))), + 'response.usage.billable': providerUsage(candidate, back, answer.billable), 'response.http.status': result.response.status, 'response.http.headers': [...result.response.headers], 'response.http.body': answer.body, @@ -110,7 +101,7 @@ interface UpstreamAnswer { readonly billable: readonly BillableEntity[]; /** Every arm hands one up, this stage's own reading included: the record holds a body as a * stream, and `failover` releases the losing attempts' by consuming that key. */ - readonly body: ReadableStream & Owned; + readonly body: (ReadableStream & Owned) | null; } /** What the upstream said, on the shape the request asked for. Which of the four this is is @@ -122,24 +113,28 @@ const readUpstream = async ( candidate: ModelCandidate, signal: AbortSignal | undefined, dump: TurnDump | null, + ownedBody: (ReadableStream & Owned) | null, + wasCalled: boolean, ): Promise => { // An upstream was called and reported nothing — which is what every arm but a read usage // block leaves standing, and is a different statement from reporting zero. - const called: readonly BillableEntity[] = [{ identity, quantities: {} }]; + const called: readonly BillableEntity[] = wasCalled ? [{ identity, quantities: {} }] : []; if (!response.ok) { // An upstream error body is JSON like any other body. Reading it here is also what // leaves a losing attempt with nothing open behind it. - return { payload: await refusal(response), streamedUsage: null, billable: called, body: spentBody(response.body) }; + spentBody(ownedBody); + return { payload: await refusal(response), streamedUsage: null, billable: called, body: spentBody(ownedBody) }; } if (!wantsStream) { + spentBody(ownedBody); const read = await readResult(response); return { payload: isFailure(read) ? read : read.value, streamedUsage: null, - billable: isFailure(read) ? called : [{ identity, quantities: billed(read.value.usage, read.value.service_tier, candidate) }], - body: spentBody(response.body), + billable: isFailure(read) || !wasCalled ? called : [{ identity, ...billed(read.value.usage, read.value.service_tier, candidate) }], + body: spentBody(ownedBody), }; } @@ -157,22 +152,17 @@ const readUpstream = async ( // what drives it. What it finds arrives with the last frame, long after this stage has // handed up, which is why the entity above carries no quantities. const metered = meterFrames(parseOpenAICompletionsStream(response.body, { signal }), identity, candidate); + setRelease(ownedBody!, async () => { for await (const _frame of metered.frames) { /* drain */ } }); return { payload: recordStream(metered.frames, dump), streamedUsage: metered.outcome, billable: called, // Releasing this body is draining those frames: they are one reader over one connection, // and a second reader is not something a `ReadableStream` allows. - body: own(response.body, async (): Promise => { for await (const _frame of metered.frames) { /* to end of stream */ } }), + body: ownedBody, }; }; -/** A body this stage has already read to the end, or one the upstream never sent. The record - * holds a body as a stream and `failover` releases the losing attempts', so every path hands - * one up; what says an answer was unusable is the failure at the payload key, not this. */ -const spentBody = (body: ReadableStream | null): ReadableStream & Owned => - own(body ?? new ReadableStream({ start: controller => controller.close() }), (): Promise => Promise.resolve()); - const refusal = async (response: Response): Promise => { const text = await response.text(); try { @@ -232,13 +222,13 @@ const meterFrames = ( // Reached however the frames ended — the terminal chunk, a client that stopped // reading, or a broken upstream — because tokens the upstream already metered are // billable whatever happened to the downstream half. - settle({ billable: [{ identity, quantities: billed(usage, tier, candidate) }], failed: !sawTerminal }); + settle({ billable: [{ identity, ...billed(usage, tier, candidate) }], failed: !sawTerminal }); } })()); return { frames, outcome }; }; -const billed = (usage: unknown, tier: string | null | undefined, candidate: ModelCandidate): UsageQuantities => { +const billed = (usage: unknown, tier: string | null | undefined, candidate: ModelCandidate): Pick => { const model = providerModelOf(candidate); const tokens = tokenUsageFromOpenAICompletionsUsage( usage, @@ -246,11 +236,6 @@ const billed = (usage: unknown, tier: string | null | undefined, candidate: Mode model.enabledFlags.has('usage-exclusive-cached-tokens'), `${candidate.provider.upstreamId}/${model.id}`, ); - // An upstream that reported nothing leaves no quantities at all, which is a different - // statement from reporting zero. - // - // The service tier survives as far as `TokenUsage.tier` and no further: a billed entity is - // an identity and a bag of quantities, and the pricing selector the tier feeds has no seat - // there. It comes back when settlement does. - return tokens === null ? {} : tokenUsageQuantities(tokens); + const measurement = tokenUsageMeasurement(tokens); + return { quantities: measurement.quantities, pricingFacts: measurement.pricingFacts }; }; diff --git a/packages/gateway/src/data-plane/openai-completions/http.ts b/packages/gateway/src/data-plane/openai-completions/http.ts index 596c0931ea..cd0a08d590 100644 --- a/packages/gateway/src/data-plane/openai-completions/http.ts +++ b/packages/gateway/src/data-plane/openai-completions/http.ts @@ -50,6 +50,5 @@ export const openaiCompletions = async (c: Context): Promise => { ? { frames: rendered } : { body: JSON.stringify(rendered), contentType: 'application/json' }; }, - facts => facts['response.openaiCompletions.streamedUsage'], ); }; diff --git a/packages/gateway/src/data-plane/openai-completions/pipeline.ts b/packages/gateway/src/data-plane/openai-completions/pipeline.ts index 993ae8fba2..10c629eccd 100644 --- a/packages/gateway/src/data-plane/openai-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-completions/pipeline.ts @@ -12,15 +12,13 @@ export const openaiCompletionsServePipeline: Pipeline< Fields<'ingress.http.headers' | 'ingress.openaiCompletions.wantsStream' | 'ingress.openaiCompletions.wantsUsageChunk' | 'request.openaiCompletions.payload' | 'serve.model'>, Fields<'response.openaiCompletions.rendered' | 'response.openaiCompletions.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> > = compose('openaiCompletionsServe', [ + writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, 'response.openaiCompletions.streamedUsage'), emitOpenAICompletions, - writeSettlement( - handedUp => isFailure((handedUp as { 'response.openaiCompletions.payload'?: unknown })['response.openaiCompletions.payload']), - handedUp => (handedUp as { 'response.openaiCompletions.streamedUsage'?: unknown })['response.openaiCompletions.streamedUsage'] !== null, - ), resolveCandidates(narrowing), failover({ failed: handedUp => isFailure((handedUp as { 'response.openaiCompletions.payload'?: unknown })['response.openaiCompletions.payload']), owns: ['response.http.body'], + pendingUsage: 'response.openaiCompletions.streamedUsage', }), callOpenAICompletionsUpstream, ]); diff --git a/packages/gateway/src/data-plane/openai-completions/usage.ts b/packages/gateway/src/data-plane/openai-completions/usage.ts index f3d4332aa5..ab99173838 100644 --- a/packages/gateway/src/data-plane/openai-completions/usage.ts +++ b/packages/gateway/src/data-plane/openai-completions/usage.ts @@ -1,5 +1,5 @@ import type { TokenUsage } from '../../repo/types.ts'; -import { foldsExclusiveCacheTokens, openAICacheTokensFromUsage, tokenUsage } from '../shared/telemetry/usage.ts'; +import { foldsExclusiveCacheTokens, openAICacheTokensFromUsage } from '../shared/telemetry/usage.ts'; import { billableServiceTier, splitInclusiveInputTokens } from '@floway-dev/protocols/common'; // `/v1/completions` shares OpenAI's CompletionUsage schema with @@ -43,11 +43,11 @@ export const tokenUsageFromOpenAICompletionsUsage = ( cacheWrite, }, identity); const split = splitInclusiveInputTokens(fold ? promptTokens + cacheRead + cacheWrite : promptTokens, cacheRead, cacheWrite); - return tokenUsage({ + return { input: split.input, input_cache_read: split.cacheRead, input_cache_write: split.cacheWrite, output: completionTokens, tier: billableServiceTier(serviceTier), - }); + }; }; diff --git a/packages/gateway/src/data-plane/openai-embeddings/call-upstream.ts b/packages/gateway/src/data-plane/openai-embeddings/call-upstream.ts index 0f89029e24..04f4b75e3b 100644 --- a/packages/gateway/src/data-plane/openai-embeddings/call-upstream.ts +++ b/packages/gateway/src/data-plane/openai-embeddings/call-upstream.ts @@ -1,14 +1,16 @@ import type { Fields } from './facts.ts'; import type { UsageQuantities } from '../../repo/types.ts'; import type { Failure } from '../pipeline/facts.ts'; +import { providerEntry } from '../pipeline/provider-entry.ts'; +import { providerUsage } from '../pipeline/provider-usage.ts'; import type { GatewayServices } from '../pipeline/services.ts'; import { dialFailure, readUpstreamBody, unreadableBody } from '../pipeline/upstream-body.ts'; import { upstreamPerformanceContext, telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; -import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; -import { defineStage, move } from '@floway-dev/pipeline'; +import { exchangeResponse } from '@floway-dev/http/pipeline'; +import { defineStage, move, setRelease } from '@floway-dev/pipeline'; import { upstreamErrorMessage, parseDecimalString } from '@floway-dev/protocols/common'; import { serializeOpenAIEmbeddingsRequest, parseOpenAIEmbeddingsResponse, type CanonicalOpenAIEmbeddingsResponse, type CanonicalOpenAIEmbeddingsUsage } from '@floway-dev/protocols/openai-embeddings'; -import { providerModelOf } from '@floway-dev/provider'; +import { type ProviderRequest, type ProviderResponse, type ProviderOperationPayloads } from '@floway-dev/provider'; /** * The ending. It dials, reads the upstream's body, and provides the canonical answer and @@ -17,55 +19,51 @@ import { providerModelOf } from '@floway-dev/provider'; */ export const callOpenAIEmbeddingsUpstream = defineStage< Fields<'request.openaiEmbeddings.canonical' | 'route.attempt' | 'ingress.http.headers' | 'serve.model'>, - Fields<'response.openaiEmbeddings.canonical' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'>, + ProviderRequest, + ProviderResponse, + Fields<'response.openaiEmbeddings.canonical' | 'response.http.status' | 'response.http.headers' | 'response.http.body' | 'response.usage.billable'>, GatewayServices >({ name: 'callOpenAIEmbeddingsUpstream', - return: { - provides: ['response.openaiEmbeddings.canonical', 'response.http.status', 'response.http.headers', 'response.usage.billable'], + into: { + request: { needs: ['request.openaiEmbeddings.canonical', 'route.attempt', 'ingress.http.headers', 'serve.model'], consumes: [], provides: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'] }, + response: { needs: ['response.http.exchange', 'response.provider.modelKey', 'response.provider.called'], consumes: ['response.http.exchange', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'], provides: ['response.openaiEmbeddings.canonical', 'response.http.status', 'response.http.headers', 'response.http.body', 'response.usage.billable'] }, }, - execute: async (facts, use) => { + execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); // Attribution is set before the dial, so an attempt that never completes still names the // candidate it was made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'embeddings'); - let result; - try { - result = await candidate.provider.instance.callOpenAIEmbeddings( - providerModelOf(candidate), - serializeOpenAIEmbeddingsRequest(facts['request.openaiEmbeddings.canonical']), - use.gateway.abortSignal, - // The client's own headers reach the upstream from the record, not from a live - // request object: what a provider is allowed to forward is filtered per provider, - // and the dump shows what was there to filter. - buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), - ); - } catch (error) { - use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); - // A dial that never completed reached no upstream, so nothing was billed and there are - // no headers to carry. What it leaves behind is the performance row settlement writes. + const pipeline = candidate.provider.pipelines.openaiEmbeddings; + if (pipeline === undefined) throw new Error(`Provider ${candidate.provider.kind} has no OpenAI Embeddings pipeline`); + const back = await next(providerEntry(facts, candidate, serializeOpenAIEmbeddingsRequest(facts['request.openaiEmbeddings.canonical'])), pipeline); + const { 'response.http.exchange': exchange, 'response.provider.modelKey': modelKey, 'response.provider.called': called, 'response.provider.previousCalls': _previousCalls, ...rest } = back; + if (exchange.type === 'transportFailure') { return move({ - ...facts, - 'response.openaiEmbeddings.canonical': dialFailure(error), + ...rest, + 'response.openaiEmbeddings.canonical': dialFailure(exchange.error), 'response.http.status': 502, 'response.http.headers': [], - 'response.usage.billable': [], + 'response.usage.billable': providerUsage(candidate, back, []), }); } + const result = { response: exchangeResponse(exchange), modelKey }; const identity = telemetryModelIdentity(candidate, result.modelKey); // What came back, unfiltered: the edge is where a client's view of it is decided. const headers = [...result.response.headers]; - const body = await readUpstreamBody(result.response); + const body = await readUpstreamBody(result.response).finally(() => { + if (exchange.body !== null) setRelease(exchange.body, async () => {}); + }); // The upstream was called and reported nothing, which is a different situation from // reporting zero — so the entity is present with no quantities. const answered = (canonical: CanonicalOpenAIEmbeddingsResponse | Failure, quantities: UsageQuantities) => move({ - ...facts, + ...rest, 'response.openaiEmbeddings.canonical': canonical, 'response.http.status': result.response.status, 'response.http.headers': headers, - 'response.usage.billable': [{ identity, quantities }], + 'response.usage.billable': providerUsage(candidate, back, called ? [{ identity, quantities }] : []), }); const reportedNothing: UsageQuantities = {}; diff --git a/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts b/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts index 7ee6f970fa..b0f928c63d 100644 --- a/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts @@ -12,12 +12,12 @@ export const openaiEmbeddingsServePipeline: Pipeline< Fields<'ingress.http.headers' | 'ingress.openaiEmbeddings.encodingFormat' | 'request.openaiEmbeddings.canonical' | 'serve.model'>, Fields<'response.openaiEmbeddings.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> > = compose('openaiEmbeddingsServe', [ + writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400), emitOpenAIEmbeddings, - writeSettlement(handedUp => isFailure((handedUp as { 'response.openaiEmbeddings.canonical'?: unknown })['response.openaiEmbeddings.canonical'])), resolveCandidates(narrowing), failover({ failed: handedUp => isFailure((handedUp as { 'response.openaiEmbeddings.canonical'?: unknown })['response.openaiEmbeddings.canonical']), - owns: [], + owns: ['response.http.body'], }), callOpenAIEmbeddingsUpstream, ]); diff --git a/packages/gateway/src/data-plane/openai-images/call-upstream.ts b/packages/gateway/src/data-plane/openai-images/call-upstream.ts index b434f13c6c..9affa0dc96 100644 --- a/packages/gateway/src/data-plane/openai-images/call-upstream.ts +++ b/packages/gateway/src/data-plane/openai-images/call-upstream.ts @@ -2,15 +2,17 @@ import type { Fields, OpenAIImagesFrames } from './facts.ts'; import { recordStream } from '../../dump/turn-dump.ts'; import type { UsageQuantities } from '../../repo/types.ts'; import type { BillableEntity, Failure } from '../pipeline/facts.ts'; +import { providerEntry } from '../pipeline/provider-entry.ts'; +import { providerUsage } from '../pipeline/provider-usage.ts'; import type { StreamOutcome } from '../pipeline/serve.ts'; import type { GatewayServices } from '../pipeline/services.ts'; -import { dialFailure, readUpstreamBody } from '../pipeline/upstream-body.ts'; +import { dialFailure, readUpstreamBody, spentBody } from '../pipeline/upstream-body.ts'; import { upstreamPerformanceContext, telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; -import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; -import { defineStage, move, own, defer, type Owned, type Deferred } from '@floway-dev/pipeline'; +import { exchangeResponse } from '@floway-dev/http/pipeline'; +import { defineStage, move, setRelease, defer, type Deferred } from '@floway-dev/pipeline'; import { upstreamErrorMessage, isEventStreamMediaType, eventFrame, mediaTypeEssence, parseDecimalString } from '@floway-dev/protocols/common'; import { parseOpenAIImagesResponse, parseOpenAIImagesStream, isOpenAIImagesTerminalEvent, parseOpenAIImagesUsage, OPENAI_IMAGES_MISSING_TERMINAL_MESSAGE, type OpenAIImagesOperation, type CanonicalOpenAIImagesResponse, type CanonicalOpenAIImagesUsage, type CanonicalOpenAIImagesEditsRequest, type OpenAIImagesEditImage, type OpenAIImageEditReference } from '@floway-dev/protocols/openai-images'; -import { providerModelOf, isBase64ImageDataUrl, type PerformanceOperation, type TelemetryModelIdentity, type OpenAIImagesEditsRequest, type OpenAIImagesEditsSource } from '@floway-dev/provider'; +import { isBase64ImageDataUrl, type ProviderRequest, type ProviderResponse, type ProviderOperationPayloads, type PerformanceOperation, type TelemetryModelIdentity, type OpenAIImagesEditsRequest, type OpenAIImagesEditsSource } from '@floway-dev/provider'; /** A stream as a value the record can hold, as a wrapper around the generator rather than the * generator itself. What the wrapper says is where the resource is: the one resource in an @@ -32,81 +34,62 @@ const PERFORMANCE_OPERATION = { */ export const callOpenAIImagesUpstream = defineStage< Fields<'ingress.openaiImages.wantsStream' | 'request.openaiImages.canonical' | 'route.attempt' | 'ingress.http.headers'>, + ProviderRequest, + ProviderResponse, Fields<'response.openaiImages.canonical' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.http.body' | 'response.usage.billable'>, GatewayServices >({ name: 'callOpenAIImagesUpstream', - return: { - provides: [ - 'response.openaiImages.canonical', - 'response.openaiImages.streamedUsage', - 'response.http.status', - 'response.http.headers', - 'response.http.body', - 'response.usage.billable', - ], + into: { + request: { needs: ['ingress.openaiImages.wantsStream', 'request.openaiImages.canonical', 'route.attempt', 'ingress.http.headers'], consumes: [], provides: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'] }, + response: { needs: ['response.http.exchange', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.http.body'], consumes: ['response.http.exchange', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'], provides: ['response.openaiImages.canonical', 'response.openaiImages.streamedUsage', 'response.http.status', 'response.http.headers', 'response.http.body', 'response.usage.billable'] }, }, - execute: async (facts, use) => { + execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); const request = facts['request.openaiImages.canonical']; - const options = buildUpstreamCallOptions( - candidate, - use.gateway, - // The client's own headers reach the upstream from the record, not from a live request - // object: what a provider is allowed to forward is filtered per provider, and the dump - // shows what was there to filter. - new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value])), - ); - const model = providerModelOf(candidate); // Attribution is set before the dial, so an attempt that never completes still names the // candidate it was made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, PERFORMANCE_OPERATION[request.operation]); - let result; - try { - // No abort signal, as on this family's endpoints from the beginning: an image the upstream - // has already begun is charged for whether or not the client waited for it, so dropping the - // call would lose the usage reading and save nothing. Reading a stream is the other half of - // that and does take the signal — what it drops there is a connection, not an image. - result = request.operation === 'generations' - ? await candidate.provider.instance.callOpenAIImagesGenerations(model, request.parameters, undefined, options) - : await candidate.provider.instance.callOpenAIImagesEdits(model, providerEditsRequest(request), undefined, options); - } catch (error) { - use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); - // A dial that never completed reached no upstream, so nothing was billed and there are - // no headers to carry. What it leaves behind is the performance row settlement writes. + const back = request.operation === 'generations' + ? await next(providerEntry(facts, candidate, request.parameters), candidate.provider.pipelines.openaiImagesGenerations!) + : await next(providerEntry(facts, candidate, providerEditsRequest(request)), candidate.provider.pipelines.openaiImagesEdits!); + const { 'response.http.exchange': exchange, 'response.provider.modelKey': modelKey, 'response.provider.called': wasCalled, 'response.provider.previousCalls': _previousCalls, ...rest } = back; + if (exchange.type === 'transportFailure') { return move({ - ...facts, - 'response.openaiImages.canonical': dialFailure(error), + ...rest, + 'response.openaiImages.canonical': dialFailure(exchange.error), 'response.openaiImages.streamedUsage': null, 'response.http.status': 502, 'response.http.headers': [], 'response.http.body': spentBody(null), - 'response.usage.billable': [], + 'response.usage.billable': providerUsage(candidate, back, []), }); } + const result = { response: exchangeResponse(exchange), modelKey }; const identity = telemetryModelIdentity(candidate, result.modelKey); // What came back, unfiltered: the edge is where a client's view of it is decided. const headers = [...result.response.headers]; // An entity with no quantities is how "the upstream was called and reported nothing" is // said, which is a different situation from reporting zero. - const called: readonly BillableEntity[] = [{ identity, quantities: {} }]; + const called: readonly BillableEntity[] = wasCalled ? [{ identity, quantities: {} }] : []; const read = (canonical: CanonicalOpenAIImagesResponse | Failure, billable: readonly BillableEntity[]) => move({ - ...facts, + ...rest, 'response.openaiImages.canonical': canonical, 'response.openaiImages.streamedUsage': null, 'response.http.status': result.response.status, 'response.http.headers': headers, - 'response.http.body': spentBody(result.response.body), - 'response.usage.billable': billable, + 'response.http.body': spentBody(exchange.body), + 'response.usage.billable': providerUsage(candidate, back, billable), }); if (!result.response.ok) { use.log.warn('upstream refused', { status: result.response.status }); // An upstream error body is JSON like any other body. Reading it here is also what // leaves a losing attempt with nothing open behind it. + spentBody(exchange.body); const body = await readUpstreamBody(result.response); return read({ status: result.response.status, @@ -129,21 +112,23 @@ export const callOpenAIImagesUpstream = defineStage< // stream is what drives it. What it finds arrives with the completed event, long after // this stage has handed up, which is why the entity above carries no quantities. const metered = meterFrames(result.response.body, identity, use.gateway.abortSignal); + setRelease(exchange.body!, async () => { for await (const _event of metered.frames) { /* drain */ } }); return move({ - ...facts, + ...rest, // This protocol's stream is bare events rather than protocol frames, so the record is // told how one becomes a frame instead of being left to assume. 'response.openaiImages.canonical': recordStream(metered.frames, use.gateway.dump, eventFrame), - 'response.openaiImages.streamedUsage': metered.outcome, + 'response.openaiImages.streamedUsage': defer(metered.outcome.then(outcome => ({ ...outcome, billable: providerUsage(candidate, back, outcome.billable) }))), 'response.http.status': result.response.status, 'response.http.headers': headers, // Releasing this body is reading those events to the end: they are one reader over one // connection, and a second reader is not something a `ReadableStream` allows. - 'response.http.body': own(result.response.body, async (): Promise => { for await (const _event of metered.frames) { /* to end of stream */ } }), - 'response.usage.billable': called, + 'response.http.body': exchange.body, + 'response.usage.billable': providerUsage(candidate, back, called), }); } + spentBody(exchange.body); const body = await readUpstreamBody(result.response); if (!('json' in body)) { // Every protocol the gateway carries is one it fully understands, so a body it cannot @@ -165,16 +150,10 @@ export const callOpenAIImagesUpstream = defineStage< use.log.warn('upstream answered with a body the OpenAI Images protocol cannot read', { message }); return read({ status: 502, message, body: body.json }, called); } - return read(canonical, [{ identity, quantities: billed(canonical.usage) }]); + return read(canonical, wasCalled ? [{ identity, quantities: billed(canonical.usage) }] : []); }, }); -/** A body this stage has already read to the end, or one the upstream never sent. The record - * holds a body as a stream and `failover` releases the losing attempts', so every path hands - * one up; what says an answer was unusable is the failure at the canonical key, not this. */ -const spentBody = (body: ReadableStream | null): ReadableStream & Owned => - own(body ?? new ReadableStream({ start: controller => controller.close() }), (): Promise => Promise.resolve()); - interface MeteredFrames { readonly frames: OpenAIImagesFrames; readonly outcome: Deferred; @@ -246,7 +225,7 @@ const providerEditsRequest = (request: CanonicalOpenAIImagesEditsRequest): OpenA const providerEditsSource = (image: OpenAIImagesEditImage): OpenAIImagesEditsSource => { if (image.kind === 'file') { - return { type: 'upload', file: new File([image.file.bytes], image.file.fileName, { type: image.file.mediaType }) }; + return { type: 'upload', file: { bytes: image.file.bytes, name: image.file.fileName, type: image.file.mediaType } }; } const { reference } = image; return typeof reference.image_url === 'string' && isBase64ImageDataUrl(reference.image_url) diff --git a/packages/gateway/src/data-plane/openai-images/http.ts b/packages/gateway/src/data-plane/openai-images/http.ts index 6ecac99d63..68fff7fbd4 100644 --- a/packages/gateway/src/data-plane/openai-images/http.ts +++ b/packages/gateway/src/data-plane/openai-images/http.ts @@ -58,7 +58,6 @@ const serveOpenAIImages = async ( const answer = facts['response.openaiImages.rendered']; return isFrames(answer) ? { frames: answer } : { body: JSON.stringify(answer), contentType: 'application/json' }; }, - facts => facts['response.openaiImages.streamedUsage'], ); }; diff --git a/packages/gateway/src/data-plane/openai-images/pipeline.ts b/packages/gateway/src/data-plane/openai-images/pipeline.ts index 5bce6d5af8..56eea13e67 100644 --- a/packages/gateway/src/data-plane/openai-images/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-images/pipeline.ts @@ -11,15 +11,13 @@ import type { CanonicalOpenAIImagesRequest } from '@floway-dev/protocols/openai- export const openaiImagesServePipeline = (request: CanonicalOpenAIImagesRequest): Pipeline => compose('openaiImagesServe', [ + writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, 'response.openaiImages.streamedUsage'), emitOpenAIImages, - writeSettlement( - handedUp => isFailure((handedUp as { 'response.openaiImages.canonical'?: unknown })['response.openaiImages.canonical']), - handedUp => (handedUp as { 'response.openaiImages.streamedUsage'?: unknown })['response.openaiImages.streamedUsage'] !== null, - ), resolveCandidates(narrowing(request)), failover({ failed: handedUp => isFailure((handedUp as { 'response.openaiImages.canonical'?: unknown })['response.openaiImages.canonical']), owns: ['response.http.body'], + pendingUsage: 'response.openaiImages.streamedUsage', }), callOpenAIImagesUpstream, ]); diff --git a/packages/gateway/src/data-plane/pipeline/candidates.ts b/packages/gateway/src/data-plane/pipeline/candidates.ts index d195850f32..40362d93c9 100644 --- a/packages/gateway/src/data-plane/pipeline/candidates.ts +++ b/packages/gateway/src/data-plane/pipeline/candidates.ts @@ -3,7 +3,13 @@ import { providerModelOf, type ModelCandidate } from '@floway-dev/provider'; export const createCandidateRegistry = () => { const live: ModelCandidate[] = []; + const resolveCandidate = (candidateId: number): ModelCandidate => { + const candidate = live[candidateId]; + if (candidate === undefined) throw new Error(`resolveAttempt: nothing live for candidate ${candidateId}; the selector did not come from this run`); + return candidate; + }; return { + resolveCandidate, rememberCandidates: (candidates: readonly ModelCandidate[]): readonly AttemptSelector[] => candidates.map(candidate => { const candidateId = live.length; live.push(candidate); @@ -15,9 +21,7 @@ export const createCandidateRegistry = () => { }; }), resolveAttempt: (selector: AttemptSelector): ModelCandidate => { - const candidate = live[selector.candidateId]; - if (candidate === undefined) throw new Error(`resolveAttempt: nothing live for candidate ${selector.candidateId}; the selector did not come from this run`); - return candidate; + return resolveCandidate(selector.candidateId); }, }; }; diff --git a/packages/gateway/src/data-plane/pipeline/facts.ts b/packages/gateway/src/data-plane/pipeline/facts.ts index da25b463cf..d00ca87005 100644 --- a/packages/gateway/src/data-plane/pipeline/facts.ts +++ b/packages/gateway/src/data-plane/pipeline/facts.ts @@ -85,9 +85,12 @@ export interface GatewayFacts { * rather than `AsyncDisposable`, because a structural type would say what the host happens * to mark rather than what this run answers for — and what a host marks differs between the * Node versions this ships on. Ownership is claimed, so the type says so too. */ - 'response.http.body': ReadableStream & Owned; + 'response.http.body': (ReadableStream & Owned) | null; /** The authoritative reading, provided closest to the upstream on the dialect it * actually spoke. Every step that changes usage re-provides it. */ 'response.usage.billable': readonly BillableEntity[]; + + /** Completed calls carried into a nested attempt so an exceptional exit can settle them. */ + 'serve.usage.prior': readonly BillableEntity[]; } diff --git a/packages/gateway/src/data-plane/pipeline/failover.ts b/packages/gateway/src/data-plane/pipeline/failover.ts index cc7193f1c5..dd85eb8e7c 100644 --- a/packages/gateway/src/data-plane/pipeline/failover.ts +++ b/packages/gateway/src/data-plane/pipeline/failover.ts @@ -1,6 +1,8 @@ +import type { BillableEntity } from './facts.ts'; +import type { StreamOutcome } from './serve.ts'; import type { GatewayServices } from './services.ts'; import type { Slice } from './stage-contracts.ts'; -import { defineStage, move, type Facts } from '@floway-dev/pipeline'; +import { defineStage, move, defer, type Deferred, type Facts } from '@floway-dev/pipeline'; /** * Runs what follows it once per candidate and returns the first that did not fail. @@ -23,40 +25,55 @@ export interface Forking { * produce and hands up makes it throw the other way. Which keys carry a resource is a * statement only the family can make. */ readonly owns: readonly string[]; + readonly pendingUsage?: string; } -export const failover = ({ failed, owns }: Forking) => defineStage< - Slice<'serve.candidates'>, - Slice<'serve.candidates' | 'route.attempt'>, +export const failover = ({ failed, owns, pendingUsage }: Forking) => defineStage< + Slice<'serve.candidates' | 'serve.usage.prior'>, + Slice<'serve.candidates' | 'route.attempt' | 'serve.usage.prior'>, Slice<'response.usage.billable'>, Slice<'response.usage.billable'>, GatewayServices >({ name: 'failover', through: { - request: { needs: ['serve.candidates'], consumes: [], provides: ['route.attempt'] }, + request: { needs: ['serve.candidates', 'serve.usage.prior'], consumes: [], provides: ['route.attempt', 'serve.usage.prior'] }, response: { - needs: ['response.usage.billable'], + needs: ['response.usage.billable', ...(pendingUsage === undefined ? [] : [pendingUsage])] as never, // Owned on the way up and handed onward: every attempt's is this stage's to release, // and the one it adopts rides up with ownership going with it. consumes: owns as never, - provides: owns as never, + provides: [...owns, 'response.usage.billable', ...(pendingUsage === undefined ? [] : [pendingUsage])] as never, }, }, execute: async (facts, next, use) => { let last: Slice<'response.usage.billable'> | undefined; + let observed: readonly BillableEntity[] = []; for (const candidate of facts['serve.candidates']) { // Per-attempt telemetry state, cleared before control leaves, so a mid-attempt throw // still attributes its performance row to the candidate that was being tried. use.gateway.attempt.timing.upstreamCallStartedAt = null; use.gateway.attempt.timing.firstOutputTokenAt = null; - last = await next({ ...facts, 'route.attempt': move(candidate) }); - if (!failed(last as Facts)) return last; + last = await next({ ...facts, 'route.attempt': move(candidate), 'serve.usage.prior': move([...facts['serve.usage.prior'], ...observed]) }); + const prior = observed; + observed = [...observed, ...last['response.usage.billable']]; + if (!failed(last as Facts)) { + if (prior.length === 0) return { ...last, 'serve.usage.prior': facts['serve.usage.prior'] }; + const pending = pendingUsage === undefined ? null : (last as Facts)[pendingUsage] as Deferred | null; + return { + ...last, + 'serve.usage.prior': facts['serve.usage.prior'], + 'response.usage.billable': move(observed), + ...(pending === null ? {} : { + [pendingUsage!]: move(defer(pending.then(outcome => ({ ...outcome, billable: [...prior, ...outcome.billable] })))), + }), + }; + } use.log.info('candidate failed, trying the next', { upstream: candidate.upstreamId }); } if (last === undefined) throw new Error('failover: assembly handed it an empty candidate list'); // Every candidate failed, and the last failure is the base — so the client sees real // upstream telemetry rather than a synthesized gateway envelope. - return last; + return { ...last, 'serve.usage.prior': facts['serve.usage.prior'], 'response.usage.billable': move(observed) }; }, }); diff --git a/packages/gateway/src/data-plane/pipeline/provider-entry.ts b/packages/gateway/src/data-plane/pipeline/provider-entry.ts new file mode 100644 index 0000000000..4ea7645f5c --- /dev/null +++ b/packages/gateway/src/data-plane/pipeline/provider-entry.ts @@ -0,0 +1,20 @@ +import type { AttemptSelector } from './facts.ts'; +import { filterInboundHeadersForProvider } from '../shared/inbound-headers.ts'; +import { move } from '@floway-dev/pipeline'; +import { providerModelFacts, providerModelOf, type ModelCandidate, type ProviderRequest } from '@floway-dev/provider'; + +export const providerEntry =

( + facts: { 'route.attempt': AttemptSelector; 'ingress.http.headers': readonly (readonly [string, string])[] }, + candidate: ModelCandidate, + payload: P, + forwardedHeaders: readonly (readonly [string, string])[] = facts['ingress.http.headers'], +): ProviderRequest

& Record => move({ + ...facts, + 'request.provider.model': providerModelFacts(providerModelOf(candidate)), + 'request.provider.payload': payload, + 'request.http.callId': facts['route.attempt'].candidateId, + 'request.http.headers': [...filterInboundHeadersForProvider( + new Headers(forwardedHeaders.map(([name, value]): [string, string] => [name, value])), + candidate.provider, + )], +}); diff --git a/packages/gateway/src/data-plane/pipeline/provider-usage.ts b/packages/gateway/src/data-plane/pipeline/provider-usage.ts new file mode 100644 index 0000000000..e96ef0556f --- /dev/null +++ b/packages/gateway/src/data-plane/pipeline/provider-usage.ts @@ -0,0 +1,25 @@ +import type { BillableEntity } from './facts.ts'; +import { telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; +import type { Facts } from '@floway-dev/pipeline'; +import type { ModelCandidate, ProviderResponse } from '@floway-dev/provider'; + +export const providerUsage = ( + candidate: ModelCandidate, + response: Pick, + finalCall: readonly BillableEntity[], +): readonly BillableEntity[] => { + const previous = response['response.provider.previousCalls']; + if (previous.length === 0) return finalCall; + return [ + ...previous.map(call => ({ identity: telemetryModelIdentity(candidate, call.modelKey), quantities: {} })), + ...finalCall, + ]; +}; + +export const providerCalls = (candidate: ModelCandidate, facts: Facts): readonly BillableEntity[] => { + if (!('response.provider.called' in facts)) return []; + const response = facts as unknown as ProviderResponse; + const current: readonly BillableEntity[] = response['response.provider.called'] + ? [{ identity: telemetryModelIdentity(candidate, response['response.provider.modelKey']), quantities: {} }] : []; + return providerUsage(candidate, response, current); +}; diff --git a/packages/gateway/src/data-plane/pipeline/serve.ts b/packages/gateway/src/data-plane/pipeline/serve.ts index a7e5ee6195..0f64a43a20 100644 --- a/packages/gateway/src/data-plane/pipeline/serve.ts +++ b/packages/gateway/src/data-plane/pipeline/serve.ts @@ -11,18 +11,18 @@ import { streamSSE } from 'hono/streaming'; import type { ContentfulStatusCode } from 'hono/utils/http-status'; import { createCandidateRegistry } from './candidates.ts'; -import type { BillableEntity, GatewayFacts } from './facts.ts'; +import type { AttemptSelector, BillableEntity, GatewayFacts } from './facts.ts'; import type { GatewayServices } from './services.ts'; -import { settleBillable } from './settlement.ts'; import { openRunDump } from '../../dump/run-sink.ts'; import { apiKeyFromContext, type AuthedContext } from '../../middleware/auth.ts'; import { internalErrorResponse } from '../../middleware/internal-error-response.ts'; import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; import { consoleLogSink } from '../../runtime/log.ts'; +import { stampUpstreamCallStart } from '../shared/attempt-timing.ts'; import { createGatewayCtxFromHono, finalizeGatewayResponse, type AttemptState, type GatewayCtx } from '../shared/gateway-ctx.ts'; import { readRequestBody, takeRequestBody, type RequestBody } from '../shared/request-body.ts'; import { writeSSEFrames } from '../shared/sse.ts'; -import { run, type Deferred, type Pipeline } from '@floway-dev/pipeline'; +import { run, getFailureFacts, type Pipeline } from '@floway-dev/pipeline'; import { sseCommentFrame, type SseFrame } from '@floway-dev/protocols/common'; type Slice = { [P in K]: GatewayFacts[P] }; @@ -89,6 +89,7 @@ export const openPrologue = ( backgroundScheduler, dump: runDump, }); + const candidates = createCandidateRegistry(); return { gateway, @@ -97,7 +98,16 @@ export const openPrologue = ( gateway, log: consoleLogSink, background: work => { gateway.backgroundScheduler(work); }, - ...createCandidateRegistry(), + ...candidates, + httpCall: callId => { + const candidate = candidates.resolveCandidate(callId); + return { + fetcher: candidate.fetcher, + signal: candidate.model.kind === 'image' ? undefined : gateway.abortSignal, + waitUntil: gateway.backgroundScheduler, + wrapUpstreamCall: stampUpstreamCallStart(gateway.attempt.timing), + }; + }, // Absent when this key has no retention configured, which is what keeps recording // conditional: the runner does none of it rather than doing it and discarding. ...(runDump === null ? {} : { dump: runDump.sink }), @@ -136,14 +146,6 @@ export interface StreamOutcome { readonly failed: boolean; } -/** What a streaming family will have been billed, and whether it got there. A stream's usage - * arrives with its last chunk, which is after the run has answered — so the run hands up a - * reading it has started and not finished, and settlement of it belongs here, after the answer - * is on its way. It is `Deferred` because that is what it is: the runner sees it in the record - * and waits for it at teardown, so a family that hands up a reading that never settles is - * reported rather than silently never billed. */ -export type DeferredUsage = (facts: Exit) => Deferred | null; - /** * Runs a family's pipeline and turns what it answered with into a response. * @@ -167,13 +169,17 @@ export const serveThrough = async < pipeline: Pipeline, entry: Entry, render: (facts: Exit) => Rendered, - deferredUsage?: DeferredUsage, ): Promise => { try { - return await serveRun(c, prologue, pipeline, entry, render, deferredUsage); + return await serveRun(c, prologue, pipeline, entry, render); } catch (error) { // `run` drains what it opened before it rethrows, so there is nothing left outstanding to // release here — only the record to close, and the reason to put on it. + const failureFacts = getFailureFacts(error); + if (failureFacts !== undefined && 'route.attempt' in failureFacts) { + const attempt = failureFacts['route.attempt'] as AttemptSelector; + prologue.gateway.dump?.error('gateway', attempt.upstreamId); + } prologue.gateway.dump?.failed(error); return finalizeGatewayResponse(prologue.gateway, internalErrorResponse(asError(error), c)); } @@ -193,22 +199,10 @@ const serveRun = async < pipeline: Pipeline, entry: Entry, render: (facts: Exit) => Rendered, - deferredUsage?: DeferredUsage, ): Promise => { const { facts, drain } = await run(pipeline, entry, prologue.services as never); const answer = render(facts); - const pending = deferredUsage?.(facts) ?? null; - // Registered while the request is still live, so the platform binds the write to it — and - // resolved only when the stream ends, which is the one moment both what it billed and - // whether it finished are known. A turn that stopped short is not recorded as one that - // produced what it said it would. - if (pending !== null) { - prologue.services.background(pending.then(outcome => { - settleBillable({ ...prologue.services, log: consoleLogSink }, outcome.billable, outcome.failed); - })); - } - const status = facts['response.http.status'] as ContentfulStatusCode; if ('frames' in answer) { // Hono's streamSSE builds the response itself, so what the client is to see has to be @@ -225,6 +219,9 @@ const serveRun = async < ? {} : { downstreamAbortController: prologue.gateway.downstreamAbortController }), }); + } catch (error) { + prologue.gateway.dump?.failed(error); + throw error; } finally { // Reading the frames to the client *is* releasing the body they came from, so the // drain waits for that to finish. Draining alongside it would take frames out of the diff --git a/packages/gateway/src/data-plane/pipeline/services.ts b/packages/gateway/src/data-plane/pipeline/services.ts index 9a429d8f49..b4d680c4c8 100644 --- a/packages/gateway/src/data-plane/pipeline/services.ts +++ b/packages/gateway/src/data-plane/pipeline/services.ts @@ -8,10 +8,11 @@ import type { AttemptSelector } from './facts.ts'; import type { GatewayCtx } from '../shared/gateway-ctx.ts'; +import type { HttpServices } from '@floway-dev/http/pipeline'; import type { Event, Logger } from '@floway-dev/pipeline'; import type { ModelCandidate } from '@floway-dev/provider'; -export interface GatewayServices { +export interface GatewayServices extends HttpServices { /** The global sink. Every stage's lines reach it, tagged with the stage's name. */ readonly log?: Logger; /** Present only when this request is being dumped, which is what keeps recording diff --git a/packages/gateway/src/data-plane/pipeline/settlement.ts b/packages/gateway/src/data-plane/pipeline/settlement.ts index 061112de92..74146421b1 100644 --- a/packages/gateway/src/data-plane/pipeline/settlement.ts +++ b/packages/gateway/src/data-plane/pipeline/settlement.ts @@ -1,29 +1,15 @@ -// Settlement, as a stage. It sits **above** the fork, so a run bills once however many -// candidates it tried — repetition passes through the stage that observes usage, and not -// through this one. -// -// It is unconditional: a run that measured rather than generated still writes, and its row -// simply names no billed entity. Emptiness is observed, not declared — which is why there -// is no "unknown" anywhere here. The situations are concrete and the list is open: the -// upstream reported zero, we did not call an upstream, we failed before reaching the -// upstream's usage, the upstream did not report. - -import type { BillableEntity, GatewayFacts } from './facts.ts'; +import type { BillableEntity, GatewayFacts, AttemptSelector } from './facts.ts'; +import { providerCalls } from './provider-usage.ts'; +import type { StreamOutcome } from './serve.ts'; import type { GatewayServices } from './services.ts'; import type { TokenUsage, UsageQuantities } from '../../repo/types.ts'; import { recordPerformance } from '../shared/telemetry/performance.ts'; import { recordUsage } from '../shared/telemetry/usage.ts'; -import { defineStage } from '@floway-dev/pipeline'; -import type { Logger } from '@floway-dev/pipeline'; +import { defineStage, getFailureFacts, move, type Logger, type Deferred, type Facts } from '@floway-dev/pipeline'; import type { BillingMetric } from '@floway-dev/protocols/common'; type Slice = { [P in K]: GatewayFacts[P] }; -/** The dump's own two columns, read back out of what was billed. - * - * The dump sums every input category into one number and keeps output separate, and it - * distinguishes "not measured" from "measured zero" — so an entity that reported nothing - * contributes no keys and stays null rather than becoming a zero the upstream never said. */ const dumpUsage = (quantities: UsageQuantities): TokenUsage => { const count = (metric: BillingMetric): number | undefined => quantities[metric] === undefined ? undefined : Number(quantities[metric]); @@ -31,23 +17,14 @@ const dumpUsage = (quantities: UsageQuantities): TokenUsage => { ...(count('input_tokens') === undefined ? {} : { input: count('input_tokens') }), ...(count('input_cache_read_tokens') === undefined ? {} : { input_cache_read: count('input_cache_read_tokens') }), ...(count('input_cache_write_tokens') === undefined ? {} : { input_cache_write: count('input_cache_write_tokens') }), + ...(count('input_cache_write_1h_tokens') === undefined ? {} : { input_cache_write_1h: count('input_cache_write_1h_tokens') }), ...(count('input_image_tokens') === undefined ? {} : { input_image: count('input_image_tokens') }), ...(count('output_tokens') === undefined ? {} : { output: count('output_tokens') }), ...(count('output_image_tokens') === undefined ? {} : { output_image: count('output_image_tokens') }), }; }; -/** - * Writes what was billed, and the performance sample that goes with it. - * - * The usage write is scheduled rather than awaited. A transient repository failure must not - * turn an upstream's already-flowing response into a 502, and the run's own answer does not - * depend on the row — so it is handed to the background scheduler, which binds it to the - * request's lifetime on both deployment targets. - * - * A streaming family settles through this too, from the prologue after the drain: its - * numbers arrive with the stream's last chunk, which is after the run has answered. - */ +/** Usage writes are bound to the request lifetime without delaying an upstream response. */ export const settleBillable = ( services: Pick & { readonly log: Logger }, billable: readonly BillableEntity[], @@ -61,48 +38,68 @@ export const settleBillable = ( entity.identity, entity.quantities, entity.pricingFacts ?? {}, - ).catch((error: unknown) => { - services.log.error('failed to record usage', { error: String(error) }); - })); + )); } - if (failed) services.gateway.dump?.failed('The upstream response failed.'); + if (failed) services.gateway.dump?.failed('The upstream response failed.', { fallback: true }); const outputTokens = billable.reduce((total, entity) => total + Number(entity.quantities.output_tokens ?? 0), 0); recordPerformance(services.gateway, services.gateway.attempt.telemetry, failed, outputTokens, finishedAt); }; -/** - * Prices what was observed and writes it, once per run. - * - * `stillReading` is how a streaming family says its numbers have not arrived yet: they come - * with the stream's last chunk, which is after this stage has handed up. Settling here as - * well would write the row twice — once for the entity that had reported nothing and once - * for what the stream turned out to say — so the run settles wherever the numbers are, and - * for a stream that is the epilogue. - */ +/** Settles a run once at its exit, including calls completed before a later stage throws. */ export const writeSettlement = ( failed: (handedUp: Record) => boolean, - stillReading?: (handedUp: Record) => boolean, + pendingUsage?: string, ) => defineStage< Record, - Record, - Slice<'response.usage.billable'>, + Slice<'serve.usage.prior'>, + Slice<'response.usage.billable' | 'serve.usage.prior'>, Slice<'response.usage.billable'>, GatewayServices >({ name: 'writeSettlement', through: { - // Nothing on the way down: what is settled is what came back, and naming a request key - // here would put it in the entry contract of a family that never resolved a model. - request: { needs: [], consumes: [], provides: [] }, - // It reads the authoritative reading and hands it on untouched: settlement is the last - // reader, not another writer, and a stage that changed usage re-provided it below. - response: { needs: ['response.usage.billable'], consumes: [], provides: [] }, + request: { needs: [], consumes: [], provides: ['serve.usage.prior'] }, + response: { needs: ['response.usage.billable'], consumes: ['serve.usage.prior'], provides: [] }, }, execute: async (facts, next, use) => { - const back = await next(facts); - if (stillReading?.(back as Record) === true) return back; - // One performance sample per run, attributed to the attempt that answered. - settleBillable(use, back['response.usage.billable'], failed(back as Record)); - return back; + const entry = { ...facts, 'serve.usage.prior': move([]) }; + let back: Slice<'response.usage.billable' | 'serve.usage.prior'>; + try { + back = await next(entry); + } catch (error) { + const atFailure = getFailureFacts(error) ?? entry; + settleExit(use, atFailure, true, pendingUsage); + throw error; + } + settleExit(use, back, failed(back as Facts), pendingUsage); + const { 'serve.usage.prior': _prior, ...rest } = back as Slice<'response.usage.billable' | 'serve.usage.prior'>; + return rest; }, }); + +const settleExit = ( + use: GatewayServices & { readonly log: Logger }, + facts: Facts, + failed: boolean, + pendingUsage: string | undefined, +): void => { + const pending = pendingUsage === undefined || !(pendingUsage in facts) + ? null : facts[pendingUsage] as Deferred | null; + if (pending !== null) { + use.background(pending.then(outcome => { + settleBillable(use, outcome.billable, failed || outcome.failed); + })); + return; + } + if ('response.usage.billable' in facts) { + settleBillable(use, facts['response.usage.billable'] as readonly BillableEntity[], failed); + return; + } + const prior = facts['serve.usage.prior'] as readonly BillableEntity[]; + if (!('response.provider.called' in facts)) { + settleBillable(use, prior, failed); + return; + } + const candidate = use.resolveAttempt(facts['route.attempt'] as AttemptSelector); + settleBillable(use, [...prior, ...providerCalls(candidate, facts)], failed); +}; diff --git a/packages/gateway/src/data-plane/pipeline/upstream-body.ts b/packages/gateway/src/data-plane/pipeline/upstream-body.ts index 84f5484345..6f6f3e1513 100644 --- a/packages/gateway/src/data-plane/pipeline/upstream-body.ts +++ b/packages/gateway/src/data-plane/pipeline/upstream-body.ts @@ -7,6 +7,7 @@ // upstream answered with something the protocol does not admit. import type { Failure } from './facts.ts'; +import { setRelease, type Owned } from '@floway-dev/pipeline'; export interface UpstreamBody { readonly text: string; @@ -35,6 +36,7 @@ export const readUpstreamBody = async (response: Response): Promise ({ status: 502, message: `The upstream answered ${response.status} with a body ${protocolName} cannot read: ${body.text.slice(0, 200)}`, + body: 'json' in body ? body.json : body.text, }); /** @@ -50,3 +52,8 @@ export const dialFailure = (error: unknown): Failure => ({ status: 502, message: error instanceof Error ? error.message : String(error), }); + +export const spentBody = (body: (ReadableStream & Owned) | null): typeof body => { + if (body !== null) setRelease(body, async () => {}); + return body; +}; diff --git a/packages/gateway/src/data-plane/rerank/call-upstream.ts b/packages/gateway/src/data-plane/rerank/call-upstream.ts index 59ee71306d..1c2a17b966 100644 --- a/packages/gateway/src/data-plane/rerank/call-upstream.ts +++ b/packages/gateway/src/data-plane/rerank/call-upstream.ts @@ -1,14 +1,17 @@ import type { Fields } from './facts.ts'; import type { UsageQuantities } from '../../repo/types.ts'; import type { BillableEntity } from '../pipeline/facts.ts'; +import { providerEntry } from '../pipeline/provider-entry.ts'; +import { providerUsage } from '../pipeline/provider-usage.ts'; import type { GatewayServices } from '../pipeline/services.ts'; -import { dialFailure, readUpstreamBody, unreadableBody } from '../pipeline/upstream-body.ts'; +import { dialFailure, readUpstreamBody, unreadableBody, spentBody } from '../pipeline/upstream-body.ts'; import { upstreamPerformanceContext, telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; -import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; +import { exchangeResponse } from '@floway-dev/http/pipeline'; import { defineStage, move } from '@floway-dev/pipeline'; import { upstreamErrorMessage, parseDecimalString } from '@floway-dev/protocols/common'; import { parseRerankUsage, parseRerankResponse, type CanonicalRerankResponse } from '@floway-dev/protocols/rerank'; import { providerModelOf } from '@floway-dev/provider'; +import type { ProviderRequest, ProviderOperationPayloads, ProviderRerankResponse } from '@floway-dev/provider'; /** * The ending. It dials, reads the upstream's body, and provides the canonical answer and @@ -17,14 +20,17 @@ import { providerModelOf } from '@floway-dev/provider'; */ export const callRerankUpstream = defineStage< Fields<'request.rerank.canonical' | 'route.attempt' | 'ingress.http.headers' | 'ingress.rerank.sourceProtocol'>, - Fields<'response.rerank.canonical' | 'response.rerank.targetProtocol' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'>, + ProviderRequest, + ProviderRerankResponse, + Fields<'response.rerank.canonical' | 'response.rerank.targetProtocol' | 'response.http.status' | 'response.http.headers' | 'response.http.body' | 'response.usage.billable'>, GatewayServices >({ name: 'callRerankUpstream', - return: { - provides: ['response.rerank.canonical', 'response.rerank.targetProtocol', 'response.http.status', 'response.http.headers', 'response.usage.billable'], + into: { + request: { needs: ['request.rerank.canonical', 'route.attempt', 'ingress.http.headers', 'ingress.rerank.sourceProtocol'], consumes: [], provides: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'] }, + response: { needs: ['response.http.exchange', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.http.body', 'response.provider.rerankTarget'], consumes: ['response.http.exchange', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.provider.previousCalls', 'response.provider.rerankTarget'], provides: ['response.rerank.canonical', 'response.rerank.targetProtocol', 'response.http.status', 'response.http.headers', 'response.http.body', 'response.usage.billable'] }, }, - execute: async (facts, use) => { + execute: async (facts, next, use) => { const candidate = use.resolveAttempt(facts['route.attempt']); const request = facts['request.rerank.canonical']; const model = providerModelOf(candidate); @@ -39,40 +45,32 @@ export const callRerankUpstream = defineStage< // candidate it was made against rather than the one tried before it. use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'rerank'); - let result; - try { - result = await candidate.provider.instance.callRerank( - model, - request, - use.gateway.abortSignal, - // The client's own headers reach the upstream from the record, not from a live - // request object: what a provider is allowed to forward is filtered per provider, - // and the dump shows what was there to filter. - buildUpstreamCallOptions(candidate, use.gateway, new Headers(facts['ingress.http.headers'].map(([name, value]): [string, string] => [name, value]))), - ); - } catch (error) { - use.log.warn('dial failed', { upstream: facts['route.attempt'].upstreamId, error: String(error) }); - // A dial that never completed reached no upstream, so nothing was billed and there are - // no headers to carry. What it leaves behind is the performance row settlement writes. + const pipeline = candidate.provider.pipelines.rerank; + if (pipeline === undefined) throw new Error(`Provider ${candidate.provider.kind} has no rerank pipeline`); + const back = await next(providerEntry(facts, candidate, request), pipeline); + const { 'response.http.exchange': exchange, 'response.provider.modelKey': modelKey, 'response.provider.called': wasCalled, 'response.provider.previousCalls': _previousCalls, 'response.provider.rerankTarget': target, ...rest } = back; + if (exchange.type === 'transportFailure') { return move({ - ...facts, - 'response.rerank.canonical': dialFailure(error), + ...rest, + 'response.rerank.canonical': dialFailure(exchange.error), 'response.rerank.targetProtocol': configuredTarget.protocol, 'response.http.status': 502, 'response.http.headers': [], - 'response.usage.billable': [], + 'response.usage.billable': providerUsage(candidate, back, []), }); } + const result = { response: exchangeResponse(exchange), modelKey, target }; const identity = telemetryModelIdentity(candidate, result.modelKey); // What came back, unfiltered: the edge is where a client's view of it is decided. const headers = [...result.response.headers]; + spentBody(exchange.body); const body = await readUpstreamBody(result.response); if (!result.response.ok) { use.log.warn('upstream refused', { status: result.response.status }); return move({ - ...facts, + ...rest, 'response.rerank.canonical': { status: result.response.status, message: upstreamErrorMessage(body.json) ?? body.text, @@ -83,18 +81,18 @@ export const callRerankUpstream = defineStage< 'response.http.headers': headers, // The upstream was called and reported nothing, which is a different situation // from reporting zero — so the entity is present with no quantities. - 'response.usage.billable': [{ identity, quantities: {} }], + 'response.usage.billable': providerUsage(candidate, back, wasCalled ? [{ identity, quantities: {} }] : []), }); } if (!('json' in body)) { return move({ - ...facts, + ...rest, 'response.rerank.canonical': unreadableBody(result.response, body, 'the rerank protocol'), 'response.rerank.targetProtocol': result.target.protocol, 'response.http.status': result.response.status, 'response.http.headers': headers, - 'response.usage.billable': [{ identity, quantities: {} }], + 'response.usage.billable': providerUsage(candidate, back, wasCalled ? [{ identity, quantities: {} }] : []), }); } @@ -129,12 +127,12 @@ export const callRerankUpstream = defineStage< if (translating) { use.log.warn('upstream answered with results the rerank protocol cannot read', { error: String(error) }); return move({ - ...facts, + ...rest, 'response.rerank.canonical': unreadableBody(result.response, body, 'the rerank protocol'), 'response.rerank.targetProtocol': result.target.protocol, 'response.http.status': result.response.status, 'response.http.headers': headers, - 'response.usage.billable': metered, + 'response.usage.billable': providerUsage(candidate, back, wasCalled ? metered : []), }); } use.log.debug('same-protocol answer carries results this gateway does not model', { error: String(error) }); @@ -142,12 +140,12 @@ export const callRerankUpstream = defineStage< } return move({ - ...facts, + ...rest, 'response.rerank.canonical': canonical, 'response.rerank.targetProtocol': result.target.protocol, 'response.http.status': result.response.status, 'response.http.headers': headers, - 'response.usage.billable': metered, + 'response.usage.billable': providerUsage(candidate, back, wasCalled ? metered : []), }); }, }); diff --git a/packages/gateway/src/data-plane/rerank/pipeline.ts b/packages/gateway/src/data-plane/rerank/pipeline.ts index 79bd32fd50..51c96432ac 100644 --- a/packages/gateway/src/data-plane/rerank/pipeline.ts +++ b/packages/gateway/src/data-plane/rerank/pipeline.ts @@ -13,12 +13,12 @@ export const rerankServePipeline = (request: CanonicalRerankRequest): Pipeline< Fields<'ingress.http.headers' | 'ingress.rerank.sourceProtocol' | 'request.rerank.canonical' | 'serve.model'>, Fields<'response.rerank.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> > => compose('rerankServe', [ + writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400), emitRerank, - writeSettlement(handedUp => isFailure((handedUp as { 'response.rerank.canonical'?: unknown })['response.rerank.canonical'])), resolveCandidates(narrowing(request)), failover({ failed: handedUp => isFailure((handedUp as { 'response.rerank.canonical'?: unknown })['response.rerank.canonical']), - owns: [], + owns: ['response.http.body'], }), callRerankUpstream, ]); diff --git a/packages/gateway/src/dump/attribution.ts b/packages/gateway/src/dump/attribution.ts index 0a32bc255e..37143d681e 100644 --- a/packages/gateway/src/dump/attribution.ts +++ b/packages/gateway/src/dump/attribution.ts @@ -44,11 +44,10 @@ export const oneLineError = (err: unknown): string => { // and cache writes; sum the present ones onto the dump's single inputTokens // column. Missing categories stay null (not measured) instead of zero so a // recorded zero genuinely means "upstream said zero". -const tokenUsageInput = (usage: TokenUsage | null): number | null => { - if (!usage) return null; - const { input, input_cache_read, input_cache_write } = usage; - if (input === undefined && input_cache_read === undefined && input_cache_write === undefined) return null; - return (input ?? 0) + (input_cache_read ?? 0) + (input_cache_write ?? 0); +const tokenTotal = (usage: TokenUsage | null, keys: readonly (keyof TokenUsage)[]): number | null => { + if (usage === null) return null; + const present = keys.map(key => usage[key]).filter((value): value is number => typeof value === 'number'); + return present.length === 0 ? null : present.reduce((sum, value) => sum + value, 0); }; const resolveUpstreamRef = async (id: string | null): Promise => { @@ -90,6 +89,7 @@ export class DumpAttribution { private inputTokens: number | null = null; private outputTokens: number | null = null; private errorMeta: DumpErrorMeta | null = null; + private settlementError: DumpErrorMeta | null = null; requestedModel(model: string): void { this.model = model; @@ -100,16 +100,19 @@ export class DumpAttribution { if (upstream !== undefined) this.upstreamId = upstream; } - failed(reason: unknown): void { - this.errorMeta = { kind: 'failed', reason: typeof reason === 'string' ? reason : oneLineError(reason) }; + failed(reason: unknown, options?: { readonly fallback: boolean }): void { + const error: DumpErrorMeta = { kind: 'failed', reason: typeof reason === 'string' ? reason : oneLineError(reason) }; + if (options?.fallback === true) this.settlementError = error; + else this.errorMeta = error; } success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void { this.model = identity.model; this.upstreamId = identity.upstream; - const input = tokenUsageInput(usage); + const input = tokenTotal(usage, ['input', 'input_cache_read', 'input_cache_write', 'input_cache_write_1h', 'input_image']); if (input !== null) this.inputTokens = (this.inputTokens ?? 0) + input; - if (usage?.output !== undefined) this.outputTokens = (this.outputTokens ?? 0) + usage.output; + const output = tokenTotal(usage, ['output', 'output_image']); + if (output !== null) this.outputTokens = (this.outputTokens ?? 0) + output; } async metadata(outcome: DumpTurnOutcome): Promise { @@ -128,7 +131,7 @@ export class DumpAttribution { responseBytes: outcome.responseBytes, durationMs: outcome.completedAt - outcome.startedAt, ttftMs: outcome.ttftMs, - error: this.errorMeta ?? outcome.fallbackError, + error: this.errorMeta ?? outcome.fallbackError ?? this.settlementError, }; } } diff --git a/packages/gateway/src/dump/run-sink.ts b/packages/gateway/src/dump/run-sink.ts index 4bd50d0629..3242b27801 100644 --- a/packages/gateway/src/dump/run-sink.ts +++ b/packages/gateway/src/dump/run-sink.ts @@ -67,8 +67,8 @@ export class RunDump { this.attribution.error(kind, upstream); } - failed(reason: unknown): void { - this.attribution.failed(reason); + failed(reason: unknown, options?: { readonly fallback: boolean }): void { + this.attribution.failed(reason, options); } /** diff --git a/packages/gateway/src/dump/turn-dump.ts b/packages/gateway/src/dump/turn-dump.ts index 2ce3277344..7be6061f4a 100644 --- a/packages/gateway/src/dump/turn-dump.ts +++ b/packages/gateway/src/dump/turn-dump.ts @@ -35,7 +35,7 @@ export interface TurnDump { requestedModel(model: string): void; success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void; error(kind: 'upstream' | 'gateway', upstream?: string): void; - failed(reason: unknown): void; + failed(reason: unknown, options?: { readonly fallback: boolean }): void; frame(frame: ProtocolFrame): void; /** Begins recording one stream. Every call is a new one, which is what lets a turn that * opens two — a sub-request beside the answer — keep them apart. */ From 86ca4ba6f4a0b814481bdfa395a48f9f3538a3d1 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 05:30:14 +0800 Subject: [PATCH 63/79] feat: inspect stage facts and changes in request records Build a stage tree from the shared object space, preserve repeated descent and folded response structure, and show request/response facts, identity-aware changes, deferred outcomes and structured logs. Keep raw events available on demand. Verified a production build in Chrome across light, dark and narrow layouts, including Monaco rendering and stage selection. --- README.md | 5 +- .../components/requests/run-model_test.ts | 54 ++++++++ .../components/requests/run-stages_test.tsx | 37 +++++ apps/web/package.json | 1 + apps/web/src/components/requests/detail.tsx | 14 +- apps/web/src/components/requests/run-model.ts | 129 ++++++++++++++++++ .../web/src/components/requests/run-render.ts | 1 + .../src/components/requests/run-stages.tsx | 62 +++++++++ apps/web/src/i18n/locales/en.ts | 8 ++ apps/web/src/i18n/locales/zh-Hans.ts | 8 ++ pnpm-lock.yaml | 3 + 11 files changed, 317 insertions(+), 5 deletions(-) create mode 100644 apps/web/__tests__/components/requests/run-model_test.ts create mode 100644 apps/web/__tests__/components/requests/run-stages_test.tsx create mode 100644 apps/web/src/components/requests/run-model.ts create mode 100644 apps/web/src/components/requests/run-stages.tsx diff --git a/README.md b/README.md index 538ece627b..77725d5599 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,10 @@ the upstream HTTP status. Billing records each actual upstream usage observation independently, including multiple model calls within one hosted-tool turn. Pricing uses each observation's -model identity and pricing facts. Request diagnostics sum the observed token +model identity and pricing facts. The request inspector shows the stage tree, both fact +directions, changes at each descent and return, and stage logs. Repeated objects retain +shared identities, while deferred outcomes and native content descriptions remain visible. +Request diagnostics sum the observed token quantities; performance uses the observed output count for TPOT and partial output failures. diff --git a/apps/web/__tests__/components/requests/run-model_test.ts b/apps/web/__tests__/components/requests/run-model_test.ts new file mode 100644 index 0000000000..7a33b12e32 --- /dev/null +++ b/apps/web/__tests__/components/requests/run-model_test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from 'vitest'; + +import { readRun } from '../../../src/components/requests/run-model'; + +describe('run stage model', () => { + it('restores inherited entries, repeated descents and folded returns', () => { + const model = readRun([ + { type: 'object', fromObjectId: 1, nodes: [{ text: 'before' }, { text: 'after' }] }, + { type: 'stage.entered', stageId: 1, name: 'fork', parentStageId: null, facts: { request: { $: 1 } } }, + { type: 'stage.entered', stageId: 2, name: 'first', parentStageId: 1 }, + { type: 'stage.leaved', stageId: 2, facts: { response: 'retry' } }, + { type: 'stage.entered', stageId: 3, name: 'second', parentStageId: 1, facts: { request: { $: 2 } } }, + { type: 'stage.leaved', stageId: 3, facts: { response: 'answer' } }, + ].map(event => JSON.stringify(event)).join('\n')); + const fork = model.roots[0]!; + expect(fork.children.map(stage => stage.id)).toEqual([2, 3]); + expect(fork.children[0]!.request).toBe(fork.request); + expect(model.state(fork.response!)).toEqual({ response: 'answer' }); + expect(model.diff(fork.request, fork.children[1]!.request)).toEqual([ + { path: ['request', 'text'], kind: 'changed', before: 'before', after: 'after' }, + ]); + }); + + it('uses shared object identities, handles cycles and decodes schema keys', () => { + const model = readRun([ + { type: 'object', fromObjectId: 1, nodes: [{ '$$ref': 'schema', self: { $: 1 } }, { '$$ref': 'next', self: { $: 2 } }] }, + { type: 'stage.entered', stageId: 1, name: 'stage', parentStageId: null, facts: { same: { $: 1 }, changed: { $: 1 } } }, + { type: 'stage.leaved', stageId: 1, facts: { same: { $: 1 }, changed: { $: 2 }, added: { $undefined: true } } }, + ].map(event => JSON.stringify(event)).join('\n')); + const stage = model.roots[0]!; + expect(model.state(stage.request)).toEqual({ + same: { $ref: 'schema', self: { $cycle: 1 } }, changed: { $ref: 'schema', self: { $cycle: 1 } }, + }); + expect(model.diff(stage.request, stage.response!)).toEqual([ + { path: ['changed', '$ref'], kind: 'changed', before: 'schema', after: 'next' }, + { path: ['added'], kind: 'added', after: { $undefined: true } }, + ]); + }); + it('reports both changed aliases and displays a late deferred outcome', () => { + const model = readRun([ + { type: 'object', fromObjectId: 1, nodes: [{ value: 'old' }, { value: 'new' }, { $deferred: true }, { status: 'fulfilled', value: { usage: 5 } }] }, + { type: 'stage.entered', stageId: 1, name: 'stage', parentStageId: null, facts: { first: { $: 1 }, second: { $: 1 } } }, + { type: 'stage.leaved', stageId: 1, facts: { first: { $: 2 }, second: { $: 2 }, pending: { $: 3 } } }, + { type: 'deferred.settled', deferred: { $: 3 }, outcome: { $: 4 } }, + ].map(event => JSON.stringify(event)).join('\n')); + const stage = model.roots[0]!; + expect(model.diff(stage.request, stage.response!).slice(0, 2)).toEqual([ + { path: ['first', 'value'], kind: 'changed', before: 'old', after: 'new' }, + { path: ['second', 'value'], kind: 'changed', before: 'old', after: 'new' }, + ]); + expect(model.state(stage.response!)).toMatchObject({ pending: { $deferred: { status: 'fulfilled', value: { usage: 5 } } } }); + }); + +}); diff --git a/apps/web/__tests__/components/requests/run-stages_test.tsx b/apps/web/__tests__/components/requests/run-stages_test.tsx new file mode 100644 index 0000000000..33ab1365cf --- /dev/null +++ b/apps/web/__tests__/components/requests/run-stages_test.tsx @@ -0,0 +1,37 @@ +import { fireEvent, screen, waitFor } from '@testing-library/react'; +import { describe, expect, it, vi } from 'vitest'; + +import { RunStages } from '../../../src/components/requests/run-stages'; +import { renderInApp } from '../../render'; + +vi.mock('../../../src/components/ui/body-editor', () => ({ + default: ({ text, toolbarStart }: { text: string; toolbarStart: React.ReactNode }) =>

{toolbarStart}
{text}
, +})); + +const ndjson = [ + { type: 'object', fromObjectId: 1, nodes: [{ text: 'before' }, { text: 'after' }] }, + { type: 'stage.entered', stageId: 1, name: 'source', parentStageId: null, facts: { request: { $: 1 } } }, + { type: 'stage.entered', stageId: 2, name: 'target', parentStageId: 1, facts: { request: { $: 2 } } }, + { type: 'stage.leaved', stageId: 2, facts: { response: 'answer' } }, +].map(event => JSON.stringify(event)).join('\n'); + +describe('run stages', () => { + it('selects a stage and restores its actual request values', async () => { + renderInApp(); + await waitFor(() => expect(screen.getByTestId('state').textContent).toContain('before')); + fireEvent.click(screen.getByRole('treeitem', { name: 'target #2' })); + await waitFor(() => expect(screen.getByTestId('state').textContent).toContain('after')); + }); + + it('compares the values on a descent and the folded response', async () => { + renderInApp(); + await screen.findByTestId('state'); + fireEvent.click(screen.getByRole('combobox', { name: 'Stage view' })); + fireEvent.click(screen.getByRole('option', { name: 'Request changes' })); + expect(screen.getByTestId('state').textContent).toContain('"before": "before"'); + expect(screen.getByTestId('state').textContent).toContain('"after": "after"'); + fireEvent.click(screen.getByRole('combobox', { name: 'Stage view' })); + fireEvent.click(screen.getByRole('option', { name: 'Response facts' })); + expect(screen.getByTestId('state').textContent).toContain('"response": "answer"'); + }); +}); diff --git a/apps/web/package.json b/apps/web/package.json index 61b54e97c2..d5941e3082 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -11,6 +11,7 @@ "typecheck": "react-router typegen && tsc --noEmit && tsc --noEmit -p tsconfig.scripts.json" }, "dependencies": { + "@floway-dev/pipeline": "workspace:*", "@floway-dev/protocols": "workspace:*", "@floway-dev/provider": "workspace:*", "@floway-dev/provider-custom": "workspace:*", diff --git a/apps/web/src/components/requests/detail.tsx b/apps/web/src/components/requests/detail.tsx index b6339b36d0..1dd1cbaa25 100644 --- a/apps/web/src/components/requests/detail.tsx +++ b/apps/web/src/components/requests/detail.tsx @@ -8,6 +8,7 @@ import { errorLabel, requestSeverity } from './format'; import { isSensitiveHeader, redactHeaderValue } from './header-redact'; import { redactRunHeaders } from './run-redact'; import { renderRunEvents } from './run-render'; +import { RunStages } from './run-stages'; import { collectKindFromTargetApi, detectCollectKind, type CollectedStream } from './stream-render'; import { fluentComponents } from '../../fluent'; import { useTranslation } from '../../i18n/translation'; @@ -179,21 +180,26 @@ function RecordTiming({ meta }: { meta: DumpMetadata }) { function RunRecordDetail({ record }: { record: DumpRunRecord }) { const { t } = useTranslation(); + const [view, setView] = useState('stages'); const redacted = useMemo(() => redactRunHeaders(record.events), [record.events]); - const events = useMemo(() => renderRunEvents(redacted).map(event => ({ + const count = useMemo(() => redacted.split('\n').filter(Boolean).length, [redacted]); + const events = useMemo(() => view === 'events' ? renderRunEvents(redacted).map(event => ({ event: `${event.type} ${event.subject ?? ''}`.trim(), text: event.text, parseError: event.parseError, - })), [redacted]); + })) : [], [redacted, view]); const failure = errorLabel(record.meta.error); return
- {t('dashboard.requests.run')} + setView(data.optionValue!)}> + + + {record.meta.status ?? t('dashboard.requests.noStatus')} } label={t('dashboard.requests.exportRecord')} onClick={() => downloadRecords([record])} />
{failure && {failure}}
- {t('dashboard.requests.events', { count: events.length })}} emptyText={t('dashboard.requests.noRunEvents')} /> + {view === 'stages' ? : {t('dashboard.requests.events', { count })}} emptyText={t('dashboard.requests.noRunEvents')} />}
; } diff --git a/apps/web/src/components/requests/run-model.ts b/apps/web/src/components/requests/run-model.ts new file mode 100644 index 0000000000..340a112906 --- /dev/null +++ b/apps/web/src/components/requests/run-model.ts @@ -0,0 +1,129 @@ +import { createRunReader, type DumpEvent } from '@floway-dev/pipeline'; + +type Stored = Extract['nodes'][number]; +type FactState = Readonly>; + +export interface RunStage { + readonly id: number; + readonly name: string; + readonly parentId: number | null; + readonly children: RunStage[]; + readonly request: FactState; + response: FactState | null; + readonly logs: Extract[]; +} + +export interface ValueChange { + readonly path: readonly (string | number)[]; + readonly kind: 'added' | 'removed' | 'changed'; + readonly before?: unknown; + readonly after?: unknown; +} + +const reference = (value: Stored): number | null => + typeof value === 'object' && value !== null && !Array.isArray(value) && '$' in value + ? value.$ as number : null; + +const decodeKey = (key: string): string => key.startsWith('$$') ? key.slice(1) : key; + +export const readRun = (ndjson: string) => { + const read = createRunReader(); + const stages = new Map(); + const outcomes = new Map(); + const roots: RunStage[] = []; + const stage = (id: number): RunStage => { + const found = stages.get(id); + if (found === undefined) throw new Error(`Run references missing stage ${id}`); + return found; + }; + const node = (id: number): Stored => { + const stored = read.node(id); + return outcomes.has(id) ? { $deferred: outcomes.get(id)! } : stored; + }; + for (const line of ndjson.split('\n').filter(Boolean)) { + const event = JSON.parse(line) as DumpEvent; + switch (event.type) { + case 'object': + read(event); + break; + case 'stage.entered': { + const parent = event.parentStageId === null ? null : stage(event.parentStageId); + const request = event.facts ?? parent?.request; + if (request === undefined) throw new Error(`Run root stage ${event.stageId} has no request facts`); + const entered: RunStage = { + id: event.stageId, name: event.name, parentId: event.parentStageId, + children: [], logs: [], request, response: null, + }; + stages.set(entered.id, entered); + (parent === null ? roots : parent.children).push(entered); + break; + } + case 'stage.leaved': stage(event.stageId).response = event.facts; break; + case 'stage.log': stage(event.stageId).logs.push(event); break; + case 'stream.frame': + case 'stream.end': break; + case 'deferred.settled': { + const id = reference(event.deferred); + if (id === null) throw new Error('Run deferred settlement has no object identity'); + outcomes.set(id, event.outcome); + break; + } + } + } + // Folded exits inherit the last descent's returned state, including repeated next(). + for (const current of [...stages.values()].reverse()) { + if (current.response === null && current.children.length > 0) { + current.response = current.children.at(-1)!.response; + } + } + + const value = (stored: Stored, ancestors: ReadonlySet = new Set()): unknown => { + const id = reference(stored); + if (id !== null) { + if (ancestors.has(id)) return { $cycle: id }; + return value(node(id), new Set([...ancestors, id])); + } + if (Array.isArray(stored)) return stored.map(child => value(child, ancestors)); + if (typeof stored !== 'object' || stored === null) return stored; + return Object.fromEntries(Object.entries(stored).map(([key, child]) => [decodeKey(key), value(child, ancestors)])); + }; + const state = (facts: FactState): unknown => value(facts); + + const diff = (before: FactState, after: FactState): ValueChange[] => { + const changes: ValueChange[] = []; + const compare = (left: Stored, right: Stored, path: readonly (string | number)[], pairs: ReadonlySet = new Set()): void => { + if (left === right) return; + const leftId = reference(left); + const rightId = reference(right); + if (leftId !== null && rightId !== null) { + if (leftId === rightId) return; + const pair = `${leftId}:${rightId}`; + if (pairs.has(pair)) return; + pairs = new Set([...pairs, pair]); + } + if (leftId !== null || rightId !== null) { + compare(leftId === null ? left : node(leftId), rightId === null ? right : node(rightId), path, pairs); + return; + } + const leftArray = Array.isArray(left); + const rightArray = Array.isArray(right); + if (typeof left === 'object' && left !== null && typeof right === 'object' && right !== null && leftArray === rightArray) { + const leftEntries = Object.entries(left); + const rightEntries = Object.entries(right); + const leftObject = Object.fromEntries(leftEntries) as Record; + const rightObject = Object.fromEntries(rightEntries) as Record; + for (const key of new Set([...leftEntries.map(([key]) => key), ...rightEntries.map(([key]) => key)])) { + const nextPath = [...path, leftArray ? Number(key) : decodeKey(key)]; + if (!(key in leftObject)) changes.push({ path: nextPath, kind: 'added', after: value(rightObject[key]!) }); + else if (!(key in rightObject)) changes.push({ path: nextPath, kind: 'removed', before: value(leftObject[key]!) }); + else compare(leftObject[key]!, rightObject[key]!, nextPath, pairs); + } + return; + } + changes.push({ path, kind: 'changed', before: value(left), after: value(right) }); + }; + compare(before, after, []); + return changes; + }; + return { roots, stages: [...stages.values()], state, diff }; +}; diff --git a/apps/web/src/components/requests/run-render.ts b/apps/web/src/components/requests/run-render.ts index 9679c0f54c..d212dee240 100644 --- a/apps/web/src/components/requests/run-render.ts +++ b/apps/web/src/components/requests/run-render.ts @@ -25,6 +25,7 @@ const subjectOf = (event: DumpEvent): string | null => { case 'object': return `#${event.fromObjectId}`; case 'stream.frame': case 'stream.end': return `#${event.streamId}`; + case 'deferred.settled': return 'deferred'; } }; diff --git a/apps/web/src/components/requests/run-stages.tsx b/apps/web/src/components/requests/run-stages.tsx new file mode 100644 index 0000000000..a428773549 --- /dev/null +++ b/apps/web/src/components/requests/run-stages.tsx @@ -0,0 +1,62 @@ +import { lazy, Suspense, useMemo, useState } from 'react'; + +import { RenderedEventList } from './events'; +import { readRun, type RunStage } from './run-model'; +import { fluentComponents } from '../../fluent'; +import { useTranslation } from '../../i18n/translation'; +import { EmptyStateLine } from '../ui/empty-state'; +import { Dropdown } from '../ui/fluent-form-controls'; +import { ScrollArea } from '../ui/scroll-area'; + +const BodyEditor = lazy(() => import('../ui/body-editor')); +const { Option, Spinner, Text, Tree, TreeItem, TreeItemLayout } = fluentComponents; + +const StageBranch = ({ stage, select }: { stage: RunStage; select: (id: number) => void }) => 0 ? 'branch' : 'leaf'} onFocus={event => { if (event.target === event.currentTarget) select(stage.id); }}> + {stage.name} #{stage.id} + {stage.children.length > 0 && {stage.children.map(child => )}} +; + +export const RunStages = ({ ndjson }: { ndjson: string }) => { + const { t } = useTranslation(); + const model = useMemo(() => readRun(ndjson), [ndjson]); + const [selected, setSelected] = useState(model.roots[0]?.id ?? null); + const [view, setView] = useState('request'); + const [descent, setDescent] = useState(0); + const labels = { + request: t('dashboard.requests.requestFacts'), response: t('dashboard.requests.responseFacts'), + down: t('dashboard.requests.requestChanges'), up: t('dashboard.requests.responseChanges'), + logs: t('dashboard.requests.stageLogs'), + }; + const current = model.stages.find(stage => stage.id === selected); + if (current === undefined) return {t('dashboard.requests.noRunEvents')}; + const child = current.children[descent]; + const text = view === 'request' ? model.state(current.request) + : view === 'response' ? current.response === null ? null : model.state(current.response) + : view === 'down' ? child === undefined ? [] : model.diff(current.request, child.request) + : child?.response === null || child === undefined || current.response === null ? [] : model.diff(child.response, current.response); + const select = (id: number) => { setSelected(id); setDescent(0); }; + const toolbar =
+ setView(data.optionValue!)}> + {Object.entries(labels).map(([value, label]) => )} + + {(view === 'down' || view === 'up') && current.children.length > 0 && setDescent(Number(data.optionValue))}> + {current.children.map((stage, index) => )} + } +
; + return
+ + stage.id)} onNavigation={(_, data) => { + if (data.type === 'Click') select(Number(data.value)); + }}> + {model.roots.map(stage => )} + + +
+ {view === 'logs' ? ({ event: log.level, text: JSON.stringify({ ...log, ...(log.fields === undefined ? {} : { fields: model.state(log.fields) }) }, null, 2), parseError: null }))} + copyText={current.logs.map(log => JSON.stringify({ ...log, ...(log.fields === undefined ? {} : { fields: model.state(log.fields) }) })).join('\n')} + toolbarStart={toolbar} + /> : }>} +
+
; +}; diff --git a/apps/web/src/i18n/locales/en.ts b/apps/web/src/i18n/locales/en.ts index e6cf14ebde..d9db91b74e 100644 --- a/apps/web/src/i18n/locales/en.ts +++ b/apps/web/src/i18n/locales/en.ts @@ -1054,6 +1054,14 @@ const en = { apiKeysLink: 'API Keys', requestBody: 'Request body', run: 'Run', + stages: 'Stages', + requestFacts: 'Request facts', + responseFacts: 'Response facts', + requestChanges: 'Request changes', + responseChanges: 'Response changes', + stageLogs: 'Stage logs', + stageView: 'Stage view', + descent: 'Descent', noRunEvents: 'This run recorded no events.', responseBody: 'Response body', emptyBody: 'Empty body', diff --git a/apps/web/src/i18n/locales/zh-Hans.ts b/apps/web/src/i18n/locales/zh-Hans.ts index 3ed81cfc5b..c9a204e279 100644 --- a/apps/web/src/i18n/locales/zh-Hans.ts +++ b/apps/web/src/i18n/locales/zh-Hans.ts @@ -1006,6 +1006,14 @@ const zhHansCN = { apiKeysLink: 'API 密钥', requestBody: '请求体', run: '运行过程', + stages: 'Stages', + requestFacts: '请求 facts', + responseFacts: '响应 facts', + requestChanges: '请求变化', + responseChanges: '响应变化', + stageLogs: 'Stage 日志', + stageView: 'Stage 视图', + descent: '分支', noRunEvents: '这次运行没有记录到任何事件。', responseBody: '响应体', emptyBody: '响应体为空', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eef958a243..515d1e7961 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -139,6 +139,9 @@ importers: apps/web: dependencies: + '@floway-dev/pipeline': + specifier: workspace:* + version: link:../../packages/pipeline '@floway-dev/protocols': specifier: workspace:* version: link:../../packages/protocols From 5d269d227ac479a45fdf6fb7d9e26dc21ffd9d1d Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 07:00:46 +0800 Subject: [PATCH 64/79] feat: stream run recordings through owned request lifetimes Persist encoded facts and protocol frames with backpressure, attach an authenticated temporary byte reader, and keep durable recording active when live storage fails. Renew live and staged-file leases while waiting for upstream and close only after run drain. Bind settlement and deferred errors to the run, retain completed call observations on later failures, preserve defined zero usage, and record client frames after filtering. Shield owned meter readers from early consumer return so teardown can drain their usage. --- README.md | 8 + .../dump-subscription-key-switch_test.tsx | 9 + .../requests/use-dump-subscription.ts | 5 +- .../__tests__/control-plane/dump_test.ts | 30 +- .../data-plane/openai-audio-pipeline_test.ts | 2 +- .../openai-completions-pipeline_test.ts | 33 ++- .../openai-completions/http_test.ts | 51 ++-- .../data-plane/pipeline/settlement_test.ts | 47 ++- .../data-plane/rerank-status_test.ts | 2 +- .../gateway/__tests__/dump/run-sink_test.ts | 154 +++++++++- .../gateway/__tests__/dump/test-fixtures.ts | 4 + .../gateway/__tests__/repo/dump-store_test.ts | 48 +++- .../__tests__/test-utils/log-stream.ts | 43 +++ packages/gateway/__tests__/vitest.setup.ts | 5 +- packages/gateway/src/control-plane/dump.ts | 11 + .../data-plane/alpha-search/call-upstream.ts | 2 +- .../alpha-search/execute-operations.ts | 2 +- .../chat/anthropic-messages/respond.ts | 4 +- .../chat/gemini-generate-content/respond.ts | 4 +- .../chat/openai-chat-completions/respond.ts | 4 +- .../chat/openai-responses/respond.ts | 6 +- .../chat/openai-responses/websocket.ts | 2 +- .../data-plane/openai-audio/call-upstream.ts | 24 +- .../openai-completions/call-upstream.ts | 18 +- .../src/data-plane/openai-completions/emit.ts | 25 +- .../openai-embeddings/call-upstream.ts | 4 +- .../data-plane/openai-images/call-upstream.ts | 11 +- .../src/data-plane/pipeline/failover.ts | 2 +- .../data-plane/pipeline/resolve-candidates.ts | 4 +- .../gateway/src/data-plane/pipeline/serve.ts | 10 +- .../src/data-plane/pipeline/services.ts | 4 +- .../src/data-plane/pipeline/settlement.ts | 19 +- .../src/data-plane/pipeline/upstream-body.ts | 5 + .../src/data-plane/rerank/call-upstream.ts | 8 +- packages/gateway/src/dump/run-sink.ts | 270 +++++++++--------- packages/gateway/src/dump/store-contract.ts | 9 + packages/gateway/src/dump/turn-dump.ts | 12 +- packages/gateway/src/repo/dump-store.ts | 43 ++- packages/gateway/src/shared/gzip.ts | 3 + 39 files changed, 695 insertions(+), 252 deletions(-) create mode 100644 packages/gateway/__tests__/test-utils/log-stream.ts diff --git a/README.md b/README.md index 77725d5599..ad61cee5d4 100644 --- a/README.md +++ b/README.md @@ -204,6 +204,14 @@ wraps rendering and writes each observed call once, including calls completed be stage throws. Streaming quantities settle at the deferred exit. Explicit protocol and transport errors take precedence over the settlement's generic failure marker. +Run dumps stream their encoded NDJSON into durable storage with backpressure while appending +the same bytes to a temporary LogStream. Live writes retry at the same byte offset; persistent +live failure leaves durable recording active. Quiet runs renew their temporary stream and +staged-file leases. Closing waits for owned readers and deferred outcomes before publishing +completed metadata. Run IDs are assigned at entry; listing remains ordered by completion time +and ID. The authenticated `GET /api/dump/keys/:keyId/records/:recordId/live?offset=0` endpoint +attaches only to an existing per-key stream and uses the platform's framed byte protocol. + Pipeline endpoints are assembled from individual stage modules. Each stage owns its request and response contract; a large stage keeps local helpers in the same file and can place independent parts in a sibling directory of the same name. diff --git a/apps/web/__tests__/components/requests/dump-subscription-key-switch_test.tsx b/apps/web/__tests__/components/requests/dump-subscription-key-switch_test.tsx index 8c9fa02c90..adaa01c69b 100644 --- a/apps/web/__tests__/components/requests/dump-subscription-key-switch_test.tsx +++ b/apps/web/__tests__/components/requests/dump-subscription-key-switch_test.tsx @@ -102,6 +102,15 @@ describe('dump subscription key switch', () => { expect(screen.getAllByRole('listitem').map(item => item.textContent)).toEqual(['/v1/key-a-2', '/v1/key-a-1']); }); + it('preserves older completed records when run IDs were assigned in a different start order', () => { + const older = { ...record('z-started-later'), completedAt: 100 }; + const removed = { ...record('a-started-earlier'), completedAt: 300 }; + renderSubscription('key-a', [removed, older]); + const recent = { ...record('m-long-running'), completedAt: 200 }; + stream.liveSource().emit('snapshot', JSON.stringify({ records: [recent] })); + expect(screen.getAllByRole('listitem').map(item => item.textContent)).toEqual(['/v1/m-long-running', '/v1/z-started-later']); + }); + it('throws when the session token is gone', () => { storage.delete(flowayTokenStorageKey); diff --git a/apps/web/src/components/requests/use-dump-subscription.ts b/apps/web/src/components/requests/use-dump-subscription.ts index c1fcd867b9..fd5bc2444f 100644 --- a/apps/web/src/components/requests/use-dump-subscription.ts +++ b/apps/web/src/components/requests/use-dump-subscription.ts @@ -74,8 +74,9 @@ export const useDumpSubscription = (keyId: string | null, initialRecords: DumpMe const snapshot = (JSON.parse((raw as MessageEvent).data) as { records: DumpMetadata[] }).records; setRecords(current => { const ids = new Set(snapshot.map(record => record.id)); - const oldest = snapshot.at(-1)?.id; - const tail = oldest ? current.filter(record => !ids.has(record.id) && record.id < oldest) : []; + const oldest = snapshot.at(-1); + const tail = oldest ? current.filter(record => !ids.has(record.id) + && (record.completedAt < oldest.completedAt || (record.completedAt === oldest.completedAt && record.id < oldest.id))) : []; const next = [...snapshot, ...tail]; seenRef.current = new Set(next.map(record => record.id)); return next; diff --git a/packages/gateway/__tests__/control-plane/dump_test.ts b/packages/gateway/__tests__/control-plane/dump_test.ts index 2bebb7dd7c..6c8314739d 100644 --- a/packages/gateway/__tests__/control-plane/dump_test.ts +++ b/packages/gateway/__tests__/control-plane/dump_test.ts @@ -1,10 +1,13 @@ -import { test } from 'vitest'; +import { test, expect, vi } from 'vitest'; import { initDumpBroker, initDumpStore } from '../../src/dump/registry.ts'; +import { runStreamId } from '../../src/dump/run-sink.ts'; import type { DumpStore } from '../../src/dump/store-contract.ts'; import type { DumpMetadata, DumpRecord, StoredDumpRecord } from '../../src/dump/types.ts'; import { fakeMeta as baseFakeMeta, fakeRecord as baseFakeRecord, fakeRunRecord, installDumpStubs } from '../dump/test-fixtures.ts'; import { requestApp, setupAppTest } from '../test-utils/app.ts'; +import { testLogStreamStore } from '../test-utils/log-stream.ts'; +import { initLogStreamStore, readLogStream } from '@floway-dev/platform'; import { assertEquals, assertExists } from '@floway-dev/test-utils'; const fakeMeta = (id: string, completedAt: number): DumpMetadata => @@ -261,3 +264,28 @@ test('GET /api/dump/keys/:keyId/stream delivers a frame appended between snapsho // it. assertEquals((parseSseFrame(appended).data as { id: string }).id, '01HZZ0000000000000000000A2'); }); + +test('live record reads are framed, seekable, key-scoped and never create an unknown stream', async () => { + const { repo, apiKey } = await setupAppTest(); + await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); + const store = testLogStreamStore(); + initLogStreamStore(store); + const first = new TextEncoder().encode('{"before":"中"}\n'); + const second = new TextEncoder().encode('{"after":"β"}\n'); + const stream = await store.open(runStreamId(apiKey.id, 'active')); + await stream.append(0, first); await stream.append(first.byteLength, second); await stream.end(); + const open = vi.spyOn(store, 'open'); + const response = await requestApp(`/api/dump/keys/${apiKey.id}/records/active/live?offset=${first.byteLength}`, { headers: { 'x-api-key': apiKey.key } }); + expect(response.status).toBe(200); + const chunks: Uint8Array[] = []; + for await (const bytes of readLogStream(response)) chunks.push(bytes); + expect(chunks.map(bytes => new TextDecoder().decode(bytes)).join('')).toBe(new TextDecoder().decode(second)); + const missing = await requestApp(`/api/dump/keys/${apiKey.id}/records/absent/live`, { headers: { 'x-api-key': apiKey.key } }); + expect(missing.status).toBe(404); + expect(open).not.toHaveBeenCalled(); + const invalid = await requestApp(`/api/dump/keys/${apiKey.id}/records/active/live?offset=-1`, { headers: { 'x-api-key': apiKey.key } }); + expect(invalid.status).toBe(400); + await store.open(runStreamId('foreign-key', 'foreign')); + const foreign = await requestApp(`/api/dump/keys/${apiKey.id}/records/foreign/live`, { headers: { 'x-api-key': apiKey.key } }); + expect(foreign.status).toBe(404); +}); diff --git a/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts b/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts index 5cface65f9..124e6bc00f 100644 --- a/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-audio-pipeline_test.ts @@ -30,7 +30,7 @@ const candidate = (callOpenAIAudioTranscriptions: () => Promise { } }); + it('finishes metering after an early consumer return', async () => { + resolves([candidate('up_a', async () => ({ response: sse(chunk('hi'), usageChunk, '[DONE]'), modelKey: 'k' }))]); + const { facts, drain } = await serve(entryFacts()); + const reader = (facts['response.openaiCompletions.rendered'] as AsyncIterable)[Symbol.asyncIterator](); + expect((await reader.next()).done).toBe(false); + await reader.return!(); + await drain(); + expect((await facts['response.openaiCompletions.streamedUsage'])!.billable[0]!.quantities).toMatchObject({ input_tokens: '5', output_tokens: '7' }); + expect(recorded.usage).toHaveLength(1); + }); + + it('invalid streamed usage rejects its deferred reading and settles the proven call', async () => { + const invalidUsage = JSON.stringify({ + id: 'cmpl_1', choices: [], usage: { + prompt_tokens: 4, completion_tokens: 1, total_tokens: 5, + prompt_tokens_details: { cached_tokens: 8 }, + }, + }); + resolves([candidate('up_a', async () => ({ response: sse(chunk('hi'), invalidUsage, '[DONE]'), modelKey: 'k' }))]); + const work: Promise[] = []; + const { facts, drain } = await run(openaiCompletionsServePipeline, move(entryFacts()), { + gateway: mockGatewayCtx({ wantsStream: true }), + background: (promise: Promise) => { work.push(promise); void promise.catch(() => {}); }, + ...createCandidateRegistry(), resolveAttempt, + }); + await expect(collect(facts['response.openaiCompletions.rendered'])).rejects.toBeInstanceOf(RangeError); + await expect(drain()).rejects.toBeInstanceOf(RangeError); + await Promise.allSettled(work); + expect(recorded.usage).toMatchObject([{ modelKey: 'k', requests: 1, metrics: [] }]); + }); + it('settles completed attempts when a later refusal body throws, preserving the original exception', async () => { const original = new Error('response socket reset', { cause: new Error('connection closed') }); resolves([ diff --git a/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts b/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts index 08466a8f8c..738be26a3f 100644 --- a/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-completions/http_test.ts @@ -5,6 +5,8 @@ import { tokenCountsFromUsage } from '../../../src/repo/usage-metrics.ts'; import { eventsOf, installDumpStubs, runRecordOf } from '../../dump/test-fixtures.ts'; import { saveUpstreamForTest } from '../../repo/upstreams.ts'; import { buildCustomUpstreamRecord, flushAsyncWork, requestApp as requestAppCold, requestAppWithWarmModels, setupAppTest, warmModelsForTest } from '../../test-utils/app.ts'; +import { createRunReader, type DumpEvent } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; @@ -399,6 +401,7 @@ test('/v1/completions streaming records usage row, performance neutral row (text await registerOpenAICompletionsUpstream(repo); const dumpStubs = installDumpStubs(initDumpStore, initDumpBroker); + let delivered = ''; await withMockedFetch( () => Promise.resolve(completionStream()), async () => { @@ -408,7 +411,7 @@ test('/v1/completions streaming records usage row, performance neutral row (text body: JSON.stringify({ model: 'davinci-002', prompt: 'hello', stream: true }), }); assertEquals(response.status, 200); - await response.text(); + delivered = await response.text(); }, ); @@ -430,31 +433,27 @@ test('/v1/completions streaming records usage row, performance neutral row (text assertEquals(dump.meta.model, 'davinci-002'); assertEquals(dump.meta.inputTokens, 4); assertEquals(dump.meta.outputTokens, 2); - // The frames the gateway saw from upstream, before the edge decided which of them the - // client is shown — recorded as `stream.frame` events in the run's own stream. The fixture - // emits two content events, one usage-only event (which this client did not opt into and - // so was stripped from what it received), and a done terminator. - const events = eventsOf(runRecordOf(dumpStubs.stored[0]!.record)); - const frames = events - .filter(event => event.type === 'stream.frame') - .flatMap(event => (event as unknown as { frames: readonly unknown[] }).frames); - assertEquals(frames.length, 4); - // The usage chunk the client did not opt into is still in the run's own record: what the - // gateway saw is not narrowed to what it forwarded. The encoder shares repeated values, so - // what is asserted is that the numbers are in the stream, not the shape they took in it. - const stored = new TextDecoder().decode(runRecordOf(dumpStubs.stored[0]!.record).events); - assertEquals(stored.includes('"prompt_tokens":4'), true); - - // The stream is named, and every frame says which stream it belongs to — the id is - // allocated per stream rather than fixed, so a run that opened two would keep them apart. - const streamIds = new Set(events.filter(event => event.type === 'stream.frame').map(event => event.streamId)); - assertEquals([...streamIds], [1]); - // And the record of it is complete. A client that stopped reading would leave the frames it - // did get and no terminator, which is how a reader tells the two apart. - assertEquals(events.filter(event => event.type === 'stream.end'), [{ type: 'stream.end', streamId: 1 }]); - // The facts that hold the stream point at it: the answer the ending provided, and the - // framing the edge built for the client over the same frames. - assertEquals(stored.includes('"$stream":1'), true); + const events = eventsOf(runRecordOf(dumpStubs.stored[0]!.record)) as unknown as DumpEvent[]; + const read = createRunReader(); + const streams = new Map[]>(); + let client: number | undefined; + for (const event of events) { + const decoded = read(event); + if (decoded?.facts && 'response.openaiCompletions.rendered' in decoded.facts) { + client = (decoded.facts['response.openaiCompletions.rendered'] as { stream: number }).stream; + } + if (event.type === 'stream.frame') { + const frames = streams.get(event.streamId) ?? []; + frames.push(...decoded!.frames as ProtocolFrame[]); + streams.set(event.streamId, frames); + } + } + assertExists(client); + const clientFrames = streams.get(client)!; + assertEquals(clientFrames.length, 3); + assertEquals(clientFrames.map(frame => `data: ${frame.type === 'done' ? '[DONE]' : JSON.stringify(frame.event)}\n\n`).join(''), delivered); + assertEquals([...streams.values()].some(frames => frames.some(frame => frame.type === 'event' && 'usage' in (frame.event as object))), true); + assertEquals(events.filter(event => event.type === 'stream.end').map(event => event.streamId).sort(), [...streams.keys()].sort()); }); // A run that threw is the turn that most needs explaining and used to be the one that left diff --git a/packages/gateway/__tests__/data-plane/pipeline/settlement_test.ts b/packages/gateway/__tests__/data-plane/pipeline/settlement_test.ts index 6f5af73df2..a0c3f497a4 100644 --- a/packages/gateway/__tests__/data-plane/pipeline/settlement_test.ts +++ b/packages/gateway/__tests__/data-plane/pipeline/settlement_test.ts @@ -1,12 +1,15 @@ import { describe, expect, it, vi } from 'vitest'; -import { settleBillable } from '../../../src/data-plane/pipeline/settlement.ts'; +import type { GatewayFacts } from '../../../src/data-plane/pipeline/facts.ts'; +import { failover } from '../../../src/data-plane/pipeline/failover.ts'; +import { settleBillable, writeSettlement } from '../../../src/data-plane/pipeline/settlement.ts'; import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts'; import { openRunDump } from '../../../src/dump/run-sink.ts'; import { initRepo } from '../../../src/repo/index.ts'; import { installDumpStubs, runRecordOf } from '../../dump/test-fixtures.ts'; import { InMemoryRepo } from '../../repo/memory.ts'; import { mockGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { compose, defineStage, move, run } from '@floway-dev/pipeline'; import { mockPerfTelemetryContext } from '@floway-dev/test-utils'; describe('pipeline settlement', () => { @@ -32,11 +35,12 @@ describe('pipeline settlement', () => { }); const identity = { model: 'm', upstream: 'u', modelKey: 'm', pricing: null }; if (failed) dump!.failed(new Error('Original source failure', { cause: new Error('socket reset') })); + const active = pending.length; settleBillable({ gateway, background, log: { debug: () => {}, info: () => {}, warn: () => {}, error: () => {} } }, [ { identity, quantities: { input_tokens: '10', output_tokens: '2' } }, { identity, quantities: { input_tokens: '20', output_tokens: '3' } }, ], failed, 500); - await Promise.all(pending); + await Promise.all(pending.slice(active)); const usage = await repo.usage.listAll(); expect(recordUsage.mock.calls.map(([row]) => row.metrics.map(metric => metric.quantity))).toEqual([['10', '2'], ['20', '3']]); expect(usage).toMatchObject([{ requests: 2 }]); @@ -51,4 +55,43 @@ describe('pipeline settlement', () => { error: failed ? { kind: 'failed', reason: 'Original source failure' } : null, }); }); + it('retains earlier candidate calls when a later stage throws after reading current usage', async () => { + const repo = new InMemoryRepo(); initRepo(repo); + const work: Promise[] = []; + const original = new Error('post-reading stage failure'); + type Entry = Pick; + type Reply = Entry & Pick; + const afterReading = defineStage({ + name: 'afterReading', + through: { request: { needs: ['route.attempt'], consumes: [], provides: [] }, response: { needs: ['response.usage.billable'], consumes: [], provides: [] } }, + execute: async (facts, next) => { + const back = await next({ ...facts }); + if (facts['route.attempt'].upstreamId === 'second') throw original; + return { ...back }; + }, + }); + const source = defineStage({ + name: 'observe', return: { provides: ['response.usage.billable', 'response.http.status'] }, + execute: async facts => move({ + ...facts, 'response.http.status': facts['route.attempt'].upstreamId === 'first' ? 429 : 200, + 'response.usage.billable': [{ + identity: { + model: 'm', upstream: facts['route.attempt'].upstreamId, modelKey: 'm', pricing: null, + }, quantities: { input_tokens: '10', output_tokens: '2' }, + }], + }), + }); + const pipeline = compose('failedAfterReading', [ + writeSettlement(facts => Number(facts['response.http.status']) >= 400), + failover({ failed: facts => Number(facts['response.http.status']) >= 400, owns: [] }), afterReading, source, + ]); + await expect(run(pipeline, move({ + 'serve.candidates': ['first', 'second'].map((upstreamId, candidateId) => ({ + upstreamId, candidateId, modelId: 'm', flags: [], + })), + }), { gateway: mockGatewayCtx({}), background: (promise: Promise) => { work.push(promise); } })).rejects.toBe(original); + await Promise.all(work); + expect((await repo.usage.listAll()).map(row => row.upstream).sort()).toEqual(['first', 'second']); + }); + }); diff --git a/packages/gateway/__tests__/data-plane/rerank-status_test.ts b/packages/gateway/__tests__/data-plane/rerank-status_test.ts index 9583da5a0d..641b2e0dc4 100644 --- a/packages/gateway/__tests__/data-plane/rerank-status_test.ts +++ b/packages/gateway/__tests__/data-plane/rerank-status_test.ts @@ -29,7 +29,7 @@ const candidate = (upstream: string, callRerank: () => Promise ({ @@ -46,7 +48,7 @@ const shout = defineStage, Pick, Pick { - use.log.info('shouting', { length: facts['in.text'].length }); + await use.log.info('shouting', { length: facts['in.text'].length }); return await next(move({ ...facts, 'in.text': `${facts['in.text']}!` })); }, }); @@ -184,3 +186,147 @@ test('a run reads the stable timing state at completion alongside its stage hist assertEquals('capture' in record, false); assertEquals(lines(record).some(event => event.type === 'stage.entered'), true); }); + +test('live bytes are visible before completion and equal the durable NDJSON artifact', async () => { + initLogStreamStore(testLogStreamStore()); + const stubs = installDumpStubs(initDumpStore, initDumpBroker); + const dump = openRunDump(apiKey(3600), turn, trackBackground, false, { upstreamCallStartedAt: null, firstOutputTokenAt: null })!; + await dump.sink({ type: 'stage.entered', stageId: 1, name: 'open', parentStageId: null, facts: move({ input: '中' }) }); + const live = await getLogStreamStore().get(runStreamId(apiKey(3600).id, dump.id)); + const reader = live!.read(0, new AbortController().signal)[Symbol.asyncIterator](); + const first = await reader.next(); + expect(new TextDecoder().decode(first.value)).toContain('stage.entered'); + expect(stubs.stored).toHaveLength(0); + dump.finalize(200, 0); + await flushBackground(); + const rest: Uint8Array[] = [first.value!]; + for (;;) { const item = await reader.next(); if (item.done) break; rest.push(item.value); } + expect(rest.map(bytes => new TextDecoder().decode(bytes)).join('')).toBe(ndjson(runRecordOf(stubs.stored[0]))); + expect(stubs.stored[0]!.record.meta.id).toBe(dump.id); +}); + +test('retries a lost live acknowledgement at the same offset without duplicating bytes', async () => { + const underlying = testLogStreamStore(); + const positions: number[] = []; + initLogStreamStore({ + get: id => underlying.get(id), + open: async id => { + const stream = await underlying.open(id); + return { + ...stream, append: async (offset, bytes) => { + positions.push(offset); + await stream.append(offset, bytes); + if (positions.length === 1) throw new Error('lost acknowledgement'); + }, + }; + }, + }); + const stubs = installDumpStubs(initDumpStore, initDumpBroker); + const dump = openRunDump(apiKey(3600), turn, trackBackground, false, { upstreamCallStartedAt: null, firstOutputTokenAt: null })!; + await dump.sink({ type: 'stage.entered', stageId: 1, name: 'retry', parentStageId: null, facts: move({ input: 'x' }) }); + dump.finalize(200, 0); + await flushBackground(); + expect(positions).toEqual([0, 0]); + const stream = await underlying.get(runStreamId(apiKey(3600).id, dump.id)); + const text: string[] = []; + for await (const bytes of stream!.read(0, new AbortController().signal)) text.push(new TextDecoder().decode(bytes)); + expect(text.join('')).toBe(ndjson(runRecordOf(stubs.stored[0]))); +}); + +test('a repeatedly failed live stream still stores the complete durable artifact', async () => { + const original = new Error('live transport unavailable'); + const append = vi.fn(async () => { throw original; }); + initLogStreamStore({ + open: async () => ({ append, end: async () => {}, read: () => (async function* () {})() }), + get: async () => null, + }); + const reported = vi.spyOn(console, 'error').mockImplementation(() => {}); + const stubs = installDumpStubs(initDumpStore, initDumpBroker); + try { + const dump = openRunDump(apiKey(3600), turn, trackBackground, false, { upstreamCallStartedAt: null, firstOutputTokenAt: null })!; + const { drain } = await run(pipeline, move({ 'in.text': 'hello' }), { dump: dump.sink }); + dump.afterRun(drain); + dump.finalize(200, 0); + await flushBackground(); + expect(append).toHaveBeenCalledTimes(3); + expect(lines(runRecordOf(stubs.stored[0])).filter(event => event.type === 'stage.entered')).toHaveLength(2); + expect(reported.mock.calls.some(call => call[1] === original)).toBe(true); + } finally { reported.mockRestore(); initLogStreamStore(testLogStreamStore()); } +}); + +test('durable backpressure blocks a producer before it can enqueue another event', async () => { + initLogStreamStore(testLogStreamStore()); + const stubs = installDumpStubs(initDumpStore, initDumpBroker); + const gate = Promise.withResolvers(); + initDumpStore({ + ...stubs.store, putRun: async (keyId, writing) => { + await gate.promise; + await stubs.store.putRun(keyId, writing); + }, + }); + const dump = openRunDump(apiKey(3600), turn, trackBackground, false, { upstreamCallStartedAt: null, firstOutputTokenAt: null })!; + let wrote = false; + const write = dump.sink({ type: 'stage.entered', stageId: 1, name: 'blocked', parentStageId: null, facts: move({}) }).then(() => { wrote = true; }); + await Promise.resolve(); await Promise.resolve(); + expect(wrote).toBe(false); + gate.resolve(); + await write; + dump.finalize(200, 0); + await flushBackground(); + expect(wrote).toBe(true); +}); + +test('closing waits for deferred outcomes and persists their settlement event', async () => { + initLogStreamStore(testLogStreamStore()); + const stubs = installDumpStubs(initDumpStore, initDumpBroker); + const result = Promise.withResolvers(); + const deferred = defer(result.promise); + const terminal = defineStage, { pending: typeof deferred }>({ + name: 'pending', return: { provides: ['pending'] }, execute: async () => move({ pending: deferred }), + }); + const dump = openRunDump(apiKey(3600), turn, trackBackground, false, { upstreamCallStartedAt: null, firstOutputTokenAt: null })!; + const { drain } = await run(compose('deferred', [terminal]), move({}), { dump: dump.sink }); + dump.afterRun(drain); + dump.finalize(200, 0); + await Promise.resolve(); + expect(stubs.stored).toHaveLength(0); + result.resolve('late value'); + await flushBackground(); + expect(lines(runRecordOf(stubs.stored[0])).some(event => event.type === 'deferred.settled')).toBe(true); + expect(ndjson(runRecordOf(stubs.stored[0]))).toContain('late value'); +}); + +test('heartbeat append waits for the in-flight event and uses its resulting byte offset', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + const underlying = testLogStreamStore(); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const positions: { offset: number; length: number }[] = []; + initLogStreamStore({ + get: id => underlying.get(id), + open: async id => { + const stream = await underlying.open(id); + return { + ...stream, append: async (offset, bytes) => { + positions.push({ offset, length: bytes.byteLength }); + if (positions.length === 1) { entered.resolve(); await release.promise; } + await stream.append(offset, bytes); + }, + }; + }, + }); + const stubs = installDumpStubs(initDumpStore, initDumpBroker); + try { + const dump = openRunDump(apiKey(3600), turn, trackBackground, false, { upstreamCallStartedAt: null, firstOutputTokenAt: null })!; + const event = dump.sink({ type: 'stage.entered', stageId: 1, name: 'slow append', parentStageId: null, facts: move({}) }); + await entered.promise; + await vi.advanceTimersByTimeAsync(LOG_STREAM_IDLE_MS / 2); + expect(positions).toHaveLength(1); + release.resolve(); + await event; + dump.finalize(200, 0); + await flushBackground(); + expect(positions).toEqual([{ offset: 0, length: positions[0]!.length }, { offset: positions[0]!.length, length: 0 }]); + expect(stubs.stored).toHaveLength(1); + } finally { release.resolve(); vi.useRealTimers(); initLogStreamStore(testLogStreamStore()); } +}); diff --git a/packages/gateway/__tests__/dump/test-fixtures.ts b/packages/gateway/__tests__/dump/test-fixtures.ts index 30e5068dd2..95641f28f0 100644 --- a/packages/gateway/__tests__/dump/test-fixtures.ts +++ b/packages/gateway/__tests__/dump/test-fixtures.ts @@ -85,6 +85,10 @@ export const installDumpStubs = ( const throws: Partial> = {}; const store: DumpStore = { + async putRun(keyId, run) { + const bytes = new Uint8Array(await new Response(run.events).arrayBuffer()); + await store.put(keyId, { shape: 'run', meta: await run.metadata, events: bytes }); + }, async prepareRequestBody(body) { return { encoding: 'identity', bytes: body, decodedByteLength: body.byteLength }; }, diff --git a/packages/gateway/__tests__/repo/dump-store_test.ts b/packages/gateway/__tests__/repo/dump-store_test.ts index 5b10d2e4fd..7c845e7753 100644 --- a/packages/gateway/__tests__/repo/dump-store_test.ts +++ b/packages/gateway/__tests__/repo/dump-store_test.ts @@ -2,13 +2,14 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; -import { expect, test } from 'vitest'; +import { expect, test, vi } from 'vitest'; import { createSqliteTestDb, mapRunChangeCount } from './test-sqlite.ts'; import { decodeDumpBodyDescriptor } from '../../src/dump/storage-codec.ts'; import type { DumpWriteEdgeRecord, StoredDumpEdgeRecord, StoredDumpRecord, StoredDumpRunRecord } from '../../src/dump/types.ts'; import { FileDumpStore } from '../../src/repo/dump-store.ts'; import { initRepo } from '../../src/repo/index.ts'; +import { SPILLED_FILE_STAGE_GRACE_MS } from '../../src/repo/spilled-files-policy.ts'; import { SqlRepo } from '../../src/repo/sql.ts'; import { collectSpilledFiles } from '../../src/scheduled/spilled-files.ts'; import { encodeRun, toNdjson, type Facts } from '@floway-dev/pipeline'; @@ -661,3 +662,48 @@ test('FileDumpStore filters all retained history before applying the page limit' expect(await store.list('key_x', { q: "' OR 1=1 --", limit: 1 })).toEqual([]); expect(await store.list('other-key', { q: 'needle', limit: 1 })).toEqual([]); }); + +test('FileDumpStore writes streaming run bytes before publishing its metadata row', async () => { + const db = await openDb(); + const files = new MemoryFileStore(); + const store = new FileDumpStore(db, files); + const metadata = Promise.withResolvers(); + const pipe = new TransformStream(); + const writer = pipe.writable.getWriter(); + const id = 'streamed-run'; + const startedAt = Date.now(); + const writing = store.putRun('key_x', { id, startedAt, events: pipe.readable, metadata: metadata.promise }); + const bytes = utf8('{"type":"stage.entered","facts":{"text":"中"}}\n'); + await writer.write(bytes); + expect(await store.get('key_x', id)).toBeNull(); + await writer.close(); + metadata.resolve({ ...baseRecord(id, Date.now()).meta, startedAt }); + await writing; + const stored = await store.get('key_x', id); + expect(stored!.shape).toBe('run'); + expect(new TextDecoder().decode((stored as StoredDumpRunRecord).events)).toBe(new TextDecoder().decode(bytes)); +}); + +test('a long active run keeps its staged file out of orphan collection', async () => { + const db = await openDb(); + const files = new MemoryFileStore(); + initRepo(new SqlRepo(db)); + initFileStore(files); + const store = new FileDumpStore(db, files); + const metadata = Promise.withResolvers(); + const pipe = new TransformStream(); + const writer = pipe.writable.getWriter(); + const id = 'long-active-run'; + const startedAt = Date.now(); + vi.useFakeTimers({ toFake: ['Date', 'setTimeout', 'clearTimeout'] }); + try { + const writing = store.putRun('key_x', { id, startedAt, events: pipe.readable, metadata: metadata.promise }); + await writer.write(utf8('{"type":"stage.entered"}\n')); + await vi.advanceTimersByTimeAsync(SPILLED_FILE_STAGE_GRACE_MS * 2 + 1); + await collectSpilledFiles(Date.now()); + await writer.close(); + metadata.resolve({ ...baseRecord(id, Date.now()).meta, startedAt }); + await writing; + expect((await store.get('key_x', id))!.shape).toBe('run'); + } finally { vi.useRealTimers(); } +}); diff --git a/packages/gateway/__tests__/test-utils/log-stream.ts b/packages/gateway/__tests__/test-utils/log-stream.ts new file mode 100644 index 0000000000..f48629313e --- /dev/null +++ b/packages/gateway/__tests__/test-utils/log-stream.ts @@ -0,0 +1,43 @@ +import type { LogStream, LogStreamStore } from '@floway-dev/platform'; + +export const testLogStreamStore = (): LogStreamStore => { + const streams = new Map(); + return { + async open(id) { + const chunks: { offset: number; bytes: Uint8Array }[] = []; + let length = 0; + let ended = false; + const listeners = new Set<() => void>(); + const wake = () => { for (const listener of listeners) listener(); }; + const stream: LogStream = { + async append(offset, bytes) { + if (ended) throw new Error('Test LogStream is ended'); + if (offset > length) throw new Error('Test LogStream append would leave a hole'); + const tail = bytes.slice(length - offset); + if (tail.byteLength > 0) { chunks.push({ offset: length, bytes: tail }); length += tail.byteLength; } + wake(); + }, + async end() { ended = true; wake(); }, + async *read(offset, signal) { + while (!signal.aborted) { + const chunk = chunks.find(chunk => chunk.offset + chunk.bytes.byteLength > offset); + if (chunk) { + const bytes = chunk.bytes.slice(Math.max(0, offset - chunk.offset)); + offset += bytes.byteLength; + yield bytes; + } else if (ended) return; + else await new Promise(resolve => { + const finish = () => { listeners.delete(finish); signal.removeEventListener('abort', finish); resolve(); }; + listeners.add(finish); + signal.addEventListener('abort', finish, { once: true }); + }); + } + signal.throwIfAborted(); + }, + }; + streams.set(id, stream); + return stream; + }, + async get(id) { return streams.get(id) ?? null; }, + }; +}; diff --git a/packages/gateway/__tests__/vitest.setup.ts b/packages/gateway/__tests__/vitest.setup.ts index f4af14b3bb..dd55d46e30 100644 --- a/packages/gateway/__tests__/vitest.setup.ts +++ b/packages/gateway/__tests__/vitest.setup.ts @@ -1,4 +1,5 @@ import { trackBackground } from './test-utils/background-tracker.ts'; +import { testLogStreamStore } from './test-utils/log-stream.ts'; import type { DumpBroker } from '../src/dump/broker.ts'; import { initDumpBroker, initDumpStore } from '../src/dump/registry.ts'; import type { DumpStore } from '../src/dump/store-contract.ts'; @@ -7,7 +8,7 @@ import { handleExecutionRequest } from '../src/execution/handler.ts'; import { initBackgroundSchedulerResolver } from '../src/runtime/background.ts'; import { initExecutionCellNamespace } from '../src/runtime/execution.ts'; import { isReplayableBody } from '@floway-dev/http'; -import { initEnv, initFetch, initRuntimeKind, initTimingSafeEqual, InProcessExecutionCellNamespace } from '@floway-dev/platform'; +import { initEnv, initFetch, initRuntimeKind, initTimingSafeEqual, InProcessExecutionCellNamespace, initLogStreamStore } from '@floway-dev/platform'; // Production always initializes the environment getter at boot. Mirror that // here with a neutral default; tests needing real values (RUNTIME_LOCATION, @@ -16,6 +17,7 @@ initEnv(() => ''); // Tests run as 'node' by default. The few tests that exercise CF-specific // runtime behaviour re-init this with 'cloudflare'. initRuntimeKind('node'); +initLogStreamStore(testLogStreamStore()); initTimingSafeEqual((a, b) => a.every((byte, index) => byte === b[index])); initFetch((url, init) => { const body = init.body; @@ -33,6 +35,7 @@ initExecutionCellNamespace(new InProcessExecutionCellNamespace(handleExecutionRe // independent of that subsystem. Dump-specific tests install real or recording // implementations. const noopStore: DumpStore = { + async putRun(_keyId, run) { await new Response(run.events).arrayBuffer(); await run.metadata; }, async prepareRequestBody(body) { return { encoding: 'identity', bytes: body, decodedByteLength: body.byteLength }; }, async put(): Promise { /* noop */ }, async list(): Promise { return []; }, diff --git a/packages/gateway/src/control-plane/dump.ts b/packages/gateway/src/control-plane/dump.ts index 50463abc94..2bab7e2614 100644 --- a/packages/gateway/src/control-plane/dump.ts +++ b/packages/gateway/src/control-plane/dump.ts @@ -5,8 +5,10 @@ import { z } from 'zod'; import { ownedKeyForUser } from './shared/owned-key.ts'; import { getDumpBroker, getDumpStore } from '../dump/registry.ts'; +import { runStreamId } from '../dump/run-sink.ts'; import { dumpRecordToWire } from '../dump/wire.ts'; import { zValidator } from '../middleware/zod-validator.ts'; +import { getLogStreamStore, serveLogStream } from '@floway-dev/platform'; const LIST_LIMIT_DEFAULT = 100; const LIST_LIMIT_MAX = 200; @@ -40,6 +42,15 @@ export const dumpRoutes = new Hono() }); return c.json({ records }); }) + .get('/keys/:keyId/records/:recordId/live', zValidator('query', z.object({ + offset: z.coerce.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).optional(), + })), async c => { + const owned = await ownedKey(c); + if (owned.id === null) return c.json({ error: owned.error }, 404); + const stream = await getLogStreamStore().get(runStreamId(owned.id, c.req.param('recordId')!)); + if (stream === null) return c.json({ error: 'Live recording is unavailable' }, 404); + return serveLogStream(stream, c.req.valid('query').offset ?? 0, c.req.raw.signal); + }) .get('/keys/:keyId/records/:recordId', async c => { const owned = await ownedKey(c); if (owned.id === null) return c.json({ error: owned.error }, 404); diff --git a/packages/gateway/src/data-plane/alpha-search/call-upstream.ts b/packages/gateway/src/data-plane/alpha-search/call-upstream.ts index c7fa7ef1ef..2b5ea0ccbb 100644 --- a/packages/gateway/src/data-plane/alpha-search/call-upstream.ts +++ b/packages/gateway/src/data-plane/alpha-search/call-upstream.ts @@ -101,7 +101,7 @@ export const callSearchUpstream = (pinned: PinnedSearchUpstream) => defineStage< spentBody(exchange.body); const raw = await result.response.text(); if (!result.response.ok) { - use.log.warn('upstream refused', { status: result.response.status }); + await use.log.warn('upstream refused', { status: result.response.status }); // The message is what came back as text and the body is the same thing parsed: a dump // reader gets the upstream's own words either way, and only the parsed form is // something the edge can serialize back out. diff --git a/packages/gateway/src/data-plane/alpha-search/execute-operations.ts b/packages/gateway/src/data-plane/alpha-search/execute-operations.ts index 5df9f7964d..80af3bcc5f 100644 --- a/packages/gateway/src/data-plane/alpha-search/execute-operations.ts +++ b/packages/gateway/src/data-plane/alpha-search/execute-operations.ts @@ -34,7 +34,7 @@ export const executeSearchOperations = defineStage< const ops = [...facts['request.search.operations']]; const batch = await startBatchFetch({ kind: 'ops', ops }, session); const blocks = await Promise.all(ops.map(op => executeOperationToText(op, session, batch))); - use.log.debug('ran the search operations', { operations: ops.length }); + await use.log.debug('ran the search operations', { operations: ops.length }); return move({ ...facts, diff --git a/packages/gateway/src/data-plane/chat/anthropic-messages/respond.ts b/packages/gateway/src/data-plane/chat/anthropic-messages/respond.ts index c46a3c547c..98ad801d17 100644 --- a/packages/gateway/src/data-plane/chat/anthropic-messages/respond.ts +++ b/packages/gateway/src/data-plane/chat/anthropic-messages/respond.ts @@ -107,7 +107,7 @@ const observeAnthropicMessagesFrames = async function* ( ctx: GatewayCtx, ) { for await (const frame of frames) { - ctx.dump?.frame(frame); + await ctx.dump?.frame(frame); const failed = frame.type === 'event' && frame.event.type === 'error'; if (failed) state.failed = true; if (isAnthropicMessagesTerminalFrame(frame) && !failed) state.completed = true; @@ -126,7 +126,7 @@ const anthropicMessagesSseFrames = async function* (frames: AsyncIterable>, state: SourceStreamState, ctx: GatewayCtx) { for await (const frame of frames) { - ctx.dump?.frame(frame); + await ctx.dump?.frame(frame); const failed = frame.type === 'event' && isGeminiGenerateContentErrorEvent(frame.event); if (failed) state.failed = true; if (isGeminiGenerateContentTerminalFrame(frame) && !failed) state.completed = true; @@ -209,7 +209,7 @@ const geminiGenerateContentSseFrames = async function* (frames: AsyncIterable>, state: SourceStreamState, ctx: GatewayCtx) { for await (const frame of frames) { - ctx.dump?.frame(frame); + await ctx.dump?.frame(frame); const failed = isOpenAIChatCompletionsFailureFrame(frame); if (failed) state.failed = true; if (isOpenAIChatCompletionsTerminalFrame(frame) && !failed) state.completed = true; @@ -124,7 +124,7 @@ const openaiChatCompletionsSseFrames = async function* (frames: AsyncIterable>, state: SourceStreamState, ctx: GatewayCtx) { for await (const frame of frames) { - ctx.dump?.frame(frame); + await ctx.dump?.frame(frame); const failed = frame.type === 'event' && (frame.event.type === 'error' || frame.event.type === 'response.failed'); if (failed) state.failed = true; if (isOpenAIResponsesTerminalFrame(frame) && !failed) state.completed = true; @@ -169,11 +169,11 @@ const openaiResponsesSseFrames = async function* (frames: AsyncIterable> { for await (const frame of frames) { - ctx.dump?.frame(frame); + await ctx.dump?.frame(frame); if (frame.type === 'event') { const event = frame.event; const failed = event.type === 'error' || event.type === 'response.failed'; diff --git a/packages/gateway/src/data-plane/openai-audio/call-upstream.ts b/packages/gateway/src/data-plane/openai-audio/call-upstream.ts index 922f2aa858..311bb40707 100644 --- a/packages/gateway/src/data-plane/openai-audio/call-upstream.ts +++ b/packages/gateway/src/data-plane/openai-audio/call-upstream.ts @@ -5,7 +5,7 @@ import type { BillableEntity, Failure } from '../pipeline/facts.ts'; import { providerEntry } from '../pipeline/provider-entry.ts'; import { providerUsage } from '../pipeline/provider-usage.ts'; import type { GatewayServices } from '../pipeline/services.ts'; -import { dialFailure, spentBody } from '../pipeline/upstream-body.ts'; +import { dialFailure, spentBody, retainReader } from '../pipeline/upstream-body.ts'; import { upstreamPerformanceContext, telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; import { exchangeResponse } from '@floway-dev/http/pipeline'; import { defineStage, move, setRelease, defer, type Owned, type Logger, type Deferred } from '@floway-dev/pipeline'; @@ -13,8 +13,6 @@ import { isEventStreamMediaType, eventFrame, parseSSEStream, parseDecimalString import { parseOpenAIAudioTranscription, parseOpenAIAudioTranscriptionUsage, isOpenAIAudioTranscriptionDoneEvent, parseOpenAIAudioTranscriptionStreamUsage, type OpenAIAudioTranscriptionResponseFormat, type CanonicalOpenAIAudioTranscription, type OpenAIAudioTranscriptionUsage } from '@floway-dev/protocols/openai-audio'; import type { ProviderRequest, ProviderOperationPayloads, ProviderResponse, TelemetryModelIdentity } from '@floway-dev/provider'; -const viewOf = (events: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => events }); - /** * The ending. It dials, reads what came back in the rendering the request asked for, and * provides the answer, the raw HTTP response beneath it, and what the call is billable for. @@ -72,7 +70,7 @@ export const callOpenAIAudioTranscriptionUpstream = defineStage< const called: readonly BillableEntity[] = wasCalled ? [{ identity, quantities: {} }] : []; if (!result.response.ok) { - use.log.warn('upstream refused', { status }); + await use.log.warn('upstream refused', { status }); spentBody(exchange.body); // An upstream error body is JSON like any other body, and reading it here is also what // leaves a losing attempt with nothing open behind it. @@ -130,7 +128,7 @@ export const callOpenAIAudioTranscriptionUpstream = defineStage< // So there is nothing here to fail over from: the reading feeds the record and the usage // row, and the bytes travel either way. spentBody(exchange.body); - const read = readTranscription(format, new Uint8Array(await result.response.arrayBuffer()), use.log); + const read = await readTranscription(format, new Uint8Array(await result.response.arrayBuffer()), use.log); return move({ ...rest, 'response.openaiAudioTranscription.canonical': read.canonical, @@ -165,32 +163,32 @@ const refusal = async (status: number, response: Response): Promise => * an upstream whose usage block this gateway cannot model still had its transcript read, and * one whose document it could not open is still billed for nothing rather than mis-billed. */ -const readTranscription = ( +const readTranscription = async ( format: OpenAIAudioTranscriptionResponseFormat, document: Uint8Array, log: Logger, -): { readonly canonical: CanonicalOpenAIAudioTranscription; readonly usage: OpenAIAudioTranscriptionUsage | undefined } => { +): Promise<{ readonly canonical: CanonicalOpenAIAudioTranscription; readonly usage: OpenAIAudioTranscriptionUsage | undefined }> => { let canonical: CanonicalOpenAIAudioTranscription; try { canonical = parseOpenAIAudioTranscription(format, document); } catch (error) { - log.warn('failed to parse 2xx upstream body for /audio/transcriptions; forwarding it as it arrived', { error: String(error) }); + await log.warn('failed to parse 2xx upstream body for /audio/transcriptions; forwarding it as it arrived', { error }); return { canonical: { document }, usage: undefined }; } // Only an object rendering states usage: `text`, `srt` and `vtt` have nowhere to put it, // and asking them for one is what would warn about every subtitle a client requests. - return { canonical, usage: canonical.raw === undefined ? undefined : readUsage(() => parseOpenAIAudioTranscriptionUsage(canonical.raw), log) }; + return { canonical, usage: canonical.raw === undefined ? undefined : await readUsage(() => parseOpenAIAudioTranscriptionUsage(canonical.raw), log) }; }; /** A transcription states its usage in two places — the body of an object rendering and the * terminal event of a stream — and either can state it in a shape this gateway cannot read. * That upstream metered something, and the request is recorded saying exactly that: the * entity, and no quantities. */ -const readUsage = (read: () => OpenAIAudioTranscriptionUsage | undefined, log: Logger): OpenAIAudioTranscriptionUsage | undefined => { +const readUsage = async (read: () => OpenAIAudioTranscriptionUsage | undefined, log: Logger): Promise => { try { return read(); } catch (error) { - log.warn('invalid usage in 2xx upstream response; recording the request only', { error: String(error) }); + await log.warn('invalid usage in 2xx upstream response; recording the request only', { error }); return undefined; } }; @@ -210,14 +208,14 @@ const meterEvents = ( // Declared as this run's own unfinished work, so the runner waits for it at teardown where // it can see it rather than the reading being started and forgotten. const outcome = defer(new Promise(resolve => { settle = resolve; })); - const events = viewOf((async function* () { + const events = retainReader((async function* () { let usage: OpenAIAudioTranscriptionUsage | undefined; let completed = false; try { for await (const frame of parseSSEStream(body, { signal })) { const event: unknown = JSON.parse(frame.data); if (isOpenAIAudioTranscriptionDoneEvent(event)) { - usage = readUsage(() => parseOpenAIAudioTranscriptionStreamUsage(event), log); + usage = await readUsage(() => parseOpenAIAudioTranscriptionStreamUsage(event), log); completed = true; yield frame; // The transcript is complete, so there is nothing further to read. An upstream that diff --git a/packages/gateway/src/data-plane/openai-completions/call-upstream.ts b/packages/gateway/src/data-plane/openai-completions/call-upstream.ts index 8fb2b5cbf3..cd4e1977f6 100644 --- a/packages/gateway/src/data-plane/openai-completions/call-upstream.ts +++ b/packages/gateway/src/data-plane/openai-completions/call-upstream.ts @@ -6,7 +6,7 @@ import { providerEntry } from '../pipeline/provider-entry.ts'; import { providerUsage } from '../pipeline/provider-usage.ts'; import type { StreamOutcome } from '../pipeline/serve.ts'; import type { GatewayServices } from '../pipeline/services.ts'; -import { dialFailure, spentBody } from '../pipeline/upstream-body.ts'; +import { dialFailure, spentBody, retainReader } from '../pipeline/upstream-body.ts'; import { upstreamPerformanceContext, telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; import { tokenUsageMeasurement } from '../shared/telemetry/usage.ts'; import { exchangeResponse } from '@floway-dev/http/pipeline'; @@ -24,7 +24,6 @@ import { providerModelOf, type ProviderRequest, type ProviderOperationPayloads, * What comes back is single-shot on purpose: a stream that has been read is a stream that is * over, and a second reader learns that rather than being lied to. */ -const view = (frames: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => frames }); /** * The ending. It dials, reads what came back on the shape the request asked for, and @@ -71,7 +70,7 @@ export const callOpenAICompletionsUpstream = defineStage< } const result = { response: exchangeResponse(exchange), modelKey }; - if (!result.response.ok) use.log.warn('upstream refused', { status: result.response.status }); + if (!result.response.ok) await use.log.warn('upstream refused', { status: result.response.status }); const answer = await readUpstream( result.response, @@ -195,14 +194,14 @@ const meterFrames = ( identity: TelemetryModelIdentity, candidate: ModelCandidate, ): MeteredFrames => { - let settle!: (outcome: StreamOutcome) => void; + const settlement = Promise.withResolvers(); // Declared as this run's own unfinished work, so the runner waits for it at teardown where // it can see it rather than the reading being started and forgotten. - const outcome = defer(new Promise(resolve => { settle = resolve; })); + const outcome = defer(settlement.promise); // Running out without the terminal frame is what "it did not finish" means, and it is known // at the same moment the usage is. let sawTerminal = false; - const frames = view((async function* () { + const frames = retainReader((async function* () { let usage: unknown; let tier: string | null | undefined; try { @@ -222,7 +221,12 @@ const meterFrames = ( // Reached however the frames ended — the terminal chunk, a client that stopped // reading, or a broken upstream — because tokens the upstream already metered are // billable whatever happened to the downstream half. - settle({ billable: [{ identity, ...billed(usage, tier, candidate) }], failed: !sawTerminal }); + try { + settlement.resolve({ billable: [{ identity, ...billed(usage, tier, candidate) }], failed: !sawTerminal }); + } catch (error) { + settlement.reject(error); + throw error; + } } })()); return { frames, outcome }; diff --git a/packages/gateway/src/data-plane/openai-completions/emit.ts b/packages/gateway/src/data-plane/openai-completions/emit.ts index 05b0080399..bd1a83db71 100644 --- a/packages/gateway/src/data-plane/openai-completions/emit.ts +++ b/packages/gateway/src/data-plane/openai-completions/emit.ts @@ -1,6 +1,7 @@ import type { OpenAICompletionsFacts, OpenAICompletionsFrames, Fields } from './facts.ts'; -import { streamReferenceOf } from '../../dump/turn-dump.ts'; +import { recordStream, streamReferenceOf, type TurnDump } from '../../dump/turn-dump.ts'; import { isFailure } from '../pipeline/facts.ts'; +import type { GatewayServices } from '../pipeline/services.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; import { defineStage, move } from '@floway-dev/pipeline'; import { renderErrorEnvelope, isOpenAIUsageOnlyEventShape, type SseFrame } from '@floway-dev/protocols/common'; @@ -22,7 +23,8 @@ export const emitOpenAICompletions = defineStage< Fields<'ingress.openaiCompletions.wantsStream' | 'ingress.openaiCompletions.wantsUsageChunk' | 'request.openaiCompletions.payload'>, Fields<'ingress.openaiCompletions.wantsStream' | 'ingress.openaiCompletions.wantsUsageChunk' | 'request.openaiCompletions.payload'>, Fields<'ingress.openaiCompletions.wantsUsageChunk' | 'response.openaiCompletions.payload' | 'response.http.status' | 'response.http.headers'>, - Fields<'response.openaiCompletions.rendered' | 'response.http.status' | 'response.http.headers'> + Fields<'response.openaiCompletions.rendered' | 'response.http.status' | 'response.http.headers'>, + GatewayServices >({ name: 'emitOpenAICompletions', through: { @@ -37,7 +39,7 @@ export const emitOpenAICompletions = defineStage< provides: ['response.openaiCompletions.rendered', 'response.http.status', 'response.http.headers'], }, }, - execute: async (facts, next) => { + execute: async (facts, next, use) => { const asked = facts['request.openaiCompletions.payload']; const back = await next({ ...facts, @@ -55,7 +57,7 @@ export const emitOpenAICompletions = defineStage< ...rest, 'response.http.headers': forwardable.length === headers.length ? headers : move(forwardable), 'response.http.status': isFailure(answer) ? answer.status : back['response.http.status'], - 'response.openaiCompletions.rendered': move(rendered(answer, back['ingress.openaiCompletions.wantsUsageChunk'])), + 'response.openaiCompletions.rendered': move(rendered(answer, back['ingress.openaiCompletions.wantsUsageChunk'], use.gateway.dump)), }; }, }); @@ -63,19 +65,28 @@ export const emitOpenAICompletions = defineStage< const rendered = ( answer: OpenAICompletionsFacts['response.openaiCompletions.payload'], wantsUsageChunk: boolean, + dump: TurnDump | null, ): OpenAICompletionsFacts['response.openaiCompletions.rendered'] => isFailure(answer) ? renderErrorEnvelope(answer.message, answer.body) - : isFrames(answer) ? renderSSE(answer, wantsUsageChunk) + : isFrames(answer) ? renderSSE(wantsUsageChunk ? answer : recordStream(withoutUsage(answer), dump)) : answer; -const renderSSE = (frames: OpenAICompletionsFrames, wantsUsageChunk: boolean): AsyncIterable => ({ +const renderSSE = (frames: OpenAICompletionsFrames): AsyncIterable => ({ // The frames the client reads are a reframing of the ones the record holds, so this key // points at that same stream rather than at nothing. ...streamReferenceOf(frames), [Symbol.asyncIterator]: () => (async function* () { for await (const frame of frames) { - if (!wantsUsageChunk && frame.type === 'event' && isOpenAIUsageOnlyEventShape(frame.event)) continue; yield openaiCompletionsProtocolFrameToSSEFrame(frame); } })(), }); + +const withoutUsage = (frames: OpenAICompletionsFrames): OpenAICompletionsFrames => ({ + async *[Symbol.asyncIterator]() { + for await (const frame of frames) { + if (frame.type === 'event' && isOpenAIUsageOnlyEventShape(frame.event)) continue; + yield frame; + } + }, +}); diff --git a/packages/gateway/src/data-plane/openai-embeddings/call-upstream.ts b/packages/gateway/src/data-plane/openai-embeddings/call-upstream.ts index 04f4b75e3b..9e7c75978c 100644 --- a/packages/gateway/src/data-plane/openai-embeddings/call-upstream.ts +++ b/packages/gateway/src/data-plane/openai-embeddings/call-upstream.ts @@ -68,7 +68,7 @@ export const callOpenAIEmbeddingsUpstream = defineStage< const reportedNothing: UsageQuantities = {}; if (!result.response.ok) { - use.log.warn('upstream refused', { status: result.response.status }); + await use.log.warn('upstream refused', { status: result.response.status }); return answered({ status: result.response.status, message: upstreamErrorMessage(body.json) ?? body.text, @@ -85,7 +85,7 @@ export const callOpenAIEmbeddingsUpstream = defineStage< try { canonical = parseOpenAIEmbeddingsResponse(body.json, facts['serve.model']); } catch (error) { - use.log.warn('upstream answered with a body the OpenAI Embeddings protocol cannot read', { error: String(error) }); + await use.log.warn('upstream answered with a body the OpenAI Embeddings protocol cannot read', { error: String(error) }); return answered(unreadableBody(result.response, body, 'the OpenAI Embeddings protocol'), reportedNothing); } return answered(canonical, billed(canonical.usage)); diff --git a/packages/gateway/src/data-plane/openai-images/call-upstream.ts b/packages/gateway/src/data-plane/openai-images/call-upstream.ts index 9affa0dc96..ac187ffc23 100644 --- a/packages/gateway/src/data-plane/openai-images/call-upstream.ts +++ b/packages/gateway/src/data-plane/openai-images/call-upstream.ts @@ -6,7 +6,7 @@ import { providerEntry } from '../pipeline/provider-entry.ts'; import { providerUsage } from '../pipeline/provider-usage.ts'; import type { StreamOutcome } from '../pipeline/serve.ts'; import type { GatewayServices } from '../pipeline/services.ts'; -import { dialFailure, readUpstreamBody, spentBody } from '../pipeline/upstream-body.ts'; +import { dialFailure, readUpstreamBody, spentBody, retainReader } from '../pipeline/upstream-body.ts'; import { upstreamPerformanceContext, telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; import { exchangeResponse } from '@floway-dev/http/pipeline'; import { defineStage, move, setRelease, defer, type Deferred } from '@floway-dev/pipeline'; @@ -18,7 +18,6 @@ import { isBase64ImageDataUrl, type ProviderRequest, type ProviderResponse, type * generator itself. What the wrapper says is where the resource is: the one resource in an * OpenAI Images run is the upstream's body at `response.http.body`, claimed with `own()`, and * this keeps a frame view from reading as another. */ -const view = (frames: AsyncGenerator): AsyncIterable => ({ [Symbol.asyncIterator]: () => frames }); const PERFORMANCE_OPERATION = { generations: 'image_generation', @@ -86,7 +85,7 @@ export const callOpenAIImagesUpstream = defineStage< }); if (!result.response.ok) { - use.log.warn('upstream refused', { status: result.response.status }); + await use.log.warn('upstream refused', { status: result.response.status }); // An upstream error body is JSON like any other body. Reading it here is also what // leaves a losing attempt with nothing open behind it. spentBody(exchange.body); @@ -134,7 +133,7 @@ export const callOpenAIImagesUpstream = defineStage< // Every protocol the gateway carries is one it fully understands, so a body it cannot // read is not handed on unread. const essence = mediaTypeEssence(mediaType) ?? 'no media type'; - use.log.warn('upstream answered with a body that is not JSON', { status: result.response.status, mediaType: essence }); + await use.log.warn('upstream answered with a body that is not JSON', { status: result.response.status, mediaType: essence }); return read({ status: 502, message: `The upstream answered ${result.response.status} with ${essence}, and the OpenAI Images protocol is JSON.`, @@ -147,7 +146,7 @@ export const callOpenAIImagesUpstream = defineStage< canonical = parseOpenAIImagesResponse(body.json); } catch (error) { const message = error instanceof Error ? error.message : String(error); - use.log.warn('upstream answered with a body the OpenAI Images protocol cannot read', { message }); + await use.log.warn('upstream answered with a body the OpenAI Images protocol cannot read', { message }); return read({ status: 502, message, body: body.json }, called); } return read(canonical, wasCalled ? [{ identity, quantities: billed(canonical.usage) }] : []); @@ -171,7 +170,7 @@ const meterFrames = ( // Running out without the completed event is what "it did not finish" means, and it is known // at the same moment the usage is. let sawTerminal = false; - const frames = view((async function* () { + const frames = retainReader((async function* () { let usage: CanonicalOpenAIImagesUsage | undefined; try { for await (const event of parseOpenAIImagesStream(body, { signal })) { diff --git a/packages/gateway/src/data-plane/pipeline/failover.ts b/packages/gateway/src/data-plane/pipeline/failover.ts index dd85eb8e7c..f3cf96d801 100644 --- a/packages/gateway/src/data-plane/pipeline/failover.ts +++ b/packages/gateway/src/data-plane/pipeline/failover.ts @@ -69,7 +69,7 @@ export const failover = ({ failed, owns, pendingUsage }: Forking) => defineStage }), }; } - use.log.info('candidate failed, trying the next', { upstream: candidate.upstreamId }); + await use.log.info('candidate failed, trying the next', { upstream: candidate.upstreamId }); } if (last === undefined) throw new Error('failover: assembly handed it an empty candidate list'); // Every candidate failed, and the last failure is the base — so the client sees real diff --git a/packages/gateway/src/data-plane/pipeline/resolve-candidates.ts b/packages/gateway/src/data-plane/pipeline/resolve-candidates.ts index b6021b00dd..f0c9465991 100644 --- a/packages/gateway/src/data-plane/pipeline/resolve-candidates.ts +++ b/packages/gateway/src/data-plane/pipeline/resolve-candidates.ts @@ -86,11 +86,11 @@ export const resolveCandidates = (narrowing: Narrowing => ({ }); export interface Prologue { + readonly runDump: RunDump | null; readonly services: GatewayServices; readonly gateway: GatewayCtx; readonly headers: readonly (readonly [string, string])[]; @@ -92,6 +94,7 @@ export const openPrologue = ( const candidates = createCandidateRegistry(); return { + runDump, gateway, headers: ingress.headers, services: { @@ -99,6 +102,7 @@ export const openPrologue = ( log: consoleLogSink, background: work => { gateway.backgroundScheduler(work); }, ...candidates, + recordProtocolFrames: frames => recordStream(frames, runDump), httpCall: callId => { const candidate = candidates.resolveCandidate(callId); return { @@ -201,7 +205,9 @@ const serveRun = async < render: (facts: Exit) => Rendered, ): Promise => { const { facts, drain } = await run(pipeline, entry, prologue.services as never); - const answer = render(facts); + prologue.runDump?.afterRun(drain); + let answer: Rendered; + try { answer = render(facts); } catch (error) { await drain(); throw error; } const status = facts['response.http.status'] as ContentfulStatusCode; if ('frames' in answer) { diff --git a/packages/gateway/src/data-plane/pipeline/services.ts b/packages/gateway/src/data-plane/pipeline/services.ts index b4d680c4c8..a80230a1d1 100644 --- a/packages/gateway/src/data-plane/pipeline/services.ts +++ b/packages/gateway/src/data-plane/pipeline/services.ts @@ -10,14 +10,16 @@ import type { AttemptSelector } from './facts.ts'; import type { GatewayCtx } from '../shared/gateway-ctx.ts'; import type { HttpServices } from '@floway-dev/http/pipeline'; import type { Event, Logger } from '@floway-dev/pipeline'; +import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { ModelCandidate } from '@floway-dev/provider'; export interface GatewayServices extends HttpServices { + readonly recordProtocolFrames: >(frames: AsyncIterable) => AsyncIterable; /** The global sink. Every stage's lines reach it, tagged with the stage's name. */ readonly log?: Logger; /** Present only when this request is being dumped, which is what keeps recording * conditional: with no sink resolved here, the runner does none of it. */ - readonly dump?: (event: Event) => void; + readonly dump?: (event: Event) => void | Promise; /** The request-scoped context the settlement and telemetry stages read. It is a service * and not a fact because it holds live handles — the scheduler, the abort signal. */ diff --git a/packages/gateway/src/data-plane/pipeline/settlement.ts b/packages/gateway/src/data-plane/pipeline/settlement.ts index 74146421b1..5578d198b9 100644 --- a/packages/gateway/src/data-plane/pipeline/settlement.ts +++ b/packages/gateway/src/data-plane/pipeline/settlement.ts @@ -49,6 +49,7 @@ export const settleBillable = ( export const writeSettlement = ( failed: (handedUp: Record) => boolean, pendingUsage?: string, + recoverCalls = true, ) => defineStage< Record, Slice<'serve.usage.prior'>, @@ -68,10 +69,10 @@ export const writeSettlement = ( back = await next(entry); } catch (error) { const atFailure = getFailureFacts(error) ?? entry; - settleExit(use, atFailure, true, pendingUsage); + settleExit(use, atFailure, true, pendingUsage, true, recoverCalls); throw error; } - settleExit(use, back, failed(back as Facts), pendingUsage); + settleExit(use, back, failed(back as Facts), pendingUsage, false, recoverCalls); const { 'serve.usage.prior': _prior, ...rest } = back as Slice<'response.usage.billable' | 'serve.usage.prior'>; return rest; }, @@ -82,21 +83,27 @@ const settleExit = ( facts: Facts, failed: boolean, pendingUsage: string | undefined, + exceptional: boolean, + recoverCalls: boolean, ): void => { + const prior = exceptional ? facts['serve.usage.prior'] as readonly BillableEntity[] : []; const pending = pendingUsage === undefined || !(pendingUsage in facts) ? null : facts[pendingUsage] as Deferred | null; if (pending !== null) { use.background(pending.then(outcome => { - settleBillable(use, outcome.billable, failed || outcome.failed); + settleBillable(use, [...prior, ...outcome.billable], failed || outcome.failed); + }, error => { + use.gateway.dump?.failed(error); + settleBillable(use, [...prior, ...facts['response.usage.billable'] as readonly BillableEntity[]], true); + throw error; })); return; } if ('response.usage.billable' in facts) { - settleBillable(use, facts['response.usage.billable'] as readonly BillableEntity[], failed); + settleBillable(use, [...prior, ...facts['response.usage.billable'] as readonly BillableEntity[]], failed); return; } - const prior = facts['serve.usage.prior'] as readonly BillableEntity[]; - if (!('response.provider.called' in facts)) { + if (!recoverCalls || !('response.provider.called' in facts)) { settleBillable(use, prior, failed); return; } diff --git a/packages/gateway/src/data-plane/pipeline/upstream-body.ts b/packages/gateway/src/data-plane/pipeline/upstream-body.ts index 6f6f3e1513..290ef56290 100644 --- a/packages/gateway/src/data-plane/pipeline/upstream-body.ts +++ b/packages/gateway/src/data-plane/pipeline/upstream-body.ts @@ -57,3 +57,8 @@ export const spentBody = (body: (ReadableStream & Owned) | null): ty if (body !== null) setRelease(body, async () => {}); return body; }; + +/** Consumer return closes its projection; the owned body still finishes this reader at drain. */ +export const retainReader = (reader: AsyncGenerator): AsyncIterable => ({ + [Symbol.asyncIterator]: () => ({ next: () => reader.next() }), +}); diff --git a/packages/gateway/src/data-plane/rerank/call-upstream.ts b/packages/gateway/src/data-plane/rerank/call-upstream.ts index 1c2a17b966..de9a1c3933 100644 --- a/packages/gateway/src/data-plane/rerank/call-upstream.ts +++ b/packages/gateway/src/data-plane/rerank/call-upstream.ts @@ -68,7 +68,7 @@ export const callRerankUpstream = defineStage< const body = await readUpstreamBody(result.response); if (!result.response.ok) { - use.log.warn('upstream refused', { status: result.response.status }); + await use.log.warn('upstream refused', { status: result.response.status }); return move({ ...rest, 'response.rerank.canonical': { @@ -104,7 +104,7 @@ export const callRerankUpstream = defineStage< try { usage = parseRerankUsage(result.target.protocol, body.json); } catch (error) { - use.log.warn('upstream reported usage the rerank protocol cannot read', { error: String(error) }); + await use.log.warn('upstream reported usage the rerank protocol cannot read', { error: String(error) }); usage = {}; } const metered: readonly BillableEntity[] = [{ @@ -125,7 +125,7 @@ export const callRerankUpstream = defineStage< canonical = parseRerankResponse(result.target.protocol, body.json); } catch (error) { if (translating) { - use.log.warn('upstream answered with results the rerank protocol cannot read', { error: String(error) }); + await use.log.warn('upstream answered with results the rerank protocol cannot read', { error: String(error) }); return move({ ...rest, 'response.rerank.canonical': unreadableBody(result.response, body, 'the rerank protocol'), @@ -135,7 +135,7 @@ export const callRerankUpstream = defineStage< 'response.usage.billable': providerUsage(candidate, back, wasCalled ? metered : []), }); } - use.log.debug('same-protocol answer carries results this gateway does not model', { error: String(error) }); + await use.log.debug('same-protocol answer carries results this gateway does not model', { error: String(error) }); canonical = { raw: body.json as Record, results: [] }; } diff --git a/packages/gateway/src/dump/run-sink.ts b/packages/gateway/src/dump/run-sink.ts index 3242b27801..5b536be745 100644 --- a/packages/gateway/src/dump/run-sink.ts +++ b/packages/gateway/src/dump/run-sink.ts @@ -1,18 +1,3 @@ -// The pipeline's half of the dump: the record of a **whole run**. -// -// A run emits events — every stage, both directions — and this is where they -// go. What the runner hands `services.dump` is `sink`; what it accumulates is -// the encoded stream, folded event by event by `createRunEncoder` so a run is -// written down as it happens rather than re-walked at the end. At the terminal -// point the stream becomes NDJSON and one record goes through `DumpStore.put`, -// which is the same contract the edge record is written under: the stream is -// one more gzipped body file, retained and swept by the same row. -// -// Recording is conditional and that is structural: with no retention configured -// there is no sink to hand to `run`, so `services.dump` is absent and the -// runner does none of the recording — not a no-op that accumulates and throws -// the result away. - import { DumpAttribution, oneLineError, streamReadError } from './attribution.ts'; import { getDumpBroker, getDumpStore } from './registry.ts'; import type { StreamRecording } from './turn-dump.ts'; @@ -21,12 +6,11 @@ import { attemptTtftMs, type AttemptTiming } from '../data-plane/shared/attempt- import type { RequestBody } from '../data-plane/shared/request-body.ts'; import type { ApiKey, TokenUsage } from '../repo/types.ts'; import { ulid } from '../shared/ulid.ts'; -import { createRunEncoder, streamFact, toNdjson, type DumpEvent, type Event } from '@floway-dev/pipeline'; -import type { BackgroundScheduler } from '@floway-dev/platform'; +import { createRunEncoder, streamFact, toNdjson, type Event } from '@floway-dev/pipeline'; +import { getLogStreamStore, LOG_STREAM_IDLE_MS, type BackgroundScheduler, type LogStream } from '@floway-dev/platform'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { TelemetryModelIdentity } from '@floway-dev/provider'; -// Only metadata is snapshotted here; request contents are facts recorded by the run. interface RequestSnapshot { readonly method: string; readonly path: string; @@ -34,10 +18,25 @@ interface RequestSnapshot { readonly streamError: string | null; } +const LIVE_APPEND_ATTEMPTS = 3; +const CHUNK_BYTES = 64 * 1024; + +export const runStreamId = (keyId: string, runId: string): string => `${keyId}/${runId}`; + export class RunDump { + readonly id: string; private readonly attribution = new DumpAttribution(); private readonly encode = createRunEncoder(); - private readonly events: DumpEvent[] = []; + private readonly metadata = Promise.withResolvers(); + private readonly writer: WritableStreamDefaultWriter; + private readonly liveReady: Promise; + private live: LogStream | null = null; + private offset = 0; + private heartbeat: ReturnType | undefined; + private heartbeatWrite: Promise = Promise.resolve(); + private closing = false; + private tail: Promise = Promise.resolve(); + private runDrain: (() => Promise) | null = null; private sentPayloadBytes = 0; private streams = 0; private answerStream: StreamRecording | undefined; @@ -49,96 +48,109 @@ export class RunDump { private readonly backgroundScheduler: BackgroundScheduler, private readonly wantsStream: boolean, private readonly timing: AttemptTiming, - ) {} + ) { + const id = ulid(startedAt); + this.id = id; + const bytes = new TransformStream(); + this.writer = bytes.writable.getWriter(); + // The metadata wait can reject before the storage reader reaches it; storage owns reporting. + void this.metadata.promise.catch(() => {}); + this.backgroundScheduler((async () => { + await getDumpStore().putRun(apiKey.id, { + id, startedAt, events: bytes.readable, metadata: this.metadata.promise, + }); + await getDumpBroker().publish(apiKey.id, await this.metadata.promise); + })().catch(async error => { + this.metadata.reject(error); + await this.writer.abort(error); + throw error; + })); + this.liveReady = getLogStreamStore().open(runStreamId(apiKey.id, id)).then(stream => { + this.live = stream; + this.scheduleHeartbeat(); + return stream; + }).catch(error => { + console.error('[dump] live stream open failed', error); + return null; + }); + } - /** What the prologue hands to `run` as `services.dump`. Bound to this - * recording, so it travels as a value. */ - readonly sink = (event: Event): void => { - for (const encoded of this.encode(event)) this.events.push(encoded); - }; + afterRun(drain: () => Promise): void { + this.runDrain = drain; + } - // --- mid-flight hooks, the same ones the edge record is stamped with --- + // One event owns its entire object batch; concurrent deferred outcomes cannot overtake it. + readonly sink = (event: Event): Promise => { + const write = this.tail.then(async () => { + const bytes = new TextEncoder().encode(toNdjson(this.encode(event))); + await this.liveReady; + for (let offset = 0; offset < bytes.byteLength; offset += CHUNK_BYTES) { + const chunk = bytes.subarray(offset, offset + CHUNK_BYTES); + await this.writer.write(chunk); + await this.appendLive(chunk); + } + }); + this.tail = write; + return write; + }; - requestedModel(model: string): void { - this.attribution.requestedModel(model); + private async appendLive(bytes: Uint8Array): Promise { + const stream = this.live; + if (stream === null) return; + const atOffset = this.offset; + for (let attempt = 0; attempt < LIVE_APPEND_ATTEMPTS; attempt += 1) { + try { + await stream.append(atOffset, bytes); + this.offset += bytes.byteLength; + return; + } catch (error) { + if (attempt + 1 === LIVE_APPEND_ATTEMPTS) { + this.live = null; + console.error('[dump] live stream unavailable; durable recording continues', error); + } + } + } } - error(kind: 'upstream' | 'gateway', upstream?: string): void { - this.attribution.error(kind, upstream); + private scheduleHeartbeat(): void { + if (this.closing || this.live === null) return; + this.heartbeat = setTimeout(() => { + const heartbeat = this.tail.then(() => this.appendLive(new Uint8Array())); + this.tail = heartbeat; + this.heartbeatWrite = heartbeat.then(() => { this.scheduleHeartbeat(); }); + }, LOG_STREAM_IDLE_MS / 2); } - failed(reason: unknown, options?: { readonly fallback: boolean }): void { - this.attribution.failed(reason, options); - } + requestedModel(model: string): void { this.attribution.requestedModel(model); } + error(kind: 'upstream' | 'gateway', upstream?: string): void { this.attribution.error(kind, upstream); } + failed(reason: unknown, options?: { readonly fallback: boolean }): void { this.attribution.failed(reason, options); } + success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void { this.attribution.success(identity, usage); } - /** - * A frame the client was sent, as the event the format names for one. - * - * The edge dump kept a frame log of its own; here a frame is content about a stream, so it - * is `stream.frame` and it folds through the same encoder as everything else. A frame pushed - * without opening a stream first belongs to the run's first one, which is what a transport - * writing single synthesized frames alongside its answer is doing. - */ - frame(frame: ProtocolFrame): void { + async frame(frame: ProtocolFrame): Promise { this.answerStream ??= this.openStream(); - this.answerStream.frame(frame); + await this.answerStream.frame(frame); } - /** - * Begins recording one stream, under an id of its own. - * - * The id is what makes the frames resolvable: the fact holding the stream carries - * `{"$stream": n}` and every frame event names the same `n`, so a run that opened two — a - * sub-request's stream beside the answer's — keeps them apart. `end` says the record of that - * stream is complete, and a client that stopped reading never reaches it. - */ openStream(): StreamRecording { const streamId = ++this.streams; return { - frame: frame => { this.sink({ type: 'stream.frame', streamId, frames: [frame] }); }, - end: () => { this.sink({ type: 'stream.end', streamId }); }, + frame: frame => this.sink({ type: 'stream.frame', streamId, frames: [frame] }), + end: () => this.sink({ type: 'stream.end', streamId }), fact: streamFact(streamId), }; } - success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void { - this.attribution.success(identity, usage); - } - - // --- terminal point --- - - // Two input shapes, matching the edge accumulator's seam: - // - // • `(status, responseBytes)` — the caller already knows what it wrote. - // • `(response)` — tees the answer so the client gets bytes flowing while a - // background reader measures the other half. Protocol contents live in the run events. - // - // The drain → encode → store put → broker publish runs on the runtime's - // BackgroundScheduler so a dump write failure cannot turn a served answer - // into a 502. - /** A transport that writes its own frames counts what it sent, because nothing downstream - * of it can. The run's own bytes are its events; this is the answer's. */ - recordSentPayloadBytes(byteLength: number): void { - this.sentPayloadBytes += byteLength; - } + recordSentPayloadBytes(byteLength: number): void { this.sentPayloadBytes += byteLength; } finalize(status: number | null, responseBytes: number): void; finalize(response: Response): Response; finalize(...args: [number | null, number] | [Response]): void | Response { if (args.length === 2) { - const [status, responseBytes] = args; - // A transport that wrote its own frames counted them as it went; what it passes here - // is whatever else it sent alongside them. - this.backgroundScheduler(this.write(status, responseBytes + this.sentPayloadBytes, null)); + this.backgroundScheduler(this.write(args[0], args[1] + this.sentPayloadBytes, null)); return; } - const [response] = args; - if (response.body === null) { - this.finalize(response.status, 0); - return response; - } - + if (response.body === null) { this.finalize(response.status, 0); return response; } const [forClient, forMeasure] = response.body.tee(); this.backgroundScheduler((async () => { const reader = forMeasure.getReader(); @@ -150,81 +162,57 @@ export class RunDump { if (done) break; payloadBytes += value.byteLength; } - } catch (err) { - streamError = oneLineError(err); + } catch (error) { + streamError = oneLineError(error); } finally { reader.releaseLock(); } await this.write(response.status, payloadBytes, streamError); })()); - - return new Response(forClient, { - status: response.status, - statusText: response.statusText, - headers: response.headers, - }); + return new Response(forClient, { status: response.status, statusText: response.statusText, headers: response.headers }); } private async write(status: number | null, responseBytes: number, responseStreamError: string | null): Promise { - // ULID-from-completedAt keeps ids increasing with row creation time; the - // random tail provides the deterministic tie-breaker for one millisecond. - const completedAt = Date.now(); - const recordId = ulid(completedAt); - const meta: DumpMetadata = await this.attribution.metadata({ - id: recordId, - startedAt: this.startedAt, - completedAt, - method: this.requestSnapshot.method, - path: this.requestSnapshot.path, - status, - requestBytes: this.requestSnapshot.bodyByteLength, - responseBytes, - ttftMs: this.wantsStream ? attemptTtftMs(this.timing) : null, - fallbackError: streamReadError(this.requestSnapshot.streamError, responseStreamError), - }); - - // Commit the row before publishing so subscribers fetching detail off the meta frame find it. try { - await getDumpStore().put(this.apiKey.id, { - shape: 'run', - meta, - events: new TextEncoder().encode(toNdjson(this.events)), + if (this.runDrain !== null) { + try { await this.runDrain(); } catch (error) { this.failed(error); } + } + await this.tail; + const completedAt = Date.now(); + const meta = await this.attribution.metadata({ + id: this.id, startedAt: this.startedAt, completedAt, + method: this.requestSnapshot.method, path: this.requestSnapshot.path, status, + requestBytes: this.requestSnapshot.bodyByteLength, responseBytes, + ttftMs: this.wantsStream ? attemptTtftMs(this.timing) : null, + fallbackError: streamReadError(this.requestSnapshot.streamError, responseStreamError), }); - await getDumpBroker().publish(this.apiKey.id, meta); - } catch (err) { - console.error(`[dump] run write failed for key=${this.apiKey.id} record=${recordId}`, oneLineError(err)); + this.metadata.resolve(meta); + await this.writer.close(); + await this.liveReady; + this.closing = true; + clearTimeout(this.heartbeat); + await this.heartbeatWrite; + if (this.live !== null) { + try { await this.live.end(); } catch (error) { console.error('[dump] live stream end failed', error); } + } + } catch (error) { + this.metadata.reject(error); + throw error; + } finally { + this.closing = true; + clearTimeout(this.heartbeat); } } } -/** - * Returns null when the api key opts out of dumps, and the absence is the - * mechanism: the prologue has nothing to put in `services.dump`, so the run - * emits nothing and accumulates nothing. - * - * `method` and `path` are passed rather than read off a request so a transport - * that carries several turns over one connection can name each one as what it - * is — the same reason the edge accumulator takes its method explicitly. - */ export const openRunDump = ( apiKey: ApiKey, turn: { readonly method: string; readonly path: string; readonly body: RequestBody }, backgroundScheduler: BackgroundScheduler, wantsStream: boolean, timing: AttemptTiming, -): RunDump | null => { - if (apiKey.dumpRetentionSeconds === null) return null; - return new RunDump( - apiKey, - { - method: turn.method, - path: turn.path, - bodyByteLength: turn.body.bytes.byteLength, - streamError: turn.body.streamError, - }, - Date.now(), - backgroundScheduler, - wantsStream, - timing, - ); -}; +): RunDump | null => apiKey.dumpRetentionSeconds === null ? null : new RunDump( + apiKey, + { method: turn.method, path: turn.path, bodyByteLength: turn.body.bytes.byteLength, streamError: turn.body.streamError }, + Date.now(), backgroundScheduler, wantsStream, timing, +); diff --git a/packages/gateway/src/dump/store-contract.ts b/packages/gateway/src/dump/store-contract.ts index ffa1a51fc7..fd09286e8e 100644 --- a/packages/gateway/src/dump/store-contract.ts +++ b/packages/gateway/src/dump/store-contract.ts @@ -11,7 +11,16 @@ export interface DumpListOptions { limit: number; } +export interface DumpRunWrite { + readonly id: string; + readonly startedAt: number; + readonly events: ReadableStream; + readonly metadata: Promise; +} + export interface DumpStore { + putRun(keyId: string, run: DumpRunWrite): Promise; + // Starts body preparation while the request is in flight. Implementations // may return identity bytes, but persistent stores compress here so the // accumulator can release the original request buffer before terminal IO. diff --git a/packages/gateway/src/dump/turn-dump.ts b/packages/gateway/src/dump/turn-dump.ts index 7be6061f4a..71bae0b305 100644 --- a/packages/gateway/src/dump/turn-dump.ts +++ b/packages/gateway/src/dump/turn-dump.ts @@ -25,8 +25,8 @@ import type { TelemetryModelIdentity } from '@floway-dev/provider'; * and it goes away with the shape. */ export interface StreamRecording { - frame(frame: ProtocolFrame): void; - end(): void; + frame(frame: ProtocolFrame): void | Promise; + end(): void | Promise; readonly fact: StreamFact | null; } @@ -36,7 +36,7 @@ export interface TurnDump { success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void; error(kind: 'upstream' | 'gateway', upstream?: string): void; failed(reason: unknown, options?: { readonly fallback: boolean }): void; - frame(frame: ProtocolFrame): void; + frame(frame: ProtocolFrame): void | Promise; /** Begins recording one stream. Every call is a new one, which is what lets a turn that * opens two — a sub-request beside the answer — keep them apart. */ openStream(): StreamRecording; @@ -64,7 +64,7 @@ export interface TurnDump { * family whose stream is bare protocol events says how one becomes a frame, because the record * cannot guess and a cast would be it guessing. */ -export function recordStream(stream: AsyncIterable>, dump: TurnDump | null): AsyncIterable>; +export function recordStream>(stream: AsyncIterable, dump: TurnDump | null): AsyncIterable; export function recordStream(stream: AsyncIterable, dump: TurnDump | null, asFrame: (value: T) => ProtocolFrame): AsyncIterable; export function recordStream( stream: AsyncIterable, @@ -80,12 +80,12 @@ export function recordStream( ...recording.fact, [Symbol.asyncIterator]: () => (async function* () { for await (const value of stream) { - recording.frame(asFrame(value)); + await recording.frame(asFrame(value)); yield value; } // Reached only where the source ran out on its own, which is what makes the record of // this stream complete. A reader that stopped early never gets here. - recording.end(); + await recording.end(); })(), }; } diff --git a/packages/gateway/src/repo/dump-store.ts b/packages/gateway/src/repo/dump-store.ts index 5b88ed72c7..f1cbd90a67 100644 --- a/packages/gateway/src/repo/dump-store.ts +++ b/packages/gateway/src/repo/dump-store.ts @@ -12,7 +12,7 @@ import { encodePersistedDumpMetadata, } from '../dump/storage-codec.ts'; import type { DumpBodyDescriptor } from '../dump/storage-codec.ts'; -import type { DumpListOptions, DumpStore } from '../dump/store-contract.ts'; +import type { DumpListOptions, DumpStore, DumpRunWrite } from '../dump/store-contract.ts'; import type { DumpMetadata, DumpRecordId, @@ -27,7 +27,7 @@ import type { StoredDumpUpstreamResponse, } from '../dump/types.ts'; import { upstreamResponseToWire } from '../dump/wire.ts'; -import { gunzipBytes, gzipBytes } from '../shared/gzip.ts'; +import { gunzipBytes, gzipBytes, gzipStream } from '../shared/gzip.ts'; import type { FileStore, SqlDatabase } from '@floway-dev/platform'; import { decodeForgivingBase64 } from '@floway-dev/protocols/common'; @@ -117,6 +117,45 @@ const NO_EDGE_HEADERS = '[]'; export class FileDumpStore implements DumpStore { constructor(private readonly db: SqlDatabase, private readonly files: FileStore) {} + async putRun(keyId: string, run: DumpRunWrite): Promise { + const fileKey = bodyPath(keyId, hourBucket(run.startedAt), run.id, 'run'); + await this.db.prepare( + `INSERT INTO spilled_files (file_key, owner_kind, owner_key, state, collect_after) + VALUES (?, 'dump-response', ?, 'staged', ?)`, + ).bind(fileKey, JSON.stringify([keyId, run.id]), Date.now() + SPILLED_FILE_STAGE_GRACE_MS).run(); + let timer: ReturnType | undefined; + let finished = false; + let renewal: Promise = Promise.resolve(); + const stopRenewing = () => { finished = true; clearTimeout(timer); }; + const scheduleRenewal = () => { + timer = setTimeout(() => { + renewal = this.db.prepare( + "UPDATE spilled_files SET collect_after = ? WHERE file_key = ? AND state = 'staged'", + ).bind(Date.now() + SPILLED_FILE_STAGE_GRACE_MS, fileKey).run().then(() => { + if (!finished) scheduleRenewal(); + }); + // Awaited before publication even if the lease write failed while upstream was quiet. + void renewal.catch(() => {}); + }, SPILLED_FILE_STAGE_GRACE_MS / 2); + }; + scheduleRenewal(); + try { + await this.files.put(fileKey, gzipStream(run.events)); + const meta = await run.metadata; + stopRenewing(); + await renewal; + await this.db.prepare( + `INSERT INTO dump_records + (key_id, id, created_at, upstream_id, meta_json, request_headers_json, response_headers_json, request_body_descriptor, response_body_descriptor, response_upstream_body_descriptor) + VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, ?, NULL)`, + ).bind( + keyId, meta.id, meta.completedAt, meta.upstream?.id ?? null, + encodePersistedDumpMetadata(meta, `dump record ${meta.id} metadata`), NO_EDGE_HEADERS, + encodeDumpBodyDescriptor({ key: fileKey, type: 'run' }, `dump record ${meta.id} run descriptor`), + ).run(); + } finally { stopRenewing(); } + } + async prepareRequestBody(body: Uint8Array): Promise { return { encoding: 'gzip', diff --git a/packages/gateway/src/shared/gzip.ts b/packages/gateway/src/shared/gzip.ts index 74bda3d0e6..77a1f8241b 100644 --- a/packages/gateway/src/shared/gzip.ts +++ b/packages/gateway/src/shared/gzip.ts @@ -34,3 +34,6 @@ export const gzipBytes = async (bytes: Uint8Array): Promise => export const gunzipBytes = async (bytes: Uint8Array): Promise => await collect(bytesStream(bytes).pipeThrough(new DecompressionStream('gzip'))); + +export const gzipStream = (stream: ReadableStream): ReadableStream => + (stream as ReadableStream).pipeThrough(new CompressionStream('gzip')); From aa7c70f8728c3907afb1ea43a3c2530a19ec1b1d Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 07:03:57 +0800 Subject: [PATCH 65/79] fix: distinguish failed stages from folded returns Keep a stage error distinct from a successful child result and show it through the reader UI. Decode the recorded object-space error and preserve the successful child facts for inspection. --- .../components/requests/run-model_test.ts | 14 ++++++++++++++ .../components/requests/run-stages_test.tsx | 12 ++++++++++++ apps/web/src/components/requests/run-model.ts | 6 ++++-- apps/web/src/components/requests/run-render.ts | 5 ++--- apps/web/src/components/requests/run-stages.tsx | 13 +++++++------ apps/web/src/i18n/locales/en.ts | 2 ++ apps/web/src/i18n/locales/zh-Hans.ts | 2 ++ 7 files changed, 43 insertions(+), 11 deletions(-) diff --git a/apps/web/__tests__/components/requests/run-model_test.ts b/apps/web/__tests__/components/requests/run-model_test.ts index 7a33b12e32..5d26527860 100644 --- a/apps/web/__tests__/components/requests/run-model_test.ts +++ b/apps/web/__tests__/components/requests/run-model_test.ts @@ -21,6 +21,20 @@ describe('run stage model', () => { ]); }); + it('does not fold a successful child response into a parent that subsequently failed', () => { + const model = readRun([ + { type: 'object', fromObjectId: 1, nodes: [{ $error: { name: 'Error', message: 'projection failed', stack: 'stack' } }] }, + { type: 'stage.entered', stageId: 1, name: 'projection', parentStageId: null, facts: {} }, + { type: 'stage.entered', stageId: 2, name: 'answer', parentStageId: 1 }, + { type: 'stage.leaved', stageId: 2, facts: { answer: 'upstream value' } }, + { type: 'stage.failed', stageId: 1, error: { $: 1 } }, + ].map(event => JSON.stringify(event)).join('\n')); + const parent = model.roots[0]!; + expect(parent.response).toBeNull(); + expect(parent.children[0]!.response).toEqual({ answer: 'upstream value' }); + expect(model.state(parent.failure!)).toMatchObject({ error: { $error: { message: 'projection failed' } } }); + }); + it('uses shared object identities, handles cycles and decodes schema keys', () => { const model = readRun([ { type: 'object', fromObjectId: 1, nodes: [{ '$$ref': 'schema', self: { $: 1 } }, { '$$ref': 'next', self: { $: 2 } }] }, diff --git a/apps/web/__tests__/components/requests/run-stages_test.tsx b/apps/web/__tests__/components/requests/run-stages_test.tsx index 33ab1365cf..b393ea1efa 100644 --- a/apps/web/__tests__/components/requests/run-stages_test.tsx +++ b/apps/web/__tests__/components/requests/run-stages_test.tsx @@ -34,4 +34,16 @@ describe('run stages', () => { fireEvent.click(screen.getByRole('option', { name: 'Response facts' })); expect(screen.getByTestId('state').textContent).toContain('"response": "answer"'); }); + + it('shows the failure separately from a child response', async () => { + renderInApp(); + await screen.findByTestId('state'); + fireEvent.click(screen.getByRole('combobox', { name: 'Stage view' })); + fireEvent.click(screen.getByRole('option', { name: 'Response facts' })); + expect(screen.getByText('This stage failed without returning facts.')).toBeTruthy(); + fireEvent.click(screen.getByRole('combobox', { name: 'Stage view' })); + fireEvent.click(screen.getByRole('option', { name: 'Stage error' })); + expect(screen.getByTestId('state').textContent).toContain('projection failed'); + }); + }); diff --git a/apps/web/src/components/requests/run-model.ts b/apps/web/src/components/requests/run-model.ts index 340a112906..c2998d2db6 100644 --- a/apps/web/src/components/requests/run-model.ts +++ b/apps/web/src/components/requests/run-model.ts @@ -10,6 +10,7 @@ export interface RunStage { readonly children: RunStage[]; readonly request: FactState; response: FactState | null; + failure: FactState | null; readonly logs: Extract[]; } @@ -52,13 +53,14 @@ export const readRun = (ndjson: string) => { if (request === undefined) throw new Error(`Run root stage ${event.stageId} has no request facts`); const entered: RunStage = { id: event.stageId, name: event.name, parentId: event.parentStageId, - children: [], logs: [], request, response: null, + children: [], logs: [], request, response: null, failure: null, }; stages.set(entered.id, entered); (parent === null ? roots : parent.children).push(entered); break; } case 'stage.leaved': stage(event.stageId).response = event.facts; break; + case 'stage.failed': stage(event.stageId).failure = { error: event.error }; break; case 'stage.log': stage(event.stageId).logs.push(event); break; case 'stream.frame': case 'stream.end': break; @@ -72,7 +74,7 @@ export const readRun = (ndjson: string) => { } // Folded exits inherit the last descent's returned state, including repeated next(). for (const current of [...stages.values()].reverse()) { - if (current.response === null && current.children.length > 0) { + if (current.failure === null && current.response === null && current.children.length > 0) { current.response = current.children.at(-1)!.response; } } diff --git a/apps/web/src/components/requests/run-render.ts b/apps/web/src/components/requests/run-render.ts index d212dee240..4fbec1fd26 100644 --- a/apps/web/src/components/requests/run-render.ts +++ b/apps/web/src/components/requests/run-render.ts @@ -15,12 +15,11 @@ export interface RenderedRunEvent { parseError: string | null; } -// Only three of the six name a stage, and each of the other three names its own -// namespace, so the subject is read off the event rather than looked up. const subjectOf = (event: DumpEvent): string | null => { switch (event.type) { case 'stage.entered': return event.name; - case 'stage.leaved': return `#${event.stageId}`; + case 'stage.leaved': + case 'stage.failed': return `#${event.stageId}`; case 'stage.log': return event.level; case 'object': return `#${event.fromObjectId}`; case 'stream.frame': diff --git a/apps/web/src/components/requests/run-stages.tsx b/apps/web/src/components/requests/run-stages.tsx index a428773549..b20bd159e8 100644 --- a/apps/web/src/components/requests/run-stages.tsx +++ b/apps/web/src/components/requests/run-stages.tsx @@ -25,15 +25,16 @@ export const RunStages = ({ ndjson }: { ndjson: string }) => { const labels = { request: t('dashboard.requests.requestFacts'), response: t('dashboard.requests.responseFacts'), down: t('dashboard.requests.requestChanges'), up: t('dashboard.requests.responseChanges'), - logs: t('dashboard.requests.stageLogs'), + logs: t('dashboard.requests.stageLogs'), error: t('dashboard.requests.stageError'), }; const current = model.stages.find(stage => stage.id === selected); if (current === undefined) return {t('dashboard.requests.noRunEvents')}; const child = current.children[descent]; - const text = view === 'request' ? model.state(current.request) - : view === 'response' ? current.response === null ? null : model.state(current.response) - : view === 'down' ? child === undefined ? [] : model.diff(current.request, child.request) - : child?.response === null || child === undefined || current.response === null ? [] : model.diff(child.response, current.response); + const text = view === 'error' ? current.failure === null ? null : model.state(current.failure) + : view === 'request' ? model.state(current.request) + : view === 'response' ? current.response === null ? null : model.state(current.response) + : view === 'down' ? child === undefined ? [] : model.diff(current.request, child.request) + : child?.response === null || child === undefined || current.response === null ? [] : model.diff(child.response, current.response); const select = (id: number) => { setSelected(id); setDescent(0); }; const toolbar =
setView(data.optionValue!)}> @@ -52,7 +53,7 @@ export const RunStages = ({ ndjson }: { ndjson: string }) => {
- {view === 'logs' ? {toolbar}{t('dashboard.requests.stageFailed')}
: view === 'logs' ? ({ event: log.level, text: JSON.stringify({ ...log, ...(log.fields === undefined ? {} : { fields: model.state(log.fields) }) }, null, 2), parseError: null }))} copyText={current.logs.map(log => JSON.stringify({ ...log, ...(log.fields === undefined ? {} : { fields: model.state(log.fields) }) })).join('\n')} toolbarStart={toolbar} diff --git a/apps/web/src/i18n/locales/en.ts b/apps/web/src/i18n/locales/en.ts index d9db91b74e..d4f01afd0a 100644 --- a/apps/web/src/i18n/locales/en.ts +++ b/apps/web/src/i18n/locales/en.ts @@ -1060,6 +1060,8 @@ const en = { requestChanges: 'Request changes', responseChanges: 'Response changes', stageLogs: 'Stage logs', + stageError: 'Stage error', + stageFailed: 'This stage failed without returning facts.', stageView: 'Stage view', descent: 'Descent', noRunEvents: 'This run recorded no events.', diff --git a/apps/web/src/i18n/locales/zh-Hans.ts b/apps/web/src/i18n/locales/zh-Hans.ts index c9a204e279..1af645412b 100644 --- a/apps/web/src/i18n/locales/zh-Hans.ts +++ b/apps/web/src/i18n/locales/zh-Hans.ts @@ -1012,6 +1012,8 @@ const zhHansCN = { requestChanges: '请求变化', responseChanges: '响应变化', stageLogs: 'Stage 日志', + stageError: 'Stage 错误', + stageFailed: '此 stage 失败，未返回 facts。', stageView: 'Stage 视图', descent: '分支', noRunEvents: '这次运行没有记录到任何事件。', From 756fe5549fb1d8e5c8fc57aaa4840804dfcb93a1 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 07:10:12 +0800 Subject: [PATCH 66/79] refactor: declare failover usage-context restoration --- packages/gateway/src/data-plane/pipeline/failover.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/gateway/src/data-plane/pipeline/failover.ts b/packages/gateway/src/data-plane/pipeline/failover.ts index f3cf96d801..da3ebfba00 100644 --- a/packages/gateway/src/data-plane/pipeline/failover.ts +++ b/packages/gateway/src/data-plane/pipeline/failover.ts @@ -32,7 +32,7 @@ export const failover = ({ failed, owns, pendingUsage }: Forking) => defineStage Slice<'serve.candidates' | 'serve.usage.prior'>, Slice<'serve.candidates' | 'route.attempt' | 'serve.usage.prior'>, Slice<'response.usage.billable'>, - Slice<'response.usage.billable'>, + Slice<'response.usage.billable' | 'serve.usage.prior'>, GatewayServices >({ name: 'failover', @@ -43,7 +43,7 @@ export const failover = ({ failed, owns, pendingUsage }: Forking) => defineStage // Owned on the way up and handed onward: every attempt's is this stage's to release, // and the one it adopts rides up with ownership going with it. consumes: owns as never, - provides: [...owns, 'response.usage.billable', ...(pendingUsage === undefined ? [] : [pendingUsage])] as never, + provides: [...owns, 'response.usage.billable', 'serve.usage.prior', ...(pendingUsage === undefined ? [] : [pendingUsage])] as never, }, }, execute: async (facts, next, use) => { From a549f04bbddcc762feb47fd3d211abe015109b5c Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 07:14:27 +0800 Subject: [PATCH 67/79] fix: observe asynchronous provider side effects Await scoped transport diagnostics and retain quota persistence rejection in its owned deferred outcome. Keep the intermediate provider factories valid under the shared lint contract. --- packages/http/src/http.ts | 2 +- packages/provider-codex/src/provider.ts | 2 +- packages/provider-copilot/src/stages/observe-quota.ts | 6 ++---- packages/provider-ollama/src/provider.ts | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/http/src/http.ts b/packages/http/src/http.ts index 1da37bed01..0b1efa5f86 100644 --- a/packages/http/src/http.ts +++ b/packages/http/src/http.ts @@ -24,7 +24,7 @@ export const http = defineStage { request: { needs: ['request.http.callId'], consumes: [], provides: [] }, response: { needs: ['response.http.exchange'], consumes: [], provides: ['response.copilot.quota'] }, }, - execute: async (facts, next, use) => { + execute: async (facts, next) => { const back = await next(move({ ...facts })); const exchange = back['response.http.exchange']; const pending: Promise[] = []; if (exchange.type === 'response') { const snapshot = parseCopilotQuotaHeaders(new Headers(exchange.headers.map(([name, value]): [string, string] => [name, value])), new Date()); - if (snapshot !== null) pending.push(putCopilotQuota(upstreamId, snapshot).catch(error => { - use.log.warn('Copilot quota snapshot persistence failed', { upstreamId, error }); - })); + if (snapshot !== null) pending.push(putCopilotQuota(upstreamId, snapshot)); } return move({ ...back, 'response.copilot.quota': defer(Promise.all(pending)) }); }, diff --git a/packages/provider-ollama/src/provider.ts b/packages/provider-ollama/src/provider.ts index c7fb5f9a54..836596b3bf 100644 --- a/packages/provider-ollama/src/provider.ts +++ b/packages/provider-ollama/src/provider.ts @@ -30,8 +30,8 @@ import { assertOllamaUpstreamRecord, type OllamaUpstreamConfig } from './config. import { OLLAMA_DEFAULT_FLAGS } from './defaults.ts'; import { fetchOllamaCatalog, type OllamaCatalog } from './fetch-models.ts'; import { ollamaFetchOpenAIAudioTranscriptions, ollamaFetchOpenAIChatCompletions, ollamaFetchOpenAICompletions, ollamaFetchOpenAIEmbeddings, ollamaFetchAnthropicMessages, ollamaFetchAnthropicMessagesCountTokens, ollamaFetchOpenAIResponses, ollamaFetchOpenAIResponsesCompact } from './fetch.ts'; -import { pricingForOllamaModelKey } from './pricing.ts'; import { createOllamaPipelines } from './pipelines.ts'; +import { pricingForOllamaModelKey } from './pricing.ts'; import { readOllamaUpstreamState } from './state.ts'; import { scheduleOllamaUsageProbe } from './usage-probe.ts'; import { parseAnthropicMessagesStream } from '@floway-dev/protocols/anthropic-messages'; From ded7d767d0a5eb213c4457f17131eee6611adbd7 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 07:42:55 +0800 Subject: [PATCH 68/79] test: exercise pipeline ownership and deferred failures Observe released streamed resources and restored usage context through a real run, exercise stage-to-events navigation, and prove quota persistence rejects drain without replacing the HTTP response. --- .../components/requests/detail_test.tsx | 6 +- .../data-plane/rerank-pipeline_test.ts | 73 +++++++------------ .../__tests__/stages/observe-quota_test.ts | 35 +++++++++ 3 files changed, 68 insertions(+), 46 deletions(-) create mode 100644 packages/provider-copilot/__tests__/stages/observe-quota_test.ts diff --git a/apps/web/__tests__/components/requests/detail_test.tsx b/apps/web/__tests__/components/requests/detail_test.tsx index 0add7c1963..96180a2b3a 100644 --- a/apps/web/__tests__/components/requests/detail_test.tsx +++ b/apps/web/__tests__/components/requests/detail_test.tsx @@ -16,12 +16,16 @@ const record: DumpEdgeRecord = { capture: { exchanges: [{ upstreamId: 'u', request: { url: 'https://upstream.test', method: 'POST', headers: [], body: { encoding: 'utf8', data: '{"upstream":"translated request"}' } }, response: { status: 200, headers: [], body: { encoding: 'utf8', data: 'data: {broken\n' }, complete: false, error: null }, error: null }], response: { body: { encoding: 'utf8', data: 'data: downstream\n' }, complete: true, error: null } }, }; -it('shows pipeline facts through the searchable event viewer with timing and export controls', () => { +it('opens stage facts and offers searchable events with timing and export controls', async () => { const run: DumpRunRecord = { shape: 'run', meta: { ...record.meta, ttftMs: 125 }, events: '{"type":"stage.entered","stageId":1,"name":"serve","parentStageId":null,"facts":{"serve.model":"m"}}\n', }; renderInApp(); + expect(screen.getByRole('treeitem', { name: 'serve #1' })).toBeTruthy(); + expect((await screen.findByTestId('body-content')).textContent).toContain('serve.model'); + fireEvent.click(screen.getByRole('combobox', { name: 'Run' })); + fireEvent.click(screen.getByRole('option', { name: 'Events (1)' })); expect(screen.getByRole('button', { name: '#1 stage.entered serve' })).toBeTruthy(); expect(screen.getByRole('button', { name: 'Search events' })).toBeTruthy(); expect(screen.getByRole('button', { name: 'Export record' })).toBeTruthy(); diff --git a/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts b/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts index 77df0d4c5e..137c0f528b 100644 --- a/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts +++ b/packages/gateway/__tests__/data-plane/rerank-pipeline_test.ts @@ -1,13 +1,9 @@ -// Rerank's pipeline, assembled. `compose` derives the entry contract and rejects an array -// that cannot work, so most of what this file establishes is established by the assembly -// succeeding at all — and what is worth writing down is the entry contract it derives, -// including the one key it cannot see. - import { describe, expect, it } from 'vitest'; import { failover } from '../../src/data-plane/pipeline/failover.ts'; import { rerankServePipeline } from '../../src/data-plane/rerank/pipeline.ts'; -import { move, run } from '@floway-dev/pipeline'; +import { mockGatewayCtx } from '../test-utils/gateway-ctx.ts'; +import { compose, defineStage, move, own, run } from '@floway-dev/pipeline'; import type { CanonicalRerankRequest } from '@floway-dev/protocols/rerank'; const request: CanonicalRerankRequest = { @@ -27,45 +23,32 @@ describe('the rerank pipeline', () => { ]); }); - // `failover` used to declare that it provides `response.http.body` for every family. Three - // of the six never produce one — they read their answer to the end — so those pipelines - // composed cleanly and then threw on the first real request, at the deepest stage, with a - // message about a key their author had never written down. - // - // What a fork owns is a statement only the family can make, so it makes it. This asserts - // the shape of that statement rather than the absence of a bug, because the absence of a - // bug is what every one of these tests asserted before and none of them caught it. - it('claims nothing on the way up when a family reads its answer to the end', () => { - const reading = failover({ failed: () => false, owns: [] }); - expect(reading.through?.response.consumes).toEqual([]); - expect(reading.through?.response.provides).toEqual(['response.usage.billable']); - - // And a family that streams claims the key it streams at, in both directions: every - // attempt's is the fork's to release, and the one it adopts rides up with ownership. - const streaming = failover({ failed: () => false, owns: ['response.http.body'] }); - expect(streaming.through?.response.consumes).toEqual(['response.http.body']); - expect(streaming.through?.response.provides).toEqual(['response.http.body', 'response.usage.billable']); - }); - - // A live handle is never a fact, and the test for that is whether it can be rendered into - // the dump. A `ModelCandidate` cannot: it carries the provider's instance, its fetcher and - // its models cache. Putting one in the record deep-freezes all three, and the SWR cache - // refresh the provider does on its own schedule then breaks — throwing under a module, - // which every file here is, and failing silently anywhere that is not strict. - it('carries a selector, so freezing the record cannot reach a live handle', () => { - const instance = { cache: null as unknown }; - const candidate = { provider: { upstreamId: 'u', instance, modelsCache: { at: 1 } } }; - - // What the record actually holds: data, and nothing that answers to a call. - move({ 'route.attempt': { upstreamId: 'u', modelId: 'm', flags: [] } }); - expect(Object.isFrozen(instance)).toBe(false); - expect(Object.isFrozen(candidate.provider.modelsCache)).toBe(false); - - // And what putting the candidate itself there would have done, so the difference is not - // hypothetical — this is the shape the six families carried until the selector split. - move({ 'route.candidate': candidate } as never); - expect(Object.isFrozen(instance)).toBe(true); - expect(() => { instance.cache = { refreshed: true }; }).toThrow(TypeError); + it('retains only streamed bodies and restores the caller usage context', async () => { + for (const streaming of [false, true]) { + let released = 0; + const body = streaming ? own({ source: 'upstream body' }, async () => { released += 1; }) : null; + const prior = move([]); + const answer = defineStage, Record>({ + name: 'answer', + return: { provides: ['response.usage.billable', ...streaming ? ['response.http.body'] : []] }, + execute: async facts => move({ + ...facts, 'response.usage.billable': [], ...body === null ? {} : { 'response.http.body': body }, + }), + }); + const pipeline = compose, Record>('readOrStream', [ + failover({ failed: () => false, owns: streaming ? ['response.http.body'] : [] }), answer, + ]); + const { facts, drain } = await run(pipeline, move({ + 'serve.candidates': [{ upstreamId: 'upstream', modelId: 'rerank', flags: [] }], + 'serve.usage.prior': prior, + }), { gateway: mockGatewayCtx() }); + expect(facts['serve.usage.prior']).toBe(prior); + if (streaming) expect(facts['response.http.body']).toBe(body); + else expect(facts).not.toHaveProperty('response.http.body'); + expect(released).toBe(0); + await drain(); + expect(released).toBe(streaming ? 1 : 0); + } }); it('names the entry key a caller did not bring, before any stage runs', async () => { diff --git a/packages/provider-copilot/__tests__/stages/observe-quota_test.ts b/packages/provider-copilot/__tests__/stages/observe-quota_test.ts new file mode 100644 index 0000000000..8b8f2b3875 --- /dev/null +++ b/packages/provider-copilot/__tests__/stages/observe-quota_test.ts @@ -0,0 +1,35 @@ +import { expect, test } from 'vitest'; + +import { observeCopilotQuota } from '../../src/stages/observe-quota.ts'; +import type { HttpRequestFacts, HttpResponseFacts } from '@floway-dev/http/pipeline'; +import { compose, defineStage, move, run } from '@floway-dev/pipeline'; +import { initProviderRepo, type ProviderServices } from '@floway-dev/provider'; +import { noopUpstreamCallOptions } from '@floway-dev/test-utils'; + +test('returns the HTTP reply while quota persistence rejects the deferred drain with its original error', async () => { + const error = new Error('quota persistence failed'); + let writes = 0; + initProviderRepo(() => ({ + upstreams: { + getById: async () => null, + saveState: async () => { writes++; throw error; }, + }, + })); + const exchange = move({ + type: 'response' as const, + status: 200, + statusText: 'OK', + headers: [['x-quota-snapshot-chat', 'ent=-1&ov=0.0&ovPerm=false&rem=100.0&rst=2026-09-01T00%3A00%3A00Z&totRem=-1']] as const, + body: null, + }); + const terminal = defineStage, HttpResponseFacts>({ + name: 'reply', + return: { provides: ['response.http.exchange', 'response.http.body'] }, + execute: async facts => move({ ...facts, 'response.http.exchange': exchange, 'response.http.body': null }), + }); + const services: ProviderServices = { httpCall: () => ({ ...noopUpstreamCallOptions(), signal: undefined }) }; + const executed = await run(compose, HttpResponseFacts>('quota', [observeCopilotQuota('up_quota'), terminal]), move({ 'request.http.callId': 1 }), services); + expect(executed.facts['response.http.exchange']).toBe(exchange); + expect(writes).toBe(1); + await expect(executed.drain()).rejects.toBe(error); +}); From aeb3312ddb7a4ebd84cc0b2368053427018fa492 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 07:42:55 +0800 Subject: [PATCH 69/79] fix: serialize client JSON before run settlement Encode the final JSON body in a response stage so codec failures retain failure facts and measured usage. Preserve canonical protocol content for diagnostics, forward encoded bytes from HTTP, and leave streams and upstream documents to their owning transport. --- README.md | 3 +- .../pipeline/serialize-client-json_test.ts | 73 +++++++++++++++++++ .../src/data-plane/openai-audio/http.ts | 8 +- .../src/data-plane/openai-audio/pipeline.ts | 4 +- .../src/data-plane/openai-completions/http.ts | 2 +- .../data-plane/openai-completions/pipeline.ts | 4 +- .../src/data-plane/openai-embeddings/http.ts | 2 +- .../data-plane/openai-embeddings/pipeline.ts | 4 +- .../src/data-plane/openai-images/facts.ts | 2 +- .../src/data-plane/openai-images/http.ts | 2 +- .../src/data-plane/openai-images/pipeline.ts | 2 + .../gateway/src/data-plane/pipeline/facts.ts | 31 ++------ .../pipeline/serialize-client-json.ts | 26 +++++++ .../gateway/src/data-plane/rerank/pipeline.ts | 4 +- .../gateway/src/data-plane/rerank/serve.ts | 2 +- 15 files changed, 127 insertions(+), 42 deletions(-) create mode 100644 packages/gateway/__tests__/data-plane/pipeline/serialize-client-json_test.ts create mode 100644 packages/gateway/src/data-plane/pipeline/serialize-client-json.ts diff --git a/README.md b/README.md index ad61cee5d4..fb2795d682 100644 --- a/README.md +++ b/README.md @@ -201,7 +201,8 @@ upload facts contain bytes and file metadata. The shared HTTP stage adopts each once, and decoders replace its release action as ownership moves through the run. Provider retry observations and failed-over attempts join the same billable collection. Settlement wraps rendering and writes each observed call once, including calls completed before a later -stage throws. Streaming quantities settle at the deferred exit. Explicit protocol and +stage throws. Client JSON serialization runs inside settlement and publishes UTF-8 body bytes +while retaining the canonical content for inspection. Streaming quantities settle at the deferred exit. Explicit protocol and transport errors take precedence over the settlement's generic failure marker. Run dumps stream their encoded NDJSON into durable storage with backpressure while appending diff --git a/packages/gateway/__tests__/data-plane/pipeline/serialize-client-json_test.ts b/packages/gateway/__tests__/data-plane/pipeline/serialize-client-json_test.ts new file mode 100644 index 0000000000..f962d22580 --- /dev/null +++ b/packages/gateway/__tests__/data-plane/pipeline/serialize-client-json_test.ts @@ -0,0 +1,73 @@ +import { expect, test } from 'vitest'; + +import type { GatewayFacts } from '../../../src/data-plane/pipeline/facts.ts'; +import { serializeClientJson } from '../../../src/data-plane/pipeline/serialize-client-json.ts'; +import { writeSettlement } from '../../../src/data-plane/pipeline/settlement.ts'; +import { initRepo } from '../../../src/repo/index.ts'; +import { InMemoryRepo } from '../../repo/memory.ts'; +import { mockGatewayCtx } from '../../test-utils/gateway-ctx.ts'; +import { compose, defineStage, getFailureFacts, move, run, type Event } from '@floway-dev/pipeline'; +import { mockPerfTelemetryContext } from '@floway-dev/test-utils'; + +const renderedKey = 'response.test.rendered'; +const answer = (value: unknown) => defineStage, Record & Pick>({ + name: 'answer', + return: { provides: [renderedKey, 'response.usage.billable', 'response.http.status'] }, + execute: async facts => move({ + ...facts, [renderedKey]: value, 'response.http.status': 200, + 'response.usage.billable': [{ + identity: { model: 'm', upstream: 'u', modelKey: 'm', pricing: null }, + quantities: { input_tokens: '3', output_tokens: '2' }, + }], + }), +}); + +test('serializes client JSON within the run while preserving the canonical object', async () => { + const value = move({ text: '中' }); + const pipeline = compose, Record>('json', [serializeClientJson(renderedKey), answer(value)]); + const executed = await run(pipeline, move({}), {}); + expect(executed.facts[renderedKey]).toBe(value); + expect(new TextDecoder().decode(executed.facts['response.http.jsonBody'] as Uint8Array)).toBe('{"text":"中"}'); + await executed.drain(); +}); + +test('rejects a client value without a JSON representation', async () => { + await expect(run(compose('invalidJson', [serializeClientJson(renderedKey), answer(undefined)]), move({}), {})) + .rejects.toThrow('Client JSON body has no JSON representation'); +}); + +test('leaves streams and upstream documents to their owning transport', async () => { + for (const value of [new Uint8Array([0, 255, 1]), { async *[Symbol.asyncIterator]() { yield 'frame'; } }]) { + const pipeline = compose, Record>('nonJson', [serializeClientJson(renderedKey), answer(value)]); + const executed = await run(pipeline, move({}), {}); + expect(executed.facts[renderedKey]).toBe(value); + expect(executed.facts['response.http.jsonBody']).toBeNull(); + await executed.drain(); + } +}); + +test('a JSON encoding fault settles measured usage as a failure and retains the failure facts', async () => { + const repo = new InMemoryRepo(); initRepo(repo); + const value = move({ extension: 1n }); + const work: Promise[] = []; + const events: Event[] = []; + const gateway = mockGatewayCtx({ + attempt: { + timing: { upstreamCallStartedAt: 50, firstOutputTokenAt: 100 }, + telemetry: mockPerfTelemetryContext(), + }, + }); + const pipeline = compose('encodingFailure', [ + writeSettlement(() => false), serializeClientJson(renderedKey), answer(value), + ]); + const error = await run(pipeline, move({}), { + gateway, background: (promise: Promise) => { work.push(promise); }, + dump: (event: Event) => { events.push(event); }, + }).then(() => { throw new Error('Expected JSON serialization to reject'); }, error => error as unknown); + expect(error).toBeInstanceOf(TypeError); + expect(getFailureFacts(error)?.[renderedKey]).toBe(value); + expect(events.find((event): event is Extract => event.type === 'stage.failed' && event.stageId === 2)?.error).toBe(error); + await Promise.all(work); + expect(await repo.usage.listAll()).toMatchObject([{ requests: 1, metrics: expect.arrayContaining([{ metric: 'input_tokens', quantity: '3', unitPrice: null }, { metric: 'output_tokens', quantity: '2', unitPrice: null }]) }]); + expect(await repo.performance.listAll()).toMatchObject([{ errorsWithOutput: 1, errorsNoOutput: 0 }]); +}); diff --git a/packages/gateway/src/data-plane/openai-audio/http.ts b/packages/gateway/src/data-plane/openai-audio/http.ts index 5e1541baa5..e88dd417dc 100644 --- a/packages/gateway/src/data-plane/openai-audio/http.ts +++ b/packages/gateway/src/data-plane/openai-audio/http.ts @@ -77,12 +77,6 @@ const prepareOpenAIAudioTranscription = async (bytes: Uint8Array, contentType: s }; }; -/** The client is sent bytes on every path, never a string, so nothing downstream of here - * guesses a media type: a `Response` built over a string is labelled `text/plain` by the - * platform, and an upstream that declared no media type has not asked for that. */ -const bodyOf = (rendered: Record | Uint8Array): Uint8Array => - rendered instanceof Uint8Array ? rendered : new TextEncoder().encode(JSON.stringify(rendered)); - /** * The epilogue: what the run answered with, as a response. * @@ -125,7 +119,7 @@ export const openaiAudioTranscriptions = async (c: Context): Promise = if (isFrames(rendered)) return { frames: rendered }; // The upstream's own media type, or none where it declared none: a document this // gateway carried rather than wrote is not one it can describe. - return { body: bodyOf(rendered) as BodyInit, contentType: facts['response.openaiAudioTranscription.mediaType'] }; + return { body: (rendered instanceof Uint8Array ? rendered : facts['response.http.jsonBody']!) as BodyInit, contentType: facts['response.openaiAudioTranscription.mediaType'] }; }, ); }; diff --git a/packages/gateway/src/data-plane/openai-audio/pipeline.ts b/packages/gateway/src/data-plane/openai-audio/pipeline.ts index de18c50944..6edc059069 100644 --- a/packages/gateway/src/data-plane/openai-audio/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-audio/pipeline.ts @@ -5,16 +5,18 @@ import { narrowing } from './target.ts'; import { isFailure, type BillableEntity } from '../pipeline/facts.ts'; import { failover } from '../pipeline/failover.ts'; import { resolveCandidates } from '../pipeline/resolve-candidates.ts'; +import { serializeClientJson } from '../pipeline/serialize-client-json.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; import { compose, type Pipeline } from '@floway-dev/pipeline'; export const openaiAudioTranscriptionServePipeline: Pipeline< Fields<'ingress.http.headers' | 'ingress.openaiAudioTranscription.responseFormat' | 'request.openaiAudioTranscription.form' | 'serve.model'>, - Fields<'response.openaiAudioTranscription.rendered' | 'response.openaiAudioTranscription.mediaType' | 'response.openaiAudioTranscription.streamedOutcome'> + Fields<'response.http.jsonBody' | 'response.openaiAudioTranscription.rendered' | 'response.openaiAudioTranscription.mediaType' | 'response.openaiAudioTranscription.streamedOutcome'> & { 'response.http.status': number; 'response.usage.billable': readonly BillableEntity[]; 'response.http.headers': readonly (readonly [string, string])[]; } > = compose('openaiAudioTranscriptionServe', [ writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, 'response.openaiAudioTranscription.streamedOutcome'), + serializeClientJson('response.openaiAudioTranscription.rendered'), emitOpenAIAudioTranscription, resolveCandidates(narrowing), failover({ diff --git a/packages/gateway/src/data-plane/openai-completions/http.ts b/packages/gateway/src/data-plane/openai-completions/http.ts index cd0a08d590..3997b90b36 100644 --- a/packages/gateway/src/data-plane/openai-completions/http.ts +++ b/packages/gateway/src/data-plane/openai-completions/http.ts @@ -48,7 +48,7 @@ export const openaiCompletions = async (c: Context): Promise => { const rendered = facts['response.openaiCompletions.rendered']; return isFrames(rendered) ? { frames: rendered } - : { body: JSON.stringify(rendered), contentType: 'application/json' }; + : { body: facts['response.http.jsonBody']!, contentType: 'application/json' }; }, ); }; diff --git a/packages/gateway/src/data-plane/openai-completions/pipeline.ts b/packages/gateway/src/data-plane/openai-completions/pipeline.ts index 10c629eccd..f28f10208e 100644 --- a/packages/gateway/src/data-plane/openai-completions/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-completions/pipeline.ts @@ -5,14 +5,16 @@ import { narrowing } from './target.ts'; import { isFailure } from '../pipeline/facts.ts'; import { failover } from '../pipeline/failover.ts'; import { resolveCandidates } from '../pipeline/resolve-candidates.ts'; +import { serializeClientJson } from '../pipeline/serialize-client-json.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; import { compose, type Pipeline } from '@floway-dev/pipeline'; export const openaiCompletionsServePipeline: Pipeline< Fields<'ingress.http.headers' | 'ingress.openaiCompletions.wantsStream' | 'ingress.openaiCompletions.wantsUsageChunk' | 'request.openaiCompletions.payload' | 'serve.model'>, - Fields<'response.openaiCompletions.rendered' | 'response.openaiCompletions.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> + Fields<'response.http.jsonBody' | 'response.openaiCompletions.rendered' | 'response.openaiCompletions.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> > = compose('openaiCompletionsServe', [ writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, 'response.openaiCompletions.streamedUsage'), + serializeClientJson('response.openaiCompletions.rendered'), emitOpenAICompletions, resolveCandidates(narrowing), failover({ diff --git a/packages/gateway/src/data-plane/openai-embeddings/http.ts b/packages/gateway/src/data-plane/openai-embeddings/http.ts index 2882e60f36..883f030639 100644 --- a/packages/gateway/src/data-plane/openai-embeddings/http.ts +++ b/packages/gateway/src/data-plane/openai-embeddings/http.ts @@ -49,6 +49,6 @@ export const openaiEmbeddings = async (c: Context): Promise => { 'request.openaiEmbeddings.canonical': request, 'serve.model': model, }) as never, - facts => ({ body: JSON.stringify(facts['response.openaiEmbeddings.rendered']), contentType: 'application/json' }), + facts => ({ body: facts['response.http.jsonBody']!, contentType: 'application/json' }), ); }; diff --git a/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts b/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts index b0f928c63d..8a7685dd51 100644 --- a/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-embeddings/pipeline.ts @@ -5,14 +5,16 @@ import { narrowing } from './target.ts'; import { isFailure } from '../pipeline/facts.ts'; import { failover } from '../pipeline/failover.ts'; import { resolveCandidates } from '../pipeline/resolve-candidates.ts'; +import { serializeClientJson } from '../pipeline/serialize-client-json.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; import { compose, type Pipeline } from '@floway-dev/pipeline'; export const openaiEmbeddingsServePipeline: Pipeline< Fields<'ingress.http.headers' | 'ingress.openaiEmbeddings.encodingFormat' | 'request.openaiEmbeddings.canonical' | 'serve.model'>, - Fields<'response.openaiEmbeddings.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> + Fields<'response.http.jsonBody' | 'response.openaiEmbeddings.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> > = compose('openaiEmbeddingsServe', [ writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400), + serializeClientJson('response.openaiEmbeddings.rendered'), emitOpenAIEmbeddings, resolveCandidates(narrowing), failover({ diff --git a/packages/gateway/src/data-plane/openai-images/facts.ts b/packages/gateway/src/data-plane/openai-images/facts.ts index 573a870002..76e8d3e1c2 100644 --- a/packages/gateway/src/data-plane/openai-images/facts.ts +++ b/packages/gateway/src/data-plane/openai-images/facts.ts @@ -43,5 +43,5 @@ export type Fields = { [P in K]: OpenAIImages export type OpenAIImagesServeEntry = Fields<'ingress.http.headers' | 'ingress.openaiImages.wantsStream' | 'request.openaiImages.canonical' | 'serve.model'>; export type OpenAIImagesServeExit = Fields< - 'response.openaiImages.rendered' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable' + 'response.http.jsonBody' | 'response.openaiImages.rendered' | 'response.openaiImages.streamedUsage' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable' >; diff --git a/packages/gateway/src/data-plane/openai-images/http.ts b/packages/gateway/src/data-plane/openai-images/http.ts index 68fff7fbd4..f1d437ac88 100644 --- a/packages/gateway/src/data-plane/openai-images/http.ts +++ b/packages/gateway/src/data-plane/openai-images/http.ts @@ -56,7 +56,7 @@ const serveOpenAIImages = async ( }) as never, facts => { const answer = facts['response.openaiImages.rendered']; - return isFrames(answer) ? { frames: answer } : { body: JSON.stringify(answer), contentType: 'application/json' }; + return isFrames(answer) ? { frames: answer } : { body: facts['response.http.jsonBody']!, contentType: 'application/json' }; }, ); }; diff --git a/packages/gateway/src/data-plane/openai-images/pipeline.ts b/packages/gateway/src/data-plane/openai-images/pipeline.ts index 56eea13e67..8e5449412a 100644 --- a/packages/gateway/src/data-plane/openai-images/pipeline.ts +++ b/packages/gateway/src/data-plane/openai-images/pipeline.ts @@ -5,6 +5,7 @@ import { narrowing } from './target.ts'; import { isFailure } from '../pipeline/facts.ts'; import { failover } from '../pipeline/failover.ts'; import { resolveCandidates } from '../pipeline/resolve-candidates.ts'; +import { serializeClientJson } from '../pipeline/serialize-client-json.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; import { compose, type Pipeline } from '@floway-dev/pipeline'; import type { CanonicalOpenAIImagesRequest } from '@floway-dev/protocols/openai-images'; @@ -12,6 +13,7 @@ import type { CanonicalOpenAIImagesRequest } from '@floway-dev/protocols/openai- export const openaiImagesServePipeline = (request: CanonicalOpenAIImagesRequest): Pipeline => compose('openaiImagesServe', [ writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400, 'response.openaiImages.streamedUsage'), + serializeClientJson('response.openaiImages.rendered'), emitOpenAIImages, resolveCandidates(narrowing(request)), failover({ diff --git a/packages/gateway/src/data-plane/pipeline/facts.ts b/packages/gateway/src/data-plane/pipeline/facts.ts index d00ca87005..401fdb9e5e 100644 --- a/packages/gateway/src/data-plane/pipeline/facts.ts +++ b/packages/gateway/src/data-plane/pipeline/facts.ts @@ -1,12 +1,4 @@ -// The keys every family's pipeline shares. A family extends this space with its own -// protocol keys by intersection and never merges into it, so a stage written here drops -// into any family's pipeline and a family's own keys are unreachable from a stage that -// was not written against them. -// -// Keys are namespaced, camelCase and family-first. `ingress.*` is what the client sent and -// stays put across a protocol switch; `serve.*` belongs to the served request as a whole -// and outlives an attempt; `request.*` and `response.*` are the two directions, mirrored -// key for key and semantically disjoint. +// Shared gateway content facts. Live providers, fetchers and model caches remain in services. import type { UsageQuantities } from '../../repo/types.ts'; import type { Secret, Owned } from '@floway-dev/pipeline'; @@ -43,8 +35,7 @@ export interface BillableEntity { export interface Failure { readonly status: number; readonly message: string; - /** The upstream's own body, when there was one. A client is not owed the upstream's - * exact bytes, but a dump reader is owed what actually came back. */ + /** Complete parsed upstream error content when available. */ readonly body?: unknown; } @@ -61,25 +52,15 @@ export interface GatewayFacts { 'serve.model': string; 'serve.candidates': readonly AttemptSelector[]; - /** Which upstream this attempt targets. Provided per attempt by the stage that forks. - * - * A **selector**, not the candidate itself. A `ModelCandidate` carries the provider's - * live instance, its fetcher and its models cache, and a live handle is never a fact — - * the test being whether it can be rendered into the dump. Putting one in the record - * deep-freezes all three, and the writes the provider relies on then fail *silently*, - * because a frozen write only throws in strict mode and the provider's own code is not - * the caller. The SWR models cache would stop refreshing with nothing to see. - * - * So the resolver is a service and the selector is the fact, which is the ruling as - * written. What travels is what identifies the attempt; what dials is injected. */ + /** A selector resolves a live candidate without freezing the provider's mutable catalog. */ 'route.attempt': AttemptSelector; - /** There is exactly one url and one headers. Headers are rewritten the whole way down, - * so the dump shows a header's entire history in one place, and a value may be secret. */ - 'request.http.url': string; + /** Admitted headers remain values until the provider shapes its authenticated HTTP request. */ 'request.http.headers': readonly (readonly [string, string | Secret])[]; 'response.http.status': number; + /** Serialized client JSON, or null for a protocol stream or an upstream document. */ + 'response.http.jsonBody': Uint8Array | null; 'response.http.headers': readonly (readonly [string, string])[]; /** The upstream's body, still open, and marked as something the run answers for. `Owned` * rather than `AsyncDisposable`, because a structural type would say what the host happens diff --git a/packages/gateway/src/data-plane/pipeline/serialize-client-json.ts b/packages/gateway/src/data-plane/pipeline/serialize-client-json.ts new file mode 100644 index 0000000000..299ac9bb0e --- /dev/null +++ b/packages/gateway/src/data-plane/pipeline/serialize-client-json.ts @@ -0,0 +1,26 @@ +import type { GatewayFacts } from './facts.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; + +export const serializeClientJson = (renderedKey: K) => defineStage< + Record, + Record, + Record, + Pick +>({ + name: 'serializeClientJson', + through: { + request: { needs: [], consumes: [], provides: [] }, + response: { needs: [renderedKey], consumes: [], provides: ['response.http.jsonBody'] }, + }, + execute: async (facts, next) => { + const back = await next(facts); + const rendered = back[renderedKey]; + if (typeof rendered === 'object' && rendered !== null + && (rendered instanceof Uint8Array || Symbol.asyncIterator in rendered)) { + return move({ ...back, 'response.http.jsonBody': null }); + } + const json = JSON.stringify(rendered); + if (json === undefined) throw new TypeError('Client JSON body has no JSON representation'); + return move({ ...back, 'response.http.jsonBody': new TextEncoder().encode(json) }); + }, +}); diff --git a/packages/gateway/src/data-plane/rerank/pipeline.ts b/packages/gateway/src/data-plane/rerank/pipeline.ts index 51c96432ac..71a22324af 100644 --- a/packages/gateway/src/data-plane/rerank/pipeline.ts +++ b/packages/gateway/src/data-plane/rerank/pipeline.ts @@ -5,15 +5,17 @@ import { narrowing } from './target.ts'; import { isFailure } from '../pipeline/facts.ts'; import { failover } from '../pipeline/failover.ts'; import { resolveCandidates } from '../pipeline/resolve-candidates.ts'; +import { serializeClientJson } from '../pipeline/serialize-client-json.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; import { compose, type Pipeline } from '@floway-dev/pipeline'; import type { CanonicalRerankRequest } from '@floway-dev/protocols/rerank'; export const rerankServePipeline = (request: CanonicalRerankRequest): Pipeline< Fields<'ingress.http.headers' | 'ingress.rerank.sourceProtocol' | 'request.rerank.canonical' | 'serve.model'>, - Fields<'response.rerank.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> + Fields<'response.http.jsonBody' | 'response.rerank.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> > => compose('rerankServe', [ writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400), + serializeClientJson('response.rerank.rendered'), emitRerank, resolveCandidates(narrowing(request)), failover({ diff --git a/packages/gateway/src/data-plane/rerank/serve.ts b/packages/gateway/src/data-plane/rerank/serve.ts index 8c5d440b41..1744d30216 100644 --- a/packages/gateway/src/data-plane/rerank/serve.ts +++ b/packages/gateway/src/data-plane/rerank/serve.ts @@ -62,6 +62,6 @@ export const rerank = (sourceProtocol: RerankSourceProtocol) => async (c: Contex 'request.rerank.canonical': request, 'serve.model': model, }) as never, - facts => ({ body: JSON.stringify(facts['response.rerank.rendered']), contentType: 'application/json' }), + facts => ({ body: facts['response.http.jsonBody']!, contentType: 'application/json' }), ); }; From 3bc1368fe8b7ec518604dda25b2308a5911d5eee Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 08:26:28 +0800 Subject: [PATCH 70/79] fix: serialize Alpha search inside settlement Forward the client JSON bytes produced by the response stage, preserving parsed search content and its observed call collection if encoding fails. The route regression compares stored body bytes with the real HTTP response. --- .../data-plane/alpha-search/routes_test.ts | 16 +++++++++------- .../gateway/src/data-plane/alpha-search/http.ts | 2 +- .../src/data-plane/alpha-search/pipeline.ts | 4 +++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/alpha-search/routes_test.ts b/packages/gateway/__tests__/data-plane/alpha-search/routes_test.ts index 7ebebbc66f..c0e0c99222 100644 --- a/packages/gateway/__tests__/data-plane/alpha-search/routes_test.ts +++ b/packages/gateway/__tests__/data-plane/alpha-search/routes_test.ts @@ -10,6 +10,8 @@ import { internalErrorResponse } from '../../../src/middleware/internal-error-re import { eventsOf, installDumpStubs, runRecordOf } from '../../dump/test-fixtures.ts'; import { saveUpstreamForTest } from '../../repo/upstreams.ts'; import { buildCustomUpstreamRecord, flushAsyncWork, setupAppTest, warmModelsForTest } from '../../test-utils/app.ts'; +import { createRunReader, type DumpEvent } from '@floway-dev/pipeline'; +import { decodeForgivingBase64 } from '@floway-dev/protocols/common'; import { withMockedFetch } from '@floway-dev/test-utils'; // Real provider construction (`createTavilyWebSearchProvider` etc.) hits the @@ -337,9 +339,6 @@ describe('/alpha/search data plane', () => { }); }); - // The endpoint is served by the pipeline that was written for it, which is what a run record - // says and nothing else does: the handler it replaced opened no run at all, so a turn on this - // path produced no record however much retention the key had. describe('the run it is served by', () => { it('records the whole run, stage by stage', async () => { const { apiKey, repo } = await setupAppTest({ webSearchConfig: TAVILY_CONFIG }); @@ -350,19 +349,22 @@ describe('/alpha/search data plane', () => { const response = await postSearch(buildAlphaSearchApp(), apiKey.key, { commands: { search_query: [{ q: 'Floway' }] } }); expect(response.status).toBe(200); - await response.json(); + const wire = await response.text(); await flushAsyncWork(); expect(dumpStubs.stored).toHaveLength(1); const record = runRecordOf(dumpStubs.stored[0]?.record); expect(record.meta.path).toBe(SEARCH_PATH); expect(record.meta.status).toBe(200); - // The chain the operator's configuration assembled: the edge, settlement, and the local - // ending's two stages. const entered = eventsOf(record) .filter(event => event.type === 'stage.entered') .map(event => event.name); - expect(entered).toEqual(['writeSettlement', 'emitAlphaSearch', 'parseSearchOperations', 'executeSearchOperations']); + expect(entered).toEqual(['writeSettlement', 'serializeClientJson', 'emitAlphaSearch', 'parseSearchOperations', 'executeSearchOperations']); + const read = createRunReader(); + const decoded = eventsOf(record).map(event => read(event as unknown as DumpEvent)); + const bytes = decoded.find(event => event?.facts && 'response.http.jsonBody' in event.facts)?.facts?.['response.http.jsonBody'] as { bytes: string } | undefined; + expect(bytes).toBeDefined(); + expect(new TextDecoder().decode(decodeForgivingBase64(bytes!.bytes))).toBe(wire); }); }); diff --git a/packages/gateway/src/data-plane/alpha-search/http.ts b/packages/gateway/src/data-plane/alpha-search/http.ts index 7d1eebe842..fe38d1c61a 100644 --- a/packages/gateway/src/data-plane/alpha-search/http.ts +++ b/packages/gateway/src/data-plane/alpha-search/http.ts @@ -92,6 +92,6 @@ export const alphaSearch = async (c: Context): Promise => { 'ingress.http.headers': prologue.headers, 'request.search.alphaSearch': read.request, }) as never, - facts => ({ body: JSON.stringify(facts['response.search.rendered']), contentType: 'application/json' }), + facts => ({ body: facts['response.http.jsonBody']!, contentType: 'application/json' }), ); }; diff --git a/packages/gateway/src/data-plane/alpha-search/pipeline.ts b/packages/gateway/src/data-plane/alpha-search/pipeline.ts index f6e699fd6a..5325fe5ce9 100644 --- a/packages/gateway/src/data-plane/alpha-search/pipeline.ts +++ b/packages/gateway/src/data-plane/alpha-search/pipeline.ts @@ -3,16 +3,18 @@ import { emitAlphaSearch } from './emit.ts'; import { executeSearchOperations } from './execute-operations.ts'; import type { SearchExecution, Fields } from './facts.ts'; import { parseSearchOperations } from './parse-operations.ts'; +import { serializeClientJson } from '../pipeline/serialize-client-json.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; import { compose, type Pipeline } from '@floway-dev/pipeline'; export const searchServePipeline = (execution: SearchExecution): Pipeline< Fields<'request.search.alphaSearch'>, - Fields<'response.search.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> + Fields<'response.http.jsonBody' | 'response.search.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> > => compose('searchServe', [ // Unconditional, as everywhere: a run that searched locally reached no upstream and bills // for none, and saying so is a row that names no billed entity rather than no row. writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400), + serializeClientJson('response.search.rendered'), emitAlphaSearch, ...(execution.kind === 'local' ? [parseSearchOperations, executeSearchOperations] From 84ebf6d76648c6a5ff15868d9b3d1c77ca63d000 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 09:17:14 +0800 Subject: [PATCH 71/79] refactor: keep run persistence within the agreed cold-recording scope Retain the encoded run object space and wait for owned readers and deferred outcomes before storing completed records. Defer live writer integration and its business reader to the later realtime work; preserve that implementation on the local archive branch. --- .../__tests__/control-plane/dump_test.ts | 30 +-- .../gateway/__tests__/dump/run-sink_test.ts | 131 +---------- .../gateway/__tests__/dump/test-fixtures.ts | 4 - .../gateway/__tests__/repo/dump-store_test.ts | 48 +--- .../__tests__/test-utils/log-stream.ts | 43 ---- packages/gateway/__tests__/vitest.setup.ts | 5 +- packages/gateway/src/control-plane/dump.ts | 11 - packages/gateway/src/dump/run-sink.ts | 211 +++++++----------- packages/gateway/src/dump/store-contract.ts | 9 - packages/gateway/src/repo/dump-store.ts | 43 +--- packages/gateway/src/shared/gzip.ts | 3 - 11 files changed, 88 insertions(+), 450 deletions(-) delete mode 100644 packages/gateway/__tests__/test-utils/log-stream.ts diff --git a/packages/gateway/__tests__/control-plane/dump_test.ts b/packages/gateway/__tests__/control-plane/dump_test.ts index 6c8314739d..2bebb7dd7c 100644 --- a/packages/gateway/__tests__/control-plane/dump_test.ts +++ b/packages/gateway/__tests__/control-plane/dump_test.ts @@ -1,13 +1,10 @@ -import { test, expect, vi } from 'vitest'; +import { test } from 'vitest'; import { initDumpBroker, initDumpStore } from '../../src/dump/registry.ts'; -import { runStreamId } from '../../src/dump/run-sink.ts'; import type { DumpStore } from '../../src/dump/store-contract.ts'; import type { DumpMetadata, DumpRecord, StoredDumpRecord } from '../../src/dump/types.ts'; import { fakeMeta as baseFakeMeta, fakeRecord as baseFakeRecord, fakeRunRecord, installDumpStubs } from '../dump/test-fixtures.ts'; import { requestApp, setupAppTest } from '../test-utils/app.ts'; -import { testLogStreamStore } from '../test-utils/log-stream.ts'; -import { initLogStreamStore, readLogStream } from '@floway-dev/platform'; import { assertEquals, assertExists } from '@floway-dev/test-utils'; const fakeMeta = (id: string, completedAt: number): DumpMetadata => @@ -264,28 +261,3 @@ test('GET /api/dump/keys/:keyId/stream delivers a frame appended between snapsho // it. assertEquals((parseSseFrame(appended).data as { id: string }).id, '01HZZ0000000000000000000A2'); }); - -test('live record reads are framed, seekable, key-scoped and never create an unknown stream', async () => { - const { repo, apiKey } = await setupAppTest(); - await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); - const store = testLogStreamStore(); - initLogStreamStore(store); - const first = new TextEncoder().encode('{"before":"中"}\n'); - const second = new TextEncoder().encode('{"after":"β"}\n'); - const stream = await store.open(runStreamId(apiKey.id, 'active')); - await stream.append(0, first); await stream.append(first.byteLength, second); await stream.end(); - const open = vi.spyOn(store, 'open'); - const response = await requestApp(`/api/dump/keys/${apiKey.id}/records/active/live?offset=${first.byteLength}`, { headers: { 'x-api-key': apiKey.key } }); - expect(response.status).toBe(200); - const chunks: Uint8Array[] = []; - for await (const bytes of readLogStream(response)) chunks.push(bytes); - expect(chunks.map(bytes => new TextDecoder().decode(bytes)).join('')).toBe(new TextDecoder().decode(second)); - const missing = await requestApp(`/api/dump/keys/${apiKey.id}/records/absent/live`, { headers: { 'x-api-key': apiKey.key } }); - expect(missing.status).toBe(404); - expect(open).not.toHaveBeenCalled(); - const invalid = await requestApp(`/api/dump/keys/${apiKey.id}/records/active/live?offset=-1`, { headers: { 'x-api-key': apiKey.key } }); - expect(invalid.status).toBe(400); - await store.open(runStreamId('foreign-key', 'foreign')); - const foreign = await requestApp(`/api/dump/keys/${apiKey.id}/records/foreign/live`, { headers: { 'x-api-key': apiKey.key } }); - expect(foreign.status).toBe(404); -}); diff --git a/packages/gateway/__tests__/dump/run-sink_test.ts b/packages/gateway/__tests__/dump/run-sink_test.ts index b719c13f89..71e8dfdc97 100644 --- a/packages/gateway/__tests__/dump/run-sink_test.ts +++ b/packages/gateway/__tests__/dump/run-sink_test.ts @@ -1,14 +1,12 @@ -import { expect, test, vi } from 'vitest'; +import { expect, test } from 'vitest'; import { installDumpStubs } from './test-fixtures.ts'; import { initDumpBroker, initDumpStore } from '../../src/dump/registry.ts'; -import { openRunDump, runStreamId } from '../../src/dump/run-sink.ts'; +import { openRunDump } from '../../src/dump/run-sink.ts'; import type { StoredDumpRecord, StoredDumpRunRecord } from '../../src/dump/types.ts'; import type { ApiKey } from '../../src/repo/types.ts'; import { flushBackground, trackBackground } from '../test-utils/background-tracker.ts'; -import { testLogStreamStore } from '../test-utils/log-stream.ts'; import { compose, defineStage, defer, move, run, type DumpEvent, type Event } from '@floway-dev/pipeline'; -import { getLogStreamStore, initLogStreamStore, LOG_STREAM_IDLE_MS } from '@floway-dev/platform'; import { assertEquals } from '@floway-dev/test-utils'; const apiKey = (dumpRetentionSeconds: number | null): ApiKey => ({ @@ -187,97 +185,7 @@ test('a run reads the stable timing state at completion alongside its stage hist assertEquals(lines(record).some(event => event.type === 'stage.entered'), true); }); -test('live bytes are visible before completion and equal the durable NDJSON artifact', async () => { - initLogStreamStore(testLogStreamStore()); - const stubs = installDumpStubs(initDumpStore, initDumpBroker); - const dump = openRunDump(apiKey(3600), turn, trackBackground, false, { upstreamCallStartedAt: null, firstOutputTokenAt: null })!; - await dump.sink({ type: 'stage.entered', stageId: 1, name: 'open', parentStageId: null, facts: move({ input: '中' }) }); - const live = await getLogStreamStore().get(runStreamId(apiKey(3600).id, dump.id)); - const reader = live!.read(0, new AbortController().signal)[Symbol.asyncIterator](); - const first = await reader.next(); - expect(new TextDecoder().decode(first.value)).toContain('stage.entered'); - expect(stubs.stored).toHaveLength(0); - dump.finalize(200, 0); - await flushBackground(); - const rest: Uint8Array[] = [first.value!]; - for (;;) { const item = await reader.next(); if (item.done) break; rest.push(item.value); } - expect(rest.map(bytes => new TextDecoder().decode(bytes)).join('')).toBe(ndjson(runRecordOf(stubs.stored[0]))); - expect(stubs.stored[0]!.record.meta.id).toBe(dump.id); -}); - -test('retries a lost live acknowledgement at the same offset without duplicating bytes', async () => { - const underlying = testLogStreamStore(); - const positions: number[] = []; - initLogStreamStore({ - get: id => underlying.get(id), - open: async id => { - const stream = await underlying.open(id); - return { - ...stream, append: async (offset, bytes) => { - positions.push(offset); - await stream.append(offset, bytes); - if (positions.length === 1) throw new Error('lost acknowledgement'); - }, - }; - }, - }); - const stubs = installDumpStubs(initDumpStore, initDumpBroker); - const dump = openRunDump(apiKey(3600), turn, trackBackground, false, { upstreamCallStartedAt: null, firstOutputTokenAt: null })!; - await dump.sink({ type: 'stage.entered', stageId: 1, name: 'retry', parentStageId: null, facts: move({ input: 'x' }) }); - dump.finalize(200, 0); - await flushBackground(); - expect(positions).toEqual([0, 0]); - const stream = await underlying.get(runStreamId(apiKey(3600).id, dump.id)); - const text: string[] = []; - for await (const bytes of stream!.read(0, new AbortController().signal)) text.push(new TextDecoder().decode(bytes)); - expect(text.join('')).toBe(ndjson(runRecordOf(stubs.stored[0]))); -}); - -test('a repeatedly failed live stream still stores the complete durable artifact', async () => { - const original = new Error('live transport unavailable'); - const append = vi.fn(async () => { throw original; }); - initLogStreamStore({ - open: async () => ({ append, end: async () => {}, read: () => (async function* () {})() }), - get: async () => null, - }); - const reported = vi.spyOn(console, 'error').mockImplementation(() => {}); - const stubs = installDumpStubs(initDumpStore, initDumpBroker); - try { - const dump = openRunDump(apiKey(3600), turn, trackBackground, false, { upstreamCallStartedAt: null, firstOutputTokenAt: null })!; - const { drain } = await run(pipeline, move({ 'in.text': 'hello' }), { dump: dump.sink }); - dump.afterRun(drain); - dump.finalize(200, 0); - await flushBackground(); - expect(append).toHaveBeenCalledTimes(3); - expect(lines(runRecordOf(stubs.stored[0])).filter(event => event.type === 'stage.entered')).toHaveLength(2); - expect(reported.mock.calls.some(call => call[1] === original)).toBe(true); - } finally { reported.mockRestore(); initLogStreamStore(testLogStreamStore()); } -}); - -test('durable backpressure blocks a producer before it can enqueue another event', async () => { - initLogStreamStore(testLogStreamStore()); - const stubs = installDumpStubs(initDumpStore, initDumpBroker); - const gate = Promise.withResolvers(); - initDumpStore({ - ...stubs.store, putRun: async (keyId, writing) => { - await gate.promise; - await stubs.store.putRun(keyId, writing); - }, - }); - const dump = openRunDump(apiKey(3600), turn, trackBackground, false, { upstreamCallStartedAt: null, firstOutputTokenAt: null })!; - let wrote = false; - const write = dump.sink({ type: 'stage.entered', stageId: 1, name: 'blocked', parentStageId: null, facts: move({}) }).then(() => { wrote = true; }); - await Promise.resolve(); await Promise.resolve(); - expect(wrote).toBe(false); - gate.resolve(); - await write; - dump.finalize(200, 0); - await flushBackground(); - expect(wrote).toBe(true); -}); - test('closing waits for deferred outcomes and persists their settlement event', async () => { - initLogStreamStore(testLogStreamStore()); const stubs = installDumpStubs(initDumpStore, initDumpBroker); const result = Promise.withResolvers(); const deferred = defer(result.promise); @@ -295,38 +203,3 @@ test('closing waits for deferred outcomes and persists their settlement event', expect(lines(runRecordOf(stubs.stored[0])).some(event => event.type === 'deferred.settled')).toBe(true); expect(ndjson(runRecordOf(stubs.stored[0]))).toContain('late value'); }); - -test('heartbeat append waits for the in-flight event and uses its resulting byte offset', async () => { - vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); - const underlying = testLogStreamStore(); - const entered = Promise.withResolvers(); - const release = Promise.withResolvers(); - const positions: { offset: number; length: number }[] = []; - initLogStreamStore({ - get: id => underlying.get(id), - open: async id => { - const stream = await underlying.open(id); - return { - ...stream, append: async (offset, bytes) => { - positions.push({ offset, length: bytes.byteLength }); - if (positions.length === 1) { entered.resolve(); await release.promise; } - await stream.append(offset, bytes); - }, - }; - }, - }); - const stubs = installDumpStubs(initDumpStore, initDumpBroker); - try { - const dump = openRunDump(apiKey(3600), turn, trackBackground, false, { upstreamCallStartedAt: null, firstOutputTokenAt: null })!; - const event = dump.sink({ type: 'stage.entered', stageId: 1, name: 'slow append', parentStageId: null, facts: move({}) }); - await entered.promise; - await vi.advanceTimersByTimeAsync(LOG_STREAM_IDLE_MS / 2); - expect(positions).toHaveLength(1); - release.resolve(); - await event; - dump.finalize(200, 0); - await flushBackground(); - expect(positions).toEqual([{ offset: 0, length: positions[0]!.length }, { offset: positions[0]!.length, length: 0 }]); - expect(stubs.stored).toHaveLength(1); - } finally { release.resolve(); vi.useRealTimers(); initLogStreamStore(testLogStreamStore()); } -}); diff --git a/packages/gateway/__tests__/dump/test-fixtures.ts b/packages/gateway/__tests__/dump/test-fixtures.ts index 95641f28f0..30e5068dd2 100644 --- a/packages/gateway/__tests__/dump/test-fixtures.ts +++ b/packages/gateway/__tests__/dump/test-fixtures.ts @@ -85,10 +85,6 @@ export const installDumpStubs = ( const throws: Partial> = {}; const store: DumpStore = { - async putRun(keyId, run) { - const bytes = new Uint8Array(await new Response(run.events).arrayBuffer()); - await store.put(keyId, { shape: 'run', meta: await run.metadata, events: bytes }); - }, async prepareRequestBody(body) { return { encoding: 'identity', bytes: body, decodedByteLength: body.byteLength }; }, diff --git a/packages/gateway/__tests__/repo/dump-store_test.ts b/packages/gateway/__tests__/repo/dump-store_test.ts index 7c845e7753..5b10d2e4fd 100644 --- a/packages/gateway/__tests__/repo/dump-store_test.ts +++ b/packages/gateway/__tests__/repo/dump-store_test.ts @@ -2,14 +2,13 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; -import { expect, test, vi } from 'vitest'; +import { expect, test } from 'vitest'; import { createSqliteTestDb, mapRunChangeCount } from './test-sqlite.ts'; import { decodeDumpBodyDescriptor } from '../../src/dump/storage-codec.ts'; import type { DumpWriteEdgeRecord, StoredDumpEdgeRecord, StoredDumpRecord, StoredDumpRunRecord } from '../../src/dump/types.ts'; import { FileDumpStore } from '../../src/repo/dump-store.ts'; import { initRepo } from '../../src/repo/index.ts'; -import { SPILLED_FILE_STAGE_GRACE_MS } from '../../src/repo/spilled-files-policy.ts'; import { SqlRepo } from '../../src/repo/sql.ts'; import { collectSpilledFiles } from '../../src/scheduled/spilled-files.ts'; import { encodeRun, toNdjson, type Facts } from '@floway-dev/pipeline'; @@ -662,48 +661,3 @@ test('FileDumpStore filters all retained history before applying the page limit' expect(await store.list('key_x', { q: "' OR 1=1 --", limit: 1 })).toEqual([]); expect(await store.list('other-key', { q: 'needle', limit: 1 })).toEqual([]); }); - -test('FileDumpStore writes streaming run bytes before publishing its metadata row', async () => { - const db = await openDb(); - const files = new MemoryFileStore(); - const store = new FileDumpStore(db, files); - const metadata = Promise.withResolvers(); - const pipe = new TransformStream(); - const writer = pipe.writable.getWriter(); - const id = 'streamed-run'; - const startedAt = Date.now(); - const writing = store.putRun('key_x', { id, startedAt, events: pipe.readable, metadata: metadata.promise }); - const bytes = utf8('{"type":"stage.entered","facts":{"text":"中"}}\n'); - await writer.write(bytes); - expect(await store.get('key_x', id)).toBeNull(); - await writer.close(); - metadata.resolve({ ...baseRecord(id, Date.now()).meta, startedAt }); - await writing; - const stored = await store.get('key_x', id); - expect(stored!.shape).toBe('run'); - expect(new TextDecoder().decode((stored as StoredDumpRunRecord).events)).toBe(new TextDecoder().decode(bytes)); -}); - -test('a long active run keeps its staged file out of orphan collection', async () => { - const db = await openDb(); - const files = new MemoryFileStore(); - initRepo(new SqlRepo(db)); - initFileStore(files); - const store = new FileDumpStore(db, files); - const metadata = Promise.withResolvers(); - const pipe = new TransformStream(); - const writer = pipe.writable.getWriter(); - const id = 'long-active-run'; - const startedAt = Date.now(); - vi.useFakeTimers({ toFake: ['Date', 'setTimeout', 'clearTimeout'] }); - try { - const writing = store.putRun('key_x', { id, startedAt, events: pipe.readable, metadata: metadata.promise }); - await writer.write(utf8('{"type":"stage.entered"}\n')); - await vi.advanceTimersByTimeAsync(SPILLED_FILE_STAGE_GRACE_MS * 2 + 1); - await collectSpilledFiles(Date.now()); - await writer.close(); - metadata.resolve({ ...baseRecord(id, Date.now()).meta, startedAt }); - await writing; - expect((await store.get('key_x', id))!.shape).toBe('run'); - } finally { vi.useRealTimers(); } -}); diff --git a/packages/gateway/__tests__/test-utils/log-stream.ts b/packages/gateway/__tests__/test-utils/log-stream.ts deleted file mode 100644 index f48629313e..0000000000 --- a/packages/gateway/__tests__/test-utils/log-stream.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { LogStream, LogStreamStore } from '@floway-dev/platform'; - -export const testLogStreamStore = (): LogStreamStore => { - const streams = new Map(); - return { - async open(id) { - const chunks: { offset: number; bytes: Uint8Array }[] = []; - let length = 0; - let ended = false; - const listeners = new Set<() => void>(); - const wake = () => { for (const listener of listeners) listener(); }; - const stream: LogStream = { - async append(offset, bytes) { - if (ended) throw new Error('Test LogStream is ended'); - if (offset > length) throw new Error('Test LogStream append would leave a hole'); - const tail = bytes.slice(length - offset); - if (tail.byteLength > 0) { chunks.push({ offset: length, bytes: tail }); length += tail.byteLength; } - wake(); - }, - async end() { ended = true; wake(); }, - async *read(offset, signal) { - while (!signal.aborted) { - const chunk = chunks.find(chunk => chunk.offset + chunk.bytes.byteLength > offset); - if (chunk) { - const bytes = chunk.bytes.slice(Math.max(0, offset - chunk.offset)); - offset += bytes.byteLength; - yield bytes; - } else if (ended) return; - else await new Promise(resolve => { - const finish = () => { listeners.delete(finish); signal.removeEventListener('abort', finish); resolve(); }; - listeners.add(finish); - signal.addEventListener('abort', finish, { once: true }); - }); - } - signal.throwIfAborted(); - }, - }; - streams.set(id, stream); - return stream; - }, - async get(id) { return streams.get(id) ?? null; }, - }; -}; diff --git a/packages/gateway/__tests__/vitest.setup.ts b/packages/gateway/__tests__/vitest.setup.ts index dd55d46e30..f4af14b3bb 100644 --- a/packages/gateway/__tests__/vitest.setup.ts +++ b/packages/gateway/__tests__/vitest.setup.ts @@ -1,5 +1,4 @@ import { trackBackground } from './test-utils/background-tracker.ts'; -import { testLogStreamStore } from './test-utils/log-stream.ts'; import type { DumpBroker } from '../src/dump/broker.ts'; import { initDumpBroker, initDumpStore } from '../src/dump/registry.ts'; import type { DumpStore } from '../src/dump/store-contract.ts'; @@ -8,7 +7,7 @@ import { handleExecutionRequest } from '../src/execution/handler.ts'; import { initBackgroundSchedulerResolver } from '../src/runtime/background.ts'; import { initExecutionCellNamespace } from '../src/runtime/execution.ts'; import { isReplayableBody } from '@floway-dev/http'; -import { initEnv, initFetch, initRuntimeKind, initTimingSafeEqual, InProcessExecutionCellNamespace, initLogStreamStore } from '@floway-dev/platform'; +import { initEnv, initFetch, initRuntimeKind, initTimingSafeEqual, InProcessExecutionCellNamespace } from '@floway-dev/platform'; // Production always initializes the environment getter at boot. Mirror that // here with a neutral default; tests needing real values (RUNTIME_LOCATION, @@ -17,7 +16,6 @@ initEnv(() => ''); // Tests run as 'node' by default. The few tests that exercise CF-specific // runtime behaviour re-init this with 'cloudflare'. initRuntimeKind('node'); -initLogStreamStore(testLogStreamStore()); initTimingSafeEqual((a, b) => a.every((byte, index) => byte === b[index])); initFetch((url, init) => { const body = init.body; @@ -35,7 +33,6 @@ initExecutionCellNamespace(new InProcessExecutionCellNamespace(handleExecutionRe // independent of that subsystem. Dump-specific tests install real or recording // implementations. const noopStore: DumpStore = { - async putRun(_keyId, run) { await new Response(run.events).arrayBuffer(); await run.metadata; }, async prepareRequestBody(body) { return { encoding: 'identity', bytes: body, decodedByteLength: body.byteLength }; }, async put(): Promise { /* noop */ }, async list(): Promise { return []; }, diff --git a/packages/gateway/src/control-plane/dump.ts b/packages/gateway/src/control-plane/dump.ts index 2bab7e2614..50463abc94 100644 --- a/packages/gateway/src/control-plane/dump.ts +++ b/packages/gateway/src/control-plane/dump.ts @@ -5,10 +5,8 @@ import { z } from 'zod'; import { ownedKeyForUser } from './shared/owned-key.ts'; import { getDumpBroker, getDumpStore } from '../dump/registry.ts'; -import { runStreamId } from '../dump/run-sink.ts'; import { dumpRecordToWire } from '../dump/wire.ts'; import { zValidator } from '../middleware/zod-validator.ts'; -import { getLogStreamStore, serveLogStream } from '@floway-dev/platform'; const LIST_LIMIT_DEFAULT = 100; const LIST_LIMIT_MAX = 200; @@ -42,15 +40,6 @@ export const dumpRoutes = new Hono() }); return c.json({ records }); }) - .get('/keys/:keyId/records/:recordId/live', zValidator('query', z.object({ - offset: z.coerce.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).optional(), - })), async c => { - const owned = await ownedKey(c); - if (owned.id === null) return c.json({ error: owned.error }, 404); - const stream = await getLogStreamStore().get(runStreamId(owned.id, c.req.param('recordId')!)); - if (stream === null) return c.json({ error: 'Live recording is unavailable' }, 404); - return serveLogStream(stream, c.req.valid('query').offset ?? 0, c.req.raw.signal); - }) .get('/keys/:keyId/records/:recordId', async c => { const owned = await ownedKey(c); if (owned.id === null) return c.json({ error: owned.error }, 404); diff --git a/packages/gateway/src/dump/run-sink.ts b/packages/gateway/src/dump/run-sink.ts index 5b536be745..95a74c831c 100644 --- a/packages/gateway/src/dump/run-sink.ts +++ b/packages/gateway/src/dump/run-sink.ts @@ -6,11 +6,12 @@ import { attemptTtftMs, type AttemptTiming } from '../data-plane/shared/attempt- import type { RequestBody } from '../data-plane/shared/request-body.ts'; import type { ApiKey, TokenUsage } from '../repo/types.ts'; import { ulid } from '../shared/ulid.ts'; -import { createRunEncoder, streamFact, toNdjson, type Event } from '@floway-dev/pipeline'; -import { getLogStreamStore, LOG_STREAM_IDLE_MS, type BackgroundScheduler, type LogStream } from '@floway-dev/platform'; +import { createRunEncoder, streamFact, toNdjson, type DumpEvent, type Event } from '@floway-dev/pipeline'; +import type { BackgroundScheduler } from '@floway-dev/platform'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { TelemetryModelIdentity } from '@floway-dev/provider'; +// Only metadata is snapshotted here; request contents are facts recorded by the run. interface RequestSnapshot { readonly method: string; readonly path: string; @@ -18,24 +19,10 @@ interface RequestSnapshot { readonly streamError: string | null; } -const LIVE_APPEND_ATTEMPTS = 3; -const CHUNK_BYTES = 64 * 1024; - -export const runStreamId = (keyId: string, runId: string): string => `${keyId}/${runId}`; - export class RunDump { - readonly id: string; private readonly attribution = new DumpAttribution(); private readonly encode = createRunEncoder(); - private readonly metadata = Promise.withResolvers(); - private readonly writer: WritableStreamDefaultWriter; - private readonly liveReady: Promise; - private live: LogStream | null = null; - private offset = 0; - private heartbeat: ReturnType | undefined; - private heartbeatWrite: Promise = Promise.resolve(); - private closing = false; - private tail: Promise = Promise.resolve(); + private readonly events: DumpEvent[] = []; private runDrain: (() => Promise) | null = null; private sentPayloadBytes = 0; private streams = 0; @@ -48,109 +35,63 @@ export class RunDump { private readonly backgroundScheduler: BackgroundScheduler, private readonly wantsStream: boolean, private readonly timing: AttemptTiming, - ) { - const id = ulid(startedAt); - this.id = id; - const bytes = new TransformStream(); - this.writer = bytes.writable.getWriter(); - // The metadata wait can reject before the storage reader reaches it; storage owns reporting. - void this.metadata.promise.catch(() => {}); - this.backgroundScheduler((async () => { - await getDumpStore().putRun(apiKey.id, { - id, startedAt, events: bytes.readable, metadata: this.metadata.promise, - }); - await getDumpBroker().publish(apiKey.id, await this.metadata.promise); - })().catch(async error => { - this.metadata.reject(error); - await this.writer.abort(error); - throw error; - })); - this.liveReady = getLogStreamStore().open(runStreamId(apiKey.id, id)).then(stream => { - this.live = stream; - this.scheduleHeartbeat(); - return stream; - }).catch(error => { - console.error('[dump] live stream open failed', error); - return null; - }); - } - - afterRun(drain: () => Promise): void { - this.runDrain = drain; - } + ) {} - // One event owns its entire object batch; concurrent deferred outcomes cannot overtake it. - readonly sink = (event: Event): Promise => { - const write = this.tail.then(async () => { - const bytes = new TextEncoder().encode(toNdjson(this.encode(event))); - await this.liveReady; - for (let offset = 0; offset < bytes.byteLength; offset += CHUNK_BYTES) { - const chunk = bytes.subarray(offset, offset + CHUNK_BYTES); - await this.writer.write(chunk); - await this.appendLive(chunk); - } - }); - this.tail = write; - return write; + afterRun(drain: () => Promise): void { this.runDrain = drain; } + readonly sink = (event: Event): void => { + for (const encoded of this.encode(event)) this.events.push(encoded); }; - private async appendLive(bytes: Uint8Array): Promise { - const stream = this.live; - if (stream === null) return; - const atOffset = this.offset; - for (let attempt = 0; attempt < LIVE_APPEND_ATTEMPTS; attempt += 1) { - try { - await stream.append(atOffset, bytes); - this.offset += bytes.byteLength; - return; - } catch (error) { - if (attempt + 1 === LIVE_APPEND_ATTEMPTS) { - this.live = null; - console.error('[dump] live stream unavailable; durable recording continues', error); - } - } - } + requestedModel(model: string): void { + this.attribution.requestedModel(model); } - private scheduleHeartbeat(): void { - if (this.closing || this.live === null) return; - this.heartbeat = setTimeout(() => { - const heartbeat = this.tail.then(() => this.appendLive(new Uint8Array())); - this.tail = heartbeat; - this.heartbeatWrite = heartbeat.then(() => { this.scheduleHeartbeat(); }); - }, LOG_STREAM_IDLE_MS / 2); + error(kind: 'upstream' | 'gateway', upstream?: string): void { + this.attribution.error(kind, upstream); } - requestedModel(model: string): void { this.attribution.requestedModel(model); } - error(kind: 'upstream' | 'gateway', upstream?: string): void { this.attribution.error(kind, upstream); } - failed(reason: unknown, options?: { readonly fallback: boolean }): void { this.attribution.failed(reason, options); } - success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void { this.attribution.success(identity, usage); } + failed(reason: unknown, options?: { readonly fallback: boolean }): void { + this.attribution.failed(reason, options); + } - async frame(frame: ProtocolFrame): Promise { + frame(frame: ProtocolFrame): void | Promise { this.answerStream ??= this.openStream(); - await this.answerStream.frame(frame); + return this.answerStream.frame(frame); } + // Stream references and frame events share one run-wide identity; end marks complete recording. openStream(): StreamRecording { const streamId = ++this.streams; return { - frame: frame => this.sink({ type: 'stream.frame', streamId, frames: [frame] }), - end: () => this.sink({ type: 'stream.end', streamId }), + frame: frame => { this.sink({ type: 'stream.frame', streamId, frames: [frame] }); }, + end: () => { this.sink({ type: 'stream.end', streamId }); }, fact: streamFact(streamId), }; } - recordSentPayloadBytes(byteLength: number): void { this.sentPayloadBytes += byteLength; } + success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void { + this.attribution.success(identity, usage); + } + + recordSentPayloadBytes(byteLength: number): void { + this.sentPayloadBytes += byteLength; + } finalize(status: number | null, responseBytes: number): void; finalize(response: Response): Response; finalize(...args: [number | null, number] | [Response]): void | Response { if (args.length === 2) { - this.backgroundScheduler(this.write(args[0], args[1] + this.sentPayloadBytes, null)); + const [status, responseBytes] = args; + this.backgroundScheduler(this.write(status, responseBytes + this.sentPayloadBytes, null)); return; } + const [response] = args; - if (response.body === null) { this.finalize(response.status, 0); return response; } + if (response.body === null) { + this.finalize(response.status, 0); + return response; + } + const [forClient, forMeasure] = response.body.tee(); this.backgroundScheduler((async () => { const reader = forMeasure.getReader(); @@ -162,46 +103,45 @@ export class RunDump { if (done) break; payloadBytes += value.byteLength; } - } catch (error) { - streamError = oneLineError(error); + } catch (err) { + streamError = oneLineError(err); } finally { reader.releaseLock(); } await this.write(response.status, payloadBytes, streamError); })()); - return new Response(forClient, { status: response.status, statusText: response.statusText, headers: response.headers }); + + return new Response(forClient, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }); } private async write(status: number | null, responseBytes: number, responseStreamError: string | null): Promise { - try { - if (this.runDrain !== null) { - try { await this.runDrain(); } catch (error) { this.failed(error); } - } - await this.tail; - const completedAt = Date.now(); - const meta = await this.attribution.metadata({ - id: this.id, startedAt: this.startedAt, completedAt, - method: this.requestSnapshot.method, path: this.requestSnapshot.path, status, - requestBytes: this.requestSnapshot.bodyByteLength, responseBytes, - ttftMs: this.wantsStream ? attemptTtftMs(this.timing) : null, - fallbackError: streamReadError(this.requestSnapshot.streamError, responseStreamError), - }); - this.metadata.resolve(meta); - await this.writer.close(); - await this.liveReady; - this.closing = true; - clearTimeout(this.heartbeat); - await this.heartbeatWrite; - if (this.live !== null) { - try { await this.live.end(); } catch (error) { console.error('[dump] live stream end failed', error); } - } - } catch (error) { - this.metadata.reject(error); - throw error; - } finally { - this.closing = true; - clearTimeout(this.heartbeat); + if (this.runDrain !== null) { + try { await this.runDrain(); } catch (error) { this.failed(error); } } + const completedAt = Date.now(); + const recordId = ulid(completedAt); + const meta: DumpMetadata = await this.attribution.metadata({ + id: recordId, + startedAt: this.startedAt, + completedAt, + method: this.requestSnapshot.method, + path: this.requestSnapshot.path, + status, + requestBytes: this.requestSnapshot.bodyByteLength, + responseBytes, + ttftMs: this.wantsStream ? attemptTtftMs(this.timing) : null, + fallbackError: streamReadError(this.requestSnapshot.streamError, responseStreamError), + }); + + // Publishing follows durable storage so metadata subscribers can immediately fetch detail. + await getDumpStore().put(this.apiKey.id, { + shape: 'run', meta, events: new TextEncoder().encode(toNdjson(this.events)), + }); + await getDumpBroker().publish(this.apiKey.id, meta); } } @@ -211,8 +151,19 @@ export const openRunDump = ( backgroundScheduler: BackgroundScheduler, wantsStream: boolean, timing: AttemptTiming, -): RunDump | null => apiKey.dumpRetentionSeconds === null ? null : new RunDump( - apiKey, - { method: turn.method, path: turn.path, bodyByteLength: turn.body.bytes.byteLength, streamError: turn.body.streamError }, - Date.now(), backgroundScheduler, wantsStream, timing, -); +): RunDump | null => { + if (apiKey.dumpRetentionSeconds === null) return null; + return new RunDump( + apiKey, + { + method: turn.method, + path: turn.path, + bodyByteLength: turn.body.bytes.byteLength, + streamError: turn.body.streamError, + }, + Date.now(), + backgroundScheduler, + wantsStream, + timing, + ); +}; diff --git a/packages/gateway/src/dump/store-contract.ts b/packages/gateway/src/dump/store-contract.ts index fd09286e8e..ffa1a51fc7 100644 --- a/packages/gateway/src/dump/store-contract.ts +++ b/packages/gateway/src/dump/store-contract.ts @@ -11,16 +11,7 @@ export interface DumpListOptions { limit: number; } -export interface DumpRunWrite { - readonly id: string; - readonly startedAt: number; - readonly events: ReadableStream; - readonly metadata: Promise; -} - export interface DumpStore { - putRun(keyId: string, run: DumpRunWrite): Promise; - // Starts body preparation while the request is in flight. Implementations // may return identity bytes, but persistent stores compress here so the // accumulator can release the original request buffer before terminal IO. diff --git a/packages/gateway/src/repo/dump-store.ts b/packages/gateway/src/repo/dump-store.ts index f1cbd90a67..5b88ed72c7 100644 --- a/packages/gateway/src/repo/dump-store.ts +++ b/packages/gateway/src/repo/dump-store.ts @@ -12,7 +12,7 @@ import { encodePersistedDumpMetadata, } from '../dump/storage-codec.ts'; import type { DumpBodyDescriptor } from '../dump/storage-codec.ts'; -import type { DumpListOptions, DumpStore, DumpRunWrite } from '../dump/store-contract.ts'; +import type { DumpListOptions, DumpStore } from '../dump/store-contract.ts'; import type { DumpMetadata, DumpRecordId, @@ -27,7 +27,7 @@ import type { StoredDumpUpstreamResponse, } from '../dump/types.ts'; import { upstreamResponseToWire } from '../dump/wire.ts'; -import { gunzipBytes, gzipBytes, gzipStream } from '../shared/gzip.ts'; +import { gunzipBytes, gzipBytes } from '../shared/gzip.ts'; import type { FileStore, SqlDatabase } from '@floway-dev/platform'; import { decodeForgivingBase64 } from '@floway-dev/protocols/common'; @@ -117,45 +117,6 @@ const NO_EDGE_HEADERS = '[]'; export class FileDumpStore implements DumpStore { constructor(private readonly db: SqlDatabase, private readonly files: FileStore) {} - async putRun(keyId: string, run: DumpRunWrite): Promise { - const fileKey = bodyPath(keyId, hourBucket(run.startedAt), run.id, 'run'); - await this.db.prepare( - `INSERT INTO spilled_files (file_key, owner_kind, owner_key, state, collect_after) - VALUES (?, 'dump-response', ?, 'staged', ?)`, - ).bind(fileKey, JSON.stringify([keyId, run.id]), Date.now() + SPILLED_FILE_STAGE_GRACE_MS).run(); - let timer: ReturnType | undefined; - let finished = false; - let renewal: Promise = Promise.resolve(); - const stopRenewing = () => { finished = true; clearTimeout(timer); }; - const scheduleRenewal = () => { - timer = setTimeout(() => { - renewal = this.db.prepare( - "UPDATE spilled_files SET collect_after = ? WHERE file_key = ? AND state = 'staged'", - ).bind(Date.now() + SPILLED_FILE_STAGE_GRACE_MS, fileKey).run().then(() => { - if (!finished) scheduleRenewal(); - }); - // Awaited before publication even if the lease write failed while upstream was quiet. - void renewal.catch(() => {}); - }, SPILLED_FILE_STAGE_GRACE_MS / 2); - }; - scheduleRenewal(); - try { - await this.files.put(fileKey, gzipStream(run.events)); - const meta = await run.metadata; - stopRenewing(); - await renewal; - await this.db.prepare( - `INSERT INTO dump_records - (key_id, id, created_at, upstream_id, meta_json, request_headers_json, response_headers_json, request_body_descriptor, response_body_descriptor, response_upstream_body_descriptor) - VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, ?, NULL)`, - ).bind( - keyId, meta.id, meta.completedAt, meta.upstream?.id ?? null, - encodePersistedDumpMetadata(meta, `dump record ${meta.id} metadata`), NO_EDGE_HEADERS, - encodeDumpBodyDescriptor({ key: fileKey, type: 'run' }, `dump record ${meta.id} run descriptor`), - ).run(); - } finally { stopRenewing(); } - } - async prepareRequestBody(body: Uint8Array): Promise { return { encoding: 'gzip', diff --git a/packages/gateway/src/shared/gzip.ts b/packages/gateway/src/shared/gzip.ts index 77a1f8241b..74bda3d0e6 100644 --- a/packages/gateway/src/shared/gzip.ts +++ b/packages/gateway/src/shared/gzip.ts @@ -34,6 +34,3 @@ export const gzipBytes = async (bytes: Uint8Array): Promise => export const gunzipBytes = async (bytes: Uint8Array): Promise => await collect(bytesStream(bytes).pipeThrough(new DecompressionStream('gzip'))); - -export const gzipStream = (stream: ReadableStream): ReadableStream => - (stream as ReadableStream).pipeThrough(new CompressionStream('gzip')); From f7cfaa4464477abf6815ee0b0e5e261e982b557e Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 09:17:23 +0800 Subject: [PATCH 72/79] feat: inspect recorded protocol frames in each pipeline stage Resolve stream references through the shared run object space, show literal decoded frames, and distinguish streams with and without their recorded end marker. Keep frame comparison algorithms and realtime business integration deferred. --- README.md | 14 +++---- .../components/requests/run-model_test.ts | 20 ++++++++++ .../components/requests/run-stages_test.tsx | 22 +++++++++++ apps/web/src/components/requests/run-model.ts | 39 +++++++++++++++++-- .../src/components/requests/run-stages.tsx | 19 +++++++-- apps/web/src/i18n/locales/en.ts | 6 +++ apps/web/src/i18n/locales/zh-Hans.ts | 6 +++ 7 files changed, 112 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index fb2795d682..0fcb6e6d23 100644 --- a/README.md +++ b/README.md @@ -100,8 +100,9 @@ the upstream HTTP status. Billing records each actual upstream usage observation independently, including multiple model calls within one hosted-tool turn. Pricing uses each observation's model identity and pricing facts. The request inspector shows the stage tree, both fact -directions, changes at each descent and return, and stage logs. Repeated objects retain +directions, changes at each descent and return, stage logs and decoded protocol frames. Repeated objects retain shared identities, while deferred outcomes and native content descriptions remain visible. +Each referenced stream shows its recorded frames and completion marker. Request diagnostics sum the observed token quantities; performance uses the observed output count for TPOT and partial output failures. @@ -205,13 +206,10 @@ stage throws. Client JSON serialization runs inside settlement and publishes UTF while retaining the canonical content for inspection. Streaming quantities settle at the deferred exit. Explicit protocol and transport errors take precedence over the settlement's generic failure marker. -Run dumps stream their encoded NDJSON into durable storage with backpressure while appending -the same bytes to a temporary LogStream. Live writes retry at the same byte offset; persistent -live failure leaves durable recording active. Quiet runs renew their temporary stream and -staged-file leases. Closing waits for owned readers and deferred outcomes before publishing -completed metadata. Run IDs are assigned at entry; listing remains ordered by completion time -and ID. The authenticated `GET /api/dump/keys/:keyId/records/:recordId/live?offset=0` endpoint -attaches only to an existing per-key stream and uses the platform's framed byte protocol. +Run dumps encode stage facts, logs and protocol frames into one object space. Closing waits +for owned readers and deferred outcomes before storing the completed NDJSON record and +publishing its metadata. LogStream infrastructure is available on both platforms; live +recording and its business readers remain outside these three PRs. Pipeline endpoints are assembled from individual stage modules. Each stage owns its request and response contract; a large stage keeps local helpers in the same diff --git a/apps/web/__tests__/components/requests/run-model_test.ts b/apps/web/__tests__/components/requests/run-model_test.ts index 5d26527860..97ebc05653 100644 --- a/apps/web/__tests__/components/requests/run-model_test.ts +++ b/apps/web/__tests__/components/requests/run-model_test.ts @@ -65,4 +65,24 @@ describe('run stage model', () => { expect(model.state(stage.response!)).toMatchObject({ pending: { $deferred: { status: 'fulfilled', value: { usage: 5 } } } }); }); + it('resolves referenced protocol streams through aliases and cycles without treating schema keys as stream tags', () => { + const model = readRun([ + { + type: 'object', fromObjectId: 1, nodes: [ + { stream: { $stream: 2 }, '$$stream': 999, self: { $: 1 } }, + { type: 'event', event: { type: 'citation', source: 'https://example.test', '$$schema': 'value' } }, + ], + }, + { type: 'stage.entered', stageId: 1, name: 'translate', parentStageId: null, facts: { source: { $: 1 } } }, + { type: 'stage.leaved', stageId: 1, facts: { source: { $: 1 }, client: { $stream: 3 } } }, + { type: 'stream.frame', streamId: 2, frames: [{ $: 2 }] }, + { type: 'stream.end', streamId: 2 }, + ].map(event => JSON.stringify(event)).join('\n')); + const stage = model.roots[0]!; + const streams = model.referencedStreams(stage.request, stage.response!); + expect(streams.map(stream => ({ id: stream.id, ended: stream.ended }))).toEqual([{ id: 2, ended: true }, { id: 3, ended: false }]); + expect(model.frameValue(streams[0]!.frames[0]!)).toEqual({ type: 'event', event: { type: 'citation', source: 'https://example.test', $schema: 'value' } }); + expect(streams[1]!.frames).toEqual([]); + }); + }); diff --git a/apps/web/__tests__/components/requests/run-stages_test.tsx b/apps/web/__tests__/components/requests/run-stages_test.tsx index b393ea1efa..9812d978f4 100644 --- a/apps/web/__tests__/components/requests/run-stages_test.tsx +++ b/apps/web/__tests__/components/requests/run-stages_test.tsx @@ -3,6 +3,7 @@ import { describe, expect, it, vi } from 'vitest'; import { RunStages } from '../../../src/components/requests/run-stages'; import { renderInApp } from '../../render'; +import { encodeRun, streamFact, toNdjson } from '@floway-dev/pipeline'; vi.mock('../../../src/components/ui/body-editor', () => ({ default: ({ text, toolbarStart }: { text: string; toolbarStart: React.ReactNode }) =>
{toolbarStart}
{text}
, @@ -46,4 +47,25 @@ describe('run stages', () => { expect(screen.getByTestId('state').textContent).toContain('projection failed'); }); + it('shows decoded protocol frames and the completion of each referenced stream', async () => { + const events = toNdjson(encodeRun([ + { type: 'stage.entered', stageId: 1, name: 'emit', parentStageId: null, facts: { source: streamFact(1) } }, + { type: 'stage.leaved', stageId: 1, facts: { source: streamFact(1), client: streamFact(2) } }, + { type: 'stream.frame', streamId: 1, frames: [{ type: 'event', event: { type: 'text', value: 'upstream content' } }] }, + { type: 'stream.end', streamId: 1 }, + { type: 'stream.frame', streamId: 2, frames: [{ type: 'event', event: { type: 'citation', source: 'https://citation.test' } }] }, + ])); + renderInApp(); + await screen.findByTestId('state'); + fireEvent.click(screen.getByRole('combobox', { name: 'Stage view' })); + fireEvent.click(screen.getByRole('option', { name: 'Protocol frames' })); + expect(screen.getByText('Recording complete')).toBeTruthy(); + expect(screen.getByText(/upstream content/)).toBeTruthy(); + fireEvent.click(screen.getByRole('combobox', { name: 'Protocol stream' })); + fireEvent.click(screen.getByRole('option', { name: '#2' })); + expect(screen.getByText('Recording incomplete')).toBeTruthy(); + expect(screen.getByText(/https:\/\/citation.test/)).toBeTruthy(); + expect(screen.queryByText(/upstream content/)).toBeNull(); + }); + }); diff --git a/apps/web/src/components/requests/run-model.ts b/apps/web/src/components/requests/run-model.ts index c2998d2db6..989ffd8e11 100644 --- a/apps/web/src/components/requests/run-model.ts +++ b/apps/web/src/components/requests/run-model.ts @@ -14,6 +14,12 @@ export interface RunStage { readonly logs: Extract[]; } +export interface RunStream { + readonly id: number; + readonly frames: Stored[]; + ended: boolean; +} + export interface ValueChange { readonly path: readonly (string | number)[]; readonly kind: 'added' | 'removed' | 'changed'; @@ -32,6 +38,15 @@ export const readRun = (ndjson: string) => { const stages = new Map(); const outcomes = new Map(); const roots: RunStage[] = []; + const streams = new Map(); + const stream = (id: number): RunStream => { + let recorded = streams.get(id); + if (recorded === undefined) { + recorded = { id, frames: [], ended: false }; + streams.set(id, recorded); + } + return recorded; + }; const stage = (id: number): RunStage => { const found = stages.get(id); if (found === undefined) throw new Error(`Run references missing stage ${id}`); @@ -62,8 +77,8 @@ export const readRun = (ndjson: string) => { case 'stage.leaved': stage(event.stageId).response = event.facts; break; case 'stage.failed': stage(event.stageId).failure = { error: event.error }; break; case 'stage.log': stage(event.stageId).logs.push(event); break; - case 'stream.frame': - case 'stream.end': break; + case 'stream.frame': stream(event.streamId).frames.push(...event.frames); break; + case 'stream.end': stream(event.streamId).ended = true; break; case 'deferred.settled': { const id = reference(event.deferred); if (id === null) throw new Error('Run deferred settlement has no object identity'); @@ -90,6 +105,24 @@ export const readRun = (ndjson: string) => { return Object.fromEntries(Object.entries(stored).map(([key, child]) => [decodeKey(key), value(child, ancestors)])); }; const state = (facts: FactState): unknown => value(facts); + const referencedStreams = (...states: FactState[]): RunStream[] => { + const ids = new Set(); + const seen = new Set(); + const visit = (stored: Stored): void => { + const id = reference(stored); + if (id !== null) { + if (seen.has(id)) return; + seen.add(id); + visit(node(id)); + } else if (Array.isArray(stored)) stored.forEach(visit); + else if (typeof stored === 'object' && stored !== null) { + if ('$stream' in stored) ids.add(stored.$stream as number); + else Object.values(stored).forEach(visit); + } + }; + states.forEach(visit); + return [...ids].map(stream); + }; const diff = (before: FactState, after: FactState): ValueChange[] => { const changes: ValueChange[] = []; @@ -127,5 +160,5 @@ export const readRun = (ndjson: string) => { compare(before, after, []); return changes; }; - return { roots, stages: [...stages.values()], state, diff }; + return { roots, stages: [...stages.values()], state, diff, referencedStreams, frameValue: value }; }; diff --git a/apps/web/src/components/requests/run-stages.tsx b/apps/web/src/components/requests/run-stages.tsx index b20bd159e8..c124fc1073 100644 --- a/apps/web/src/components/requests/run-stages.tsx +++ b/apps/web/src/components/requests/run-stages.tsx @@ -22,24 +22,37 @@ export const RunStages = ({ ndjson }: { ndjson: string }) => { const [selected, setSelected] = useState(model.roots[0]?.id ?? null); const [view, setView] = useState('request'); const [descent, setDescent] = useState(0); + const [streamIndex, setStreamIndex] = useState(0); const labels = { request: t('dashboard.requests.requestFacts'), response: t('dashboard.requests.responseFacts'), down: t('dashboard.requests.requestChanges'), up: t('dashboard.requests.responseChanges'), logs: t('dashboard.requests.stageLogs'), error: t('dashboard.requests.stageError'), + frames: t('dashboard.requests.protocolFrames'), }; const current = model.stages.find(stage => stage.id === selected); if (current === undefined) return {t('dashboard.requests.noRunEvents')}; const child = current.children[descent]; - const text = view === 'error' ? current.failure === null ? null : model.state(current.failure) + const streams = view === 'frames' ? model.referencedStreams(current.request, ...current.response === null ? [] : [current.response]) : []; + const stream = streams[streamIndex]; + const frames = stream?.frames.map(stored => { + const frame = model.frameValue(stored) as { type: string; event?: unknown }; + const event = frame.type === 'event' && typeof frame.event === 'object' && frame.event !== null + && 'type' in frame.event && typeof frame.event.type === 'string' ? frame.event.type : frame.type; + return { event, text: JSON.stringify(frame, null, 2), parseError: null }; + }) ?? []; + const text = view === 'frames' || view === 'logs' ? null : view === 'error' ? current.failure === null ? null : model.state(current.failure) : view === 'request' ? model.state(current.request) : view === 'response' ? current.response === null ? null : model.state(current.response) : view === 'down' ? child === undefined ? [] : model.diff(current.request, child.request) : child?.response === null || child === undefined || current.response === null ? [] : model.diff(child.response, current.response); - const select = (id: number) => { setSelected(id); setDescent(0); }; + const select = (id: number) => { setSelected(id); setDescent(0); setStreamIndex(0); }; const toolbar =
setView(data.optionValue!)}> {Object.entries(labels).map(([value, label]) => )} + {view === 'frames' && stream !== undefined && <> setStreamIndex(Number(data.optionValue))}> + {streams.map((recorded, index) => )} + {t(stream.ended ? 'dashboard.requests.recordingComplete' : 'dashboard.requests.recordingIncomplete')}} {(view === 'down' || view === 'up') && current.children.length > 0 && setDescent(Number(data.optionValue))}> {current.children.map((stage, index) => )} } @@ -53,7 +66,7 @@ export const RunStages = ({ ndjson }: { ndjson: string }) => {
- {view === 'response' && current.failure !== null ?
{toolbar}{t('dashboard.requests.stageFailed')}
: view === 'logs' ? frame.text).join('\n')} toolbarStart={toolbar} emptyText={t(stream === undefined ? 'dashboard.requests.noProtocolStreams' : 'dashboard.requests.noProtocolFrames')} /> : view === 'response' && current.failure !== null ?
{toolbar}{t('dashboard.requests.stageFailed')}
: view === 'logs' ? ({ event: log.level, text: JSON.stringify({ ...log, ...(log.fields === undefined ? {} : { fields: model.state(log.fields) }) }, null, 2), parseError: null }))} copyText={current.logs.map(log => JSON.stringify({ ...log, ...(log.fields === undefined ? {} : { fields: model.state(log.fields) }) })).join('\n')} toolbarStart={toolbar} diff --git a/apps/web/src/i18n/locales/en.ts b/apps/web/src/i18n/locales/en.ts index d4f01afd0a..d77347db60 100644 --- a/apps/web/src/i18n/locales/en.ts +++ b/apps/web/src/i18n/locales/en.ts @@ -1062,6 +1062,12 @@ const en = { stageLogs: 'Stage logs', stageError: 'Stage error', stageFailed: 'This stage failed without returning facts.', + protocolFrames: 'Protocol frames', + protocolStream: 'Protocol stream', + noProtocolStreams: 'No protocol streams in this stage.', + noProtocolFrames: 'No protocol frames recorded.', + recordingComplete: 'Recording complete', + recordingIncomplete: 'Recording incomplete', stageView: 'Stage view', descent: 'Descent', noRunEvents: 'This run recorded no events.', diff --git a/apps/web/src/i18n/locales/zh-Hans.ts b/apps/web/src/i18n/locales/zh-Hans.ts index 1af645412b..9461e455ca 100644 --- a/apps/web/src/i18n/locales/zh-Hans.ts +++ b/apps/web/src/i18n/locales/zh-Hans.ts @@ -1014,6 +1014,12 @@ const zhHansCN = { stageLogs: 'Stage 日志', stageError: 'Stage 错误', stageFailed: '此 stage 失败，未返回 facts。', + protocolFrames: '协议帧', + protocolStream: '协议流', + noProtocolStreams: '此 stage 没有协议流。', + noProtocolFrames: '没有记录到协议帧。', + recordingComplete: '记录完整', + recordingIncomplete: '记录不完整', stageView: 'Stage 视图', descent: '分支', noRunEvents: '这次运行没有记录到任何事件。', From 5b17649051b4e0d598ea50f8436d5d4f2be1dc3c Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 10:10:43 +0800 Subject: [PATCH 73/79] fix: settle streamed image usage conversion failures Reject the deferred outcome with the original quantity conversion error instead of leaving run drain pending after terminal output. The real HTTP regression supplies overflowing numeric usage, verifies background completion, retains the actual call observation and records failed diagnostics. --- .../data-plane/openai-images/stream_test.ts | 34 ++++++++++++++++++- .../data-plane/openai-images/call-upstream.ts | 11 ++++-- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/openai-images/stream_test.ts b/packages/gateway/__tests__/data-plane/openai-images/stream_test.ts index acae744d47..6d959bf258 100644 --- a/packages/gateway/__tests__/data-plane/openai-images/stream_test.ts +++ b/packages/gateway/__tests__/data-plane/openai-images/stream_test.ts @@ -1,5 +1,7 @@ -import { test, vi } from 'vitest'; +import { expect, test, vi } from 'vitest'; +import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts'; +import { installDumpStubs, runRecordOf } from '../../dump/test-fixtures.ts'; import type { InMemoryRepo } from '../../repo/memory.ts'; import { saveUpstreamForTest } from '../../repo/upstreams.ts'; import { buildCustomUpstreamRecord, flushAsyncWork, requestAppWithWarmModels as requestApp, setupAppTest } from '../../test-utils/app.ts'; @@ -280,3 +282,33 @@ test('/v1/images/edits streams a multipart request, whose stream field arrives a { metric: 'output_tokens', quantity: '12' }, ]); }); + +test('rejects invalid streamed image quantities and completes owned background work', async () => { + const { apiKey, repo } = await setupAppTest(); + await repo.apiKeys.save({ ...apiKey, dumpRetentionSeconds: 3600 }); + await registerOpenAIImagesModel(repo); + const dumps = installDumpStubs(initDumpStore, initDumpBroker); + const reported = vi.spyOn(console, 'error').mockImplementation(() => {}); + try { + await withMockedFetch( + () => Promise.resolve(sseResponse('data: {"type":"image_generation.completed","b64_json":"RklO","usage":{"input_tokens":1e999,"output_tokens":3}}\n\n')), + async () => { + const response = await requestApp('/v1/images/generations', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + body: JSON.stringify({ model: 'gpt-image-2', prompt: 'image', stream: true }), + }); + expect(await response.text()).toContain('image_generation.completed'); + await flushAsyncWork(); + const errors = reported.mock.calls.flat().filter(value => value instanceof TypeError && value.message.includes('Infinity')); + expect(errors.length).toBeGreaterThan(0); + expect(new Set(errors).size).toBe(1); + }, + ); + const [usage] = await repo.usage.listAll(); + expect(usage).toMatchObject({ requests: 1, metrics: [] }); + const [performance] = await repo.performance.listAll(); + expect(performance).toMatchObject({ requests: 1, errorsNoOutput: 1 }); + expect(runRecordOf(dumps.stored[0]?.record).meta).toMatchObject({ status: 200, upstream: { id: 'up_images' }, error: { kind: 'failed', reason: expect.stringContaining('Infinity') } }); + } finally { reported.mockRestore(); } +}); diff --git a/packages/gateway/src/data-plane/openai-images/call-upstream.ts b/packages/gateway/src/data-plane/openai-images/call-upstream.ts index ac187ffc23..c22550c813 100644 --- a/packages/gateway/src/data-plane/openai-images/call-upstream.ts +++ b/packages/gateway/src/data-plane/openai-images/call-upstream.ts @@ -163,10 +163,10 @@ const meterFrames = ( identity: TelemetryModelIdentity, signal: AbortSignal | undefined, ): MeteredFrames => { - let settle!: (outcome: StreamOutcome) => void; + const settlement = Promise.withResolvers(); // Declared as this run's own unfinished work, so the runner waits for it at teardown where // it can see it rather than the reading being started and forgotten. - const outcome = defer(new Promise(resolve => { settle = resolve; })); + const outcome = defer(settlement.promise); // Running out without the completed event is what "it did not finish" means, and it is known // at the same moment the usage is. let sawTerminal = false; @@ -189,7 +189,12 @@ const meterFrames = ( // Reached however the events ended — the completed one, a client that stopped reading, // or a broken upstream — because what the upstream already metered is billable whatever // happened to the downstream half. - settle({ billable: [{ identity, quantities: billed(usage) }], failed: !sawTerminal }); + try { + settlement.resolve({ billable: [{ identity, quantities: billed(usage) }], failed: !sawTerminal }); + } catch (error) { + settlement.reject(error); + throw error; + } } // Only an upstream that ended its body without ever completing the image reaches here: the // arm above returns on the terminal event. A client has been sent partial images and no From 41287429d1f036a0904c78741c5ddc7fe914defc Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 13:51:20 +0800 Subject: [PATCH 74/79] refactor: compose non-chat recording through the dump package Delegate event encoding and stream references to the shared recorder through explicit storage and broker ports. Keep admission, attribution, drain, HTTP measurement and dual-shape persistence in the gateway adapter. Narrow pipelined request contexts to run recorders and import metadata directly from its owning package. Preserve domain behavior and archival metadata; 145 focused tests, gateway/web typechecks and scoped lint pass. --- README.md | 11 ++-- .../components/requests/dump-stream-stub.ts | 2 +- .../dump-subscription-key-switch_test.tsx | 2 +- .../components/requests/ttft_test.tsx | 3 +- apps/web/package.json | 1 + apps/web/src/components/requests/detail.tsx | 3 +- apps/web/src/components/requests/format.ts | 2 +- apps/web/src/components/requests/list.tsx | 2 +- .../web/src/components/requests/run-redact.ts | 2 +- .../web/src/components/requests/run-render.ts | 2 +- .../requests/use-dump-subscription.ts | 2 +- .../src/routes/dashboard-monitor-requests.tsx | 3 +- .../__tests__/control-plane/dump_test.ts | 3 +- .../data-plane/pipeline/settlement_test.ts | 11 ++-- packages/gateway/__tests__/vitest.setup.ts | 3 +- .../data-plane/openai-audio/call-upstream.ts | 2 +- .../openai-completions/call-upstream.ts | 5 +- .../src/data-plane/openai-completions/emit.ts | 5 +- .../data-plane/openai-images/call-upstream.ts | 2 +- .../src/data-plane/openai-images/emit.ts | 2 +- .../gateway/src/data-plane/pipeline/serve.ts | 25 +++++---- .../src/data-plane/pipeline/services.ts | 9 ++-- packages/gateway/src/dump/accumulator.ts | 11 +--- packages/gateway/src/dump/attribution.ts | 2 +- packages/gateway/src/dump/run-sink.ts | 42 +++++---------- packages/gateway/src/dump/storage-codec.ts | 3 +- packages/gateway/src/dump/store-contract.ts | 3 +- packages/gateway/src/dump/turn-dump.ts | 53 +------------------ packages/gateway/src/dump/types.ts | 4 -- packages/gateway/src/repo/dump-store.ts | 15 +----- pnpm-lock.yaml | 3 ++ 31 files changed, 88 insertions(+), 150 deletions(-) diff --git a/README.md b/README.md index 0fcb6e6d23..8e159795f2 100644 --- a/README.md +++ b/README.md @@ -206,10 +206,13 @@ stage throws. Client JSON serialization runs inside settlement and publishes UTF while retaining the canonical content for inspection. Streaming quantities settle at the deferred exit. Explicit protocol and transport errors take precedence over the settlement's generic failure marker. -Run dumps encode stage facts, logs and protocol frames into one object space. Closing waits -for owned readers and deferred outcomes before storing the completed NDJSON record and -publishing its metadata. LogStream infrastructure is available on both platforms; live -recording and its business readers remain outside these three PRs. +Pipeline owns the stage-event format and object-space encoding. `@floway-dev/dump` owns +run recording, stream references, completed NDJSON artifacts and metadata publication. +Gateway supplies storage/broker ports and applies request admission, model attribution, +HTTP byte measurement and retention policy. Closing waits for owned readers and deferred +outcomes before storing the completed record and publishing its metadata. LogStream +infrastructure is available on both platforms; live recording and its business readers +remain outside these three PRs. Pipeline endpoints are assembled from individual stage modules. Each stage owns its request and response contract; a large stage keeps local helpers in the same diff --git a/apps/web/__tests__/components/requests/dump-stream-stub.ts b/apps/web/__tests__/components/requests/dump-stream-stub.ts index 21d1b814af..057ea82afa 100644 --- a/apps/web/__tests__/components/requests/dump-stream-stub.ts +++ b/apps/web/__tests__/components/requests/dump-stream-stub.ts @@ -1,7 +1,7 @@ import { act } from '@testing-library/react'; import { afterEach, beforeEach, expect } from 'vitest'; -import type { DumpMetadata } from '@floway-dev/gateway/dump-types'; +import type { DumpMetadata } from '@floway-dev/dump/types'; export const record = (id: string): DumpMetadata => ({ id, diff --git a/apps/web/__tests__/components/requests/dump-subscription-key-switch_test.tsx b/apps/web/__tests__/components/requests/dump-subscription-key-switch_test.tsx index adaa01c69b..f5b2b51118 100644 --- a/apps/web/__tests__/components/requests/dump-subscription-key-switch_test.tsx +++ b/apps/web/__tests__/components/requests/dump-subscription-key-switch_test.tsx @@ -7,7 +7,7 @@ import { flowayTokenStorageKey } from '../../../src/auth/session'; import { useDumpSubscription } from '../../../src/components/requests/use-dump-subscription'; import { stubLocalStorage } from '../../local-storage-stub'; import { renderInApp } from '../../render'; -import type { DumpMetadata } from '@floway-dev/gateway/dump-types'; +import type { DumpMetadata } from '@floway-dev/dump/types'; interface Commit { ids: string[]; keyId: string | null } interface Selection { keyId: string | null; seed: DumpMetadata[] } diff --git a/apps/web/__tests__/components/requests/ttft_test.tsx b/apps/web/__tests__/components/requests/ttft_test.tsx index 65a95266ec..2cc4831271 100644 --- a/apps/web/__tests__/components/requests/ttft_test.tsx +++ b/apps/web/__tests__/components/requests/ttft_test.tsx @@ -7,7 +7,8 @@ import type { ApiKey } from '../../../src/api/types'; import { RequestDetailPanel } from '../../../src/components/requests/detail'; import { RequestListPanel } from '../../../src/components/requests/list'; import { renderInApp } from '../../render'; -import type { DumpMetadata, DumpEdgeRecord } from '@floway-dev/gateway/dump-types'; +import type { DumpMetadata } from '@floway-dev/dump/types'; +import type { DumpEdgeRecord } from '@floway-dev/gateway/dump-types'; vi.mock('../../../src/components/ui/body-editor', () => ({ default: ({ text, toolbarStart }: { text: string; toolbarStart?: ReactNode }) => ( diff --git a/apps/web/package.json b/apps/web/package.json index d5941e3082..114e3006b0 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -47,6 +47,7 @@ "zustand": "^5.0.14" }, "devDependencies": { + "@floway-dev/dump": "workspace:*", "@floway-dev/gateway": "workspace:*", "@react-router/dev": "8.3.0", "@testing-library/react": "^16.3.0", diff --git a/apps/web/src/components/requests/detail.tsx b/apps/web/src/components/requests/detail.tsx index 1dd1cbaa25..9e48c4a8e9 100644 --- a/apps/web/src/components/requests/detail.tsx +++ b/apps/web/src/components/requests/detail.tsx @@ -21,7 +21,8 @@ import { OutcomeMessageBar } from '../ui/outcome-message-bar'; import { PANEL_BAND_CLASS } from '../ui/panel'; import { TooltipIconButton } from '../ui/tooltip-icon-button'; import { copyOutcomeIcon, useCopyLabel, useCopyToClipboard } from '../ui/use-copy-to-clipboard'; -import type { DumpMetadata, DumpRecord, DumpResponseBody, DumpRunRecord, DumpEdgeRecord } from '@floway-dev/gateway/dump-types'; +import type { DumpMetadata } from '@floway-dev/dump/types'; +import type { DumpRecord, DumpResponseBody, DumpRunRecord, DumpEdgeRecord } from '@floway-dev/gateway/dump-types'; const BodyEditor = lazy(() => import('../ui/body-editor')); const { Button, DialogActions, DialogTitle, Option, Spinner, Text, Tooltip } = fluentComponents; diff --git a/apps/web/src/components/requests/format.ts b/apps/web/src/components/requests/format.ts index 1fb41bff6f..017dff6cad 100644 --- a/apps/web/src/components/requests/format.ts +++ b/apps/web/src/components/requests/format.ts @@ -1,4 +1,4 @@ -import type { DumpErrorMeta, DumpMetadata } from '@floway-dev/gateway/dump-types'; +import type { DumpErrorMeta, DumpMetadata } from '@floway-dev/dump/types'; export type RequestSeverity = 'success' | 'warning' | 'error'; diff --git a/apps/web/src/components/requests/list.tsx b/apps/web/src/components/requests/list.tsx index e4cbc4f3f5..2b81717180 100644 --- a/apps/web/src/components/requests/list.tsx +++ b/apps/web/src/components/requests/list.tsx @@ -31,7 +31,7 @@ import { useRouteAddress } from '../ui/route-link'; import { useScrollAreaHost } from '../ui/scroll-area'; import { TruncationTooltip } from '../ui/truncation-tooltip'; import { ProviderBadge } from '../upstreams/provider-badge'; -import type { DumpMetadata } from '@floway-dev/gateway/dump-types'; +import type { DumpMetadata } from '@floway-dev/dump/types'; const { Option, Text, Tooltip, makeStyles, mergeClasses } = fluentComponents; const ROW_HEIGHT = 84; diff --git a/apps/web/src/components/requests/run-redact.ts b/apps/web/src/components/requests/run-redact.ts index 2a82dd0dbf..15fff3ee77 100644 --- a/apps/web/src/components/requests/run-redact.ts +++ b/apps/web/src/components/requests/run-redact.ts @@ -1,5 +1,5 @@ import { isSensitiveHeader, redactHeaderValue } from './header-redact'; -import type { DumpEvent } from '@floway-dev/gateway/dump-types'; +import type { DumpEvent } from '@floway-dev/pipeline'; // Header values can be shared string nodes. Redacting only the tuple would // leave the original credential in the object event that defined that node. diff --git a/apps/web/src/components/requests/run-render.ts b/apps/web/src/components/requests/run-render.ts index 4fbec1fd26..32c1174941 100644 --- a/apps/web/src/components/requests/run-render.ts +++ b/apps/web/src/components/requests/run-render.ts @@ -1,5 +1,5 @@ import { errorMessage } from '../../lib/error-message'; -import type { DumpEvent } from '@floway-dev/gateway/dump-types'; +import type { DumpEvent } from '@floway-dev/pipeline'; // A pipelined turn is recorded as its whole run: one NDJSON line per event, in // the order the run emitted them. A line is also one SSE `data:` payload, so diff --git a/apps/web/src/components/requests/use-dump-subscription.ts b/apps/web/src/components/requests/use-dump-subscription.ts index fd5bc2444f..9c915a09a4 100644 --- a/apps/web/src/components/requests/use-dump-subscription.ts +++ b/apps/web/src/components/requests/use-dump-subscription.ts @@ -5,7 +5,7 @@ import { api, callApi } from '../../api/client'; import { getSessionToken } from '../../auth/session'; import { useTranslation } from '../../i18n/translation'; import { errorMessage } from '../../lib/error-message'; -import type { DumpMetadata } from '@floway-dev/gateway/dump-types'; +import type { DumpMetadata } from '@floway-dev/dump/types'; const PAGE_LIMIT = 100; diff --git a/apps/web/src/routes/dashboard-monitor-requests.tsx b/apps/web/src/routes/dashboard-monitor-requests.tsx index 2e72838208..8366870cbd 100644 --- a/apps/web/src/routes/dashboard-monitor-requests.tsx +++ b/apps/web/src/routes/dashboard-monitor-requests.tsx @@ -25,7 +25,8 @@ import { fluentComponents } from '../fluent'; import { dashboardWorkspaceHandle } from '../lib/dashboard-route-handle'; import { useEntryRewrite } from '../lib/page-navigation'; import { useMediaQuery } from '../lib/use-media-query'; -import type { DumpMetadata, DumpRecord } from '@floway-dev/gateway/dump-types'; +import type { DumpMetadata } from '@floway-dev/dump/types'; +import type { DumpRecord } from '@floway-dev/gateway/dump-types'; export const handle = dashboardWorkspaceHandle; diff --git a/packages/gateway/__tests__/control-plane/dump_test.ts b/packages/gateway/__tests__/control-plane/dump_test.ts index 2bebb7dd7c..babe7fb711 100644 --- a/packages/gateway/__tests__/control-plane/dump_test.ts +++ b/packages/gateway/__tests__/control-plane/dump_test.ts @@ -2,9 +2,10 @@ import { test } from 'vitest'; import { initDumpBroker, initDumpStore } from '../../src/dump/registry.ts'; import type { DumpStore } from '../../src/dump/store-contract.ts'; -import type { DumpMetadata, DumpRecord, StoredDumpRecord } from '../../src/dump/types.ts'; +import type { DumpRecord, StoredDumpRecord } from '../../src/dump/types.ts'; import { fakeMeta as baseFakeMeta, fakeRecord as baseFakeRecord, fakeRunRecord, installDumpStubs } from '../dump/test-fixtures.ts'; import { requestApp, setupAppTest } from '../test-utils/app.ts'; +import type { DumpMetadata } from '@floway-dev/dump/types'; import { assertEquals, assertExists } from '@floway-dev/test-utils'; const fakeMeta = (id: string, completedAt: number): DumpMetadata => diff --git a/packages/gateway/__tests__/data-plane/pipeline/settlement_test.ts b/packages/gateway/__tests__/data-plane/pipeline/settlement_test.ts index a0c3f497a4..dc25b65eed 100644 --- a/packages/gateway/__tests__/data-plane/pipeline/settlement_test.ts +++ b/packages/gateway/__tests__/data-plane/pipeline/settlement_test.ts @@ -29,10 +29,13 @@ describe('pipeline settlement', () => { const dump = openRunDump(key, { method: 'POST', path: '/v1/responses', body: { bytes: new Uint8Array(), streamError: null }, }, background, true, timing); - const gateway = mockGatewayCtx({ - dump, backgroundScheduler: background, - attempt: { timing, telemetry: mockPerfTelemetryContext({ keyId: key.id }) }, - }); + const gateway = { + ...mockGatewayCtx({ + backgroundScheduler: background, + attempt: { timing, telemetry: mockPerfTelemetryContext({ keyId: key.id }) }, + }), + dump, + }; const identity = { model: 'm', upstream: 'u', modelKey: 'm', pricing: null }; if (failed) dump!.failed(new Error('Original source failure', { cause: new Error('socket reset') })); const active = pending.length; diff --git a/packages/gateway/__tests__/vitest.setup.ts b/packages/gateway/__tests__/vitest.setup.ts index 5ca7278aff..0f06a9ce47 100644 --- a/packages/gateway/__tests__/vitest.setup.ts +++ b/packages/gateway/__tests__/vitest.setup.ts @@ -1,11 +1,12 @@ import { trackBackground } from './test-utils/background-tracker.ts'; import { initDumpBroker, initDumpStore } from '../src/dump/registry.ts'; import type { DumpStore } from '../src/dump/store-contract.ts'; -import type { DumpMetadata, StoredDumpRecord, DumpRecordId } from '../src/dump/types.ts'; +import type { StoredDumpRecord } from '../src/dump/types.ts'; import { handleExecutionRequest } from '../src/execution/handler.ts'; import { initBackgroundSchedulerResolver } from '../src/runtime/background.ts'; import { initExecutionCellNamespace } from '../src/runtime/execution.ts'; import type { DumpBroker } from '@floway-dev/dump'; +import type { DumpMetadata, DumpRecordId } from '@floway-dev/dump/types'; import { isReplayableBody } from '@floway-dev/http'; import { initEnv, initFetch, initRuntimeKind, initTimingSafeEqual, InProcessExecutionCellNamespace } from '@floway-dev/platform'; diff --git a/packages/gateway/src/data-plane/openai-audio/call-upstream.ts b/packages/gateway/src/data-plane/openai-audio/call-upstream.ts index 311bb40707..1d658dc480 100644 --- a/packages/gateway/src/data-plane/openai-audio/call-upstream.ts +++ b/packages/gateway/src/data-plane/openai-audio/call-upstream.ts @@ -1,5 +1,4 @@ import type { Fields, OpenAIAudioTranscriptionEvents, OpenAIAudioTranscriptionStreamOutcome } from './facts.ts'; -import { recordStream } from '../../dump/turn-dump.ts'; import type { UsageQuantities } from '../../repo/types.ts'; import type { BillableEntity, Failure } from '../pipeline/facts.ts'; import { providerEntry } from '../pipeline/provider-entry.ts'; @@ -7,6 +6,7 @@ import { providerUsage } from '../pipeline/provider-usage.ts'; import type { GatewayServices } from '../pipeline/services.ts'; import { dialFailure, spentBody, retainReader } from '../pipeline/upstream-body.ts'; import { upstreamPerformanceContext, telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; +import { recordStream } from '@floway-dev/dump'; import { exchangeResponse } from '@floway-dev/http/pipeline'; import { defineStage, move, setRelease, defer, type Owned, type Logger, type Deferred } from '@floway-dev/pipeline'; import { isEventStreamMediaType, eventFrame, parseSSEStream, parseDecimalString } from '@floway-dev/protocols/common'; diff --git a/packages/gateway/src/data-plane/openai-completions/call-upstream.ts b/packages/gateway/src/data-plane/openai-completions/call-upstream.ts index cd4e1977f6..467ff78c6f 100644 --- a/packages/gateway/src/data-plane/openai-completions/call-upstream.ts +++ b/packages/gateway/src/data-plane/openai-completions/call-upstream.ts @@ -1,6 +1,6 @@ import type { Fields, OpenAICompletionsFacts, OpenAICompletionsFrames } from './facts.ts'; import { tokenUsageFromOpenAICompletionsUsage } from './usage.ts'; -import { recordStream, type TurnDump } from '../../dump/turn-dump.ts'; +import type { RunDump } from '../../dump/run-sink.ts'; import { isFailure, type BillableEntity, type Failure } from '../pipeline/facts.ts'; import { providerEntry } from '../pipeline/provider-entry.ts'; import { providerUsage } from '../pipeline/provider-usage.ts'; @@ -9,6 +9,7 @@ import type { GatewayServices } from '../pipeline/services.ts'; import { dialFailure, spentBody, retainReader } from '../pipeline/upstream-body.ts'; import { upstreamPerformanceContext, telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; import { tokenUsageMeasurement } from '../shared/telemetry/usage.ts'; +import { recordStream } from '@floway-dev/dump'; import { exchangeResponse } from '@floway-dev/http/pipeline'; import { defineStage, move, setRelease, defer, type Owned, type Deferred } from '@floway-dev/pipeline'; import { isOpenAIUsageOnlyEventShape, type ProtocolFrame } from '@floway-dev/protocols/common'; @@ -111,7 +112,7 @@ const readUpstream = async ( identity: TelemetryModelIdentity, candidate: ModelCandidate, signal: AbortSignal | undefined, - dump: TurnDump | null, + dump: RunDump | null, ownedBody: (ReadableStream & Owned) | null, wasCalled: boolean, ): Promise => { diff --git a/packages/gateway/src/data-plane/openai-completions/emit.ts b/packages/gateway/src/data-plane/openai-completions/emit.ts index bd1a83db71..224625d53e 100644 --- a/packages/gateway/src/data-plane/openai-completions/emit.ts +++ b/packages/gateway/src/data-plane/openai-completions/emit.ts @@ -1,8 +1,9 @@ import type { OpenAICompletionsFacts, OpenAICompletionsFrames, Fields } from './facts.ts'; -import { recordStream, streamReferenceOf, type TurnDump } from '../../dump/turn-dump.ts'; +import type { RunDump } from '../../dump/run-sink.ts'; import { isFailure } from '../pipeline/facts.ts'; import type { GatewayServices } from '../pipeline/services.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; +import { recordStream, streamReferenceOf } from '@floway-dev/dump'; import { defineStage, move } from '@floway-dev/pipeline'; import { renderErrorEnvelope, isOpenAIUsageOnlyEventShape, type SseFrame } from '@floway-dev/protocols/common'; import { openaiCompletionsProtocolFrameToSSEFrame } from '@floway-dev/protocols/openai-completions'; @@ -65,7 +66,7 @@ export const emitOpenAICompletions = defineStage< const rendered = ( answer: OpenAICompletionsFacts['response.openaiCompletions.payload'], wantsUsageChunk: boolean, - dump: TurnDump | null, + dump: RunDump | null, ): OpenAICompletionsFacts['response.openaiCompletions.rendered'] => isFailure(answer) ? renderErrorEnvelope(answer.message, answer.body) : isFrames(answer) ? renderSSE(wantsUsageChunk ? answer : recordStream(withoutUsage(answer), dump)) diff --git a/packages/gateway/src/data-plane/openai-images/call-upstream.ts b/packages/gateway/src/data-plane/openai-images/call-upstream.ts index c22550c813..f2fd323a0b 100644 --- a/packages/gateway/src/data-plane/openai-images/call-upstream.ts +++ b/packages/gateway/src/data-plane/openai-images/call-upstream.ts @@ -1,5 +1,4 @@ import type { Fields, OpenAIImagesFrames } from './facts.ts'; -import { recordStream } from '../../dump/turn-dump.ts'; import type { UsageQuantities } from '../../repo/types.ts'; import type { BillableEntity, Failure } from '../pipeline/facts.ts'; import { providerEntry } from '../pipeline/provider-entry.ts'; @@ -8,6 +7,7 @@ import type { StreamOutcome } from '../pipeline/serve.ts'; import type { GatewayServices } from '../pipeline/services.ts'; import { dialFailure, readUpstreamBody, spentBody, retainReader } from '../pipeline/upstream-body.ts'; import { upstreamPerformanceContext, telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; +import { recordStream } from '@floway-dev/dump'; import { exchangeResponse } from '@floway-dev/http/pipeline'; import { defineStage, move, setRelease, defer, type Deferred } from '@floway-dev/pipeline'; import { upstreamErrorMessage, isEventStreamMediaType, eventFrame, mediaTypeEssence, parseDecimalString } from '@floway-dev/protocols/common'; diff --git a/packages/gateway/src/data-plane/openai-images/emit.ts b/packages/gateway/src/data-plane/openai-images/emit.ts index 35154cd14a..cb910c3af0 100644 --- a/packages/gateway/src/data-plane/openai-images/emit.ts +++ b/packages/gateway/src/data-plane/openai-images/emit.ts @@ -1,7 +1,7 @@ import type { OpenAIImagesFrames, Fields, OpenAIImagesFacts } from './facts.ts'; -import { streamReferenceOf } from '../../dump/turn-dump.ts'; import { isFailure } from '../pipeline/facts.ts'; import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; +import { streamReferenceOf } from '@floway-dev/dump'; import { defineStage, move } from '@floway-dev/pipeline'; import { renderErrorEnvelope, type SseFrame } from '@floway-dev/protocols/common'; import { renderOpenAIImagesResponse, openaiImagesStreamEventToSSEFrame, type CanonicalOpenAIImagesResponse } from '@floway-dev/protocols/openai-images'; diff --git a/packages/gateway/src/data-plane/pipeline/serve.ts b/packages/gateway/src/data-plane/pipeline/serve.ts index 806a4c17a2..002118c6e3 100644 --- a/packages/gateway/src/data-plane/pipeline/serve.ts +++ b/packages/gateway/src/data-plane/pipeline/serve.ts @@ -12,17 +12,17 @@ import type { ContentfulStatusCode } from 'hono/utils/http-status'; import { createCandidateRegistry } from './candidates.ts'; import type { AttemptSelector, BillableEntity, GatewayFacts } from './facts.ts'; -import type { GatewayServices } from './services.ts'; +import type { GatewayServices, RunGatewayCtx } from './services.ts'; import { openRunDump, type RunDump } from '../../dump/run-sink.ts'; -import { recordStream } from '../../dump/turn-dump.ts'; import { apiKeyFromContext, type AuthedContext } from '../../middleware/auth.ts'; import { internalErrorResponse } from '../../middleware/internal-error-response.ts'; import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; import { consoleLogSink } from '../../runtime/log.ts'; import { stampUpstreamCallStart } from '../shared/attempt-timing.ts'; -import { createGatewayCtxFromHono, finalizeGatewayResponse, type AttemptState, type GatewayCtx } from '../shared/gateway-ctx.ts'; +import { createGatewayCtxFromHono, finalizeGatewayResponse, type AttemptState } from '../shared/gateway-ctx.ts'; import { readRequestBody, takeRequestBody, type RequestBody } from '../shared/request-body.ts'; import { writeSSEFrames } from '../shared/sse.ts'; +import { recordStream } from '@floway-dev/dump'; import { run, getFailureFacts, type Pipeline } from '@floway-dev/pipeline'; import { sseCommentFrame, type SseFrame } from '@floway-dev/protocols/common'; @@ -51,7 +51,7 @@ export const readIngress = async (c: Context): Promise => ({ export interface Prologue { readonly runDump: RunDump | null; readonly services: GatewayServices; - readonly gateway: GatewayCtx; + readonly gateway: RunGatewayCtx; readonly headers: readonly (readonly [string, string])[]; } @@ -83,14 +83,17 @@ export const openPrologue = ( options.wantsStream, attempt.timing, ); - const gateway = createGatewayCtxFromHono(c, { - wantsStream: options.wantsStream, - attempt, - ...(options.model === undefined ? {} : { model: options.model }), - requestBody: takeRequestBody(ingress.body), - backgroundScheduler, + const gateway: RunGatewayCtx = { + ...createGatewayCtxFromHono(c, { + wantsStream: options.wantsStream, + attempt, + ...(options.model === undefined ? {} : { model: options.model }), + requestBody: takeRequestBody(ingress.body), + backgroundScheduler, + dump: runDump, + }), dump: runDump, - }); + }; const candidates = createCandidateRegistry(); return { diff --git a/packages/gateway/src/data-plane/pipeline/services.ts b/packages/gateway/src/data-plane/pipeline/services.ts index a80230a1d1..325766070c 100644 --- a/packages/gateway/src/data-plane/pipeline/services.ts +++ b/packages/gateway/src/data-plane/pipeline/services.ts @@ -3,16 +3,19 @@ // next segment's facts — if it also supplied capabilities, the same pipeline value would // run with different capabilities depending on who called it. // -// Everything in facts is dumpable, and that is the test: a live handle dumps as nothing, -// so a live handle is never a fact. +// Facts carry content and explicitly tagged run resources. Service handles bind application +// behavior and stay outside those values. import type { AttemptSelector } from './facts.ts'; +import type { RunDump } from '../../dump/run-sink.ts'; import type { GatewayCtx } from '../shared/gateway-ctx.ts'; import type { HttpServices } from '@floway-dev/http/pipeline'; import type { Event, Logger } from '@floway-dev/pipeline'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { ModelCandidate } from '@floway-dev/provider'; +export type RunGatewayCtx = GatewayCtx & { readonly dump: RunDump | null }; + export interface GatewayServices extends HttpServices { readonly recordProtocolFrames: >(frames: AsyncIterable) => AsyncIterable; /** The global sink. Every stage's lines reach it, tagged with the stage's name. */ @@ -23,7 +26,7 @@ export interface GatewayServices extends HttpServices { /** The request-scoped context the settlement and telemetry stages read. It is a service * and not a fact because it holds live handles — the scheduler, the abort signal. */ - readonly gateway: GatewayCtx; + readonly gateway: RunGatewayCtx; /** Turns a selector back into the thing that dials. The resolver is the service and the * selector is the fact: a per-upstream transport is not a fact and is not pinned at the * prologue either, so what is injected is the thing that resolves one and what travels diff --git a/packages/gateway/src/dump/accumulator.ts b/packages/gateway/src/dump/accumulator.ts index 1e1c6cf9df..213adaf3e8 100644 --- a/packages/gateway/src/dump/accumulator.ts +++ b/packages/gateway/src/dump/accumulator.ts @@ -9,21 +9,14 @@ import type { Context } from 'hono'; import { HttpCapture } from './http-capture.ts'; import { getDumpBroker, getDumpStore } from './registry.ts'; import type { StreamRecording } from './turn-dump.ts'; -import type { - DumpErrorMeta, - DumpMetadata, - DumpStreamEvent, - DumpUpstreamRef, - DumpWriteRecord, - PreparedDumpRequestBody, - StoredDumpResponseBody, -} from './types.ts'; +import type { DumpStreamEvent, DumpWriteRecord, PreparedDumpRequestBody, StoredDumpResponseBody } from './types.ts'; import { encodeBodyForWire } from './wire.ts'; import { attemptTtftMs, type AttemptTiming } from '../data-plane/shared/attempt-timing.ts'; import type { RequestBody } from '../data-plane/shared/request-body.ts'; import { getRepo } from '../repo/index.ts'; import type { ApiKey, TokenUsage } from '../repo/types.ts'; import { ulid } from '../shared/ulid.ts'; +import type { DumpErrorMeta, DumpMetadata, DumpUpstreamRef } from '@floway-dev/dump/types'; import type { BackgroundScheduler } from '@floway-dev/platform'; import { type ProtocolFrame } from '@floway-dev/protocols/common'; import type { ChatTargetApi, TelemetryModelIdentity } from '@floway-dev/provider'; diff --git a/packages/gateway/src/dump/attribution.ts b/packages/gateway/src/dump/attribution.ts index 37143d681e..f9d3d438eb 100644 --- a/packages/gateway/src/dump/attribution.ts +++ b/packages/gateway/src/dump/attribution.ts @@ -30,9 +30,9 @@ // `requestedModel`-set model survives across both error variants so even an // outright-failed turn carries model attribution. -import type { DumpErrorMeta, DumpMetadata, DumpUpstreamRef } from './types.ts'; import { getRepo } from '../repo/index.ts'; import type { TokenUsage } from '../repo/types.ts'; +import type { DumpErrorMeta, DumpMetadata, DumpUpstreamRef } from '@floway-dev/dump/types'; import type { TelemetryModelIdentity } from '@floway-dev/provider'; export const oneLineError = (err: unknown): string => { diff --git a/packages/gateway/src/dump/run-sink.ts b/packages/gateway/src/dump/run-sink.ts index 95a74c831c..5d5c0ab632 100644 --- a/packages/gateway/src/dump/run-sink.ts +++ b/packages/gateway/src/dump/run-sink.ts @@ -1,12 +1,10 @@ import { DumpAttribution, oneLineError, streamReadError } from './attribution.ts'; import { getDumpBroker, getDumpStore } from './registry.ts'; -import type { StreamRecording } from './turn-dump.ts'; -import type { DumpMetadata } from './types.ts'; import { attemptTtftMs, type AttemptTiming } from '../data-plane/shared/attempt-timing.ts'; import type { RequestBody } from '../data-plane/shared/request-body.ts'; import type { ApiKey, TokenUsage } from '../repo/types.ts'; import { ulid } from '../shared/ulid.ts'; -import { createRunEncoder, streamFact, toNdjson, type DumpEvent, type Event } from '@floway-dev/pipeline'; +import { createRunRecorder, type DumpMetadata, type RunRecorder, type StreamRecording } from '@floway-dev/dump'; import type { BackgroundScheduler } from '@floway-dev/platform'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { TelemetryModelIdentity } from '@floway-dev/provider'; @@ -21,26 +19,27 @@ interface RequestSnapshot { export class RunDump { private readonly attribution = new DumpAttribution(); - private readonly encode = createRunEncoder(); - private readonly events: DumpEvent[] = []; + private readonly recorder: RunRecorder; + readonly sink: RunRecorder['sink']; private runDrain: (() => Promise) | null = null; private sentPayloadBytes = 0; - private streams = 0; - private answerStream: StreamRecording | undefined; constructor( - private readonly apiKey: ApiKey, + apiKey: ApiKey, private readonly requestSnapshot: RequestSnapshot, private readonly startedAt: number, private readonly backgroundScheduler: BackgroundScheduler, private readonly wantsStream: boolean, private readonly timing: AttemptTiming, - ) {} + ) { + this.recorder = createRunRecorder({ + write: record => getDumpStore().put(apiKey.id, { shape: 'run', ...record }), + publish: meta => getDumpBroker().publish(apiKey.id, meta), + }); + this.sink = this.recorder.sink; + } afterRun(drain: () => Promise): void { this.runDrain = drain; } - readonly sink = (event: Event): void => { - for (const encoded of this.encode(event)) this.events.push(encoded); - }; requestedModel(model: string): void { this.attribution.requestedModel(model); @@ -55,19 +54,10 @@ export class RunDump { } frame(frame: ProtocolFrame): void | Promise { - this.answerStream ??= this.openStream(); - return this.answerStream.frame(frame); + return this.recorder.frame(frame); } - // Stream references and frame events share one run-wide identity; end marks complete recording. - openStream(): StreamRecording { - const streamId = ++this.streams; - return { - frame: frame => { this.sink({ type: 'stream.frame', streamId, frames: [frame] }); }, - end: () => { this.sink({ type: 'stream.end', streamId }); }, - fact: streamFact(streamId), - }; - } + openStream(): StreamRecording { return this.recorder.openStream(); } success(identity: TelemetryModelIdentity, usage: TokenUsage | null): void { this.attribution.success(identity, usage); @@ -137,11 +127,7 @@ export class RunDump { fallbackError: streamReadError(this.requestSnapshot.streamError, responseStreamError), }); - // Publishing follows durable storage so metadata subscribers can immediately fetch detail. - await getDumpStore().put(this.apiKey.id, { - shape: 'run', meta, events: new TextEncoder().encode(toNdjson(this.events)), - }); - await getDumpBroker().publish(this.apiKey.id, meta); + await this.recorder.finish(meta); } } diff --git a/packages/gateway/src/dump/storage-codec.ts b/packages/gateway/src/dump/storage-codec.ts index c4387befa4..576a876fd6 100644 --- a/packages/gateway/src/dump/storage-codec.ts +++ b/packages/gateway/src/dump/storage-codec.ts @@ -6,7 +6,8 @@ import { persistedDumpMetadataSchema, dumpStreamEventsSchema, } from './schemas.ts'; -import type { DumpMetadata, DumpStreamEvent } from './types.ts'; +import type { DumpStreamEvent } from './types.ts'; +import type { DumpMetadata } from '@floway-dev/dump/types'; export type DumpBodyDescriptor = z.infer; type PersistedDumpMetadata = z.infer; diff --git a/packages/gateway/src/dump/store-contract.ts b/packages/gateway/src/dump/store-contract.ts index ffa1a51fc7..b652fa8e71 100644 --- a/packages/gateway/src/dump/store-contract.ts +++ b/packages/gateway/src/dump/store-contract.ts @@ -1,4 +1,5 @@ -import type { DumpMetadata, DumpRecordId, DumpWriteRecord, PreparedDumpRequestBody, StoredDumpRecord } from './types.ts'; +import type { DumpWriteRecord, PreparedDumpRequestBody, StoredDumpRecord } from './types.ts'; +import type { DumpMetadata, DumpRecordId } from '@floway-dev/dump/types'; // Per-API-key request dump storage contract: metadata in SQL, bodies in the // FileStore. Request bytes are prepared before the terminal write; reads diff --git a/packages/gateway/src/dump/turn-dump.ts b/packages/gateway/src/dump/turn-dump.ts index 71bae0b305..dc00d0250a 100644 --- a/packages/gateway/src/dump/turn-dump.ts +++ b/packages/gateway/src/dump/turn-dump.ts @@ -7,7 +7,7 @@ import type { HttpCapture } from './http-capture.ts'; import type { TokenUsage } from '../repo/types.ts'; -import { isStreamFact, type StreamFact } from '@floway-dev/pipeline'; +import type { StreamFact } from '@floway-dev/pipeline'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { TelemetryModelIdentity } from '@floway-dev/provider'; @@ -48,54 +48,3 @@ export interface TurnDump { finalize(status: number | null, responseBytes: number | readonly unknown[]): void; finalize(response: Response): Response; } - -/** - * Records a stream's frames as they are read, and hands them on untouched. - * - * What the record holds is the stream as the client is served it, so the tee goes outside - * whatever shapes the frames and inside whatever frames them for a transport. Reading is what - * records: a losing attempt nobody read contributes nothing, because the release path drains - * the stream underneath this rather than through it. - * - * The value handed back *is* the stream reference, so the fact that holds it encodes as - * `{"$stream": n}` and the frames that arrive afterwards say which stream they belong to. - * - * A record holds protocol frames, which is what most families' streams already carry. The - * family whose stream is bare protocol events says how one becomes a frame, because the record - * cannot guess and a cast would be it guessing. - */ -export function recordStream>(stream: AsyncIterable, dump: TurnDump | null): AsyncIterable; -export function recordStream(stream: AsyncIterable, dump: TurnDump | null, asFrame: (value: T) => ProtocolFrame): AsyncIterable; -export function recordStream( - stream: AsyncIterable, - dump: TurnDump | null, - asFrame: (value: T) => ProtocolFrame = value => value as ProtocolFrame, -): AsyncIterable { - // No recording configured hands the same iterable back, so a record shows no step where - // nothing happened and the stream is not wrapped for nobody. - if (dump === null) return stream; - - const recording = dump.openStream(); - return { - ...recording.fact, - [Symbol.asyncIterator]: () => (async function* () { - for await (const value of stream) { - await recording.frame(asFrame(value)); - yield value; - } - // Reached only where the source ran out on its own, which is what makes the record of - // this stream complete. A reader that stopped early never gets here. - await recording.end(); - })(), - }; -} - -/** - * The reference a value carries to the stream the record holds, for a wrapper to carry across. - * - * A stream is framed again on its way out — protocol frames become SSE — and what the client - * is handed is a different object over the same frames. Carrying the reference onto it is what - * lets the fact that produced the stream and the fact that framed it point at one record. - */ -export const streamReferenceOf = (value: unknown): StreamFact | Record => - isStreamFact(value) ? { ...value } : {}; diff --git a/packages/gateway/src/dump/types.ts b/packages/gateway/src/dump/types.ts index e84050a181..d7fa6c869b 100644 --- a/packages/gateway/src/dump/types.ts +++ b/packages/gateway/src/dump/types.ts @@ -16,10 +16,6 @@ import type { dumpStreamEventSchema, } from './schemas.ts'; import type { DumpMetadata } from '@floway-dev/dump/types'; -import type { DumpEvent } from '@floway-dev/pipeline'; - -export type { DumpEvent }; -export type { DumpRecordId, DumpErrorMeta, DumpMetadata, DumpUpstreamRef } from '@floway-dev/dump/types'; export type DumpCapture = z.infer; diff --git a/packages/gateway/src/repo/dump-store.ts b/packages/gateway/src/repo/dump-store.ts index 5b88ed72c7..102bd166cc 100644 --- a/packages/gateway/src/repo/dump-store.ts +++ b/packages/gateway/src/repo/dump-store.ts @@ -13,21 +13,10 @@ import { } from '../dump/storage-codec.ts'; import type { DumpBodyDescriptor } from '../dump/storage-codec.ts'; import type { DumpListOptions, DumpStore } from '../dump/store-contract.ts'; -import type { - DumpMetadata, - DumpRecordId, - DumpUpstreamRef, - DumpWriteRecord, - PreparedDumpRequestBody, - StoredDumpRecord, - StoredDumpEdgeRecord, - StoredDumpRequest, - StoredDumpResponse, - StoredDumpResponseBody, - StoredDumpUpstreamResponse, -} from '../dump/types.ts'; +import type { DumpWriteRecord, PreparedDumpRequestBody, StoredDumpRecord, StoredDumpEdgeRecord, StoredDumpRequest, StoredDumpResponse, StoredDumpResponseBody, StoredDumpUpstreamResponse } from '../dump/types.ts'; import { upstreamResponseToWire } from '../dump/wire.ts'; import { gunzipBytes, gzipBytes } from '../shared/gzip.ts'; +import type { DumpMetadata, DumpRecordId, DumpUpstreamRef } from '@floway-dev/dump/types'; import type { FileStore, SqlDatabase } from '@floway-dev/platform'; import { decodeForgivingBase64 } from '@floway-dev/protocols/common'; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9196b679f4..f8863dbe4f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -248,6 +248,9 @@ importers: specifier: ^5.0.14 version: 5.0.14(@types/react@19.2.17)(react@19.2.8)(use-sync-external-store@1.6.0(react@19.2.8)) devDependencies: + '@floway-dev/dump': + specifier: workspace:* + version: link:../../packages/dump '@floway-dev/gateway': specifier: workspace:* version: link:../../packages/gateway From 09c478519193761bb3ebef1f49e3ba56283a2800 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 1 Oct 2026 13:56:41 +0800 Subject: [PATCH 75/79] fix: replace the legacy dump type in run contexts Omit the legacy dump field before supplying the concrete recorder, so pipelined services expose only the supported run methods and finalize signatures. --- packages/gateway/src/data-plane/pipeline/services.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gateway/src/data-plane/pipeline/services.ts b/packages/gateway/src/data-plane/pipeline/services.ts index 325766070c..55eab63a43 100644 --- a/packages/gateway/src/data-plane/pipeline/services.ts +++ b/packages/gateway/src/data-plane/pipeline/services.ts @@ -14,7 +14,7 @@ import type { Event, Logger } from '@floway-dev/pipeline'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { ModelCandidate } from '@floway-dev/provider'; -export type RunGatewayCtx = GatewayCtx & { readonly dump: RunDump | null }; +export type RunGatewayCtx = Omit & { readonly dump: RunDump | null }; export interface GatewayServices extends HttpServices { readonly recordProtocolFrames: >(frames: AsyncIterable) => AsyncIterable; From ac28cbcbb9f915ed5c985ed12de4871646f243aa Mon Sep 17 00:00:00 2001 From: Menci Date: Sat, 14 Nov 2026 09:23:21 +0800 Subject: [PATCH 76/79] fix(alpha-search): resolve the pinned upstream before the ending dials An ending declares no early return, so the pinned-upstream refusal moves into its own stage that provides the attempt the ending dials, matching how every other family resolves candidates above its ending. --- .../data-plane/alpha-search/call-upstream.ts | 37 +++----------- .../src/data-plane/alpha-search/pipeline.ts | 3 +- .../alpha-search/resolve-upstream.ts | 50 +++++++++++++++++++ 3 files changed, 59 insertions(+), 31 deletions(-) create mode 100644 packages/gateway/src/data-plane/alpha-search/resolve-upstream.ts diff --git a/packages/gateway/src/data-plane/alpha-search/call-upstream.ts b/packages/gateway/src/data-plane/alpha-search/call-upstream.ts index 778bdc0f4d..fbe6456737 100644 --- a/packages/gateway/src/data-plane/alpha-search/call-upstream.ts +++ b/packages/gateway/src/data-plane/alpha-search/call-upstream.ts @@ -1,14 +1,13 @@ -import type { PinnedSearchUpstream, Fields } from './facts.ts'; +import type { Fields } from './facts.ts'; import { parseAlphaSearchResponse } from './protocol.ts'; import { providerEntry } from '../pipeline/provider-entry.ts'; import { providerUsage } from '../pipeline/provider-usage.ts'; import type { GatewayServices } from '../pipeline/services.ts'; import { dialFailure, spentBody } from '../pipeline/upstream-body.ts'; import { telemetryModelIdentity } from '../shared/telemetry/attribution.ts'; -import { AlphaSearchModelNotFoundError, resolveAlphaSearchCandidate } from '../tools/web-search/alpha-search/upstream.ts'; import { exchangeResponse } from '@floway-dev/http/pipeline'; import { defineStage, move } from '@floway-dev/pipeline'; -import { type ModelCandidate, type ProviderRequest, type ProviderResponse, type ProviderOperationPayloads } from '@floway-dev/provider'; +import { type ProviderRequest, type ProviderResponse, type ProviderOperationPayloads } from '@floway-dev/provider'; /** Codex projects one per-turn metadata snapshot onto several surfaces, and `SearchRequest` * carries no field for it — so on this endpoint the header is the whole of that surface, and @@ -36,8 +35,8 @@ const asJson = (raw: string): unknown => { * what the call is billable for. A failure is a value here as everywhere, even though this * family has nothing that fails over it: the edge is what turns one into a status and a body. */ -export const callSearchUpstream = (pinned: PinnedSearchUpstream) => defineStage< - Fields<'request.search.alphaSearch' | 'ingress.http.headers'>, +export const callSearchUpstream = defineStage< + Fields<'request.search.alphaSearch' | 'ingress.http.headers' | 'route.attempt'>, ProviderRequest, ProviderResponse, Fields<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status' | 'response.http.body'>, @@ -45,41 +44,19 @@ export const callSearchUpstream = (pinned: PinnedSearchUpstream) => defineStage< >({ name: 'callSearchUpstream', into: { - request: { needs: ['request.search.alphaSearch', 'ingress.http.headers'], consumes: [], provides: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'] }, + request: { needs: ['request.search.alphaSearch', 'ingress.http.headers', 'route.attempt'], consumes: [], provides: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'] }, response: { needs: ['response.http.exchange', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.http.body'], consumes: ['response.http.exchange', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'], provides: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers', 'response.http.status', 'response.http.body'] }, }, execute: async (facts, next, use) => { - // A misconfigured pin is the operator's to see with its stack, the way the gateway - // surfaces any internal failure; only a model the key cannot reach is the client's answer. - let candidate: ModelCandidate; - try { - candidate = await resolveAlphaSearchCandidate({ - config: pinned, - subject: use.gateway.user, - upstreamIds: use.gateway.upstreamIds, - scheduler: use.gateway.backgroundScheduler, - runtimeLocation: use.gateway.runtimeLocation, - }); - } catch (error) { - if (!(error instanceof AlphaSearchModelNotFoundError)) throw error; - return move({ - ...facts, - 'response.search.alphaSearch': { status: 404, message: error.message }, - 'response.usage.billable': [], - 'response.http.headers': [], - 'response.http.status': 404, - 'response.http.body': null, - }); - } + const candidate = use.resolveAttempt(facts['route.attempt']); // The caller's model is dropped: this endpoint is pinned to the operator's model, and the // provider stamps that one on the way out. // TODO: pin SearchRequest.id to one provider account when Codex upstreams support account // pools. The current Codex provider has one active account. const { model: _named, ...request } = facts['request.search.alphaSearch']; - const selector = use.rememberCandidates([candidate])[0]!; const pipeline = candidate.provider.pipelines.alphaSearch; if (pipeline === undefined) throw new Error(`Provider ${candidate.provider.kind} has no Alpha Search pipeline`); - const back = await next(providerEntry({ ...facts, 'route.attempt': selector }, candidate, request, [...turnMetadataHeaders(facts['ingress.http.headers'])]), pipeline); + const back = await next(providerEntry(facts, candidate, request, [...turnMetadataHeaders(facts['ingress.http.headers'])]), pipeline); const { 'response.http.exchange': exchange, 'response.provider.modelKey': modelKey, 'response.provider.called': wasCalled, 'response.provider.previousCalls': _previousCalls, ...rest } = back; if (exchange.type === 'transportFailure') { return move({ ...rest, 'response.search.alphaSearch': dialFailure(exchange.error), 'response.usage.billable': providerUsage(candidate, back, []), 'response.http.headers': [], 'response.http.status': 502 }); diff --git a/packages/gateway/src/data-plane/alpha-search/pipeline.ts b/packages/gateway/src/data-plane/alpha-search/pipeline.ts index 5325fe5ce9..326eb76218 100644 --- a/packages/gateway/src/data-plane/alpha-search/pipeline.ts +++ b/packages/gateway/src/data-plane/alpha-search/pipeline.ts @@ -3,6 +3,7 @@ import { emitAlphaSearch } from './emit.ts'; import { executeSearchOperations } from './execute-operations.ts'; import type { SearchExecution, Fields } from './facts.ts'; import { parseSearchOperations } from './parse-operations.ts'; +import { resolveSearchUpstream } from './resolve-upstream.ts'; import { serializeClientJson } from '../pipeline/serialize-client-json.ts'; import { writeSettlement } from '../pipeline/settlement.ts'; import { compose, type Pipeline } from '@floway-dev/pipeline'; @@ -18,5 +19,5 @@ export const searchServePipeline = (execution: SearchExecution): Pipeline< emitAlphaSearch, ...(execution.kind === 'local' ? [parseSearchOperations, executeSearchOperations] - : [callSearchUpstream(execution)]), + : [resolveSearchUpstream(execution), callSearchUpstream]), ]); diff --git a/packages/gateway/src/data-plane/alpha-search/resolve-upstream.ts b/packages/gateway/src/data-plane/alpha-search/resolve-upstream.ts new file mode 100644 index 0000000000..f91a2def52 --- /dev/null +++ b/packages/gateway/src/data-plane/alpha-search/resolve-upstream.ts @@ -0,0 +1,50 @@ +import type { PinnedSearchUpstream, Fields } from './facts.ts'; +import type { GatewayServices } from '../pipeline/services.ts'; +import { AlphaSearchModelNotFoundError, resolveAlphaSearchCandidate } from '../tools/web-search/alpha-search/upstream.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import type { ModelCandidate } from '@floway-dev/provider'; + +/** + * Resolves the operator's pinned search upstream into the attempt the ending dials. A + * misconfigured pin is the operator's to see with its stack, the way the gateway surfaces any + * internal failure; only a model the key cannot reach is the client's answer, and that answer + * is the same whether the model is missing or withheld. + */ +export const resolveSearchUpstream = (pinned: PinnedSearchUpstream) => defineStage< + Fields, + Fields<'route.attempt'>, + Fields<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status'>, + Fields<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status'>, + Fields<'response.search.alphaSearch' | 'response.usage.billable' | 'response.http.headers' | 'response.http.status'>, + GatewayServices +>({ + name: 'resolveSearchUpstream', + through: { + request: { needs: [], consumes: [], provides: ['route.attempt'] }, + response: { needs: [], consumes: [], provides: [] }, + }, + return: { provides: ['response.search.alphaSearch', 'response.usage.billable', 'response.http.headers', 'response.http.status'] }, + execute: async (facts, next, use) => { + let candidate: ModelCandidate; + try { + candidate = await resolveAlphaSearchCandidate({ + config: pinned, + subject: use.gateway.user, + upstreamIds: use.gateway.upstreamIds, + scheduler: use.gateway.backgroundScheduler, + runtimeLocation: use.gateway.runtimeLocation, + }); + } catch (error) { + if (!(error instanceof AlphaSearchModelNotFoundError)) throw error; + return move({ + ...facts, + 'response.search.alphaSearch': { status: 404, message: error.message }, + // Nothing was called, so nothing is billable and there are no upstream headers. + 'response.usage.billable': [], + 'response.http.headers': [], + 'response.http.status': 404, + }); + } + return await next({ ...facts, 'route.attempt': move(use.rememberCandidates([candidate])[0]!) }); + }, +}); From a57a3a8b087e636fc5746e27c0f58cfa32e90843 Mon Sep 17 00:00:00 2001 From: Menci Date: Sat, 14 Nov 2026 09:33:21 +0800 Subject: [PATCH 77/79] feat(decision): serve decision models through pipelines Decision becomes a pipeline family like rerank: the route names the client's dialect as an ingress fact, candidates narrow on the decision endpoint and the provider-derived dialect, the ending reads the upstream body with its number tokens preserved, and the edge renders the upstream's own document for a same-dialect client or translates it for the other dialect. Usage is billed on input tokens only. Azure, Ollama, Copilot and Custom serve decision through provider pipelines, so callDecision and the per-provider decision fetchers go. Copilot's pipelines are parameterized by operation and now also carry image generations and edits. A same-dialect 2xx body that is not JSON is answered with the gateway's own 502, as for every other JSON protocol. --- .../decision/{serve_test.ts => http_test.ts} | 11 +- .../src/data-plane/decision/attempt.ts | 42 ----- .../src/data-plane/decision/call-upstream.ts | 124 +++++++++++++ .../gateway/src/data-plane/decision/emit.ts | 63 +++++++ .../gateway/src/data-plane/decision/facts.ts | 26 +++ .../gateway/src/data-plane/decision/http.ts | 63 +++++++ .../src/data-plane/decision/pipeline.ts | 25 +++ .../gateway/src/data-plane/decision/serve.ts | 170 ------------------ .../gateway/src/data-plane/decision/target.ts | 14 ++ .../src/data-plane/pipeline/upstream-body.ts | 6 +- packages/gateway/src/data-plane/routes.ts | 2 +- packages/provider-azure/src/fetch.ts | 6 +- packages/provider-azure/src/pipelines.ts | 1 + packages/provider-azure/src/provider.ts | 16 +- .../src/stages/prepare-request.ts | 18 +- packages/provider-claude-code/src/provider.ts | 1 - packages/provider-codex/src/provider.ts | 1 - .../__tests__/provider_test.ts | 34 ++-- packages/provider-copilot/src/auth.ts | 17 +- packages/provider-copilot/src/fetch.ts | 5 - .../provider-copilot/src/pipeline-facts.ts | 10 +- packages/provider-copilot/src/pipelines.ts | 17 +- packages/provider-copilot/src/provider.ts | 16 +- .../src/stages/authenticate.ts | 8 +- .../src/stages/prepare-request.ts | 66 +++++-- packages/provider-custom/src/fetch.ts | 3 - packages/provider-custom/src/pipelines.ts | 1 + packages/provider-custom/src/provider.ts | 16 +- .../src/stages/prepare-request.ts | 10 +- packages/provider-ollama/src/fetch.ts | 2 - packages/provider-ollama/src/pipelines.ts | 1 + packages/provider-ollama/src/provider.ts | 16 +- .../src/stages/prepare-request.ts | 27 ++- packages/provider/src/index.ts | 1 - packages/provider/src/pipeline.ts | 2 + packages/provider/src/provider.ts | 10 +- packages/test-utils/src/stubs.ts | 1 - 37 files changed, 481 insertions(+), 371 deletions(-) rename packages/gateway/__tests__/data-plane/decision/{serve_test.ts => http_test.ts} (96%) delete mode 100644 packages/gateway/src/data-plane/decision/attempt.ts create mode 100644 packages/gateway/src/data-plane/decision/call-upstream.ts create mode 100644 packages/gateway/src/data-plane/decision/emit.ts create mode 100644 packages/gateway/src/data-plane/decision/facts.ts create mode 100644 packages/gateway/src/data-plane/decision/http.ts create mode 100644 packages/gateway/src/data-plane/decision/pipeline.ts delete mode 100644 packages/gateway/src/data-plane/decision/serve.ts create mode 100644 packages/gateway/src/data-plane/decision/target.ts diff --git a/packages/gateway/__tests__/data-plane/decision/serve_test.ts b/packages/gateway/__tests__/data-plane/decision/http_test.ts similarity index 96% rename from packages/gateway/__tests__/data-plane/decision/serve_test.ts rename to packages/gateway/__tests__/data-plane/decision/http_test.ts index c38cde5403..ffa6e16746 100644 --- a/packages/gateway/__tests__/data-plane/decision/serve_test.ts +++ b/packages/gateway/__tests__/data-plane/decision/http_test.ts @@ -305,7 +305,7 @@ test('/v1/decisions surfaces a cross-dialect upstream body it cannot parse', asy ); }); -test('/v1/systemone forwards an unparseable same-dialect body verbatim', async () => { +test('/v1/systemone answers an unreadable same-dialect body with its own error', async () => { const { apiKey, repo } = await setupAppTest(); await saveDecisionUpstream(repo, { protocol: 'typesafe-systemone' }); @@ -317,9 +317,12 @@ test('/v1/systemone forwards an unparseable same-dialect body verbatim', async ( headers: requestHeaders(apiKey.key), body: JSON.stringify(choiceRequest), }); - // Same dialect: the body needs no translation, so it passes through. - assertEquals(response.status, 200); - assertEquals(await response.text(), 'opaque upstream bytes'); + // The protocol requires JSON, so a body that is not JSON is not an answer the gateway + // can carry — even to a client on the upstream's own dialect. + assertEquals(response.status, 502); + assertEquals(await response.json(), { + error: { message: 'The upstream answered 200 with a body the decision protocol cannot read: opaque upstream bytes', type: 'api_error' }, + }); }, ); }); diff --git a/packages/gateway/src/data-plane/decision/attempt.ts b/packages/gateway/src/data-plane/decision/attempt.ts deleted file mode 100644 index 20ca9a25ce..0000000000 --- a/packages/gateway/src/data-plane/decision/attempt.ts +++ /dev/null @@ -1,42 +0,0 @@ -import type { Context } from 'hono'; - -import type { GatewayCtx } from '../shared/gateway-ctx.ts'; -import { inboundHeaders } from '../shared/inbound-headers.ts'; -import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; -import { buildUpstreamCallOptions } from '../shared/upstream-call-options.ts'; -import type { DecisionTarget } from '@floway-dev/protocols/common'; -import type { CanonicalDecisionRequest } from '@floway-dev/protocols/decision'; -import { providerModelOf } from '@floway-dev/provider'; -import type { ModelCandidate, PerformanceTelemetryContext, ProviderDecisionCallResult, TelemetryModelIdentity } from '@floway-dev/provider'; - -export interface DecisionAttemptResult { - readonly type: 'plain'; - readonly status: number; - readonly response: Response; - readonly target: DecisionTarget; - readonly performance: PerformanceTelemetryContext; - readonly identity: TelemetryModelIdentity; -} - -export const decisionAttempt = async ( - c: Context, - ctx: GatewayCtx, - candidate: ModelCandidate, - request: CanonicalDecisionRequest, -): Promise => { - const model = providerModelOf(candidate); - const result: ProviderDecisionCallResult = await candidate.provider.instance.callDecision( - model, - request, - ctx.abortSignal, - buildUpstreamCallOptions(candidate, ctx, inboundHeaders(c)), - ); - return { - type: 'plain', - status: result.response.status, - response: result.response, - target: result.target, - performance: upstreamPerformanceContext(ctx, candidate, 'decision'), - identity: telemetryModelIdentity(candidate, result.modelKey), - }; -}; diff --git a/packages/gateway/src/data-plane/decision/call-upstream.ts b/packages/gateway/src/data-plane/decision/call-upstream.ts new file mode 100644 index 0000000000..81df8a9e6d --- /dev/null +++ b/packages/gateway/src/data-plane/decision/call-upstream.ts @@ -0,0 +1,124 @@ +import type { Fields } from './facts.ts'; +import type { UsageQuantities } from '../../repo/types.ts'; +import type { BillableEntity } from '../pipeline/facts.ts'; +import { providerEntry } from '../pipeline/provider-entry.ts'; +import { providerUsage } from '../pipeline/provider-usage.ts'; +import type { GatewayServices } from '../pipeline/services.ts'; +import { dialFailure, readUpstreamBody, spentBody, unreadableBody } from '../pipeline/upstream-body.ts'; +import { telemetryModelIdentity, upstreamPerformanceContext } from '../shared/telemetry/attribution.ts'; +import { exchangeResponse } from '@floway-dev/http/pipeline'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { isJsonObject, parseDecimalString, parseJSONWithRawNumbers, upstreamErrorMessage } from '@floway-dev/protocols/common'; +import { parseDecisionUsage } from '@floway-dev/protocols/decision'; +import { providerModelOf } from '@floway-dev/provider'; +import type { ProviderOperationPayloads, ProviderRequest, ProviderResponse } from '@floway-dev/provider'; + +/** Decision models are billed on input tokens only, which is also the one pricing input. */ +const metered = (usage: { inputTokens?: number }): Pick => { + if (usage.inputTokens === undefined) return { quantities: {} }; + const quantities: UsageQuantities = { input_tokens: parseDecimalString(String(usage.inputTokens)) }; + return { quantities, pricingFacts: { inputTokens: usage.inputTokens } }; +}; + +/** + * The ending. It dials, reads the upstream's body with its numbers kept as their source tokens, + * and provides the answer and what the call is billable for. A failure is a value, so an + * earlier stage can fail over it. + */ +export const callDecisionUpstream = defineStage< + Fields<'request.decision.canonical' | 'route.attempt' | 'ingress.http.headers'>, + ProviderRequest, + ProviderResponse, + Fields<'response.decision.answer' | 'response.decision.targetProtocol' | 'response.http.status' | 'response.http.headers' | 'response.http.body' | 'response.usage.billable'>, + GatewayServices +>({ + name: 'callDecisionUpstream', + into: { + request: { needs: ['request.decision.canonical', 'route.attempt', 'ingress.http.headers'], consumes: [], provides: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'] }, + response: { + needs: ['response.http.exchange', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls', 'response.http.body'], + consumes: ['response.http.exchange', 'response.provider.modelKey', 'response.provider.called', 'response.provider.previousCalls'], + provides: ['response.decision.answer', 'response.decision.targetProtocol', 'response.http.status', 'response.http.headers', 'response.http.body', 'response.usage.billable'], + }, + }, + execute: async (facts, next, use) => { + const candidate = use.resolveAttempt(facts['route.attempt']); + // The narrowing kept only candidates whose provider named a dialect, so this holds for + // every attempt; it is what the request was serialized in and what the answer is read as. + const target = providerModelOf(candidate).decisionTarget; + if (target === undefined) { + throw new Error(`${candidate.provider.upstreamId} serves decision for ${candidate.model.id} without a target dialect`); + } + // Attribution is set before the dial, so an attempt that never completes still names the + // candidate it was made against rather than the one tried before it. + use.gateway.attempt.telemetry = upstreamPerformanceContext(use.gateway, candidate, 'decision'); + + const pipeline = candidate.provider.pipelines.decision; + if (pipeline === undefined) throw new Error(`Provider ${candidate.provider.kind} has no decision pipeline`); + const back = await next(providerEntry(facts, candidate, facts['request.decision.canonical']), pipeline); + const { 'response.http.exchange': exchange, 'response.provider.modelKey': modelKey, 'response.provider.called': wasCalled, 'response.provider.previousCalls': _previousCalls, ...rest } = back; + if (exchange.type === 'transportFailure') { + return move({ + ...rest, + 'response.decision.answer': dialFailure(exchange.error), + 'response.decision.targetProtocol': target.protocol, + 'response.http.status': 502, + 'response.http.headers': [], + 'response.usage.billable': providerUsage(candidate, back, []), + }); + } + const response = exchangeResponse(exchange); + const identity = telemetryModelIdentity(candidate, modelKey); + const headers = [...response.headers]; + spentBody(exchange.body); + const body = await readUpstreamBody(response, parseJSONWithRawNumbers); + // The upstream was called and reported nothing, which is a different situation from + // reporting zero — so the entity is present with no quantities. + const unreported = wasCalled ? [{ identity, quantities: {} }] : []; + + if (!response.ok) { + await use.log.warn('upstream refused', { status: response.status }); + return move({ + ...rest, + 'response.decision.answer': { + status: response.status, + message: upstreamErrorMessage(body.json) ?? body.text, + ...('json' in body ? { body: body.json } : {}), + }, + 'response.decision.targetProtocol': target.protocol, + 'response.http.status': response.status, + 'response.http.headers': headers, + 'response.usage.billable': providerUsage(candidate, back, unreported), + }); + } + + if (!isJsonObject(body.json)) { + return move({ + ...rest, + 'response.decision.answer': unreadableBody(response, body, 'the decision protocol'), + 'response.decision.targetProtocol': target.protocol, + 'response.http.status': response.status, + 'response.http.headers': headers, + 'response.usage.billable': providerUsage(candidate, back, unreported), + }); + } + + // A usage block the reader cannot make sense of is a report we cannot parse, which from + // here is no report; it does not make the answer itself unreadable. + let usage: { inputTokens?: number }; + try { + usage = parseDecisionUsage(body.json); + } catch (error) { + await use.log.warn('upstream reported usage the decision protocol cannot read', { error: String(error) }); + usage = {}; + } + return move({ + ...rest, + 'response.decision.answer': { body: body.json }, + 'response.decision.targetProtocol': target.protocol, + 'response.http.status': response.status, + 'response.http.headers': headers, + 'response.usage.billable': providerUsage(candidate, back, wasCalled ? [{ identity, ...metered(usage) }] : []), + }); + }, +}); diff --git a/packages/gateway/src/data-plane/decision/emit.ts b/packages/gateway/src/data-plane/decision/emit.ts new file mode 100644 index 0000000000..918bc67749 --- /dev/null +++ b/packages/gateway/src/data-plane/decision/emit.ts @@ -0,0 +1,63 @@ +import type { Fields } from './facts.ts'; +import { isFailure } from '../pipeline/facts.ts'; +import { isForwardableUpstreamHeader } from '../shared/upstream-response.ts'; +import { defineStage, move } from '@floway-dev/pipeline'; +import { renderErrorEnvelope, type JsonObject } from '@floway-dev/protocols/common'; +import { parseDecisionResponse, renderDecisionResponse } from '@floway-dev/protocols/decision'; + +/** + * The outermost edge. A client on the upstream's own dialect is answered with the upstream's + * own document, vendor extensions and number tokens included; a client on the other dialect is + * answered with the canonical reading of it, rendered into its own. + */ +export const emitDecision = defineStage< + Fields<'ingress.decision.sourceProtocol'>, + Fields<'ingress.decision.sourceProtocol'>, + Fields<'ingress.decision.sourceProtocol' | 'response.decision.answer' | 'response.decision.targetProtocol' | 'response.http.status' | 'response.http.headers'>, + Fields<'response.decision.rendered' | 'response.http.status' | 'response.http.headers'> +>({ + name: 'emitDecision', + through: { + request: { needs: ['ingress.decision.sourceProtocol'], consumes: [], provides: [] }, + response: { + needs: ['response.decision.answer', 'response.http.headers', 'response.http.status'], + consumes: ['response.decision.answer', 'response.decision.targetProtocol', 'response.http.headers'], + provides: ['response.decision.rendered', 'response.http.status', 'response.http.headers'], + }, + }, + execute: async (facts, next) => { + const back = await next(facts); + const { 'response.decision.answer': answer, 'response.decision.targetProtocol': target, 'response.http.headers': headers, ...rest } = back; + // A filter that removed nothing hands the same array on, so the record shows no change + // where none happened. + const forwardable = headers.filter(([name]) => isForwardableUpstreamHeader(name)); + const forClient = forwardable.length === headers.length ? headers : move(forwardable); + if (isFailure(answer)) { + return { + ...rest, + 'response.http.headers': forClient, + 'response.decision.rendered': move(renderErrorEnvelope(answer.message, answer.body)), + 'response.http.status': answer.status, + }; + } + const source = back['ingress.decision.sourceProtocol']; + if (source === target) { + return { ...rest, 'response.http.headers': forClient, 'response.http.status': back['response.http.status'], 'response.decision.rendered': answer.body }; + } + // The upstream answered and the gateway cannot put that answer in the client's dialect — + // a refusal one dialect can express and the other cannot, or an answer missing the detail + // both contracts require. That is the gateway failing to serve, so it is 502. + let rendered: JsonObject; + try { + rendered = renderDecisionResponse(source, parseDecisionResponse(target, answer.body)); + } catch (error) { + return { + ...rest, + 'response.http.headers': forClient, + 'response.decision.rendered': move(renderErrorEnvelope(error instanceof Error ? error.message : String(error))), + 'response.http.status': 502, + }; + } + return { ...rest, 'response.http.headers': forClient, 'response.http.status': back['response.http.status'], 'response.decision.rendered': move(rendered) }; + }, +}); diff --git a/packages/gateway/src/data-plane/decision/facts.ts b/packages/gateway/src/data-plane/decision/facts.ts new file mode 100644 index 0000000000..3e009e6706 --- /dev/null +++ b/packages/gateway/src/data-plane/decision/facts.ts @@ -0,0 +1,26 @@ +import type { GatewayFacts, Failure } from '../pipeline/facts.ts'; +import type { DecisionProtocol, JsonObject } from '@floway-dev/protocols/common'; +import type { CanonicalDecisionRequest } from '@floway-dev/protocols/decision'; + +/** What an upstream answered, as JSON whose numbers keep their source tokens. It is held whole + * rather than parsed into the canonical form, because a same-dialect client is answered with + * the upstream's own document and only a translation needs the canonical one. */ +export interface DecisionAnswer { + readonly body: JsonObject; +} + +/** Decision's own keys. They extend the shared space and never merge into it, so a stage + * written against the gateway alone cannot name one. */ +export interface DecisionFacts extends GatewayFacts { + /** Which of the two decision dialects the client spoke. It belongs to the ingress and stays + * put: the answer is rendered back into it whatever the upstream spoke. */ + 'ingress.decision.sourceProtocol': DecisionProtocol; + 'request.decision.canonical': CanonicalDecisionRequest; + 'response.decision.answer': DecisionAnswer | Failure; + /** Which dialect the attempt spoke, so the edge knows what it is rendering from. */ + 'response.decision.targetProtocol': DecisionProtocol; + /** What the client is actually sent, in its own dialect. */ + 'response.decision.rendered': JsonObject; +} + +export type Fields = { [P in K]: DecisionFacts[P] }; diff --git a/packages/gateway/src/data-plane/decision/http.ts b/packages/gateway/src/data-plane/decision/http.ts new file mode 100644 index 0000000000..d24b293f60 --- /dev/null +++ b/packages/gateway/src/data-plane/decision/http.ts @@ -0,0 +1,63 @@ +// POST /v1/systemone and POST /v1/decisions, served through the pipeline. +// +// One family behind two routes. Which dialect the client spoke is the route it arrived on, so +// the mount passes it in and the handler carries it into the record as an ingress fact. +// Everything after the parse is stages. + +import type { Context } from 'hono'; + +import { decisionServePipeline } from './pipeline.ts'; +import { openPrologue, readIngress, serveThrough } from '../pipeline/serve.ts'; +import { finalizeGatewayResponse } from '../shared/gateway-ctx.ts'; +import { move } from '@floway-dev/pipeline'; +import { parseJSONWithRawNumbers, type DecisionProtocol } from '@floway-dev/protocols/common'; +import { parseDecisionRequest, type ParsedDecisionRequest } from '@floway-dev/protocols/decision'; + +// Raw-number parsing keeps a caller's own digits, so a same-dialect upstream is sent the +// numbers the caller wrote. +const readRequest = ( + sourceProtocol: DecisionProtocol, + bytes: Uint8Array, +): { type: 'ok'; parsed: ParsedDecisionRequest } | { type: 'invalid'; message: string } => { + let body: unknown; + try { + body = parseJSONWithRawNumbers(new TextDecoder().decode(bytes)); + } catch { + return { type: 'invalid', message: 'Decision request body must be valid JSON' }; + } + try { + return { type: 'ok', parsed: parseDecisionRequest(sourceProtocol, body) }; + } catch (error) { + return { type: 'invalid', message: error instanceof Error ? error.message : String(error) }; + } +}; + +export const decision = (sourceProtocol: DecisionProtocol) => async (c: Context): Promise => { + const ingress = await readIngress(c); + const result = readRequest(sourceProtocol, ingress.body.bytes); + if (result.type === 'invalid') { + // A request the gateway could not read never reaches a pipeline: there is no model to + // resolve and no attempt to make, so there is nothing for a run to record. + const refused = openPrologue(c, ingress, { wantsStream: false }); + refused.gateway.dump?.error('gateway'); + return finalizeGatewayResponse( + refused.gateway, + Response.json({ error: { message: result.message, type: 'api_error' } }, { status: 400 }), + ); + } + + const { model, request } = result.parsed; + const prologue = openPrologue(c, ingress, { wantsStream: false, model }); + return await serveThrough( + c, + prologue, + decisionServePipeline, + move({ + 'ingress.http.headers': prologue.headers, + 'ingress.decision.sourceProtocol': sourceProtocol, + 'request.decision.canonical': request, + 'serve.model': model, + }) as never, + facts => ({ body: facts['response.http.jsonBody']!, contentType: 'application/json' }), + ); +}; diff --git a/packages/gateway/src/data-plane/decision/pipeline.ts b/packages/gateway/src/data-plane/decision/pipeline.ts new file mode 100644 index 0000000000..713afe0a42 --- /dev/null +++ b/packages/gateway/src/data-plane/decision/pipeline.ts @@ -0,0 +1,25 @@ +import { callDecisionUpstream } from './call-upstream.ts'; +import { emitDecision } from './emit.ts'; +import type { Fields } from './facts.ts'; +import { narrowing } from './target.ts'; +import { isFailure } from '../pipeline/facts.ts'; +import { failover } from '../pipeline/failover.ts'; +import { resolveCandidates } from '../pipeline/resolve-candidates.ts'; +import { serializeClientJson } from '../pipeline/serialize-client-json.ts'; +import { writeSettlement } from '../pipeline/settlement.ts'; +import { compose, type Pipeline } from '@floway-dev/pipeline'; + +export const decisionServePipeline: Pipeline< + Fields<'ingress.http.headers' | 'ingress.decision.sourceProtocol' | 'request.decision.canonical' | 'serve.model'>, + Fields<'response.http.jsonBody' | 'response.decision.rendered' | 'response.http.status' | 'response.http.headers' | 'response.usage.billable'> +> = compose('decisionServe', [ + writeSettlement(handedUp => Number(handedUp['response.http.status']) >= 400), + serializeClientJson('response.decision.rendered'), + emitDecision, + resolveCandidates(narrowing), + failover({ + failed: handedUp => isFailure((handedUp as { 'response.decision.answer'?: unknown })['response.decision.answer']), + owns: ['response.http.body'], + }), + callDecisionUpstream, +]); diff --git a/packages/gateway/src/data-plane/decision/serve.ts b/packages/gateway/src/data-plane/decision/serve.ts deleted file mode 100644 index 2627713d74..0000000000 --- a/packages/gateway/src/data-plane/decision/serve.ts +++ /dev/null @@ -1,170 +0,0 @@ -import type { Context } from 'hono'; -import type { ContentfulStatusCode } from 'hono/utils/http-status'; - -import { decisionAttempt, type DecisionAttemptResult } from './attempt.ts'; -import type { UsageQuantities } from '../../repo/types.ts'; -import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; -import { enumerateModelCandidates } from '../providers/resolution.ts'; -import { appendFailedUpstreams } from '../shared/failed-upstreams.ts'; -import { createGatewayCtxFromHono, finalizeGatewayResponse, type GatewayCtx } from '../shared/gateway-ctx.ts'; -import { iterateCandidates } from '../shared/iterate-candidates.ts'; -import { readRequestBody, takeRequestBody } from '../shared/request-body.ts'; -import { recordFailedRequest, recordPerformance, type PerformanceTelemetryContext } from '../shared/telemetry/performance.ts'; -import { recordUsage } from '../shared/telemetry/usage.ts'; -import { forwardUpstreamResponse } from '../shared/upstream-response.ts'; -import { parseDecimalString, parseJSONWithRawNumbers, type DecisionProtocol } from '@floway-dev/protocols/common'; -import { parseDecisionRequest, parseDecisionResponse, parseDecisionUsage, renderDecisionResponse, type ParsedDecisionRequest } from '@floway-dev/protocols/decision'; -import { httpResponseToResponse, ProviderModelsUnavailableError, providerModelOf, toInternalDebugError } from '@floway-dev/provider'; -import type { TelemetryModelIdentity } from '@floway-dev/provider'; - -const apiError = (c: Context, message: string, status: ContentfulStatusCode): Response => - c.json({ error: { message, type: 'api_error' } }, status); - -// Raw-number parsing keeps a caller's own digits — a same-dialect forward -// re-emits `raw` verbatim, so parsing to a number first would rewrite them. -const parseJson = (bytes: Uint8Array): unknown => { - try { - return parseJSONWithRawNumbers(new TextDecoder().decode(bytes)); - } catch { - throw new Error('Decision request body must be valid JSON'); - } -}; - -// Floway bills decision models on input tokens only. A provider that reports no -// usage still records the request through recordPerformance. -const settleDecision = ( - ctx: GatewayCtx, - performanceContext: PerformanceTelemetryContext, - identity: TelemetryModelIdentity, - usage: { inputTokens?: number } | undefined, - failed: boolean, -): void => { - const quantities: UsageQuantities = {}; - if (usage?.inputTokens !== undefined) quantities.input_tokens = parseDecimalString(String(usage.inputTokens)); - const pricingFacts = usage?.inputTokens === undefined ? {} : { inputTokens: usage.inputTokens }; - ctx.backgroundScheduler(recordUsage(ctx.apiKeyId, identity, quantities, pricingFacts).catch(error => { - console.error('Failed to record decision usage:', error); - })); - recordPerformance(ctx, performanceContext, failed, 0, performance.now()); -}; - -const unsupportedMessage = (model: string): string => `Model ${model} does not support decision.`; - -export const decision = (sourceProtocol: DecisionProtocol) => async (c: Context): Promise => { - const requestBody = await readRequestBody(c); - let parsedRequest: ParsedDecisionRequest; - try { - parsedRequest = parseDecisionRequest(sourceProtocol, parseJson(requestBody.bytes)); - } catch (error) { - const ctx = createGatewayCtxFromHono(c, { - wantsStream: false, - requestBody: takeRequestBody(requestBody), - backgroundScheduler: backgroundSchedulerFromContext(c), - }); - ctx.dump?.error('gateway'); - return finalizeGatewayResponse(ctx, apiError(c, error instanceof Error ? error.message : String(error), 400)); - } - - const { model, request } = parsedRequest; - const ctx = createGatewayCtxFromHono(c, { - wantsStream: false, - model, - requestBody: takeRequestBody(requestBody), - backgroundScheduler: backgroundSchedulerFromContext(c), - }); - - let terminal: DecisionAttemptResult | undefined; - let measuredUsage: { inputTokens?: number } | undefined; - let usageSettled = false; - try { - const { candidates, sawModel, failedUpstreams } = await enumerateModelCandidates({ - subject: ctx.user, - upstreamIds: ctx.upstreamIds, - randomizeUpstreams: ctx.randomizeUpstreams, - model, - kind: 'decision', - scheduler: ctx.backgroundScheduler, - runtimeLocation: ctx.runtimeLocation, - }); - if (candidates.length === 0) { - ctx.dump?.error('gateway'); - const message = sawModel - ? unsupportedMessage(model) - : `Model ${model} is not available on any configured upstream.`; - return finalizeGatewayResponse(ctx, apiError(c, appendFailedUpstreams(message, failedUpstreams), sawModel ? 400 : 404)); - } - - const routable = candidates.flatMap(candidate => { - const providerModel = providerModelOf(candidate); - return candidate.model.endpoints.decision === undefined || providerModel.decisionTarget === undefined - ? [] - : [{ candidate, target: providerModel.decisionTarget }]; - }); - if (routable.length === 0) { - ctx.dump?.error('gateway'); - return finalizeGatewayResponse(ctx, apiError(c, appendFailedUpstreams(unsupportedMessage(model), failedUpstreams), 400)); - } - terminal = await iterateCandidates( - routable.map(({ candidate }) => candidate), - 'decision', - ctx, - 'decision', - candidate => decisionAttempt(c, ctx, candidate, request), - ); - - if (!terminal.response.ok) { - ctx.dump?.error('upstream', terminal.identity.upstream); - settleDecision(ctx, terminal.performance, terminal.identity, undefined, true); - usageSettled = true; - return finalizeGatewayResponse(ctx, forwardUpstreamResponse(terminal.response)); - } - - const sameProtocol = sourceProtocol === terminal.target.protocol; - let upstreamBody: Record; - try { - // Raw-number parsing keeps each probability's source token, so a - // cross-dialect render writes the digits back unchanged. - upstreamBody = parseJSONWithRawNumbers(await terminal.response.clone().text()) as Record; - } catch (error) { - // A cross-dialect client cannot read the upstream's own body, so a - // translation failure must surface instead of being forwarded as success. - if (!sameProtocol) throw error; - console.warn( - `decision: failed to parse 2xx upstream body for ${model}; usage row will be request-only`, - error instanceof Error ? error.message : String(error), - ); - ctx.dump?.success(terminal.identity, null); - settleDecision(ctx, terminal.performance, terminal.identity, undefined, false); - usageSettled = true; - return finalizeGatewayResponse(ctx, forwardUpstreamResponse(terminal.response)); - } - measuredUsage = parseDecisionUsage(upstreamBody); - if (sameProtocol) { - ctx.dump?.success(terminal.identity, null); - settleDecision(ctx, terminal.performance, terminal.identity, measuredUsage, false); - usageSettled = true; - return finalizeGatewayResponse(ctx, forwardUpstreamResponse(terminal.response)); - } - const canonical = parseDecisionResponse(terminal.target.protocol, upstreamBody); - const rendered = renderDecisionResponse(sourceProtocol, canonical); - ctx.dump?.success(terminal.identity, null); - settleDecision(ctx, terminal.performance, terminal.identity, measuredUsage, false); - usageSettled = true; - return finalizeGatewayResponse(ctx, forwardUpstreamResponse(terminal.response, { body: JSON.stringify(rendered) })); - } catch (error) { - if (terminal !== undefined && !usageSettled) { - settleDecision(ctx, terminal.performance, terminal.identity, measuredUsage, true); - } else if (terminal === undefined) { - recordFailedRequest(ctx, ctx.attempt.telemetry); - } - if (error instanceof ProviderModelsUnavailableError) { - const forwarded = httpResponseToResponse(error.httpResponse); - if (forwarded) { - ctx.dump?.error('upstream'); - return finalizeGatewayResponse(ctx, forwarded); - } - } - ctx.dump?.failed(error); - return finalizeGatewayResponse(ctx, c.json({ error: toInternalDebugError(error) }, 502)); - } -}; diff --git a/packages/gateway/src/data-plane/decision/target.ts b/packages/gateway/src/data-plane/decision/target.ts new file mode 100644 index 0000000000..3926626901 --- /dev/null +++ b/packages/gateway/src/data-plane/decision/target.ts @@ -0,0 +1,14 @@ +import { providerModelOf, type ModelCandidate } from '@floway-dev/provider'; + +/** A candidate without a decision endpoint, or without a dialect to speak on it, is not a + * candidate. The dialect belongs to the provider, so a model that has none cannot be served. */ +export const narrowing = { + kind: 'decision' as const, + reject: (candidate: ModelCandidate) => + candidate.model.endpoints.decision === undefined || providerModelOf(candidate).decisionTarget === undefined + ? 'the upstream does not expose a decision endpoint' + : null, + unsupported: (model: string) => `Model ${model} does not support decision.`, + refuse: (status: number, message: string) => ({ 'response.decision.answer': { status, message } }), + refuses: ['response.decision.answer'] as const, +}; diff --git a/packages/gateway/src/data-plane/pipeline/upstream-body.ts b/packages/gateway/src/data-plane/pipeline/upstream-body.ts index 290ef56290..2e12edb8df 100644 --- a/packages/gateway/src/data-plane/pipeline/upstream-body.ts +++ b/packages/gateway/src/data-plane/pipeline/upstream-body.ts @@ -16,10 +16,12 @@ export interface UpstreamBody { readonly json?: unknown; } -export const readUpstreamBody = async (response: Response): Promise => { +/** `parse` is the protocol's own JSON reader, for a protocol whose numbers must keep their + * source tokens through a round trip. */ +export const readUpstreamBody = async (response: Response, parse: (text: string) => unknown = JSON.parse): Promise => { const text = await response.text(); try { - return { text, json: JSON.parse(text) as unknown }; + return { text, json: parse(text) }; } catch { return { text }; } diff --git a/packages/gateway/src/data-plane/routes.ts b/packages/gateway/src/data-plane/routes.ts index 84fc0ffb0f..fd6a7c0ab2 100644 --- a/packages/gateway/src/data-plane/routes.ts +++ b/packages/gateway/src/data-plane/routes.ts @@ -3,7 +3,7 @@ import type { Hono } from 'hono'; import { mountAlphaSearchRoutes } from './alpha-search/routes.ts'; import { mountChatRoutes } from './chat/routes.ts'; import { mountCodexRoutes } from './codex/routes.ts'; -import { decision } from './decision/serve.ts'; +import { decision } from './decision/http.ts'; import { serveGeminiModelInfo } from './models/gemini.ts'; import { serveModels } from './models/http.ts'; import { openaiAudioTranscriptions } from './openai-audio/http.ts'; diff --git a/packages/provider-azure/src/fetch.ts b/packages/provider-azure/src/fetch.ts index c59b88e0bd..9d4572b7ff 100644 --- a/packages/provider-azure/src/fetch.ts +++ b/packages/provider-azure/src/fetch.ts @@ -1,5 +1,5 @@ import type { AzureUpstreamConfig } from './config.ts'; -import { azureAnthropicBaseUrl, azureOpenAiV1BaseUrl, azureResourceRootUrl } from './endpoint.ts'; +import { azureAnthropicBaseUrl, azureOpenAiV1BaseUrl } from './endpoint.ts'; import { type FetchInit, type UpstreamFetchOptions, joinBaseAndPath } from '@floway-dev/provider'; const azureFetchUrl = async ( @@ -79,10 +79,6 @@ export const azureFetchOpenAIImagesEdits = (config: AzureUpstreamConfig, init: F // https://github.com/Azure/azure-rest-api-specs/blob/928047803788f7377fa003a26ba2bdc2e0fcccc0/specification/cognitiveservices/OpenAI.Inference/routes/audio_transcription.tsp#L19-L49 export const azureFetchOpenAIAudioTranscriptions = (config: AzureUpstreamConfig, deployment: string, init: FetchInit, options: UpstreamFetchOptions): Promise => azureFetchUrl(config, 'openai', azureDeploymentScopedOpenAIAudioTranscriptionUrl(config, deployment), init, options); -// The decision contract is resource-scoped, so the path is absolute from the -// resource host rather than relative to the OpenAI v1 base. -export const azureFetchDecision = (config: AzureUpstreamConfig, path: string, init: FetchInit, options: UpstreamFetchOptions): Promise => - azureFetchUrl(config, 'openai', joinBaseAndPath(azureResourceRootUrl(config.endpoint), path), init, options); export const azureFetchAnthropicMessages = (config: AzureUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => azureFetchInternal(config, 'anthropic', '/v1/messages', init, options); export const azureFetchAnthropicMessagesCountTokens = (config: AzureUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => diff --git a/packages/provider-azure/src/pipelines.ts b/packages/provider-azure/src/pipelines.ts index 61803e73d2..8dd3c64e48 100644 --- a/packages/provider-azure/src/pipelines.ts +++ b/packages/provider-azure/src/pipelines.ts @@ -10,4 +10,5 @@ export const createAzurePipelines = (config: AzureUpstreamConfig): ProviderPipel openaiImagesGenerations: compose, ProviderOperationResponse<'openaiImagesGenerations'>>('azure.openaiImagesGenerations', [prepareAzureRequest(config, 'openaiImagesGenerations'), http]), openaiImagesEdits: compose, ProviderOperationResponse<'openaiImagesEdits'>>('azure.openaiImagesEdits', [prepareAzureRequest(config, 'openaiImagesEdits'), http]), openaiAudioTranscriptions: compose, ProviderOperationResponse<'openaiAudioTranscriptions'>>('azure.openaiAudioTranscriptions', [prepareAzureRequest(config, 'openaiAudioTranscriptions'), http]), + decision: compose, ProviderOperationResponse<'decision'>>('azure.decision', [prepareAzureRequest(config, 'decision'), http]), }); diff --git a/packages/provider-azure/src/provider.ts b/packages/provider-azure/src/provider.ts index 18757dd605..75d9738fe2 100644 --- a/packages/provider-azure/src/provider.ts +++ b/packages/provider-azure/src/provider.ts @@ -1,13 +1,12 @@ import { assertAzureUpstreamRecord } from './config.ts'; import { AZURE_DEFAULT_FLAGS } from './defaults.ts'; -import { azureFetchDecision, azureFetchOpenAIAudioTranscriptions, azureFetchOpenAIChatCompletions, azureFetchOpenAICompletions, azureFetchOpenAIEmbeddings, azureFetchOpenAIImagesEdits, azureFetchOpenAIImagesGenerations, azureFetchAnthropicMessages, azureFetchAnthropicMessagesCountTokens, azureFetchOpenAIResponses, azureFetchOpenAIResponsesCompact } from './fetch.ts'; +import { azureFetchOpenAIAudioTranscriptions, azureFetchOpenAIChatCompletions, azureFetchOpenAICompletions, azureFetchOpenAIEmbeddings, azureFetchOpenAIImagesEdits, azureFetchOpenAIImagesGenerations, azureFetchAnthropicMessages, azureFetchAnthropicMessagesCountTokens, azureFetchOpenAIResponses, azureFetchOpenAIResponsesCompact } from './fetch.ts'; import { AZURE_OPENAI_RESPONSES_BOUNDARY } from './interceptors/openai-responses/index.ts'; import type { OpenAIResponsesBoundaryCtx } from './interceptors/openai-responses/types.ts'; import { createAzurePipelines } from './pipelines.ts'; import { runInterceptors } from '@floway-dev/interceptor'; import { parseAnthropicMessagesStream } from '@floway-dev/protocols/anthropic-messages'; import { kindForEndpoints } from '@floway-dev/protocols/common'; -import { DEFAULT_DECISION_PATHS, serializeDecisionRequest, type CanonicalDecisionRequest } from '@floway-dev/protocols/decision'; import { parseOpenAIChatCompletionsStream } from '@floway-dev/protocols/openai-chat-completions'; import { parseOpenAIResponsesStream, type OpenAIResponsesCompactionResultEx, toCompactPayloadShape } from '@floway-dev/protocols/openai-responses'; import { headersForAnthropicMessagesCall, jsonRequestBody, serializeModelPathOpenAIAudioTranscriptionRequest, serializeOpenAIImagesEditsRequest, type FetchInit, type HttpHeaderLines, type ProviderInstance, type Provider, type ProviderModel, type ProviderOpenAIResponsesResult, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord, publicModelId, resolveEffectiveFlags, streamingProviderCall } from '@floway-dev/provider'; @@ -127,19 +126,6 @@ export const createAzureProvider = (record: UpstreamRecord): Provider => { return { response, modelKey: upstreamModelId }; }, callRerank: () => Promise.reject(new Error('Azure provider does not support callRerank')), - callDecision: async (model: ProviderModel, request: CanonicalDecisionRequest, signal: AbortSignal | undefined, opts: UpstreamCallOptions) => { - const target = model.decisionTarget; - if (target === undefined) throw new Error(`Decision model ${model.id} has no outbound target`); - const upstreamModelId = upstreamModelIdOf(model); - const body = serializeDecisionRequest(target.protocol, upstreamModelId, request); - const response = await azureFetchDecision( - azure.config, - target.path ?? DEFAULT_DECISION_PATHS[target.protocol], - { method: 'POST', body: jsonRequestBody(body), signal }, - { extraHeaders: [...opts.headers], fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, - ); - return { response, modelKey: upstreamModelId, target }; - }, }; return { diff --git a/packages/provider-azure/src/stages/prepare-request.ts b/packages/provider-azure/src/stages/prepare-request.ts index b0b1ed9ae4..c4ce681dfa 100644 --- a/packages/provider-azure/src/stages/prepare-request.ts +++ b/packages/provider-azure/src/stages/prepare-request.ts @@ -1,17 +1,18 @@ import type { AzureUpstreamConfig } from '../config.ts'; -import { azureOpenAiV1BaseUrl } from '../endpoint.ts'; +import { azureOpenAiV1BaseUrl, azureResourceRootUrl } from '../endpoint.ts'; import { type HttpRequestFacts, type HttpResponseFacts } from '@floway-dev/http/pipeline'; import { multipartBody, type HttpBody, type HttpBodyEncoding } from '@floway-dev/http/request-content'; import { defineStage, move, secret } from '@floway-dev/pipeline'; +import { DEFAULT_DECISION_PATHS, serializeDecisionRequest } from '@floway-dev/protocols/decision'; import { joinBaseAndPath, mergeHttpHeaders, prepareOpenAIImagesEditsBody, withHttpContentType, type ProviderOperationPayloads, type ProviderRequest, type ProviderResponse, type ProviderServices } from '@floway-dev/provider'; -type AzureOperation = 'openaiCompletions' | 'openaiEmbeddings' | 'openaiImagesGenerations' | 'openaiImagesEdits' | 'openaiAudioTranscriptions'; +type AzureOperation = 'openaiCompletions' | 'openaiEmbeddings' | 'openaiImagesGenerations' | 'openaiImagesEdits' | 'openaiAudioTranscriptions' | 'decision'; interface AzureHttpFacts extends HttpRequestFacts { 'request.azure.deployment': string; } -const PATHS: Record, string> = { +const PATHS: Record, string> = { openaiCompletions: '/completions', openaiEmbeddings: '/embeddings', openaiImagesGenerations: '/images/generations', @@ -20,7 +21,7 @@ const PATHS: Record, string export const prepareAzureRequest = (config: AzureUpstreamConfig, operation: O) => { const prepare = defineStage, AzureHttpFacts, HttpResponseFacts, ProviderResponse, ProviderServices>({ - name: `prepareAzure${operation.replace(/^openai/, 'OpenAI').replace(/^alpha/, 'Alpha').replace(/^rerank/, 'Rerank')}`, + name: `prepareAzure${operation.replace(/^openai/, 'OpenAI').replace(/^alpha/, 'Alpha').replace(/^rerank/, 'Rerank').replace(/^decision/, 'Decision')}`, through: { request: { needs: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'], @@ -44,8 +45,15 @@ export const prepareAzureRequest = (config: AzureUpstr url.hash = ''; // https://github.com/Azure/azure-rest-api-specs/blob/928047803788f7377fa003a26ba2bdc2e0fcccc0/specification/cognitiveservices/OpenAI.Inference/routes/audio_transcription.tsp#L19-L49 url.searchParams.set('api-version', '2025-04-01-preview'); + } else if (operation === 'decision') { + const target = model.decisionTarget; + if (target === undefined) throw new Error(`Decision model ${model.id} has no outbound target`); + // The decision contract is resource-scoped, so the path is absolute from the resource + // host rather than relative to the OpenAI v1 base. + url = new URL(joinBaseAndPath(azureResourceRootUrl(config.endpoint), target.path ?? DEFAULT_DECISION_PATHS[target.protocol])); + body = serializeDecisionRequest(target.protocol, deployment, payload as ProviderOperationPayloads['decision']); } else { - url = new URL(joinBaseAndPath(azureOpenAiV1BaseUrl(config.endpoint), PATHS[operation as Exclude])); + url = new URL(joinBaseAndPath(azureOpenAiV1BaseUrl(config.endpoint), PATHS[operation as Exclude])); if (operation === 'openaiImagesEdits') { const prepared = await prepareOpenAIImagesEditsBody(payload as ProviderOperationPayloads['openaiImagesEdits'], deployment); body = prepared.body; diff --git a/packages/provider-claude-code/src/provider.ts b/packages/provider-claude-code/src/provider.ts index b5b07b6eba..4dca265200 100644 --- a/packages/provider-claude-code/src/provider.ts +++ b/packages/provider-claude-code/src/provider.ts @@ -118,7 +118,6 @@ export const createClaudeCodeProvider = (record: UpstreamRecord): Provider => { callOpenAIImagesEdits: rejectUnsupported('callOpenAIImagesEdits'), callOpenAIAudioTranscriptions: rejectUnsupported('callOpenAIAudioTranscriptions'), callRerank: rejectUnsupported('callRerank'), - callDecision: rejectUnsupported('callDecision'), }; return { diff --git a/packages/provider-codex/src/provider.ts b/packages/provider-codex/src/provider.ts index 1028509893..ed8b0e80c5 100644 --- a/packages/provider-codex/src/provider.ts +++ b/packages/provider-codex/src/provider.ts @@ -162,7 +162,6 @@ export const createCodexProvider = (record: UpstreamRecord): Provider => { }, callOpenAIAudioTranscriptions: () => unsupportedCallResult(), callRerank: () => Promise.reject(new Error('Codex provider does not support callRerank')), - callDecision: () => Promise.reject(new Error('Codex provider does not support callDecision')), }; return { diff --git a/packages/provider-copilot/__tests__/provider_test.ts b/packages/provider-copilot/__tests__/provider_test.ts index 316e3f3c05..1f8286bb62 100644 --- a/packages/provider-copilot/__tests__/provider_test.ts +++ b/packages/provider-copilot/__tests__/provider_test.ts @@ -1350,7 +1350,7 @@ const imageModelCatalog = () => copilotModelsWithCapabilities([ const setupImageModelTest = async () => { const harness = await setupCopilotTest(); - const provider = createCopilotProvider(harness.copilotUpstream).instance; + const provider = createCopilotProvider(harness.copilotUpstream); return { harness, provider }; }; @@ -1374,8 +1374,8 @@ test('Copilot image generations POSTs JSON to the v1 path with the raw model id' return jsonResponse({ created: 1, data: [{ b64_json: 'AAAA' }] }); }, async () => { - const [model] = await provider.getProvidedModels(directFetcher); - const result = await provider.callOpenAIImagesGenerations(model, { prompt: 'a red square' }, undefined, noopUpstreamCallOptions()); + const [model] = await provider.instance.getProvidedModels(directFetcher); + const result = await callProviderPipeline(provider, 'openaiImagesGenerations', model, { prompt: 'a red square' }); assertEquals(result.response.status, 200); }, ); @@ -1386,11 +1386,9 @@ test('Copilot image generations POSTs JSON to the v1 path with the raw model id' assertEquals(captured?.body.model, 'gpt-image-2.5-flare'); }); -// The regression this guards: `copilotAuthedFetch` used to pin -// `Content-Type: application/json` unconditionally, which made Copilot answer -// a multipart edit with `400 Failed to decode image edit request` before -// reading a single part. A FormData body must reach the runtime untouched so -// it can set its own boundary. +// Copilot answers a multipart edit carrying `Content-Type: application/json` +// with `400 Failed to decode image edit request` before reading a single part, +// so an upload must be sent with its own multipart boundary. test('Copilot image edits keeps multipart uploads free of a JSON content-type', async () => { const { provider } = await setupImageModelTest(); let captured: { path: string; contentType: string | null; body: string } | undefined; @@ -1411,9 +1409,9 @@ test('Copilot image edits keeps multipart uploads free of a JSON content-type', return jsonResponse({ created: 1, data: [{ b64_json: 'AAAA' }] }); }, async () => { - const [model] = await provider.getProvidedModels(directFetcher); + const [model] = await provider.instance.getProvidedModels(directFetcher); const file = { bytes: new Uint8Array([0x89, 0x50, 0x4E, 0x47]), name: 'in.png', type: 'image/png' }; - const result = await provider.callOpenAIImagesEdits(model, { images: [{ type: 'upload', file }], parameters: { prompt: 'make it blue' } }, undefined, noopUpstreamCallOptions()); + const result = await callProviderPipeline(provider, 'openaiImagesEdits', model, { images: [{ type: 'upload', file }], parameters: { prompt: 'make it blue' } }); assertEquals(result.response.status, 200); }, ); @@ -1607,8 +1605,7 @@ test('Copilot provider merges the -fast raw variant and reaches it through servi test('Copilot provider derives decision endpoints and forwards the System One body', async () => { const { copilotUpstream } = await setupCopilotTest(); - const instance = createCopilotProvider(copilotUpstream); - const provider = instance.instance; + const provider = createCopilotProvider(copilotUpstream); const sent: Record[] = []; await withMockedFetch( @@ -1646,7 +1643,7 @@ test('Copilot provider derives decision endpoints and forwards the System One bo throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const models = await provider.getProvidedModels(directFetcher); + const models = await provider.instance.getProvidedModels(directFetcher); const decision = models.find(model => model.id === 'microsoft-decision-1')!; assertEquals(decision.kind, 'decision'); assertEquals(decision.endpoints, { decision: {} }); @@ -1657,8 +1654,7 @@ test('Copilot provider derives decision endpoints and forwards the System One bo state: 'The API returns 500 on every call.', questions: { team: { type: 'choice', instructions: 'Which team?', criteria: { billing: 'Charges', engineering: 'Bugs' } } }, }); - const result = await provider.callDecision(decision, parsed.request, undefined, noopUpstreamCallOptions()); - assertEquals(result.target, { protocol: 'typesafe-systemone' }); + const result = await callProviderPipeline(provider, 'decision', decision, parsed.request); assertEquals(result.response.status, 200); }, ); @@ -1671,7 +1667,7 @@ test('Copilot provider derives decision endpoints and forwards the System One bo test('Copilot sends an OpenAI Decisions model to /v1/decisions in its own dialect', async () => { const { copilotUpstream } = await setupCopilotTest(); - const provider = createCopilotProvider(copilotUpstream).instance; + const provider = createCopilotProvider(copilotUpstream); const sent: { path: string; body: Record }[] = []; await withMockedFetch( @@ -1695,14 +1691,14 @@ test('Copilot sends an OpenAI Decisions model to /v1/decisions in its own dialec throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const decision = (await provider.getProvidedModels(directFetcher))[0]!; + const decision = (await provider.instance.getProvidedModels(directFetcher))[0]!; const parsed = parseDecisionRequest('openai-decisions', { model: 'gpt-6-luna-decisions', input: 'The API returns 500 on every call.', questions: [{ type: 'predicate', name: 'p', instructions: 'Is this a bug?' }], }); - const result = await provider.callDecision(decision, parsed.request, undefined, noopUpstreamCallOptions()); - assertEquals(result.target, { protocol: 'openai-decisions' }); + assertEquals(decision.decisionTarget, { protocol: 'openai-decisions' }); + const result = await callProviderPipeline(provider, 'decision', decision, parsed.request); assertEquals(result.response.status, 200); }, ); diff --git a/packages/provider-copilot/src/auth.ts b/packages/provider-copilot/src/auth.ts index a2c7cdd285..d1f6964cd5 100644 --- a/packages/provider-copilot/src/auth.ts +++ b/packages/provider-copilot/src/auth.ts @@ -225,19 +225,13 @@ export interface CopilotAuth { githubToken: string; } -export const copilotRequestHeaders = (entry: CopilotTokenEntry, initHeaders: HeadersInit | undefined, attached: HttpHeaderLines | undefined, jsonBody: boolean): Headers => { +export const copilotRequestHeaders = (entry: CopilotTokenEntry, initHeaders: HeadersInit | undefined, attached: HttpHeaderLines | undefined): Headers => { // x-request-id and x-agent-task-id share a single per-call UUID, mirroring // VSCode Copilot Chat's "one id ties the request to its background task" pattern. const requestId = crypto.randomUUID(); const headers = new Headers(initHeaders); headers.set('Authorization', `Bearer ${entry.token}`); - // Content-Type follows the body. A multipart body has to reach the runtime - // untouched so it can generate the boundary itself — pinning - // `application/json` over it makes Copilot reject `/v1/images/edits` with - // `400 Failed to decode image edit request` before it reads a single part. - // A bodyless GET (the catalog) carries no Content-Type, matching VSCode. - if (jsonBody) headers.set('Content-Type', 'application/json'); headers.set('editor-version', EDITOR_VERSION); headers.set('editor-plugin-version', EDITOR_PLUGIN_VERSION); headers.set('editor-device-id', getEditorDeviceId()); @@ -281,8 +275,15 @@ export async function copilotAuthedFetch(path: string, init: FetchInit, auth: Co const entry = await getCopilotToken(auth.id, auth.githubHost, auth.githubToken, options.fetcher, signal); if (ownedInit === undefined) throw new Error('Copilot request ownership missing before dispatch'); + // Content-Type follows the body. A `FormData` body has to reach the runtime + // untouched so it can generate the multipart boundary itself — pinning + // `application/json` over it makes Copilot reject `/v1/images/edits` with + // `400 Failed to decode image edit request` before it reads a single part. + // A bodyless GET (the catalog) carries no Content-Type, matching VSCode. + const initHeaders = new Headers(ownedInit.headers); const body = ownedInit.body; - const headers = copilotRequestHeaders(entry, ownedInit.headers, options.headers, body !== undefined && body !== null && !(body instanceof FormData)); + if (body !== undefined && body !== null && !(body instanceof FormData)) initHeaders.set('Content-Type', 'application/json'); + const headers = copilotRequestHeaders(entry, initHeaders, options.headers); const request = { ...ownedInit, headers }; ownedInit = undefined; diff --git a/packages/provider-copilot/src/fetch.ts b/packages/provider-copilot/src/fetch.ts index 9200911bfe..c6d6912743 100644 --- a/packages/provider-copilot/src/fetch.ts +++ b/packages/provider-copilot/src/fetch.ts @@ -84,8 +84,3 @@ export const copilotFetchOpenAIImagesEdits = (config: CopilotFetchConfig, init: copilotFetchInternal(config, '/v1/images/edits', init, options); export const copilotFetchModels = (config: CopilotFetchConfig, init: FetchInit, options: CopilotDataPlaneFetchOptions): Promise => copilotFetchInternal(config, '/models', init, options); -// Copilot serves decision models at the path each one advertises in -// `supported_endpoints` — `/v1/systemone` or `/v1/decisions` — and the body is -// forwarded in that model's own dialect, unchanged. -export const copilotFetchDecision = (config: CopilotFetchConfig, path: string, init: FetchInit, options: CopilotDataPlaneFetchOptions): Promise => - copilotFetchInternal(config, path, init, options); diff --git a/packages/provider-copilot/src/pipeline-facts.ts b/packages/provider-copilot/src/pipeline-facts.ts index cf269f3767..69f3345311 100644 --- a/packages/provider-copilot/src/pipeline-facts.ts +++ b/packages/provider-copilot/src/pipeline-facts.ts @@ -2,15 +2,19 @@ import type { HttpRequestFacts, HttpResponseFacts } from '@floway-dev/http/pipel import type { Deferred, Secret } from '@floway-dev/pipeline'; import type { ProviderOperationPayloads, ProviderRequest } from '@floway-dev/provider'; -export type EmbeddingsRequest = ProviderRequest; +/** The operations Copilot serves through pipelines. Each name is also the catalog endpoint key + * its raw model variant is selected by. */ +export type CopilotOperation = 'openaiEmbeddings' | 'openaiImagesGenerations' | 'openaiImagesEdits' | 'decision'; + +export type CopilotOperationRequest = ProviderRequest; export interface CopilotSession { readonly token: Secret; readonly expiresAt: number; readonly baseUrl: string; } -export interface CopilotRequest extends EmbeddingsRequest { +export type CopilotRequest = CopilotOperationRequest & { 'request.copilot.session': CopilotSession; -} +}; export interface CopilotHttpFacts extends HttpRequestFacts { 'request.copilot.modelKey': string; } diff --git a/packages/provider-copilot/src/pipelines.ts b/packages/provider-copilot/src/pipelines.ts index 73636dc2d8..e47695dfd3 100644 --- a/packages/provider-copilot/src/pipelines.ts +++ b/packages/provider-copilot/src/pipelines.ts @@ -1,12 +1,19 @@ import type { CopilotAuth } from './auth.ts'; -import type { EmbeddingsRequest } from './pipeline-facts.ts'; +import type { CopilotOperation, CopilotOperationRequest } from './pipeline-facts.ts'; import { authenticateCopilot } from './stages/authenticate.ts'; import { observeCopilotQuota } from './stages/observe-quota.ts'; -import { prepareCopilotEmbeddings } from './stages/prepare-request.ts'; +import { prepareCopilotRequest } from './stages/prepare-request.ts'; import { http } from '@floway-dev/http/pipeline'; import { compose } from '@floway-dev/pipeline'; import type { ProviderPipelines, ProviderResponse } from '@floway-dev/provider'; -export const createCopilotPipelines = (auth: CopilotAuth): ProviderPipelines => ({ - openaiEmbeddings: compose('copilot.openaiEmbeddings', [authenticateCopilot(auth), prepareCopilotEmbeddings(), observeCopilotQuota(auth.id), http]), -}); +export const createCopilotPipelines = (auth: CopilotAuth): ProviderPipelines => { + const pipeline = (operation: O) => + compose, ProviderResponse>(`copilot.${operation}`, [authenticateCopilot(auth, operation), prepareCopilotRequest(operation), observeCopilotQuota(auth.id), http]); + return { + openaiEmbeddings: pipeline('openaiEmbeddings'), + openaiImagesGenerations: pipeline('openaiImagesGenerations'), + openaiImagesEdits: pipeline('openaiImagesEdits'), + decision: pipeline('decision'), + }; +}; diff --git a/packages/provider-copilot/src/provider.ts b/packages/provider-copilot/src/provider.ts index 233c396388..e77c273476 100644 --- a/packages/provider-copilot/src/provider.ts +++ b/packages/provider-copilot/src/provider.ts @@ -3,7 +3,7 @@ import { COMPACTION_TRIGGER, compactionResponse } from './compaction.ts'; import { assertCopilotUpstreamRecord } from './config.ts'; import { COPILOT_DEFAULT_FLAGS, defaultFlagsForCopilotModel } from './defaults.ts'; import { fetchCopilotModels } from './fetch-models.ts'; -import { copilotFetchOpenAIChatCompletions, copilotFetchOpenAIEmbeddings, copilotFetchAnthropicMessages, copilotFetchAnthropicMessagesCountTokens, copilotFetchOpenAIImagesEdits, copilotFetchOpenAIImagesGenerations, copilotFetchOpenAIResponses, copilotFetchDecision, type CopilotDataPlaneFetchOptions } from './fetch.ts'; +import { copilotFetchOpenAIChatCompletions, copilotFetchOpenAIEmbeddings, copilotFetchAnthropicMessages, copilotFetchAnthropicMessagesCountTokens, copilotFetchOpenAIImagesEdits, copilotFetchOpenAIImagesGenerations, copilotFetchOpenAIResponses, type CopilotDataPlaneFetchOptions } from './fetch.ts'; import { COPILOT_ANTHROPIC_MESSAGES_BOUNDARY, COPILOT_ANTHROPIC_MESSAGES_COUNT_TOKENS_BOUNDARY } from './interceptors/anthropic-messages/index.ts'; import type { AnthropicMessagesBoundaryCtx } from './interceptors/anthropic-messages/types.ts'; import { COPILOT_OPENAI_CHAT_COMPLETIONS_BOUNDARY } from './interceptors/openai-chat-completions/index.ts'; @@ -23,7 +23,6 @@ import type { CopilotRawModel } from './types.ts'; import { runInterceptors } from '@floway-dev/interceptor'; import { parseAnthropicMessagesStream, type AnthropicMessagesPayload, type AnthropicMessagesStreamEventEx } from '@floway-dev/protocols/anthropic-messages'; import { type ModelEndpoints, type ProtocolFrame, isFastServiceTier, kindForEndpoints } from '@floway-dev/protocols/common'; -import { DEFAULT_DECISION_PATHS, serializeDecisionRequest, type CanonicalDecisionRequest } from '@floway-dev/protocols/decision'; import { parseOpenAIChatCompletionsStream, type OpenAIChatCompletionsPayload, type OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import { parseOpenAIResponsesStream, type CanonicalOpenAIResponsesPayload, type OpenAIResponsesResultEx } from '@floway-dev/protocols/openai-responses'; import { eventResult, getProviderRepo, headersForAnthropicMessagesCall, jsonRequestBody, serializeOpenAIImagesEditsRequest, readUpstreamApiError, streamingProviderCall, apiErrorToResponse, resolveEffectiveFlags, type ExecuteResult, type FetchInit, type FlagOverrides, type HttpHeaderLines, type ProviderInstance, type Provider, type ProviderCallResult, type ProviderModel, type ProviderOpenAIResponsesResult, type ProviderStreamResult, type TelemetryModelIdentity, type UpstreamCallOptions, type UpstreamRecord } from '@floway-dev/provider'; @@ -459,19 +458,6 @@ export const createCopilotProvider = (record: UpstreamRecord): Provider => { }, callOpenAIAudioTranscriptions: rejectUnsupported('callOpenAIAudioTranscriptions'), callRerank: rejectUnsupported('callRerank'), - callDecision: async (model: ProviderModel, request: CanonicalDecisionRequest, signal: AbortSignal | undefined, opts: UpstreamCallOptions) => { - const target = model.decisionTarget; - if (target === undefined) throw new Error(`Decision model ${model.id} has no outbound target`); - const rawModel = rawModelFor(model, 'decision'); - const body = serializeDecisionRequest(target.protocol, rawModel.id, request); - const response = await copilotFetchDecision( - upstreamConfig, - target.path ?? DEFAULT_DECISION_PATHS[target.protocol], - { method: 'POST', body: jsonRequestBody(body), signal }, - { extraHeaders: [...opts.headers], fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, - ); - return { response, modelKey: rawModel.id, target }; - }, }; return { diff --git a/packages/provider-copilot/src/stages/authenticate.ts b/packages/provider-copilot/src/stages/authenticate.ts index 179f117e21..13a7c47037 100644 --- a/packages/provider-copilot/src/stages/authenticate.ts +++ b/packages/provider-copilot/src/stages/authenticate.ts @@ -1,12 +1,12 @@ import { getCopilotToken, isCopilotTokenFetchError, type CopilotAuth } from '../auth.ts'; import { rawModelFor } from '../operation-model.ts'; -import type { EmbeddingsRequest, CopilotRequest } from '../pipeline-facts.ts'; +import type { CopilotOperation, CopilotOperationRequest, CopilotRequest } from '../pipeline-facts.ts'; import { takeHttpResponse } from '@floway-dev/http/pipeline'; import { defineStage, move, secret, type Handed } from '@floway-dev/pipeline'; import type { ProviderResponse, ProviderServices } from '@floway-dev/provider'; -export const authenticateCopilot = (auth: CopilotAuth) => { - const authenticate = defineStage({ +export const authenticateCopilot = (auth: CopilotAuth, operation: O) => { + const authenticate = defineStage, CopilotRequest, ProviderResponse, ProviderResponse, ProviderResponse, ProviderServices>({ name: 'authenticateCopilot', through: { request: { @@ -29,7 +29,7 @@ export const authenticateCopilot = (auth: CopilotAuth) => { ...facts, 'response.http.exchange': exchange, 'response.http.body': exchange.body, - 'response.provider.modelKey': rawModelFor(facts['request.provider.model'], 'openaiEmbeddings').id, + 'response.provider.modelKey': rawModelFor(facts['request.provider.model'], operation).id, 'response.provider.called': false, 'response.provider.previousCalls': [], }); diff --git a/packages/provider-copilot/src/stages/prepare-request.ts b/packages/provider-copilot/src/stages/prepare-request.ts index cdb8cbdd55..b58b1afd3a 100644 --- a/packages/provider-copilot/src/stages/prepare-request.ts +++ b/packages/provider-copilot/src/stages/prepare-request.ts @@ -1,13 +1,56 @@ import { copilotRequestHeaders } from '../auth.ts'; import { copilotOpenAIEmbeddingsBody, rawModelFor } from '../operation-model.ts'; -import type { CopilotRequest, CopilotHttpFacts } from '../pipeline-facts.ts'; +import type { CopilotHttpFacts, CopilotOperation, CopilotRequest } from '../pipeline-facts.ts'; import type { HttpResponseFacts } from '@floway-dev/http/pipeline'; +import type { HttpBody, HttpBodyEncoding } from '@floway-dev/http/request-content'; import { defineStage, isSecret, move, secret, type Secret } from '@floway-dev/pipeline'; -import type { ProviderResponse, ProviderServices } from '@floway-dev/provider'; +import { DEFAULT_DECISION_PATHS, serializeDecisionRequest } from '@floway-dev/protocols/decision'; +import { prepareOpenAIImagesEditsBody, withHttpContentType, type ProviderOperationPayloads, type ProviderResponse, type ProviderServices } from '@floway-dev/provider'; -export const prepareCopilotEmbeddings = () => { - const prepare = defineStage({ - name: 'prepareCopilotOpenAIEmbeddings', +interface PreparedCopilotRequest { + readonly path: string; + readonly body: HttpBody; + readonly encoding: HttpBodyEncoding; +} + +// The image endpoints advertise `/v1/...` paths and only answer there: the +// unprefixed `/images/edits` the other data-plane surfaces use returns a bare +// `404 page not found`. `/v1/images/generations` accepts JSON only, while +// `/v1/images/edits` accepts JSON references and multipart uploads; it refuses +// a `reference` carrying a remote URL with `400 external image URLs are not +// supported`, which reaches the client as Copilot's own answer. +// Decision models answer at the path each one advertises in +// `supported_endpoints`, in that model's own dialect. +const prepareBody = async ( + operation: O, + model: CopilotRequest['request.provider.model'], + payload: ProviderOperationPayloads[O], + modelKey: string, +): Promise => { + switch (operation) { + case 'openaiEmbeddings': + return { path: '/embeddings', body: { ...copilotOpenAIEmbeddingsBody(payload as ProviderOperationPayloads['openaiEmbeddings']), model: modelKey }, encoding: 'json' }; + case 'openaiImagesGenerations': + return { path: '/v1/images/generations', body: { ...(payload as ProviderOperationPayloads['openaiImagesGenerations']), model: modelKey }, encoding: 'json' }; + case 'openaiImagesEdits': + return { path: '/v1/images/edits', ...await prepareOpenAIImagesEditsBody(payload as ProviderOperationPayloads['openaiImagesEdits'], modelKey) }; + case 'decision': { + const target = model.decisionTarget; + if (target === undefined) throw new Error(`Decision model ${model.id} has no outbound target`); + return { + path: target.path ?? DEFAULT_DECISION_PATHS[target.protocol], + body: serializeDecisionRequest(target.protocol, modelKey, payload as ProviderOperationPayloads['decision']), + encoding: 'json', + }; + } + default: + throw new Error(`Copilot has no pipeline for ${String(operation)}`); + } +}; + +export const prepareCopilotRequest = (operation: O) => { + const prepare = defineStage, CopilotHttpFacts, HttpResponseFacts, ProviderResponse, ProviderServices>({ + name: `prepareCopilot${operation.replace(/^openai/, 'OpenAI').replace(/^decision/, 'Decision')}`, through: { request: { needs: ['request.provider.model', 'request.provider.payload', 'request.http.headers', 'request.http.callId', 'request.copilot.session'], @@ -18,18 +61,19 @@ export const prepareCopilotEmbeddings = () => { }, execute: async (facts, next) => { const { 'request.provider.model': model, 'request.provider.payload': payload, 'request.copilot.session': session, ...rest } = facts; - const modelKey = rawModelFor(model, 'openaiEmbeddings').id; + const modelKey = rawModelFor(model, operation).id; + const prepared = await prepareBody(operation, model, payload, modelKey); const admitted = facts['request.http.headers'].map(([name, value]): [string, string] => [name, isSecret(value) ? value.reveal() : value]); - const headers = [...copilotRequestHeaders({ ...session, token: session.token.reveal() }, undefined, admitted, true)] + const identity = [...copilotRequestHeaders({ ...session, token: session.token.reveal() }, undefined, admitted)] .map(([name, value]): [string, string | Secret] => [name, name === 'authorization' ? secret(value) : value]); const back = await next(move({ ...rest, 'request.copilot.modelKey': modelKey, - 'request.http.url': `${session.baseUrl}/embeddings`, + 'request.http.url': `${session.baseUrl}${prepared.path}`, 'request.http.method': 'POST', - 'request.http.headers': headers, - 'request.http.body': { ...copilotOpenAIEmbeddingsBody(payload), model: modelKey }, - 'request.http.encoding': 'json' as const, + 'request.http.headers': withHttpContentType(identity, prepared.body, prepared.encoding), + 'request.http.body': prepared.body, + 'request.http.encoding': prepared.encoding, })); return move({ ...back, 'response.provider.modelKey': modelKey, 'response.provider.called': back['response.http.exchange'].type === 'response', 'response.provider.previousCalls': [] }); }, diff --git a/packages/provider-custom/src/fetch.ts b/packages/provider-custom/src/fetch.ts index 74af4184bb..5df603f6f9 100644 --- a/packages/provider-custom/src/fetch.ts +++ b/packages/provider-custom/src/fetch.ts @@ -59,9 +59,6 @@ const customFetchInternal = async ( export const customFetchRerank = (config: CustomUpstreamConfig, path: string, init: FetchInit, options: UpstreamFetchOptions): Promise => customFetchInternal(config, path, init, options); -export const customFetchDecision = (config: CustomUpstreamConfig, path: string, init: FetchInit, options: UpstreamFetchOptions): Promise => - customFetchInternal(config, path, init, options); - export const customFetchOpenAIChatCompletions = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => customFetchInternal(config, pathOverrideFor(config, '/chat/completions'), init, options); export const customFetchOpenAIResponses = (config: CustomUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => diff --git a/packages/provider-custom/src/pipelines.ts b/packages/provider-custom/src/pipelines.ts index ce3438c8d0..f8a00b6824 100644 --- a/packages/provider-custom/src/pipelines.ts +++ b/packages/provider-custom/src/pipelines.ts @@ -12,4 +12,5 @@ export const createCustomPipelines = (config: CustomUpstreamConfig): ProviderPip openaiImagesEdits: compose, ProviderOperationResponse<'openaiImagesEdits'>>('custom.openaiImagesEdits', [prepareCustomRequest(config, 'openaiImagesEdits'), http]), openaiAudioTranscriptions: compose, ProviderOperationResponse<'openaiAudioTranscriptions'>>('custom.openaiAudioTranscriptions', [prepareCustomRequest(config, 'openaiAudioTranscriptions'), http]), rerank: compose, ProviderOperationResponse<'rerank'>>('custom.rerank', [prepareCustomRequest(config, 'rerank'), http]), + decision: compose, ProviderOperationResponse<'decision'>>('custom.decision', [prepareCustomRequest(config, 'decision'), http]), }); diff --git a/packages/provider-custom/src/provider.ts b/packages/provider-custom/src/provider.ts index 3e9201f4e0..1a0a04dd77 100644 --- a/packages/provider-custom/src/provider.ts +++ b/packages/provider-custom/src/provider.ts @@ -1,12 +1,11 @@ import { assertCustomUpstreamRecord, type CustomUpstreamConfig } from './config.ts'; import { CUSTOM_DEFAULT_FLAGS } from './defaults.ts'; import { fetchCustomModels, type CustomModelsResponse, type CustomRawModel } from './fetch-models.ts'; -import { customFetchAlphaSearch, customFetchOpenAIAudioTranscriptions, customFetchOpenAIChatCompletions, customFetchOpenAICompletions, customFetchDecision, customFetchOpenAIEmbeddings, customFetchOpenAIImagesEdits, customFetchOpenAIImagesGenerations, customFetchAnthropicMessages, customFetchAnthropicMessagesCountTokens, customFetchRerank, customFetchOpenAIResponses, customFetchOpenAIResponsesCompact } from './fetch.ts'; +import { customFetchAlphaSearch, customFetchOpenAIAudioTranscriptions, customFetchOpenAIChatCompletions, customFetchOpenAICompletions, customFetchOpenAIEmbeddings, customFetchOpenAIImagesEdits, customFetchOpenAIImagesGenerations, customFetchAnthropicMessages, customFetchAnthropicMessagesCountTokens, customFetchRerank, customFetchOpenAIResponses, customFetchOpenAIResponsesCompact } from './fetch.ts'; import { inferEndpointsFromModelId } from './infer-endpoints.ts'; import { createCustomPipelines } from './pipelines.ts'; import { parseAnthropicMessagesStream } from '@floway-dev/protocols/anthropic-messages'; import { type ModelEndpoints, kindForEndpoints } from '@floway-dev/protocols/common'; -import { DEFAULT_DECISION_PATHS, serializeDecisionRequest, type CanonicalDecisionRequest } from '@floway-dev/protocols/decision'; import { parseOpenAIChatCompletionsStream } from '@floway-dev/protocols/openai-chat-completions'; import { parseOpenAIResponsesStream, type OpenAIResponsesCompactionResultEx, toCompactPayloadShape } from '@floway-dev/protocols/openai-responses'; import { DEFAULT_RERANK_PATHS, serializeRerankRequest } from '@floway-dev/protocols/rerank'; @@ -279,19 +278,6 @@ export const createCustomProvider = (record: UpstreamRecord): Provider => { ); return { response, modelKey: rawModelId, target }; }, - callDecision: async (model: ProviderModel, request: CanonicalDecisionRequest, signal: AbortSignal | undefined, opts: UpstreamCallOptions) => { - const target = model.decisionTarget; - if (target === undefined) throw new Error(`Decision model ${model.id} has no outbound target`); - const rawModelId = rawModelIdOf(model); - const body = serializeDecisionRequest(target.protocol, rawModelId, request); - const response = await customFetchDecision( - config, - target.path ?? DEFAULT_DECISION_PATHS[target.protocol], - { method: 'POST', body: jsonRequestBody(body), signal }, - { extraHeaders: headersForCall(opts.headers), fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, - ); - return { response, modelKey: rawModelId, target }; - }, }; return { diff --git a/packages/provider-custom/src/stages/prepare-request.ts b/packages/provider-custom/src/stages/prepare-request.ts index b8c7f5dfc9..488564cea9 100644 --- a/packages/provider-custom/src/stages/prepare-request.ts +++ b/packages/provider-custom/src/stages/prepare-request.ts @@ -3,6 +3,7 @@ import { type HttpHeaders, type HttpRequestFacts, type HttpResponseFacts } from import { multipartBody, type HttpBody, type HttpBodyEncoding } from '@floway-dev/http/request-content'; import { defineStage, isSecret, move, secret, type Secret } from '@floway-dev/pipeline'; import type { RerankTarget } from '@floway-dev/protocols/common'; +import { DEFAULT_DECISION_PATHS, serializeDecisionRequest } from '@floway-dev/protocols/decision'; import { DEFAULT_RERANK_PATHS, serializeRerankRequest } from '@floway-dev/protocols/rerank'; import { joinBaseAndPath, mergeHttpHeaders, prepareOpenAIImagesEditsBody, replaceHttpHeader, withHttpContentType, type ProviderOperation, type ProviderOperationPayloads, type ProviderRequest, type ProviderResponse, type ProviderServices } from '@floway-dev/provider'; @@ -41,7 +42,7 @@ const resolvedHeaders = (config: CustomUpstreamConfig, incoming: HttpHeaders): H export const prepareCustomRequest = (config: CustomUpstreamConfig, operation: O) => { const prepare = defineStage, CustomHttpFacts, HttpResponseFacts, CustomResponse, ProviderServices>({ - name: `prepareCustom${operation.replace(/^openai/, 'OpenAI').replace(/^alpha/, 'Alpha').replace(/^rerank/, 'Rerank')}`, + name: `prepareCustom${operation.replace(/^openai/, 'OpenAI').replace(/^alpha/, 'Alpha').replace(/^rerank/, 'Rerank').replace(/^decision/, 'Decision')}`, through: { request: { needs: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'], @@ -68,6 +69,13 @@ export const prepareCustomRequest = (config: Custom path = target.path ?? DEFAULT_RERANK_PATHS[target.protocol]; body = serializeRerankRequest(target.protocol, modelKey, payload as ProviderOperationPayloads['rerank']); break; + case 'decision': { + const decisionTarget = model.decisionTarget; + if (decisionTarget === undefined) throw new Error(`Decision model ${model.id} has no outbound target`); + path = decisionTarget.path ?? DEFAULT_DECISION_PATHS[decisionTarget.protocol]; + body = serializeDecisionRequest(decisionTarget.protocol, modelKey, payload as ProviderOperationPayloads['decision']); + break; + } case 'openaiImagesEdits': { const prepared = await prepareOpenAIImagesEditsBody(payload as ProviderOperationPayloads['openaiImagesEdits'], modelKey); body = prepared.body; diff --git a/packages/provider-ollama/src/fetch.ts b/packages/provider-ollama/src/fetch.ts index a688ca6fa1..285e634cde 100644 --- a/packages/provider-ollama/src/fetch.ts +++ b/packages/provider-ollama/src/fetch.ts @@ -27,8 +27,6 @@ const ollamaFetchInternal = async ( export const ollamaFetchOpenAIChatCompletions = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => ollamaFetchInternal(config, '/v1/chat/completions', init, options); -export const ollamaFetchDecision = (config: OllamaUpstreamConfig, path: string, init: FetchInit, options: UpstreamFetchOptions): Promise => - ollamaFetchInternal(config, path, init, options); export const ollamaFetchOpenAIResponses = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => ollamaFetchInternal(config, '/v1/responses', init, options); export const ollamaFetchOpenAIResponsesCompact = (config: OllamaUpstreamConfig, init: FetchInit, options: UpstreamFetchOptions): Promise => diff --git a/packages/provider-ollama/src/pipelines.ts b/packages/provider-ollama/src/pipelines.ts index 495674a2ab..b7e020f18e 100644 --- a/packages/provider-ollama/src/pipelines.ts +++ b/packages/provider-ollama/src/pipelines.ts @@ -12,5 +12,6 @@ export const createOllamaPipelines = (upstreamId: string, config: OllamaUpstream openaiCompletions: compose, ProviderOperationResponse<'openaiCompletions'>>('ollama.openaiCompletions', [observe, prepareOllamaRequest(config, 'openaiCompletions'), http]), openaiEmbeddings: compose, ProviderOperationResponse<'openaiEmbeddings'>>('ollama.openaiEmbeddings', [observe, prepareOllamaRequest(config, 'openaiEmbeddings'), http]), openaiAudioTranscriptions: compose, ProviderOperationResponse<'openaiAudioTranscriptions'>>('ollama.openaiAudioTranscriptions', [observe, prepareOllamaRequest(config, 'openaiAudioTranscriptions'), http]), + decision: compose, ProviderOperationResponse<'decision'>>('ollama.decision', [prepareOllamaRequest(config, 'decision'), http]), }; }; diff --git a/packages/provider-ollama/src/provider.ts b/packages/provider-ollama/src/provider.ts index f0f6364d5d..90ce7d2a71 100644 --- a/packages/provider-ollama/src/provider.ts +++ b/packages/provider-ollama/src/provider.ts @@ -29,14 +29,13 @@ import { chatFromOllamaRaw } from './chat-from-raw.ts'; import { assertOllamaUpstreamRecord, type OllamaUpstreamConfig } from './config.ts'; import { OLLAMA_DEFAULT_FLAGS } from './defaults.ts'; import { fetchOllamaCatalog, type OllamaCatalog } from './fetch-models.ts'; -import { ollamaFetchDecision, ollamaFetchOpenAIAudioTranscriptions, ollamaFetchOpenAIChatCompletions, ollamaFetchOpenAICompletions, ollamaFetchOpenAIEmbeddings, ollamaFetchAnthropicMessages, ollamaFetchAnthropicMessagesCountTokens, ollamaFetchOpenAIResponses, ollamaFetchOpenAIResponsesCompact } from './fetch.ts'; +import { ollamaFetchOpenAIAudioTranscriptions, ollamaFetchOpenAIChatCompletions, ollamaFetchOpenAICompletions, ollamaFetchOpenAIEmbeddings, ollamaFetchAnthropicMessages, ollamaFetchAnthropicMessagesCountTokens, ollamaFetchOpenAIResponses, ollamaFetchOpenAIResponsesCompact } from './fetch.ts'; import { createOllamaPipelines } from './pipelines.ts'; import { pricingForOllamaModelKey } from './pricing.ts'; import { readOllamaUpstreamState } from './state.ts'; import { scheduleOllamaUsageProbe, runOllamaScheduledTask } from './usage-probe.ts'; import { parseAnthropicMessagesStream } from '@floway-dev/protocols/anthropic-messages'; import { type ModelEndpoints, kindForEndpoints } from '@floway-dev/protocols/common'; -import { DEFAULT_DECISION_PATHS, serializeDecisionRequest, type CanonicalDecisionRequest } from '@floway-dev/protocols/decision'; import { parseOpenAIChatCompletionsStream } from '@floway-dev/protocols/openai-chat-completions'; import { parseOpenAIResponsesStream, type OpenAIResponsesCompactionResultEx, toCompactPayloadShape } from '@floway-dev/protocols/openai-responses'; import { headersForAnthropicMessagesCall, jsonRequestBody, publicModelId, resolveEffectiveFlags, serializeModelFieldOpenAIAudioTranscriptionRequest, streamingProviderCall, type FetchInit, type FlagId, type HttpHeaderLines, type ProviderInstance, type Provider, type ProviderCallResult, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord } from '@floway-dev/provider'; @@ -224,19 +223,6 @@ export const createOllamaProvider = (record: UpstreamRecord): Provider => { return { response, modelKey: rawModelId }; }, callRerank: rejectUnsupported('callRerank'), - callDecision: async (model: ProviderModel, request: CanonicalDecisionRequest, signal: AbortSignal | undefined, opts: UpstreamCallOptions) => { - const target = model.decisionTarget; - if (target === undefined) throw new Error(`Decision model ${model.id} has no outbound target`); - const rawModelId = rawModelIdOf(model); - const body = serializeDecisionRequest(target.protocol, rawModelId, request); - const response = await ollamaFetchDecision( - config, - target.path ?? DEFAULT_DECISION_PATHS[target.protocol], - { method: 'POST', body: jsonRequestBody(body), signal }, - { extraHeaders: [...opts.headers], fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, - ); - return { response, modelKey: rawModelId, target }; - }, }; return { diff --git a/packages/provider-ollama/src/stages/prepare-request.ts b/packages/provider-ollama/src/stages/prepare-request.ts index 075563c12d..96e7f05b00 100644 --- a/packages/provider-ollama/src/stages/prepare-request.ts +++ b/packages/provider-ollama/src/stages/prepare-request.ts @@ -1,16 +1,17 @@ import type { OllamaUpstreamConfig } from '../config.ts'; import type { HttpRequestFacts, HttpResponseFacts } from '@floway-dev/http/pipeline'; -import { multipartBody } from '@floway-dev/http/request-content'; +import { multipartBody, type HttpBody } from '@floway-dev/http/request-content'; import { defineStage, move, secret } from '@floway-dev/pipeline'; +import { DEFAULT_DECISION_PATHS, serializeDecisionRequest } from '@floway-dev/protocols/decision'; import { joinBaseAndPath, mergeHttpHeaders, withHttpContentType, type ProviderOperationPayloads, type ProviderRequest, type ProviderResponse, type ProviderServices } from '@floway-dev/provider'; -type OllamaOperation = 'openaiCompletions' | 'openaiEmbeddings' | 'openaiAudioTranscriptions'; +type OllamaOperation = 'openaiCompletions' | 'openaiEmbeddings' | 'openaiAudioTranscriptions' | 'decision'; interface OllamaHttpFacts extends HttpRequestFacts { 'request.ollama.modelKey': string; } -const PATHS: Record = { +const PATHS: Record, string> = { openaiCompletions: '/v1/completions', openaiEmbeddings: '/v1/embeddings', // https://github.com/ollama/ollama/blob/573386c35eac76124ffce571f4b0fefa0a7fe13c/server/routes.go#L1916-L1922 @@ -19,7 +20,7 @@ const PATHS: Record = { export const prepareOllamaRequest = (config: OllamaUpstreamConfig, operation: O) => { const prepare = defineStage, OllamaHttpFacts, HttpResponseFacts, ProviderResponse, ProviderServices>({ - name: `prepareOllama${operation.replace(/^openai/, 'OpenAI').replace(/^alpha/, 'Alpha').replace(/^rerank/, 'Rerank')}`, + name: `prepareOllama${operation.replace(/^openai/, 'OpenAI').replace(/^alpha/, 'Alpha').replace(/^rerank/, 'Rerank').replace(/^decision/, 'Decision')}`, through: { request: { needs: ['request.provider.model', 'request.provider.payload', 'request.http.callId', 'request.http.headers'], @@ -32,15 +33,25 @@ export const prepareOllamaRequest = (config: OllamaUp const { 'request.provider.model': model, 'request.provider.payload': payload, ...rest } = facts; const modelKey = model.providerData as string; const encoding = operation === 'openaiAudioTranscriptions' ? 'multipart' as const : 'json' as const; - const body = operation === 'openaiAudioTranscriptions' - ? multipartBody((payload as ProviderOperationPayloads['openaiAudioTranscriptions']).entries.map(entry => entry.name === 'model' ? { ...entry, value: modelKey } : entry)) - : { ...payload, model: modelKey }; + let path: string; + let body: HttpBody; + if (operation === 'decision') { + const target = model.decisionTarget; + if (target === undefined) throw new Error(`Decision model ${model.id} has no outbound target`); + path = target.path ?? DEFAULT_DECISION_PATHS[target.protocol]; + body = serializeDecisionRequest(target.protocol, modelKey, payload as ProviderOperationPayloads['decision']); + } else { + path = PATHS[operation as Exclude]; + body = operation === 'openaiAudioTranscriptions' + ? multipartBody((payload as ProviderOperationPayloads['openaiAudioTranscriptions']).entries.map(entry => entry.name === 'model' ? { ...entry, value: modelKey } : entry)) + : { ...payload, model: modelKey }; + } const base = config.apiKey ? [['Authorization', secret(`Bearer ${config.apiKey}`)] as const] : []; const headers = mergeHttpHeaders(withHttpContentType(base, body, encoding), facts['request.http.headers']); const back = await next(move({ ...rest, 'request.ollama.modelKey': modelKey, - 'request.http.url': joinBaseAndPath(config.baseUrl, PATHS[operation]), + 'request.http.url': joinBaseAndPath(config.baseUrl, path), 'request.http.method': 'POST', 'request.http.headers': headers, 'request.http.body': body, diff --git a/packages/provider/src/index.ts b/packages/provider/src/index.ts index e145e785ef..181cf97714 100644 --- a/packages/provider/src/index.ts +++ b/packages/provider/src/index.ts @@ -54,7 +54,6 @@ export type { ProviderInstance, ProviderCallResult, ProviderRerankCallResult, - ProviderDecisionCallResult, ProviderModule, AnthropicMessagesUpstreamCallOptions, ProviderOpenAIResponsesResult, diff --git a/packages/provider/src/pipeline.ts b/packages/provider/src/pipeline.ts index 2dfaaedee9..400aaacf90 100644 --- a/packages/provider/src/pipeline.ts +++ b/packages/provider/src/pipeline.ts @@ -5,6 +5,7 @@ import type { ProviderModel } from './model.ts'; import type { HttpResponseFacts, HttpHeaders, HttpServices } from '@floway-dev/http/pipeline'; import type { Pipeline } from '@floway-dev/pipeline'; import type { RerankTarget } from '@floway-dev/protocols/common'; +import type { CanonicalDecisionRequest } from '@floway-dev/protocols/decision'; import type { OpenAICompletionsPayload } from '@floway-dev/protocols/openai-completions'; import type { OpenAIEmbeddingsPayload } from '@floway-dev/protocols/openai-embeddings'; import type { OpenAIImagesGenerationsPayload } from '@floway-dev/protocols/openai-images'; @@ -18,6 +19,7 @@ export interface ProviderOperationPayloads { openaiImagesEdits: OpenAIImagesEditsRequest; openaiAudioTranscriptions: OpenAIAudioTranscriptionRequest; rerank: CanonicalRerankRequest; + decision: CanonicalDecisionRequest; } export type ProviderOperation = keyof ProviderOperationPayloads; diff --git a/packages/provider/src/provider.ts b/packages/provider/src/provider.ts index 030a8b1045..af73e8aeb7 100644 --- a/packages/provider/src/provider.ts +++ b/packages/provider/src/provider.ts @@ -7,8 +7,7 @@ import type { Fetcher } from './options.ts'; import type { ProviderPipelines } from './pipeline.ts'; import type { ProviderScheduledOptions } from './scheduled.ts'; import type { AnthropicMessagesPayload, AnthropicMessagesStreamEventEx } from '@floway-dev/protocols/anthropic-messages'; -import type { DecisionTarget, ProtocolFrame, RerankTarget } from '@floway-dev/protocols/common'; -import type { CanonicalDecisionRequest } from '@floway-dev/protocols/decision'; +import type { ProtocolFrame, RerankTarget } from '@floway-dev/protocols/common'; import type { OpenAIChatCompletionsPayload, OpenAIChatCompletionsStreamEvent } from '@floway-dev/protocols/openai-chat-completions'; import type { OpenAICompletionsPayload } from '@floway-dev/protocols/openai-completions'; import type { OpenAIEmbeddingsPayload } from '@floway-dev/protocols/openai-embeddings'; @@ -58,12 +57,6 @@ export interface ProviderRerankCallResult extends ProviderCallResult { target: RerankTarget; } -// The target rides back so the serve layer knows which dialect shaped the -// response and can pick the matching parser. -export interface ProviderDecisionCallResult extends ProviderCallResult { - target: DecisionTarget; -} - // Streaming endpoints (Anthropic Messages / OpenAI Responses / OpenAI Chat Completions) return decoded // protocol frames directly — the provider drives the upstream fetch, parses // the SSE wire via @floway-dev/protocols, and emits the typed event stream. @@ -163,7 +156,6 @@ export interface ProviderInstance { callOpenAIImagesEdits(model: ProviderModel, request: OpenAIImagesEditsRequest, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; callOpenAIAudioTranscriptions(model: ProviderModel, request: OpenAIAudioTranscriptionRequest, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; callRerank(model: ProviderModel, request: CanonicalRerankRequest, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; - callDecision(model: ProviderModel, request: CanonicalDecisionRequest, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; } // Static, module-shaped surface each provider package exports. The gateway diff --git a/packages/test-utils/src/stubs.ts b/packages/test-utils/src/stubs.ts index ec98b591b8..29ac77ee76 100644 --- a/packages/test-utils/src/stubs.ts +++ b/packages/test-utils/src/stubs.ts @@ -90,7 +90,6 @@ export const stubProvider = (overrides: Partial = {}): Provide callOpenAIImagesEdits: overrides.callOpenAIImagesEdits ?? (() => Promise.reject(new Error('stubProvider.callOpenAIImagesEdits was called'))), callOpenAIAudioTranscriptions: overrides.callOpenAIAudioTranscriptions ?? (() => Promise.reject(new Error('stubProvider.callOpenAIAudioTranscriptions was called'))), callRerank: overrides.callRerank ?? (() => Promise.reject(new Error('stubProvider.callRerank was called'))), - callDecision: overrides.callDecision ?? (() => Promise.reject(new Error('stubProvider.callDecision was called'))), }); // Stitches together a candidate whose `model.providerModels` map carries an From 44e181aee3ab11faa8f7321ebb00812b59dac8e6 Mon Sep 17 00:00:00 2001 From: Menci Date: Sat, 14 Nov 2026 09:37:23 +0800 Subject: [PATCH 78/79] fix(dump): redact upstream credentials before serving a run A run records header values as plain strings unless a stage marked them secret, so an operator's fixed header or an upstream's set-cookie reached whoever could read the key's dumps. Serving a run now rewrites every sensitive upstream header in the node that defines it, matching the edge records; the client's own request headers stay as the key owner sent them. The sensitive-header set lives once in the dump package. --- .../web/src/components/requests/run-redact.ts | 3 +- packages/dump/__tests__/redact_test.ts | 43 ++++++++++ packages/dump/src/index.ts | 1 + packages/dump/src/redact.ts | 86 +++++++++++++++++++ packages/dump/src/wire.ts | 3 +- packages/gateway/src/dump/wire.ts | 20 +---- 6 files changed, 136 insertions(+), 20 deletions(-) create mode 100644 packages/dump/__tests__/redact_test.ts create mode 100644 packages/dump/src/redact.ts diff --git a/apps/web/src/components/requests/run-redact.ts b/apps/web/src/components/requests/run-redact.ts index 40659544be..198a2326e5 100644 --- a/apps/web/src/components/requests/run-redact.ts +++ b/apps/web/src/components/requests/run-redact.ts @@ -1,4 +1,4 @@ -import { isSensitiveHeader, redactHeaderValue } from './header-redact'; +import { isSensitiveHeader, REDACTED_HEADER_VALUE, redactHeaderValue } from './header-redact'; import type { DumpEvent } from '@floway-dev/pipeline/browser'; // Header values can be shared string nodes. Redacting only the tuple would @@ -48,6 +48,7 @@ export const redactRunHeaders = (ndjson: string): string => { if (typeof secret === 'object' && secret !== null && '$secret' in secret) continue; throw new Error(`Run export has invalid header value at ${key}`); } + if (secret === REDACTED_HEADER_VALUE) continue; const redacted = redactHeaderValue(secret); if (id === null) pair[1] = redacted; else { diff --git a/packages/dump/__tests__/redact_test.ts b/packages/dump/__tests__/redact_test.ts new file mode 100644 index 0000000000..6c87304d33 --- /dev/null +++ b/packages/dump/__tests__/redact_test.ts @@ -0,0 +1,43 @@ +import { expect, test } from 'vitest'; + +import { REDACTED_HEADER_VALUE, redactRunCredentials } from '../src/index.ts'; +import { createRunReader, encodeRun, move, secret, toNdjson, type DumpEvent, type Event } from '@floway-dev/pipeline'; + +const exchange = { + type: 'response', + status: 200, + statusText: 'OK', + headers: [['Set-Cookie', 'session=operator'], ['x-request-id', 'trace']], + body: null, +}; + +const run = (): Event[] => [ + { + type: 'stage.entered', + stageId: 1, + name: 'call', + parentStageId: null, + facts: move({ + 'ingress.http.headers': [['authorization', 'Bearer client-key']], + 'request.http.headers': [['authorization', secret('Bearer operator-token')], ['x-api-key', 'operator-fixed-key'], ['accept', 'application/json']], + }), + }, + { type: 'stage.leaved', stageId: 1, facts: move({ 'response.http.exchange': exchange, 'response.http.headers': exchange.headers }) }, +]; + +const read = (ndjson: string) => { + const reader = createRunReader(); + return ndjson.split('\n').filter(Boolean).map(line => reader(JSON.parse(line) as DumpEvent)).flatMap(event => event === null || !('facts' in event) ? [] : [event.facts]); +}; + +test('upstream credential headers are redacted in every node that holds them', () => { + const redacted = redactRunCredentials(toNdjson(encodeRun(run()))); + const [entered, leaved] = read(redacted); + expect(entered!['ingress.http.headers']).toEqual([['authorization', 'Bearer client-key']]); + const upstream = entered!['request.http.headers'] as unknown[][]; + expect(upstream[1]).toEqual(['x-api-key', REDACTED_HEADER_VALUE]); + expect(upstream[2]).toEqual(['accept', 'application/json']); + expect((leaved!['response.http.exchange'] as typeof exchange).headers).toEqual([['Set-Cookie', REDACTED_HEADER_VALUE], ['x-request-id', 'trace']]); + expect(leaved!['response.http.headers']).toEqual([['Set-Cookie', REDACTED_HEADER_VALUE], ['x-request-id', 'trace']]); + expect(redacted).not.toContain('operator'); +}); diff --git a/packages/dump/src/index.ts b/packages/dump/src/index.ts index 6893b349b1..ee1714cf09 100644 --- a/packages/dump/src/index.ts +++ b/packages/dump/src/index.ts @@ -4,3 +4,4 @@ export * from './codec.ts'; export * from './recorder.ts'; export * from './stream.ts'; export * from './wire.ts'; +export * from './redact.ts'; diff --git a/packages/dump/src/redact.ts b/packages/dump/src/redact.ts new file mode 100644 index 0000000000..54ba0a2644 --- /dev/null +++ b/packages/dump/src/redact.ts @@ -0,0 +1,86 @@ +import type { DumpEvent } from '@floway-dev/pipeline'; + +const SENSITIVE_HEADERS = new Set([ + 'api-key', + 'authorization', + 'cookie', + 'proxy-authorization', + 'set-cookie', + 'x-api-key', + 'x-goog-api-key', +]); + +export const REDACTED_HEADER_VALUE = '[redacted]'; + +export const isSensitiveHeader = (name: string): boolean => SENSITIVE_HEADERS.has(name.toLowerCase()); + +// The owner of a key may read the dump, but the upstream credentials in it +// belong to an operator who granted Model.Use, not to that key owner. +export const redactSensitiveHeaders = (headers: ReadonlyArray): Array<[string, string]> => + headers.map(([name, value]) => [name, isSensitiveHeader(name) ? REDACTED_HEADER_VALUE : value]); + +/** Header lists a run records about its upstream exchanges. The client's own request headers + * are the key owner's, so `ingress.http.headers` keeps them. */ +const upstreamHeaderLists = (key: string, value: unknown, resolve: (value: unknown) => unknown): unknown[] => { + if (key === 'ingress.http.headers') return []; + if (key.endsWith('.http.headers')) return [value]; + if (key !== 'response.http.exchange') return []; + const exchange = resolve(value); + if (typeof exchange !== 'object' || exchange === null || !('headers' in exchange)) return []; + return [exchange.headers]; +}; + +/** + * The run's recorded events with every upstream credential header replaced by its redacted + * form. A value a stage marked secret is already recorded redacted; this covers the plain + * strings an upstream sent back or an operator configured as a fixed header. Header strings are + * interned, so a value is rewritten in the object event that defined it — rewriting the tuple + * alone would leave the credential in that node. + */ +export const redactRunCredentials = (ndjson: string): string => { + const events = ndjson.split('\n').filter(line => line.length > 0).map(line => JSON.parse(line) as DumpEvent); + const nodes = new Map(); + for (const event of events) { + if (event.type !== 'object') continue; + event.nodes.forEach((_, index) => nodes.set(event.fromObjectId + index, { values: event.nodes as unknown[], index })); + } + const reference = (value: unknown): number | null => + typeof value === 'object' && value !== null && '$' in value && typeof value.$ === 'number' ? value.$ : null; + const slot = (id: number) => { + const found = nodes.get(id); + if (found === undefined) throw new Error(`Run events reference missing object ${id}`); + return found; + }; + const resolve = (value: unknown): unknown => { + const id = reference(value); + if (id === null) return value; + const found = slot(id); + return found.values[found.index]; + }; + const redactValue = (pair: unknown[]): void => { + const id = reference(pair[1]); + if (id === null) { + if (typeof pair[1] === 'string') pair[1] = REDACTED_HEADER_VALUE; + return; + } + const found = slot(id); + if (typeof found.values[found.index] === 'string') found.values[found.index] = REDACTED_HEADER_VALUE; + }; + for (const event of events) { + if (event.type !== 'stage.entered' && event.type !== 'stage.leaved') continue; + for (const [key, value] of Object.entries(event.facts ?? {})) { + for (const list of upstreamHeaderLists(key, value, resolve)) { + const headers = resolve(list); + if (!Array.isArray(headers)) throw new Error(`Run events record invalid headers at ${key}`); + for (const entry of headers) { + const pair = resolve(entry); + if (!Array.isArray(pair) || pair.length !== 2) throw new Error(`Run events record an invalid header at ${key}`); + const name = resolve(pair[0]); + if (typeof name !== 'string') throw new Error(`Run events record an invalid header name at ${key}`); + if (isSensitiveHeader(name)) redactValue(pair); + } + } + } + } + return events.map(event => `${JSON.stringify(event)}\n`).join(''); +}; diff --git a/packages/dump/src/wire.ts b/packages/dump/src/wire.ts index 023ac20ce1..608a65e30a 100644 --- a/packages/dump/src/wire.ts +++ b/packages/dump/src/wire.ts @@ -1,6 +1,7 @@ +import { redactRunCredentials } from './redact.ts'; import type { DumpRecord, StoredDumpRecord } from './types.ts'; export const dumpRecordToWire = (record: StoredDumpRecord): DumpRecord => ({ meta: record.meta, - events: new TextDecoder('utf-8', { fatal: true }).decode(record.events), + events: redactRunCredentials(new TextDecoder('utf-8', { fatal: true }).decode(record.events)), }); diff --git a/packages/gateway/src/dump/wire.ts b/packages/gateway/src/dump/wire.ts index 5d61500279..448a58bb57 100644 --- a/packages/gateway/src/dump/wire.ts +++ b/packages/gateway/src/dump/wire.ts @@ -7,28 +7,12 @@ import type { StoredDumpResponseBody, StoredDumpUpstreamResponse, } from './types.ts'; +import { redactRunCredentials, redactSensitiveHeaders } from '@floway-dev/dump'; import { encodeBase64, isTextualMediaType } from '@floway-dev/protocols/common'; const contentTypeOf = (headers: ReadonlyArray): string => headers.find(([name]) => name.toLowerCase() === 'content-type')?.[1] ?? ''; -const SENSITIVE_HEADERS = new Set([ - 'api-key', - 'authorization', - 'cookie', - 'proxy-authorization', - 'set-cookie', - 'x-api-key', - 'x-goog-api-key', -]); - -const REDACTED_HEADER_VALUE = '[redacted]'; - -// The owner of a key may read the dump, but the upstream credentials in it -// belong to an operator who granted Model.Use, not to that key owner. -const redactSensitiveHeaders = (headers: ReadonlyArray): Array<[string, string]> => - headers.map(([name, value]) => [name, SENSITIVE_HEADERS.has(name.toLowerCase()) ? REDACTED_HEADER_VALUE : value]); - // Wire encoding decision: textual content-types try UTF-8 first and fall // back to base64 when the bytes do not decode cleanly (a content-type // that lied about being text). @@ -68,7 +52,7 @@ export const dumpRecordToWire = (record: StoredDumpRecord): DumpRecord => { return { shape: 'run', meta: record.meta, - events: new TextDecoder('utf-8', { fatal: true }).decode(record.events), + events: redactRunCredentials(new TextDecoder('utf-8', { fatal: true }).decode(record.events)), }; } return { From c3a80d4af0b90909f9c1b03082b7ed8de30da445 Mon Sep 17 00:00:00 2001 From: Menci Date: Sat, 14 Nov 2026 09:39:12 +0800 Subject: [PATCH 79/79] fix(web): drop the locale left unused by the extracted record timing --- apps/web/src/components/requests/detail.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/web/src/components/requests/detail.tsx b/apps/web/src/components/requests/detail.tsx index 8aaaf20ec0..49baa975fb 100644 --- a/apps/web/src/components/requests/detail.tsx +++ b/apps/web/src/components/requests/detail.tsx @@ -76,7 +76,6 @@ export function RequestDetailPanel({ collected, upstreamCollected, error, record function RecordDetail({ record, collected, upstreamCollected }: { record: DumpEdgeRecord; collected: CollectedStream | null; upstreamCollected: CollectedStream | null }) { const { t } = useTranslation(); - const locale = useLocale(); const [source, setSource] = useState('response'); const [view, setView] = useState('collected'); const [detailsOpen, setDetailsOpen] = useState(false);