diff --git a/lib/server/proxy/responses.ts b/lib/server/proxy/responses.ts index 07b5292..b82583e 100644 --- a/lib/server/proxy/responses.ts +++ b/lib/server/proxy/responses.ts @@ -1046,6 +1046,83 @@ const normalizeTranscriptMessageToolNames = ( }); }; +const toResponsesUsageNumber = (value: unknown): number => { + const numeric = + typeof value === 'number' ? value : Number.parseFloat(String(value ?? '')); + + if (!Number.isFinite(numeric) || numeric < 0) { + return 0; + } + + return numeric; +}; + +const mapChatUsageToResponses = (usage: unknown): Record => { + if (!usage || typeof usage !== 'object') { + return { + input_tokens: 0, + input_tokens_details: { cached_tokens: 0 }, + output_tokens: 0, + output_tokens_details: { reasoning_tokens: 0 }, + total_tokens: 0, + }; + } + + const value = usage as { + cache_creation_input_tokens?: unknown; + cache_read_input_tokens?: unknown; + completion_tokens?: unknown; + completion_tokens_details?: { reasoning_tokens?: unknown }; + completion_thinking_tokens?: unknown; + input_tokens_details?: { cached_tokens?: unknown }; + prompt_cache_hit_tokens?: unknown; + prompt_cache_miss_tokens?: unknown; + prompt_cache_write_tokens?: unknown; + prompt_tokens?: unknown; + prompt_tokens_details?: { + cache_creation_tokens?: unknown; + cached_tokens?: unknown; + }; + total_tokens?: unknown; + }; + const outputTokens = toResponsesUsageNumber(value.completion_tokens); + const cachedTokens = toResponsesUsageNumber( + value.prompt_tokens_details?.cached_tokens ?? + value.input_tokens_details?.cached_tokens ?? + value.cache_read_input_tokens ?? + value.prompt_cache_hit_tokens, + ); + const cacheCreationTokens = toResponsesUsageNumber( + value.prompt_tokens_details?.cache_creation_tokens ?? + value.cache_creation_input_tokens ?? + value.prompt_cache_write_tokens, + ); + const reasoningTokens = toResponsesUsageNumber( + value.completion_tokens_details?.reasoning_tokens ?? + value.completion_thinking_tokens, + ); + // Chat usage is the single source of truth for both shapes. Keep the + // Responses counters faithful to it so clients never see zeroed metrics. + // prompt_tokens already covers its cached and created subsets, so the + // split counters are only summed when prompt_tokens is missing. Otherwise + // cached tokens would exceed the reported input total. + const inputTokens = toResponsesUsageNumber( + value.prompt_tokens ?? + toResponsesUsageNumber(value.prompt_cache_miss_tokens) + + cachedTokens + + cacheCreationTokens, + ); + + return { + input_tokens: inputTokens, + input_tokens_details: { cached_tokens: cachedTokens }, + output_tokens: outputTokens, + output_tokens_details: { reasoning_tokens: reasoningTokens }, + total_tokens: + toResponsesUsageNumber(value.total_tokens) || inputTokens + outputTokens, + }; +}; + const mapChatResponseToResponsesPayload = async ( accessKeyId: string | null, credentialFilename: string | null, @@ -1127,7 +1204,7 @@ const mapChatResponseToResponsesPayload = async ( model, output, output_text: outputText, - usage: upstreamPayload.usage ?? null, + usage: mapChatUsageToResponses(upstreamPayload.usage), metadata: defaults.metadata ?? {}, previous_response_id: previousResponseId, }; @@ -1180,6 +1257,7 @@ const createResponsesEventStream = async ( const toolCallStates = new Map(); const toolCallStateKeys = new Map(); let nextToolCallOutputIndex = 1; + let latestUsage: unknown = null; let reader: ReadableStreamDefaultReader | null = null; let cancelled = false; const releaseReader = (): void => { @@ -1405,6 +1483,7 @@ const createResponsesEventStream = async ( status: 'completed', output_text: outputText, previous_response_id: previousResponseId, + usage: mapChatUsageToResponses(latestUsage), output: [ ...(outputText ? [buildStreamingMessageItem('completed')] @@ -1464,7 +1543,13 @@ const createResponsesEventStream = async ( tool_calls?: ChatResponseToolCall[]; }; }>; + usage?: unknown; }; + // The final upstream chunk carries the aggregated usage, so + // remember it for the downstream response.completed event. + if (payload.usage !== undefined) { + latestUsage = payload.usage; + } const delta = payload.choices?.[0]?.delta; if (delta?.content) { diff --git a/tests/server/responses-memory.test.ts b/tests/server/responses-memory.test.ts index 32b4487..851be01 100644 --- a/tests/server/responses-memory.test.ts +++ b/tests/server/responses-memory.test.ts @@ -339,6 +339,148 @@ describe('Responses memory bounds', () => { expect(writePgSession).not.toHaveBeenCalled(); }); + it('maps Chat usage onto Responses usage for non-streamed requests', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce( + new Response( + JSON.stringify({ + choices: [{ message: { content: 'usage answer' } }], + usage: { + completion_tokens: 506, + completion_tokens_details: { reasoning_tokens: 12 }, + prompt_tokens: 281734, + prompt_tokens_details: { cached_tokens: 281408 }, + total_tokens: 282240, + }, + }), + { headers: { 'Content-Type': 'application/json' } }, + ), + ); + + const response = await handleResponsesRequest(makeRequest(), { + input: 'usage please', + model: 'gpt-5.5', + }); + const payload = (await response.json()) as Record; + + expect(payload.usage).toEqual({ + input_tokens: 281734, + input_tokens_details: { cached_tokens: 281408 }, + output_tokens: 506, + output_tokens_details: { reasoning_tokens: 12 }, + total_tokens: 282240, + }); + }); + + it('emits mapped usage in streamed response.completed events', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce( + new Response( + 'data: {"choices":[{"delta":{"content":"usage "}}]}\n\n' + + 'data: {"choices":[{"delta":{"content":"stream"}}]}\n\n' + + 'data: {"choices":[],"usage":{"prompt_tokens":281734,"completion_tokens":506,"total_tokens":282240,"prompt_tokens_details":{"cached_tokens":281408},"prompt_cache_hit_tokens":281408,"prompt_cache_miss_tokens":326,"completion_tokens_details":{"reasoning_tokens":12}}}\n\n' + + 'data: [DONE]\n\n', + { headers: { 'Content-Type': 'text/event-stream' } }, + ), + ); + + const response = await handleResponsesRequest(makeRequest(), { + input: 'stream usage please', + model: 'gpt-5.5', + stream: true, + }); + const body = await response.text(); + + expect(body).toContain('"output_text":"usage stream"'); + expect(body).toContain('"input_tokens":281734'); + expect(body).toContain('"cached_tokens":281408'); + expect(body).toContain('"output_tokens":506'); + expect(body).toContain('"reasoning_tokens":12'); + expect(body).toContain('"total_tokens":282240'); + }); + + it('reports zeroed Responses usage when upstream omits usage', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce( + makeChatResponse('no usage here'), + ); + + const response = await handleResponsesRequest(makeRequest(), { + input: 'missing usage', + model: 'gpt-5.5', + }); + const payload = (await response.json()) as Record; + + expect(payload.usage).toEqual({ + input_tokens: 0, + input_tokens_details: { cached_tokens: 0 }, + output_tokens: 0, + output_tokens_details: { reasoning_tokens: 0 }, + total_tokens: 0, + }); + }); + + it('does not double-count cache creation in the fallback total', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce( + new Response( + JSON.stringify({ + choices: [{ message: { content: 'derived totals' } }], + usage: { + completion_tokens: 3, + prompt_tokens: 10, + prompt_tokens_details: { cache_creation_tokens: 2 }, + }, + }), + { headers: { 'Content-Type': 'application/json' } }, + ), + ); + + const response = await handleResponsesRequest(makeRequest(), { + input: 'derive totals', + model: 'gpt-5.5', + }); + const payload = (await response.json()) as Record; + + // prompt_tokens already includes cache creation, so it must not be + // added again when computing the fallback total. + expect(payload.usage).toEqual({ + input_tokens: 10, + input_tokens_details: { cached_tokens: 0 }, + output_tokens: 3, + output_tokens_details: { reasoning_tokens: 0 }, + total_tokens: 13, + }); + }); + + it('sums split cache counters when upstream omits prompt_tokens', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce( + new Response( + JSON.stringify({ + choices: [{ message: { content: 'split counters' } }], + usage: { + completion_tokens: 506, + prompt_cache_hit_tokens: 281408, + prompt_cache_miss_tokens: 326, + }, + }), + { headers: { 'Content-Type': 'application/json' } }, + ), + ); + + const response = await handleResponsesRequest(makeRequest(), { + input: 'split counters', + model: 'gpt-5.5', + }); + const payload = (await response.json()) as Record; + + // Without prompt_tokens, the split counters must be summed so cached + // tokens never exceed the reported input total. + expect(payload.usage).toEqual({ + input_tokens: 281734, + input_tokens_details: { cached_tokens: 281408 }, + output_tokens: 506, + output_tokens_details: { reasoning_tokens: 0 }, + total_tokens: 282240, + }); + }); + it('bounds incomplete SSE frames in every proxy stream', async () => { const oversizedFrame = 'x'.repeat(1_000_001); const fetchMock = vi