Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 86 additions & 1 deletion lib/server/proxy/responses.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1046,6 +1046,83 @@ const normalizeTranscriptMessageToolNames = (
});
};

const toResponsesUsageNumber = (value: unknown): number => {
const numeric =
typeof value === 'number' ? value : Number.parseFloat(String(value ?? ''));

if (!Number.isFinite(numeric) || numeric < 0) {
return 0;
}

return numeric;
};

const mapChatUsageToResponses = (usage: unknown): Record<string, unknown> => {
if (!usage || typeof usage !== 'object') {
return {
input_tokens: 0,
input_tokens_details: { cached_tokens: 0 },
output_tokens: 0,
output_tokens_details: { reasoning_tokens: 0 },
total_tokens: 0,
};
}

const value = usage as {
cache_creation_input_tokens?: unknown;
cache_read_input_tokens?: unknown;
completion_tokens?: unknown;
completion_tokens_details?: { reasoning_tokens?: unknown };
completion_thinking_tokens?: unknown;
input_tokens_details?: { cached_tokens?: unknown };
prompt_cache_hit_tokens?: unknown;
prompt_cache_miss_tokens?: unknown;
prompt_cache_write_tokens?: unknown;
prompt_tokens?: unknown;
prompt_tokens_details?: {
cache_creation_tokens?: unknown;
cached_tokens?: unknown;
};
total_tokens?: unknown;
};
const outputTokens = toResponsesUsageNumber(value.completion_tokens);
const cachedTokens = toResponsesUsageNumber(
value.prompt_tokens_details?.cached_tokens ??
value.input_tokens_details?.cached_tokens ??
value.cache_read_input_tokens ??
value.prompt_cache_hit_tokens,
);
const cacheCreationTokens = toResponsesUsageNumber(
value.prompt_tokens_details?.cache_creation_tokens ??
value.cache_creation_input_tokens ??
value.prompt_cache_write_tokens,
);
const reasoningTokens = toResponsesUsageNumber(
value.completion_tokens_details?.reasoning_tokens ??
value.completion_thinking_tokens,
);
// Chat usage is the single source of truth for both shapes. Keep the
// Responses counters faithful to it so clients never see zeroed metrics.
// prompt_tokens already covers its cached and created subsets, so the
// split counters are only summed when prompt_tokens is missing. Otherwise
// cached tokens would exceed the reported input total.
const inputTokens = toResponsesUsageNumber(
value.prompt_tokens ??
toResponsesUsageNumber(value.prompt_cache_miss_tokens) +
cachedTokens +
cacheCreationTokens,
);

return {
input_tokens: inputTokens,
input_tokens_details: { cached_tokens: cachedTokens },
output_tokens: outputTokens,
output_tokens_details: { reasoning_tokens: reasoningTokens },
total_tokens:
toResponsesUsageNumber(value.total_tokens) || inputTokens + outputTokens,
};
};

const mapChatResponseToResponsesPayload = async (
accessKeyId: string | null,
credentialFilename: string | null,
Expand Down Expand Up @@ -1127,7 +1204,7 @@ const mapChatResponseToResponsesPayload = async (
model,
output,
output_text: outputText,
usage: upstreamPayload.usage ?? null,
usage: mapChatUsageToResponses(upstreamPayload.usage),
metadata: defaults.metadata ?? {},
previous_response_id: previousResponseId,
};
Expand Down Expand Up @@ -1180,6 +1257,7 @@ const createResponsesEventStream = async (
const toolCallStates = new Map<string, StreamingToolCallState>();
const toolCallStateKeys = new Map<string, string>();
let nextToolCallOutputIndex = 1;
let latestUsage: unknown = null;
let reader: ReadableStreamDefaultReader<Uint8Array> | null = null;
let cancelled = false;
const releaseReader = (): void => {
Expand Down Expand Up @@ -1405,6 +1483,7 @@ const createResponsesEventStream = async (
status: 'completed',
output_text: outputText,
previous_response_id: previousResponseId,
usage: mapChatUsageToResponses(latestUsage),
output: [
...(outputText
? [buildStreamingMessageItem('completed')]
Expand Down Expand Up @@ -1464,7 +1543,13 @@ const createResponsesEventStream = async (
tool_calls?: ChatResponseToolCall[];
};
}>;
usage?: unknown;
};
// The final upstream chunk carries the aggregated usage, so
// remember it for the downstream response.completed event.
if (payload.usage !== undefined) {
latestUsage = payload.usage;
}
const delta = payload.choices?.[0]?.delta;

if (delta?.content) {
Expand Down
142 changes: 142 additions & 0 deletions tests/server/responses-memory.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,148 @@ describe('Responses memory bounds', () => {
expect(writePgSession).not.toHaveBeenCalled();
});

it('maps Chat usage onto Responses usage for non-streamed requests', async () => {
vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(
new Response(
JSON.stringify({
choices: [{ message: { content: 'usage answer' } }],
usage: {
completion_tokens: 506,
completion_tokens_details: { reasoning_tokens: 12 },
prompt_tokens: 281734,
prompt_tokens_details: { cached_tokens: 281408 },
total_tokens: 282240,
},
}),
{ headers: { 'Content-Type': 'application/json' } },
),
);

const response = await handleResponsesRequest(makeRequest(), {
input: 'usage please',
model: 'gpt-5.5',
});
const payload = (await response.json()) as Record<string, unknown>;

expect(payload.usage).toEqual({
input_tokens: 281734,
input_tokens_details: { cached_tokens: 281408 },
output_tokens: 506,
output_tokens_details: { reasoning_tokens: 12 },
total_tokens: 282240,
});
});

it('emits mapped usage in streamed response.completed events', async () => {
vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(
new Response(
'data: {"choices":[{"delta":{"content":"usage "}}]}\n\n' +
'data: {"choices":[{"delta":{"content":"stream"}}]}\n\n' +
'data: {"choices":[],"usage":{"prompt_tokens":281734,"completion_tokens":506,"total_tokens":282240,"prompt_tokens_details":{"cached_tokens":281408},"prompt_cache_hit_tokens":281408,"prompt_cache_miss_tokens":326,"completion_tokens_details":{"reasoning_tokens":12}}}\n\n' +
'data: [DONE]\n\n',
{ headers: { 'Content-Type': 'text/event-stream' } },
),
);

const response = await handleResponsesRequest(makeRequest(), {
input: 'stream usage please',
model: 'gpt-5.5',
stream: true,
});
const body = await response.text();

expect(body).toContain('"output_text":"usage stream"');
expect(body).toContain('"input_tokens":281734');
expect(body).toContain('"cached_tokens":281408');
expect(body).toContain('"output_tokens":506');
expect(body).toContain('"reasoning_tokens":12');
expect(body).toContain('"total_tokens":282240');
});

it('reports zeroed Responses usage when upstream omits usage', async () => {
vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(
makeChatResponse('no usage here'),
);

const response = await handleResponsesRequest(makeRequest(), {
input: 'missing usage',
model: 'gpt-5.5',
});
const payload = (await response.json()) as Record<string, unknown>;

expect(payload.usage).toEqual({
input_tokens: 0,
input_tokens_details: { cached_tokens: 0 },
output_tokens: 0,
output_tokens_details: { reasoning_tokens: 0 },
total_tokens: 0,
});
});

it('does not double-count cache creation in the fallback total', async () => {
vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(
new Response(
JSON.stringify({
choices: [{ message: { content: 'derived totals' } }],
usage: {
completion_tokens: 3,
prompt_tokens: 10,
prompt_tokens_details: { cache_creation_tokens: 2 },
},
}),
{ headers: { 'Content-Type': 'application/json' } },
),
);

const response = await handleResponsesRequest(makeRequest(), {
input: 'derive totals',
model: 'gpt-5.5',
});
const payload = (await response.json()) as Record<string, unknown>;

// prompt_tokens already includes cache creation, so it must not be
// added again when computing the fallback total.
expect(payload.usage).toEqual({
input_tokens: 10,
input_tokens_details: { cached_tokens: 0 },
output_tokens: 3,
output_tokens_details: { reasoning_tokens: 0 },
total_tokens: 13,
});
});

it('sums split cache counters when upstream omits prompt_tokens', async () => {
vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(
new Response(
JSON.stringify({
choices: [{ message: { content: 'split counters' } }],
usage: {
completion_tokens: 506,
prompt_cache_hit_tokens: 281408,
prompt_cache_miss_tokens: 326,
},
}),
{ headers: { 'Content-Type': 'application/json' } },
),
);

const response = await handleResponsesRequest(makeRequest(), {
input: 'split counters',
model: 'gpt-5.5',
});
const payload = (await response.json()) as Record<string, unknown>;

// Without prompt_tokens, the split counters must be summed so cached
// tokens never exceed the reported input total.
expect(payload.usage).toEqual({
input_tokens: 281734,
input_tokens_details: { cached_tokens: 281408 },
output_tokens: 506,
output_tokens_details: { reasoning_tokens: 0 },
total_tokens: 282240,
});
});

it('bounds incomplete SSE frames in every proxy stream', async () => {
const oversizedFrame = 'x'.repeat(1_000_001);
const fetchMock = vi
Expand Down
Loading