From ee075cf6eb3e96cc815a808fdc0d9f1f6b02c85f Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2026 11:08:55 +0000 Subject: [PATCH 1/5] feat(server-utils): Record time to first chunk on streamed gen_ai spans Set `gen_ai.response.time_to_first_chunk` on streamed chat spans of the pi-ai, OpenAI, Anthropic, Google GenAI, Mistral, Workers AI, Vercel AI, and LangChain integrations, and map Mastra's `completionStartTime`. Only chunks with output count, following the OpenTelemetry GenAI definition. Co-Authored-By: Francesco Gringl-Novy <2411343+mydea@users.noreply.github.com> --- .../suites/tracing/pi-ai/test.ts | 3 + .../src/ai/anthropic-ai/streaming.ts | 19 +- packages/server-utils/src/ai/core/utils.ts | 27 ++- .../src/ai/google-genai/streaming.ts | 13 +- .../server-utils/src/ai/langchain/index.ts | 19 +- packages/server-utils/src/ai/mastra/types.ts | 2 + packages/server-utils/src/ai/mastra/utils.ts | 4 + .../server-utils/src/ai/mistral/streaming.ts | 21 +- .../server-utils/src/ai/openai/streaming.ts | 21 +- packages/server-utils/src/ai/pi-ai/chat.ts | 39 ++- packages/server-utils/src/ai/pi-ai/types.ts | 1 + .../src/ai/workers-ai/streaming.ts | 19 +- .../server-utils/src/integrations/pi-ai.ts | 3 +- .../src/integrations/vercel-ai/util.ts | 32 +++ .../vercel-ai/vercel-ai-dc-subscriber.ts | 1 + .../vercel-ai-orchestrion-subscriber.ts | 1 + .../tracing/google-genai-streaming.test.ts | 1 + .../openai-streaming-tool-calls.test.ts | 1 + .../test/ai/lib/tracing/stream-errors.test.ts | 1 + .../lib/tracing/time-to-first-chunk.test.ts | 228 ++++++++++++++++++ .../lib/tracing/workers-ai-streaming.test.ts | 1 + .../test/integrations/pi-ai.test.ts | 36 +++ 22 files changed, 475 insertions(+), 18 deletions(-) create mode 100644 packages/server-utils/test/ai/lib/tracing/time-to-first-chunk.test.ts diff --git a/dev-packages/node-integration-tests/suites/tracing/pi-ai/test.ts b/dev-packages/node-integration-tests/suites/tracing/pi-ai/test.ts index 995bcb7858fa..1bff015fffba 100644 --- a/dev-packages/node-integration-tests/suites/tracing/pi-ai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/pi-ai/test.ts @@ -7,6 +7,7 @@ import { GEN_AI_REQUEST_MODEL, GEN_AI_RESPONSE_FINISH_REASONS, GEN_AI_RESPONSE_STREAMING, + GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK, GEN_AI_SYSTEM_INSTRUCTIONS, GEN_AI_TOOL_DEFINITIONS, GEN_AI_USAGE_INPUT_TOKENS, @@ -60,6 +61,7 @@ describe.runIf(supports({ min: 22 }))('pi-ai integration', () => { expect(toolCall!.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('faux'); expect(toolCall!.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('faux-model'); expect(toolCall!.attributes[GEN_AI_RESPONSE_STREAMING]?.value).toBe(true); + expect(toolCall!.attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]?.value).toBeGreaterThan(0); expect(toolCall!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]?.value).toBe('You are a weather assistant.'); expect(toolCall!.attributes[GEN_AI_TOOL_DEFINITIONS]?.value).toContain('get_weather'); expect(toolCall!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value).toBe('["tool_call"]'); @@ -75,6 +77,7 @@ describe.runIf(supports({ min: 22 }))('pi-ai integration', () => { ]); expect(answer!.attributes[GEN_AI_RESPONSE_STREAMING]).toBeUndefined(); + expect(answer!.attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]).toBeUndefined(); expect(answer!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value).toBe('["stop"]'); expect( JSON.parse(String(answer!.attributes[GEN_AI_INPUT_MESSAGES]?.value)).map( diff --git a/packages/server-utils/src/ai/anthropic-ai/streaming.ts b/packages/server-utils/src/ai/anthropic-ai/streaming.ts index 8931d6098b8d..950ae09b610c 100644 --- a/packages/server-utils/src/ai/anthropic-ai/streaming.ts +++ b/packages/server-utils/src/ai/anthropic-ai/streaming.ts @@ -1,6 +1,6 @@ import { captureException, SPAN_STATUS_ERROR } from '@sentry/core'; import type { Span } from '@sentry/core'; -import { endStreamSpan, getErrorStatus } from '../core/utils'; +import { endStreamSpan, getErrorStatus, recordTimeToFirstChunk } from '../core/utils'; import type { AnthropicAiStreamingEvent } from './types'; import { mapAnthropicErrorToStatusMessage } from './utils'; @@ -186,6 +186,19 @@ function handleContentBlockStop(event: AnthropicAiStreamingEvent, state: Streami delete state.activeToolBlocks[event.index]; } +// A text block opens empty, so only its deltas are output. A tool use block opens with the tool name. +// A `signature_delta` only signs the thinking that came before it. +function hasOutput(event: AnthropicAiStreamingEvent): boolean { + if (event.type === 'content_block_start') { + return !!event.content_block?.name; + } + if (event.type === 'content_block_delta' && event.delta) { + const delta = event.delta as { text?: unknown; partial_json?: unknown; thinking?: unknown }; + return !!(delta.text || delta.partial_json || delta.thinking); + } + return false; +} + /** * Processes an event * @param event - The event to process @@ -206,6 +219,10 @@ function processEvent( const isError = isErrorEvent(event, span); if (isError) return; + if (hasOutput(event)) { + recordTimeToFirstChunk(span); + } + handleMessageMetadata(event, state); // Tool call events are sent via 3 separate events: diff --git a/packages/server-utils/src/ai/core/utils.ts b/packages/server-utils/src/ai/core/utils.ts index 816c023ab08e..fc052b1435e6 100644 --- a/packages/server-utils/src/ai/core/utils.ts +++ b/packages/server-utils/src/ai/core/utils.ts @@ -2,7 +2,15 @@ /** * Shared utils for AI integrations (OpenAI, Anthropic, Verce.AI, etc.) */ -import { getClient, isObjectLike, isThenable, SPAN_STATUS_ERROR, stringify } from '@sentry/core'; +import { + getClient, + isObjectLike, + isThenable, + SPAN_STATUS_ERROR, + spanToJSON, + stringify, + timestampInSeconds, +} from '@sentry/core'; import type { Span, SpanStatus } from '@sentry/core'; import { GEN_AI_OUTPUT_MESSAGES, @@ -11,6 +19,7 @@ import { GEN_AI_RESPONSE_MODEL, GEN_AI_RESPONSE_STREAMING, GEN_AI_RESPONSE_TEXT, + GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK, GEN_AI_RESPONSE_TOOL_CALLS, GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, @@ -264,6 +273,22 @@ export function markCancelledIfAborted(span: Span, abortSignal: AbortSignal | un } } +const spansWithFirstChunk = new WeakSet(); + +/** + * Sets `gen_ai.response.time_to_first_chunk` on a streamed span, in seconds from the span start. + * Call it for each chunk that carries new output (text, reasoning, refusals, or tool-call names or + * arguments), not for chunks with only metadata, usage, or an empty block start. Only the first call + * per span records a value. + */ +export function recordTimeToFirstChunk(span: Span): void { + if (spansWithFirstChunk.has(span)) { + return; + } + spansWithFirstChunk.add(span); + span.setAttribute(GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK, timestampInSeconds() - spanToJSON(span).start_timestamp); +} + /** * Ends a streaming span by setting all accumulated response attributes and ending the span. * Shared across OpenAI, Anthropic, and Google GenAI streaming implementations. diff --git a/packages/server-utils/src/ai/google-genai/streaming.ts b/packages/server-utils/src/ai/google-genai/streaming.ts index 2a822e3a08c2..54625d69ac6b 100644 --- a/packages/server-utils/src/ai/google-genai/streaming.ts +++ b/packages/server-utils/src/ai/google-genai/streaming.ts @@ -1,7 +1,7 @@ import { captureException, SPAN_STATUS_ERROR } from '@sentry/core'; import type { Span } from '@sentry/core'; -import { endStreamSpan, getErrorStatus } from '../core/utils'; -import type { GoogleGenAIResponse } from './types'; +import { endStreamSpan, getErrorStatus, recordTimeToFirstChunk } from '../core/utils'; +import type { ContentPart, GoogleGenAIResponse } from './types'; import type { MessagePart } from './utils'; import { candidatesToMessageParts, setOutputMessagesAttribute } from './utils'; @@ -102,6 +102,12 @@ function handleCandidateContent(chunk: GoogleGenAIResponse, state: StreamingStat } } +// A part carries text (also thoughts), a function call, inline data such as audio or images, or +// generated code. Parts with only a thought signature carry no output. +function hasOutput(part: ContentPart | undefined): boolean { + return !!part && !!(part.text || part.functionCall || part.inlineData || part.executableCode); +} + /** * Processes a single chunk from the Google GenAI stream * @param chunk - The chunk to process @@ -111,6 +117,9 @@ function handleCandidateContent(chunk: GoogleGenAIResponse, state: StreamingStat */ function processChunk(chunk: GoogleGenAIResponse, state: StreamingState, recordOutputs: boolean, span: Span): void { if (!chunk || isErrorChunk(chunk, span)) return; + if (chunk.candidates?.some(candidate => candidate?.content?.parts?.some(hasOutput))) { + recordTimeToFirstChunk(span); + } handleResponseMetadata(chunk, state); handleCandidateContent(chunk, state, recordOutputs); } diff --git a/packages/server-utils/src/ai/langchain/index.ts b/packages/server-utils/src/ai/langchain/index.ts index e05ea6f38a81..42de5c8b5695 100644 --- a/packages/server-utils/src/ai/langchain/index.ts +++ b/packages/server-utils/src/ai/langchain/index.ts @@ -28,7 +28,7 @@ import { } from '@sentry/conventions/attributes'; import { GEN_AI_CHAT, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op'; import { DEBUG_BUILD } from '../../debug-build'; -import { resolveAIRecordingOptions } from '../core/utils'; +import { recordTimeToFirstChunk, resolveAIRecordingOptions } from '../core/utils'; import { LANGCHAIN_ORIGIN, MAX_TRACKED_LANGCHAIN_RUNS } from './constants'; import { addTypeSafeClassifierResponseAttributes, @@ -269,6 +269,23 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): ); }, + // Called for each streamed chunk. A tool call chunk arrives with an empty token. + handleLLMNewToken( + token: string, + _idx: unknown, + runId: string, + _parentRunId?: string, + _tags?: string[], + fields?: unknown, + ) { + const span = runs.get(runId)?.span; + const toolCallChunks = (fields as { chunk?: { message?: { tool_call_chunks?: unknown[] } } } | undefined)?.chunk + ?.message?.tool_call_chunks; + if (span && (token || toolCallChunks?.length)) { + recordTimeToFirstChunk(span); + } + }, + // LLM End Handler - note: handleLLMEnd with capital LLM (used by both LLMs and chat models!) handleLLMEnd( output: unknown, diff --git a/packages/server-utils/src/ai/mastra/types.ts b/packages/server-utils/src/ai/mastra/types.ts index 5de6ad91efcc..56bd69fbd257 100644 --- a/packages/server-utils/src/ai/mastra/types.ts +++ b/packages/server-utils/src/ai/mastra/types.ts @@ -48,6 +48,8 @@ export interface MastraSpanAttributes { responseId?: string; finishReason?: string; streaming?: boolean; + /** When the first streamed chunk arrived. */ + completionStartTime?: Date; usage?: MastraUsageStats; parameters?: { temperature?: number; diff --git a/packages/server-utils/src/ai/mastra/utils.ts b/packages/server-utils/src/ai/mastra/utils.ts index 3f6d32ed8424..ddf66db60968 100644 --- a/packages/server-utils/src/ai/mastra/utils.ts +++ b/packages/server-utils/src/ai/mastra/utils.ts @@ -22,6 +22,7 @@ import { GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_MODEL, GEN_AI_RESPONSE_STREAMING, + GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK, GEN_AI_RESPONSE_TEXT, GEN_AI_SYSTEM_INSTRUCTIONS, GEN_AI_TOOL_CALL_ARGUMENTS, @@ -212,6 +213,9 @@ function addModelAttributes( attributes[GEN_AI_RESPONSE_MODEL] = attrs.responseModel; attributes[GEN_AI_RESPONSE_ID] = attrs.responseId; attributes[GEN_AI_RESPONSE_STREAMING] = attrs.streaming; + attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK] = attrs.completionStartTime + ? (attrs.completionStartTime.getTime() - span.startTime.getTime()) / 1000 + : undefined; attributes[GEN_AI_RESPONSE_FINISH_REASONS] = attrs.finishReason ? JSON.stringify([attrs.finishReason]) : undefined; attributes[GEN_AI_AGENT_NAME] = span.entityName; diff --git a/packages/server-utils/src/ai/mistral/streaming.ts b/packages/server-utils/src/ai/mistral/streaming.ts index 74acc6e43b70..fabb835f6ad9 100644 --- a/packages/server-utils/src/ai/mistral/streaming.ts +++ b/packages/server-utils/src/ai/mistral/streaming.ts @@ -1,6 +1,6 @@ import type { Span } from '@sentry/core'; import { SPAN_STATUS_ERROR } from '@sentry/core'; -import { endStreamSpan, setOutputMessagesAttribute } from '../core/utils'; +import { endStreamSpan, recordTimeToFirstChunk, setOutputMessagesAttribute } from '../core/utils'; import type { MistralCompletionChunk, MistralToolCall } from './types'; import { contentToString } from './utils'; @@ -74,7 +74,10 @@ function processToolCalls(toolCalls: MistralToolCall[], state: StreamingState): } } -function processChunk(chunk: MistralCompletionChunk, state: StreamingState, recordOutputs: boolean): void { +function processChunk(chunk: MistralCompletionChunk, state: StreamingState, recordOutputs: boolean, span: Span): void { + if (chunk.choices?.some(({ delta }) => !!delta && (!!contentToString(delta.content) || !!delta.toolCalls?.length))) { + recordTimeToFirstChunk(span); + } state.responseId = chunk.id ?? state.responseId; state.responseModel = chunk.model ?? state.responseModel; @@ -105,10 +108,10 @@ function processChunk(chunk: MistralCompletionChunk, state: StreamingState, reco } /** Mistral yields `CompletionEvent` objects that wrap the chunk under `data`. */ -function processEvent(event: unknown, state: StreamingState, recordOutputs: boolean): void { +function processEvent(event: unknown, state: StreamingState, recordOutputs: boolean, span: Span): void { const chunk = (event as { data?: MistralCompletionChunk } | undefined)?.data; if (chunk && typeof chunk === 'object') { - processChunk(chunk, state, recordOutputs); + processChunk(chunk, state, recordOutputs, span); } } @@ -116,13 +119,14 @@ async function* instrumentIterator( iterate: () => AsyncIterator, state: StreamingState, recordOutputs: boolean, + span: Span, claim: (consumer: StreamConsumer) => boolean, settle: (error?: unknown) => void, ): AsyncGenerator { try { for await (const event of { [Symbol.asyncIterator]: iterate }) { if (claim('iterator')) { - processEvent(event, state, recordOutputs); + processEvent(event, state, recordOutputs, span); } yield event; } @@ -138,6 +142,7 @@ function wrapReader( reader: StreamReaderLike, state: StreamingState, recordOutputs: boolean, + span: Span, claim: (consumer: StreamConsumer) => boolean, settle: (error?: unknown) => void, markCancelled: () => void, @@ -160,7 +165,7 @@ function wrapReader( if (result.done) { settle(); } else if (claim('reader')) { - processEvent(result.value, state, recordOutputs); + processEvent(result.value, state, recordOutputs, span); } return result; } catch (error) { @@ -282,7 +287,7 @@ export function instrumentEventStream(stream: unknown, span: Span, recordOutputs }; const iterate = stream[Symbol.asyncIterator].bind(stream); - const instrumented = instrumentIterator(iterate, state, recordOutputs, claim, settle); + const instrumented = instrumentIterator(iterate, state, recordOutputs, span, claim, settle); stream[Symbol.asyncIterator] = () => instrumented; const readable = stream as ReadableStreamLike; @@ -290,7 +295,7 @@ export function instrumentEventStream(stream: unknown, span: Span, recordOutputs if (typeof readable.getReader === 'function') { const getReader = readable.getReader.bind(readable); readable.getReader = (...args: unknown[]) => - wrapReader(getReader(...args), state, recordOutputs, claim, settle, markCancelled); + wrapReader(getReader(...args), state, recordOutputs, span, claim, settle, markCancelled); } if (typeof readable.cancel === 'function') { diff --git a/packages/server-utils/src/ai/openai/streaming.ts b/packages/server-utils/src/ai/openai/streaming.ts index dcd25abbf612..084d89e527d2 100644 --- a/packages/server-utils/src/ai/openai/streaming.ts +++ b/packages/server-utils/src/ai/openai/streaming.ts @@ -1,7 +1,7 @@ import { captureException, SPAN_STATUS_ERROR } from '@sentry/core'; import type { Span } from '@sentry/core'; import { getToolCallKey } from '../core/streaming-tool-calls'; -import { endStreamSpan } from '../core/utils'; +import { endStreamSpan, recordTimeToFirstChunk } from '../core/utils'; import { RESPONSE_EVENT_TYPES } from './constants'; import type { ChatCompletionChunk, @@ -209,6 +209,23 @@ function processResponsesApiEvent( } } +function hasChatCompletionOutput(chunk: ChatCompletionChunk): boolean { + return (chunk.choices ?? []).some( + ({ delta }) => + !!delta && (!!delta.content || !!delta.refusal || !!delta.tool_calls?.length || !!delta.function_call), + ); +} + +// Output arrives on the `*.delta` events (text, refusals, reasoning, tool-call arguments, audio). A +// function call's name arrives on the `response.output_item.added` event that opens it. +function hasResponsesApiOutput(event: { type: string }): boolean { + const { type, delta, item } = event as { type: string; delta?: unknown; item?: { type?: unknown } }; + if (type === 'response.output_item.added') { + return item?.type === 'function_call'; + } + return type.endsWith('.delta') && typeof delta === 'string' && delta.length > 0; +} + /** * Instruments a stream of OpenAI events, updating the provided span with relevant attributes and * optionally recording output text. This function yields each event from the input stream as it is processed. @@ -242,8 +259,10 @@ export async function* instrumentStream( try { for await (const event of stream) { if (isChatCompletionChunk(event)) { + if (hasChatCompletionOutput(event)) recordTimeToFirstChunk(span); processChatCompletionChunk(event, state, recordOutputs); } else if (isResponsesApiStreamEvent(event)) { + if (hasResponsesApiOutput(event)) recordTimeToFirstChunk(span); processResponsesApiEvent(event, state, recordOutputs, span); } yield event; diff --git a/packages/server-utils/src/ai/pi-ai/chat.ts b/packages/server-utils/src/ai/pi-ai/chat.ts index c7f8b10dc511..a6357ee62fc9 100644 --- a/packages/server-utils/src/ai/pi-ai/chat.ts +++ b/packages/server-utils/src/ai/pi-ai/chat.ts @@ -28,6 +28,7 @@ import { SERVER_PORT, } from '@sentry/conventions/attributes'; import { GEN_AI_CHAT } from '@sentry/conventions/op'; +import { recordTimeToFirstChunk } from '../core/utils'; import { PI_AI_ORIGIN } from './constants'; import type { PiAiContext } from './messages'; import { @@ -37,7 +38,7 @@ import { piAiSystemInstructions, piAiToolDefinitions, } from './messages'; -import type { PiAssistantMessage, PiModel, PiStreamOptions } from './types'; +import type { PiAssistantMessage, PiEventStream, PiModel, PiStreamOptions } from './types'; import { setPiAiUsageAttributes } from './usage'; interface PiAiChatRequest { @@ -118,3 +119,39 @@ function getRequestContentAttributes(context: PiAiContext): Record | undefined): void { + const push = events?.push; + if (!events || typeof push !== 'function') { + return; + } + events.push = function (this: unknown, event: unknown) { + if (isPiAiOutputEvent(event)) { + recordTimeToFirstChunk(span); + events.push = push; + } + return push.call(this, event); + }; +} + +// `text_start` and `thinking_start` open an empty block, so they are not output. A tool call +// start carries the tool name, which is. +function isPiAiOutputEvent(event: unknown): boolean { + if (!event || typeof event !== 'object') { + return false; + } + const { type, delta } = event as { type?: unknown; delta?: unknown }; + if (type === 'toolcall_start') { + return true; + } + return ( + (type === 'text_delta' || type === 'thinking_delta' || type === 'toolcall_delta') && + typeof delta === 'string' && + delta.length > 0 + ); +} diff --git a/packages/server-utils/src/ai/pi-ai/types.ts b/packages/server-utils/src/ai/pi-ai/types.ts index f2355d1044d8..37791a0fcce2 100644 --- a/packages/server-utils/src/ai/pi-ai/types.ts +++ b/packages/server-utils/src/ai/pi-ai/types.ts @@ -32,4 +32,5 @@ export interface PiStreamOptions { export interface PiEventStream { result(): Promise; + push(event: unknown): void; } diff --git a/packages/server-utils/src/ai/workers-ai/streaming.ts b/packages/server-utils/src/ai/workers-ai/streaming.ts index 93884ffbbb94..ca95a982294d 100644 --- a/packages/server-utils/src/ai/workers-ai/streaming.ts +++ b/packages/server-utils/src/ai/workers-ai/streaming.ts @@ -1,7 +1,7 @@ import { SPAN_STATUS_ERROR } from '@sentry/core'; import type { Span } from '@sentry/core'; import { getToolCallKey } from '../core/streaming-tool-calls'; -import { endStreamSpan, type StreamResponseState } from '../core/utils'; +import { endStreamSpan, recordTimeToFirstChunk, type StreamResponseState } from '../core/utils'; import type { WorkersAiUsage } from './types'; import { setOutputMessagesAttribute } from './utils'; @@ -94,6 +94,7 @@ function processLine( state: StreamResponseState, recordOutputs: boolean, toolCallAccumulator: ToolCallAccumulator, + span: Span, ): void { const trimmed = line.trim(); if (!trimmed.startsWith('data:')) { @@ -112,6 +113,10 @@ function processLine( return; } + if (hasOutput(parsed)) { + recordTimeToFirstChunk(span); + } + if (parsed.usage) { if (typeof parsed.usage.prompt_tokens === 'number') { state.promptTokens = parsed.usage.prompt_tokens; @@ -154,6 +159,16 @@ function processLine( } } +// Native models stream text on `response`; OpenAI-compatible models stream it on `choices[].delta`. +function hasOutput(chunk: WorkersAiStreamChunk): boolean { + return ( + (typeof chunk.response === 'string' && chunk.response.length > 0) || + (Array.isArray(chunk.tool_calls) && chunk.tool_calls.length > 0) || + (Array.isArray(chunk.choices) && + chunk.choices.some(({ delta }) => !!delta && (!!delta.content || !!delta.tool_calls?.length))) + ); +} + /** * Wrap a Workers AI streaming response (a server-sent-events `ReadableStream`) so we can * accumulate the response text and token usage while passing the original bytes through untouched. @@ -215,7 +230,7 @@ export function instrumentWorkersAiStream( // Keep the last (potentially incomplete) line in the buffer unless the stream is done. buffer = isDone ? '' : (lines.pop() ?? ''); for (const line of lines) { - processLine(line, state, recordOutputs, toolCallAccumulator); + processLine(line, state, recordOutputs, toolCallAccumulator, span); } }; diff --git a/packages/server-utils/src/integrations/pi-ai.ts b/packages/server-utils/src/integrations/pi-ai.ts index 68bc1bdca1d0..d49f22037f87 100644 --- a/packages/server-utils/src/integrations/pi-ai.ts +++ b/packages/server-utils/src/integrations/pi-ai.ts @@ -2,7 +2,7 @@ import * as diagnosticsChannel from '../utils/diagnosticsChannel'; import type { IntegrationFn, Span } from '@sentry/core'; import { defineIntegration, isThenable, startInactiveSpan } from '@sentry/core'; import type { GenAiOptions } from '../ai/core/utils'; -import { getPiAiChatSpanOptions, setPiAiResponseAttributes } from '../ai/pi-ai/chat'; +import { getPiAiChatSpanOptions, observePiAiFirstChunk, setPiAiResponseAttributes } from '../ai/pi-ai/chat'; import { PI_AI_INTEGRATION_NAME } from '../ai/pi-ai/constants'; import type { PiAiContext } from '../ai/pi-ai/messages'; import { isInsidePiAiSpan } from '../ai/pi-ai/providers'; @@ -66,6 +66,7 @@ function instrumentPiAi(options: PiAiOptions): void { if (!isThenable(settled)) { return false; } + observePiAiFirstChunk(span, events); settled.then(message => { setResponse(span, data, message); end(); diff --git a/packages/server-utils/src/integrations/vercel-ai/util.ts b/packages/server-utils/src/integrations/vercel-ai/util.ts index 6034749cafaf..c5ac12b66062 100644 --- a/packages/server-utils/src/integrations/vercel-ai/util.ts +++ b/packages/server-utils/src/integrations/vercel-ai/util.ts @@ -1,6 +1,8 @@ /** Shared, state-free helpers for the Vercel AI (`ai`) channel subscribers, plus the streamed model-call tap. */ +import type { Span } from '@sentry/core'; import { isObjectLike } from '@sentry/core'; +import { recordTimeToFirstChunk } from '../../ai/core/utils'; /** Narrow to a string, or `undefined` for anything else. */ export function asString(value: unknown): string | undefined { @@ -39,6 +41,8 @@ interface StreamChunk { delta?: unknown; // v4 text-delta chunks carry the text on `textDelta`; v5+ uses `delta`. textDelta?: unknown; + // v4 tool-call-delta chunks carry argument text on `argsTextDelta`. + argsTextDelta?: unknown; id?: unknown; modelId?: unknown; toolCallId?: unknown; @@ -75,6 +79,7 @@ export { isReadableStream } from '../../ai/core/utils'; * mid-stream failure would be worse than a slightly-less-enriched one. */ export function tapModelCallStream( + span: Span, stream: ReadableStream, onFinal: (result: StreamedModelCallResult) => void, onError: (error: unknown) => void, @@ -113,6 +118,9 @@ export function tapModelCallStream( return; } + if (isOutputChunk(value)) { + recordTimeToFirstChunk(span); + } text += accumulateChunk(state, value) ?? ''; controller.enqueue(value); } catch (error) { @@ -130,6 +138,30 @@ export function tapModelCallStream( }); } +// Text, reasoning, tool-call, and file stream parts carry output. `tool-input-start` carries the tool +// name. Metadata, usage, finish, and the other `*-start`/`*-end` parts do not. +function isOutputChunk(chunk: unknown): boolean { + if (!isObjectLike(chunk)) { + return false; + } + const { type, delta, textDelta, argsTextDelta } = chunk as StreamChunk; + switch (type) { + case 'text-delta': + case 'reasoning-delta': + case 'tool-input-delta': + case 'reasoning': + return !!(delta || textDelta); + case 'tool-call-delta': + return !!argsTextDelta; + case 'tool-input-start': + case 'tool-call': + case 'file': + return true; + default: + return false; + } +} + /** * Fold a single streamed chunk into the running aggregate. Returns any text delta so the caller can * accumulate it (kept out of `state` until the end to avoid re-joining on every chunk). diff --git a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts index d19111cb39be..52930d1271b4 100644 --- a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts +++ b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts @@ -348,6 +348,7 @@ function deferStreamedModelCallEnd( const callId = asString(data.event.callId); const { recordOutputs } = getRecordingOptions(data.event, options, data.result === undefined); result.stream = tapModelCallStream( + span, stream, final => { // Reshape the aggregate into the result `enrichSpanOnEnd` expects, then let `end` run it (it calls diff --git a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-orchestrion-subscriber.ts b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-orchestrion-subscriber.ts index 84fb1ac96c70..207c4d73cc4a 100644 --- a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-orchestrion-subscriber.ts +++ b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-orchestrion-subscriber.ts @@ -565,6 +565,7 @@ function patchModelMethod( // separately by `deferStreamTextOperationEnd`, which awaits the operation's own result promises. if (method === 'doStream' && isObjectLike(value) && isReadableStream(value.stream)) { value.stream = tapModelCallStream( + span, value.stream, final => { message.result = { ...value, ...streamedResultToChannelResult(final) }; diff --git a/packages/server-utils/test/ai/lib/tracing/google-genai-streaming.test.ts b/packages/server-utils/test/ai/lib/tracing/google-genai-streaming.test.ts index 763667555672..4523f49ac376 100644 --- a/packages/server-utils/test/ai/lib/tracing/google-genai-streaming.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/google-genai-streaming.test.ts @@ -15,6 +15,7 @@ function createMockSpan(): { span: Span; attributes: Record } { let isEnded = false; const span = { isRecording: () => !isEnded, + spanContext: () => ({ traceId: '0', spanId: '0', traceFlags: 0 }), setAttribute: (key: string, value: unknown) => { attributes[key] = value; }, diff --git a/packages/server-utils/test/ai/lib/tracing/openai-streaming-tool-calls.test.ts b/packages/server-utils/test/ai/lib/tracing/openai-streaming-tool-calls.test.ts index 36aca7b8a9c7..9ba3e572a306 100644 --- a/packages/server-utils/test/ai/lib/tracing/openai-streaming-tool-calls.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/openai-streaming-tool-calls.test.ts @@ -9,6 +9,7 @@ function createMockSpan(): { span: Span; attributes: Record } { let isEnded = false; const span = { isRecording: () => !isEnded, + spanContext: () => ({ traceId: '0', spanId: '0', traceFlags: 0 }), setAttribute: (key: string, value: unknown) => { attributes[key] = value; }, diff --git a/packages/server-utils/test/ai/lib/tracing/stream-errors.test.ts b/packages/server-utils/test/ai/lib/tracing/stream-errors.test.ts index 8e7d1f490169..4ad6b2ce6881 100644 --- a/packages/server-utils/test/ai/lib/tracing/stream-errors.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/stream-errors.test.ts @@ -14,6 +14,7 @@ function createMockSpan(): { span: Span; attributes: Record; se let isEnded = false; const span = { isRecording: () => !isEnded, + spanContext: () => ({ traceId: '0', spanId: '0', traceFlags: 0 }), setAttribute: (key: string, value: unknown) => { attributes[key] = value; }, diff --git a/packages/server-utils/test/ai/lib/tracing/time-to-first-chunk.test.ts b/packages/server-utils/test/ai/lib/tracing/time-to-first-chunk.test.ts new file mode 100644 index 000000000000..4243693fd56a --- /dev/null +++ b/packages/server-utils/test/ai/lib/tracing/time-to-first-chunk.test.ts @@ -0,0 +1,228 @@ +import { GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK } from '@sentry/conventions/attributes'; +import type { Span } from '@sentry/core'; +import { getMainCarrier, setCurrentClient, spanToJSON, startInactiveSpan } from '@sentry/core'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { instrumentAsyncIterableStream } from '../../../../src/ai/anthropic-ai/streaming'; +import type { AnthropicAiStreamingEvent } from '../../../../src/ai/anthropic-ai/types'; +import { instrumentStream as instrumentGoogleGenAIStream } from '../../../../src/ai/google-genai/streaming'; +import type { GoogleGenAIResponse } from '../../../../src/ai/google-genai/types'; +import { createLangChainCallbackHandler } from '../../../../src/ai/langchain'; +import { SentryMastraExporter } from '../../../../src/ai/mastra'; +import { instrumentEventStream as instrumentMistralStream } from '../../../../src/ai/mistral/streaming'; +import { instrumentStream as instrumentOpenAIStream } from '../../../../src/ai/openai/streaming'; +import { instrumentWorkersAiStream } from '../../../../src/ai/workers-ai/streaming'; +import { tapModelCallStream } from '../../../../src/integrations/vercel-ai/util'; +import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; + +const DELAY_MS = 50; + +const wait = (): Promise => new Promise(resolve => setTimeout(resolve, DELAY_MS)); + +// Yields the chunks that carry no output, waits, then yields the output chunks. +async function* delayedChunks(before: T[], after: T[]): AsyncGenerator { + yield* before; + await wait(); + yield* after; +} + +function toReadableStream(chunks: AsyncIterable): ReadableStream { + const iterator = chunks[Symbol.asyncIterator](); + return new ReadableStream({ + async pull(controller) { + const { done, value } = await iterator.next(); + if (done) { + controller.close(); + } else { + controller.enqueue(value); + } + }, + }); +} + +async function drain(stream: AsyncIterable | ReadableStream): Promise { + if (stream instanceof ReadableStream) { + const reader = stream.getReader(); + while (!(await reader.read()).done) { + // drain + } + return; + } + for await (const _ of stream) { + // drain + } +} + +function timeToFirstChunk(span: Span): unknown { + return spanToJSON(span).attributes?.[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]; +} + +describe('gen_ai.response.time_to_first_chunk', () => { + let endedSpans: Span[]; + + beforeEach(() => { + getMainCarrier().__SENTRY__ = undefined; + const client = new TestClient( + getDefaultTestClientOptions({ dsn: 'https://public@dsn.ingest.sentry.io/1337', tracesSampleRate: 1 }), + ); + setCurrentClient(client); + client.init(); + endedSpans = []; + client.on('spanEnd', span => endedSpans.push(span)); + }); + + afterEach(() => { + getMainCarrier().__SENTRY__ = undefined; + }); + + const streams: Array<[string, (span: Span) => AsyncIterable | ReadableStream]> = [ + [ + 'OpenAI chat completions', + span => { + const chunk = (delta: Record): unknown => ({ + id: 'chatcmpl-1', + object: 'chat.completion.chunk', + model: 'gpt-4o', + choices: [{ index: 0, delta }], + }); + return instrumentOpenAIStream( + delayedChunks([chunk({ role: 'assistant', content: '' })], [chunk({ content: 'Hi' })]), + span, + false, + ); + }, + ], + [ + 'OpenAI responses', + span => + instrumentOpenAIStream( + delayedChunks( + [ + { type: 'response.created', response: { id: 'resp_1', model: 'gpt-4o' } }, + { type: 'response.output_text.delta', delta: '' }, + ], + [{ type: 'response.output_item.added', item: { type: 'function_call', name: 'get_weather' } }], + ), + span, + false, + ), + ], + [ + 'Anthropic', + span => + instrumentAsyncIterableStream( + delayedChunks( + [ + { type: 'message_start', message: { id: 'msg_1', model: 'claude' } } as AnthropicAiStreamingEvent, + { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, + ], + [{ type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'Hi' } }], + ), + span, + false, + ), + ], + [ + 'Google GenAI', + span => + instrumentGoogleGenAIStream( + delayedChunks( + [{ candidates: [{ content: { parts: [{ thoughtSignature: 'sig' }] } }] } as GoogleGenAIResponse], + [{ candidates: [{ content: { parts: [{ text: 'Hi' }] } }] } as GoogleGenAIResponse], + ), + span, + false, + ), + ], + [ + 'Mistral', + span => { + const stream = delayedChunks( + [ + { + data: { + id: 'cmpl-1', + model: 'mistral', + choices: [{ index: 0, delta: { role: 'assistant', content: '' } }], + }, + }, + ], + [{ data: { choices: [{ index: 0, delta: { content: 'Hi' } }] } }], + ); + instrumentMistralStream(stream, span, false); + return stream; + }, + ], + [ + 'Workers AI', + span => { + const encoder = new TextEncoder(); + return instrumentWorkersAiStream( + toReadableStream( + delayedChunks( + [encoder.encode('data: {"response":""}\n\n')], + [encoder.encode('data: {"response":"Hi"}\n\ndata: [DONE]\n\n')], + ), + ), + span, + false, + ); + }, + ], + [ + 'Vercel AI', + span => + tapModelCallStream( + span, + toReadableStream( + delayedChunks( + [ + { type: 'response-metadata', id: 'resp_1', modelId: 'gpt-4o' }, + { type: 'text-start', id: '0' }, + ], + [{ type: 'text-delta', id: '0', delta: 'Hi' }], + ), + ), + () => span.end(), + () => span.end(), + ), + ], + ]; + + it.each(streams)('records the time to the first output chunk of a %s stream', async (_name, instrument) => { + const span = startInactiveSpan({ name: 'chat' }); + + await drain(instrument(span)); + span.end(); + + expect(timeToFirstChunk(span)).toBeGreaterThanOrEqual((DELAY_MS - 10) / 1000); + }); + + it('records the time to the first LangChain token or tool call chunk', async () => { + const handler = createLangChainCallbackHandler(); + handler.handleChatModelStart?.({ id: ['langchain', 'chat_models', 'ChatOpenAI'] }, [[]], 'run-1'); + handler.handleLLMNewToken?.('', undefined, 'run-1'); + await wait(); + handler.handleLLMNewToken?.('', undefined, 'run-1', undefined, undefined, { + chunk: { message: { tool_call_chunks: [{ name: 'get_weather' }] } }, + }); + handler.handleLLMEnd?.({ generations: [] }, 'run-1'); + + expect(timeToFirstChunk(endedSpans[0]!)).toBeGreaterThanOrEqual((DELAY_MS - 10) / 1000); + }); + + it('maps the completion start time of a Mastra model generation', async () => { + const exportedSpan = { + id: 'span-1', + name: 'llm', + type: 'model_generation' as const, + startTime: new Date('2026-01-01T00:00:00Z'), + endTime: new Date('2026-01-01T00:00:02Z'), + attributes: { model: 'gpt-4o', streaming: true, completionStartTime: new Date('2026-01-01T00:00:01.5Z') }, + }; + const exporter = new SentryMastraExporter(); + await exporter.exportTracingEvent({ type: 'span_started', exportedSpan }); + await exporter.exportTracingEvent({ type: 'span_ended', exportedSpan }); + + expect(timeToFirstChunk(endedSpans[0]!)).toBe(1.5); + }); +}); diff --git a/packages/server-utils/test/ai/lib/tracing/workers-ai-streaming.test.ts b/packages/server-utils/test/ai/lib/tracing/workers-ai-streaming.test.ts index 33f0ceacba83..75986b2280d0 100644 --- a/packages/server-utils/test/ai/lib/tracing/workers-ai-streaming.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/workers-ai-streaming.test.ts @@ -18,6 +18,7 @@ function createMockSpan(): { span: Span; attributes: Record; en let isEnded = false; const span = { isRecording: () => !isEnded, + spanContext: () => ({ traceId: '0', spanId: '0', traceFlags: 0 }), setAttribute: (key: string, value: unknown) => { attributes[key] = value; }, diff --git a/packages/server-utils/test/integrations/pi-ai.test.ts b/packages/server-utils/test/integrations/pi-ai.test.ts index 260ab4423de8..1a0f0f6f86e2 100644 --- a/packages/server-utils/test/integrations/pi-ai.test.ts +++ b/packages/server-utils/test/integrations/pi-ai.test.ts @@ -23,6 +23,7 @@ import { GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_MODEL, GEN_AI_RESPONSE_STREAMING, + GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK, GEN_AI_SYSTEM_INSTRUCTIONS, GEN_AI_TOOL_DEFINITIONS, GEN_AI_USAGE_INPUT_TOKENS, @@ -122,6 +123,41 @@ describe('piAiIntegration', () => { expect(endedSpans).toHaveLength(1); }); + it('records the time to the first output event that the provider pushes', async () => { + let settle!: (message: unknown) => void; + const result = new Promise(resolve => { + settle = resolve; + }); + const pushed: unknown[] = []; + const events = stream(() => ({ result: () => result, push: (event: unknown) => pushed.push(event) }), MODEL, { + messages: [], + }); + + events.push({ type: 'start' }); + events.push({ type: 'text_start' }); + events.push({ type: 'text_delta', delta: '' }); + await new Promise(resolve => setTimeout(resolve, 50)); + events.push({ type: 'text_delta', delta: 'Par' }); + events.push({ type: 'text_delta', delta: 'is.' }); + settle(ANSWER); + await events.result(); + + expect(pushed).toHaveLength(5); + const { attributes, start_timestamp, end_timestamp } = spanToJSON(endedSpans[0]!); + const timeToFirstChunk = attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK] as number; + expect(timeToFirstChunk).toBeGreaterThanOrEqual(0.04); + expect(timeToFirstChunk).toBeLessThanOrEqual(end_timestamp! - start_timestamp); + }); + + it('records no time to first chunk for a stream without output', async () => { + const events = stream(() => ({ result: async () => ANSWER, push: () => {} }), MODEL, { messages: [] }); + events.push({ type: 'start' }); + events.push({ type: 'done' }); + await events.result(); + + expect(spanToJSON(endedSpans[0]!).attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]).toBeUndefined(); + }); + it.each([true, false])('traces a request with its options and response identity (streaming: %s)', async streaming => { const message = { ...ANSWER, From 4571076401179b8c474812c9078314d6a2146ea7 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2026 11:26:34 +0000 Subject: [PATCH 2/5] test(cloudflare): Expect time to first chunk on the streamed Workers AI span --- .../suites/tracing/workers-ai/test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts index 18d5f5a72543..01b6a7cdc0f3 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts @@ -8,6 +8,7 @@ import { GEN_AI_RESPONSE_MODEL, GEN_AI_RESPONSE_STREAMING, GEN_AI_RESPONSE_TEXT, + GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK, GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, @@ -122,6 +123,7 @@ it('traces a streaming Workers AI text generation request', async ({ signal }) = type: 'string', value: 'The capital of France is Paris.', }, + [GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]: { value: expect.any(Number), type: 'double' }, [SENTRY_IS_LOCALHOST]: { value: true, type: 'boolean' }, [SENTRY_TRACE_LIFECYCLE]: { value: 'stream', type: 'string' }, [SENTRY_SEGMENT_NAME]: { value: segmentSpan!.name, type: 'string' }, From 6f9fdfcd77ad37fb81f724aa0ca75fd4331ebddb Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2026 12:35:57 +0000 Subject: [PATCH 3/5] test(server-utils): Cover time to first chunk in the node integration tests Assert the attribute on the streamed spans of the existing OpenAI, Anthropic, Google GenAI, Mistral, Vercel AI, and Mastra suites, and drop the unit tests for it. --- .../suites/tracing/anthropic/test.ts | 2 + .../suites/tracing/google-genai/test.ts | 2 + .../suites/tracing/mastra/test.ts | 2 + .../suites/tracing/mistral/test.ts | 2 + .../suites/tracing/openai/test.ts | 7 + .../suites/tracing/vercelai/v6_v7/test.ts | 4 + .../lib/tracing/time-to-first-chunk.test.ts | 228 ------------------ .../test/integrations/pi-ai.test.ts | 36 --- 8 files changed, 19 insertions(+), 264 deletions(-) delete mode 100644 packages/server-utils/test/ai/lib/tracing/time-to-first-chunk.test.ts diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts index 54dd27321cd1..3657b275a680 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts @@ -11,6 +11,7 @@ import { GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_MODEL, GEN_AI_RESPONSE_STREAMING, + GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK, GEN_AI_RESPONSE_TEXT, GEN_AI_RESPONSE_TOOL_CALLS, GEN_AI_SYSTEM_INSTRUCTIONS, @@ -116,6 +117,7 @@ describe('Anthropic integration', () => { expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); expect(streamingSpan!.status).toBe('ok'); expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(streamingSpan!.attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]?.value).toEqual(expect.any(Number)); }); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts b/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts index afc273ab86c5..644422aead61 100644 --- a/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts @@ -14,6 +14,7 @@ import { GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_MODEL, GEN_AI_RESPONSE_STREAMING, + GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK, GEN_AI_RESPONSE_TEXT, GEN_AI_RESPONSE_TOOL_CALLS, GEN_AI_SYSTEM_INSTRUCTIONS, @@ -338,6 +339,7 @@ describe('Google GenAI integration', () => { expect(generateContentSpan!.status).toBe('ok'); expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); expect(generateContentSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(generateContentSpan!.attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]?.value).toEqual(expect.any(Number)); expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.9); expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts index 0c3778daa368..433a300f791a 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts @@ -8,6 +8,7 @@ import { GEN_AI_PROVIDER_NAME, GEN_AI_REQUEST_MODEL, GEN_AI_RESPONSE_MODEL, + GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK, GEN_AI_SYSTEM_INSTRUCTIONS, GEN_AI_TOOL_CALL_ARGUMENTS, GEN_AI_TOOL_CALL_RESULT, @@ -458,6 +459,7 @@ describe.runIf(supports({ min: 22 }))('Mastra integration', () => { const cacheGetParentIds = loadSpans.map(span => span.parent_span_id); const chat = container.items.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_CHAT)!; expect(chat).toBeDefined(); + expect(chat.attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]?.value).toEqual(expect.any(Number)); const chatSpanId = chat.span_id; const executeTool = container.items.find( diff --git a/dev-packages/node-integration-tests/suites/tracing/mistral/test.ts b/dev-packages/node-integration-tests/suites/tracing/mistral/test.ts index 8b5373b22921..6bdd9253abfe 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mistral/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/mistral/test.ts @@ -11,6 +11,7 @@ import { GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_MODEL, GEN_AI_RESPONSE_STREAMING, + GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK, GEN_AI_OUTPUT_MESSAGES, GEN_AI_RESPONSE_TEXT, GEN_AI_RESPONSE_TOOL_CALLS, @@ -70,6 +71,7 @@ describe('Mistral integration', () => { expect(streamSpan!.name).toBe('chat mistral-large-latest'); expect(streamSpan!.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('chat'); expect(streamSpan!.attributes[GEN_AI_RESPONSE_STREAMING]?.value).toBe(true); + expect(streamSpan!.attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]?.value).toEqual(expect.any(Number)); expect(streamSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(30); // `chat.stream()` takes no `stream` request field, so the flag has to come from the method. diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts index 98b9f8755573..e6980ff41b71 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts @@ -12,6 +12,7 @@ import { GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_MODEL, GEN_AI_RESPONSE_STREAMING, + GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK, GEN_AI_RESPONSE_TEXT, GEN_AI_RESPONSE_TOOL_CALLS, GEN_AI_SYSTEM_INSTRUCTIONS, @@ -241,6 +242,9 @@ describe('OpenAI integration', () => { type: 'boolean', value: true, }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]?.value).toEqual( + expect.any(Number), + ); const streamingResponsesSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', @@ -299,6 +303,9 @@ describe('OpenAI integration', () => { type: 'boolean', value: true, }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]?.value).toEqual( + expect.any(Number), + ); const streamingErrorSpan = spans.find( span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts index d4774f9d34d4..39380585a6e4 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -10,6 +10,7 @@ import { GEN_AI_PROVIDER_NAME, GEN_AI_REQUEST_MODEL, GEN_AI_RESPONSE_FINISH_REASONS, + GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK, GEN_AI_RESPONSE_MODEL, GEN_AI_SYSTEM_INSTRUCTIONS, GEN_AI_TOOL_CALL_ARGUMENTS, @@ -828,6 +829,9 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe expect(generateContent.attributes[GEN_AI_OUTPUT_MESSAGES]?.value).toBe( '[{"role":"assistant","parts":[{"type":"text","content":"Stream response!"}],"finish_reason":"stop"}]', ); + expect(generateContent.attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]?.value).toEqual( + expect.any(Number), + ); // The summed usage and output also land on the parent invoke_agent span, whose own // channel result is otherwise undefined for a stream. diff --git a/packages/server-utils/test/ai/lib/tracing/time-to-first-chunk.test.ts b/packages/server-utils/test/ai/lib/tracing/time-to-first-chunk.test.ts deleted file mode 100644 index 4243693fd56a..000000000000 --- a/packages/server-utils/test/ai/lib/tracing/time-to-first-chunk.test.ts +++ /dev/null @@ -1,228 +0,0 @@ -import { GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK } from '@sentry/conventions/attributes'; -import type { Span } from '@sentry/core'; -import { getMainCarrier, setCurrentClient, spanToJSON, startInactiveSpan } from '@sentry/core'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { instrumentAsyncIterableStream } from '../../../../src/ai/anthropic-ai/streaming'; -import type { AnthropicAiStreamingEvent } from '../../../../src/ai/anthropic-ai/types'; -import { instrumentStream as instrumentGoogleGenAIStream } from '../../../../src/ai/google-genai/streaming'; -import type { GoogleGenAIResponse } from '../../../../src/ai/google-genai/types'; -import { createLangChainCallbackHandler } from '../../../../src/ai/langchain'; -import { SentryMastraExporter } from '../../../../src/ai/mastra'; -import { instrumentEventStream as instrumentMistralStream } from '../../../../src/ai/mistral/streaming'; -import { instrumentStream as instrumentOpenAIStream } from '../../../../src/ai/openai/streaming'; -import { instrumentWorkersAiStream } from '../../../../src/ai/workers-ai/streaming'; -import { tapModelCallStream } from '../../../../src/integrations/vercel-ai/util'; -import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; - -const DELAY_MS = 50; - -const wait = (): Promise => new Promise(resolve => setTimeout(resolve, DELAY_MS)); - -// Yields the chunks that carry no output, waits, then yields the output chunks. -async function* delayedChunks(before: T[], after: T[]): AsyncGenerator { - yield* before; - await wait(); - yield* after; -} - -function toReadableStream(chunks: AsyncIterable): ReadableStream { - const iterator = chunks[Symbol.asyncIterator](); - return new ReadableStream({ - async pull(controller) { - const { done, value } = await iterator.next(); - if (done) { - controller.close(); - } else { - controller.enqueue(value); - } - }, - }); -} - -async function drain(stream: AsyncIterable | ReadableStream): Promise { - if (stream instanceof ReadableStream) { - const reader = stream.getReader(); - while (!(await reader.read()).done) { - // drain - } - return; - } - for await (const _ of stream) { - // drain - } -} - -function timeToFirstChunk(span: Span): unknown { - return spanToJSON(span).attributes?.[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]; -} - -describe('gen_ai.response.time_to_first_chunk', () => { - let endedSpans: Span[]; - - beforeEach(() => { - getMainCarrier().__SENTRY__ = undefined; - const client = new TestClient( - getDefaultTestClientOptions({ dsn: 'https://public@dsn.ingest.sentry.io/1337', tracesSampleRate: 1 }), - ); - setCurrentClient(client); - client.init(); - endedSpans = []; - client.on('spanEnd', span => endedSpans.push(span)); - }); - - afterEach(() => { - getMainCarrier().__SENTRY__ = undefined; - }); - - const streams: Array<[string, (span: Span) => AsyncIterable | ReadableStream]> = [ - [ - 'OpenAI chat completions', - span => { - const chunk = (delta: Record): unknown => ({ - id: 'chatcmpl-1', - object: 'chat.completion.chunk', - model: 'gpt-4o', - choices: [{ index: 0, delta }], - }); - return instrumentOpenAIStream( - delayedChunks([chunk({ role: 'assistant', content: '' })], [chunk({ content: 'Hi' })]), - span, - false, - ); - }, - ], - [ - 'OpenAI responses', - span => - instrumentOpenAIStream( - delayedChunks( - [ - { type: 'response.created', response: { id: 'resp_1', model: 'gpt-4o' } }, - { type: 'response.output_text.delta', delta: '' }, - ], - [{ type: 'response.output_item.added', item: { type: 'function_call', name: 'get_weather' } }], - ), - span, - false, - ), - ], - [ - 'Anthropic', - span => - instrumentAsyncIterableStream( - delayedChunks( - [ - { type: 'message_start', message: { id: 'msg_1', model: 'claude' } } as AnthropicAiStreamingEvent, - { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, - ], - [{ type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'Hi' } }], - ), - span, - false, - ), - ], - [ - 'Google GenAI', - span => - instrumentGoogleGenAIStream( - delayedChunks( - [{ candidates: [{ content: { parts: [{ thoughtSignature: 'sig' }] } }] } as GoogleGenAIResponse], - [{ candidates: [{ content: { parts: [{ text: 'Hi' }] } }] } as GoogleGenAIResponse], - ), - span, - false, - ), - ], - [ - 'Mistral', - span => { - const stream = delayedChunks( - [ - { - data: { - id: 'cmpl-1', - model: 'mistral', - choices: [{ index: 0, delta: { role: 'assistant', content: '' } }], - }, - }, - ], - [{ data: { choices: [{ index: 0, delta: { content: 'Hi' } }] } }], - ); - instrumentMistralStream(stream, span, false); - return stream; - }, - ], - [ - 'Workers AI', - span => { - const encoder = new TextEncoder(); - return instrumentWorkersAiStream( - toReadableStream( - delayedChunks( - [encoder.encode('data: {"response":""}\n\n')], - [encoder.encode('data: {"response":"Hi"}\n\ndata: [DONE]\n\n')], - ), - ), - span, - false, - ); - }, - ], - [ - 'Vercel AI', - span => - tapModelCallStream( - span, - toReadableStream( - delayedChunks( - [ - { type: 'response-metadata', id: 'resp_1', modelId: 'gpt-4o' }, - { type: 'text-start', id: '0' }, - ], - [{ type: 'text-delta', id: '0', delta: 'Hi' }], - ), - ), - () => span.end(), - () => span.end(), - ), - ], - ]; - - it.each(streams)('records the time to the first output chunk of a %s stream', async (_name, instrument) => { - const span = startInactiveSpan({ name: 'chat' }); - - await drain(instrument(span)); - span.end(); - - expect(timeToFirstChunk(span)).toBeGreaterThanOrEqual((DELAY_MS - 10) / 1000); - }); - - it('records the time to the first LangChain token or tool call chunk', async () => { - const handler = createLangChainCallbackHandler(); - handler.handleChatModelStart?.({ id: ['langchain', 'chat_models', 'ChatOpenAI'] }, [[]], 'run-1'); - handler.handleLLMNewToken?.('', undefined, 'run-1'); - await wait(); - handler.handleLLMNewToken?.('', undefined, 'run-1', undefined, undefined, { - chunk: { message: { tool_call_chunks: [{ name: 'get_weather' }] } }, - }); - handler.handleLLMEnd?.({ generations: [] }, 'run-1'); - - expect(timeToFirstChunk(endedSpans[0]!)).toBeGreaterThanOrEqual((DELAY_MS - 10) / 1000); - }); - - it('maps the completion start time of a Mastra model generation', async () => { - const exportedSpan = { - id: 'span-1', - name: 'llm', - type: 'model_generation' as const, - startTime: new Date('2026-01-01T00:00:00Z'), - endTime: new Date('2026-01-01T00:00:02Z'), - attributes: { model: 'gpt-4o', streaming: true, completionStartTime: new Date('2026-01-01T00:00:01.5Z') }, - }; - const exporter = new SentryMastraExporter(); - await exporter.exportTracingEvent({ type: 'span_started', exportedSpan }); - await exporter.exportTracingEvent({ type: 'span_ended', exportedSpan }); - - expect(timeToFirstChunk(endedSpans[0]!)).toBe(1.5); - }); -}); diff --git a/packages/server-utils/test/integrations/pi-ai.test.ts b/packages/server-utils/test/integrations/pi-ai.test.ts index 1a0f0f6f86e2..260ab4423de8 100644 --- a/packages/server-utils/test/integrations/pi-ai.test.ts +++ b/packages/server-utils/test/integrations/pi-ai.test.ts @@ -23,7 +23,6 @@ import { GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_MODEL, GEN_AI_RESPONSE_STREAMING, - GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK, GEN_AI_SYSTEM_INSTRUCTIONS, GEN_AI_TOOL_DEFINITIONS, GEN_AI_USAGE_INPUT_TOKENS, @@ -123,41 +122,6 @@ describe('piAiIntegration', () => { expect(endedSpans).toHaveLength(1); }); - it('records the time to the first output event that the provider pushes', async () => { - let settle!: (message: unknown) => void; - const result = new Promise(resolve => { - settle = resolve; - }); - const pushed: unknown[] = []; - const events = stream(() => ({ result: () => result, push: (event: unknown) => pushed.push(event) }), MODEL, { - messages: [], - }); - - events.push({ type: 'start' }); - events.push({ type: 'text_start' }); - events.push({ type: 'text_delta', delta: '' }); - await new Promise(resolve => setTimeout(resolve, 50)); - events.push({ type: 'text_delta', delta: 'Par' }); - events.push({ type: 'text_delta', delta: 'is.' }); - settle(ANSWER); - await events.result(); - - expect(pushed).toHaveLength(5); - const { attributes, start_timestamp, end_timestamp } = spanToJSON(endedSpans[0]!); - const timeToFirstChunk = attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK] as number; - expect(timeToFirstChunk).toBeGreaterThanOrEqual(0.04); - expect(timeToFirstChunk).toBeLessThanOrEqual(end_timestamp! - start_timestamp); - }); - - it('records no time to first chunk for a stream without output', async () => { - const events = stream(() => ({ result: async () => ANSWER, push: () => {} }), MODEL, { messages: [] }); - events.push({ type: 'start' }); - events.push({ type: 'done' }); - await events.result(); - - expect(spanToJSON(endedSpans[0]!).attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]).toBeUndefined(); - }); - it.each([true, false])('traces a request with its options and response identity (streaming: %s)', async streaming => { const message = { ...ANSWER, From 71976f43d17efdb1d993371f75ea295b10fb8e43 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:03:17 +0000 Subject: [PATCH 4/5] ref(server-utils): Record time to first chunk on the first received chunk Drop the per-provider checks for output-carrying chunks and record the time when the first chunk of the response stream arrives, as the OTel definition says. This cuts most of the bundle size the feature added. Co-Authored-By: Francesco Gringl-Novy <2411343+mydea@users.noreply.github.com> --- .../src/ai/anthropic-ai/streaming.ts | 17 +---------- packages/server-utils/src/ai/core/utils.ts | 4 +-- .../src/ai/google-genai/streaming.ts | 12 ++------ .../server-utils/src/ai/langchain/index.ts | 15 ++-------- .../server-utils/src/ai/mistral/streaming.ts | 21 ++++++-------- .../server-utils/src/ai/openai/streaming.ts | 20 +------------ packages/server-utils/src/ai/pi-ai/chat.ts | 22 ++------------ .../src/ai/workers-ai/streaming.ts | 18 ++---------- .../src/integrations/vercel-ai/util.ts | 29 ++----------------- .../ai/lib/tracing/anthropic-ai-usage.test.ts | 7 ++++- .../ai/lib/tracing/cache-token-usage.test.ts | 11 +++++-- 11 files changed, 39 insertions(+), 137 deletions(-) diff --git a/packages/server-utils/src/ai/anthropic-ai/streaming.ts b/packages/server-utils/src/ai/anthropic-ai/streaming.ts index 950ae09b610c..134a764eafc1 100644 --- a/packages/server-utils/src/ai/anthropic-ai/streaming.ts +++ b/packages/server-utils/src/ai/anthropic-ai/streaming.ts @@ -186,19 +186,6 @@ function handleContentBlockStop(event: AnthropicAiStreamingEvent, state: Streami delete state.activeToolBlocks[event.index]; } -// A text block opens empty, so only its deltas are output. A tool use block opens with the tool name. -// A `signature_delta` only signs the thinking that came before it. -function hasOutput(event: AnthropicAiStreamingEvent): boolean { - if (event.type === 'content_block_start') { - return !!event.content_block?.name; - } - if (event.type === 'content_block_delta' && event.delta) { - const delta = event.delta as { text?: unknown; partial_json?: unknown; thinking?: unknown }; - return !!(delta.text || delta.partial_json || delta.thinking); - } - return false; -} - /** * Processes an event * @param event - The event to process @@ -219,9 +206,7 @@ function processEvent( const isError = isErrorEvent(event, span); if (isError) return; - if (hasOutput(event)) { - recordTimeToFirstChunk(span); - } + recordTimeToFirstChunk(span); handleMessageMetadata(event, state); diff --git a/packages/server-utils/src/ai/core/utils.ts b/packages/server-utils/src/ai/core/utils.ts index fc052b1435e6..50bf1701a680 100644 --- a/packages/server-utils/src/ai/core/utils.ts +++ b/packages/server-utils/src/ai/core/utils.ts @@ -277,9 +277,7 @@ const spansWithFirstChunk = new WeakSet(); /** * Sets `gen_ai.response.time_to_first_chunk` on a streamed span, in seconds from the span start. - * Call it for each chunk that carries new output (text, reasoning, refusals, or tool-call names or - * arguments), not for chunks with only metadata, usage, or an empty block start. Only the first call - * per span records a value. + * Call it for each received chunk. Only the first call per span records a value. */ export function recordTimeToFirstChunk(span: Span): void { if (spansWithFirstChunk.has(span)) { diff --git a/packages/server-utils/src/ai/google-genai/streaming.ts b/packages/server-utils/src/ai/google-genai/streaming.ts index 54625d69ac6b..61d122521b80 100644 --- a/packages/server-utils/src/ai/google-genai/streaming.ts +++ b/packages/server-utils/src/ai/google-genai/streaming.ts @@ -1,7 +1,7 @@ import { captureException, SPAN_STATUS_ERROR } from '@sentry/core'; import type { Span } from '@sentry/core'; import { endStreamSpan, getErrorStatus, recordTimeToFirstChunk } from '../core/utils'; -import type { ContentPart, GoogleGenAIResponse } from './types'; +import type { GoogleGenAIResponse } from './types'; import type { MessagePart } from './utils'; import { candidatesToMessageParts, setOutputMessagesAttribute } from './utils'; @@ -102,12 +102,6 @@ function handleCandidateContent(chunk: GoogleGenAIResponse, state: StreamingStat } } -// A part carries text (also thoughts), a function call, inline data such as audio or images, or -// generated code. Parts with only a thought signature carry no output. -function hasOutput(part: ContentPart | undefined): boolean { - return !!part && !!(part.text || part.functionCall || part.inlineData || part.executableCode); -} - /** * Processes a single chunk from the Google GenAI stream * @param chunk - The chunk to process @@ -117,9 +111,7 @@ function hasOutput(part: ContentPart | undefined): boolean { */ function processChunk(chunk: GoogleGenAIResponse, state: StreamingState, recordOutputs: boolean, span: Span): void { if (!chunk || isErrorChunk(chunk, span)) return; - if (chunk.candidates?.some(candidate => candidate?.content?.parts?.some(hasOutput))) { - recordTimeToFirstChunk(span); - } + recordTimeToFirstChunk(span); handleResponseMetadata(chunk, state); handleCandidateContent(chunk, state, recordOutputs); } diff --git a/packages/server-utils/src/ai/langchain/index.ts b/packages/server-utils/src/ai/langchain/index.ts index 42de5c8b5695..58d5501ad0b8 100644 --- a/packages/server-utils/src/ai/langchain/index.ts +++ b/packages/server-utils/src/ai/langchain/index.ts @@ -269,19 +269,10 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): ); }, - // Called for each streamed chunk. A tool call chunk arrives with an empty token. - handleLLMNewToken( - token: string, - _idx: unknown, - runId: string, - _parentRunId?: string, - _tags?: string[], - fields?: unknown, - ) { + // Called for each streamed chunk. + handleLLMNewToken(_token: string, _idx: unknown, runId: string) { const span = runs.get(runId)?.span; - const toolCallChunks = (fields as { chunk?: { message?: { tool_call_chunks?: unknown[] } } } | undefined)?.chunk - ?.message?.tool_call_chunks; - if (span && (token || toolCallChunks?.length)) { + if (span) { recordTimeToFirstChunk(span); } }, diff --git a/packages/server-utils/src/ai/mistral/streaming.ts b/packages/server-utils/src/ai/mistral/streaming.ts index fabb835f6ad9..b0d3bc22bcc5 100644 --- a/packages/server-utils/src/ai/mistral/streaming.ts +++ b/packages/server-utils/src/ai/mistral/streaming.ts @@ -74,10 +74,7 @@ function processToolCalls(toolCalls: MistralToolCall[], state: StreamingState): } } -function processChunk(chunk: MistralCompletionChunk, state: StreamingState, recordOutputs: boolean, span: Span): void { - if (chunk.choices?.some(({ delta }) => !!delta && (!!contentToString(delta.content) || !!delta.toolCalls?.length))) { - recordTimeToFirstChunk(span); - } +function processChunk(chunk: MistralCompletionChunk, state: StreamingState, recordOutputs: boolean): void { state.responseId = chunk.id ?? state.responseId; state.responseModel = chunk.model ?? state.responseModel; @@ -108,10 +105,10 @@ function processChunk(chunk: MistralCompletionChunk, state: StreamingState, reco } /** Mistral yields `CompletionEvent` objects that wrap the chunk under `data`. */ -function processEvent(event: unknown, state: StreamingState, recordOutputs: boolean, span: Span): void { +function processEvent(event: unknown, state: StreamingState, recordOutputs: boolean): void { const chunk = (event as { data?: MistralCompletionChunk } | undefined)?.data; if (chunk && typeof chunk === 'object') { - processChunk(chunk, state, recordOutputs, span); + processChunk(chunk, state, recordOutputs); } } @@ -119,14 +116,13 @@ async function* instrumentIterator( iterate: () => AsyncIterator, state: StreamingState, recordOutputs: boolean, - span: Span, claim: (consumer: StreamConsumer) => boolean, settle: (error?: unknown) => void, ): AsyncGenerator { try { for await (const event of { [Symbol.asyncIterator]: iterate }) { if (claim('iterator')) { - processEvent(event, state, recordOutputs, span); + processEvent(event, state, recordOutputs); } yield event; } @@ -142,7 +138,6 @@ function wrapReader( reader: StreamReaderLike, state: StreamingState, recordOutputs: boolean, - span: Span, claim: (consumer: StreamConsumer) => boolean, settle: (error?: unknown) => void, markCancelled: () => void, @@ -165,7 +160,7 @@ function wrapReader( if (result.done) { settle(); } else if (claim('reader')) { - processEvent(result.value, state, recordOutputs, span); + processEvent(result.value, state, recordOutputs); } return result; } catch (error) { @@ -249,7 +244,9 @@ export function instrumentEventStream(stream: unknown, span: Span, recordOutputs let settled = false; let cancelled = false; + // Called for each received event. const claim = (candidate: StreamConsumer): boolean => { + recordTimeToFirstChunk(span); consumer ??= candidate; return consumer === candidate; }; @@ -287,7 +284,7 @@ export function instrumentEventStream(stream: unknown, span: Span, recordOutputs }; const iterate = stream[Symbol.asyncIterator].bind(stream); - const instrumented = instrumentIterator(iterate, state, recordOutputs, span, claim, settle); + const instrumented = instrumentIterator(iterate, state, recordOutputs, claim, settle); stream[Symbol.asyncIterator] = () => instrumented; const readable = stream as ReadableStreamLike; @@ -295,7 +292,7 @@ export function instrumentEventStream(stream: unknown, span: Span, recordOutputs if (typeof readable.getReader === 'function') { const getReader = readable.getReader.bind(readable); readable.getReader = (...args: unknown[]) => - wrapReader(getReader(...args), state, recordOutputs, span, claim, settle, markCancelled); + wrapReader(getReader(...args), state, recordOutputs, claim, settle, markCancelled); } if (typeof readable.cancel === 'function') { diff --git a/packages/server-utils/src/ai/openai/streaming.ts b/packages/server-utils/src/ai/openai/streaming.ts index 084d89e527d2..55549bd5c6eb 100644 --- a/packages/server-utils/src/ai/openai/streaming.ts +++ b/packages/server-utils/src/ai/openai/streaming.ts @@ -209,23 +209,6 @@ function processResponsesApiEvent( } } -function hasChatCompletionOutput(chunk: ChatCompletionChunk): boolean { - return (chunk.choices ?? []).some( - ({ delta }) => - !!delta && (!!delta.content || !!delta.refusal || !!delta.tool_calls?.length || !!delta.function_call), - ); -} - -// Output arrives on the `*.delta` events (text, refusals, reasoning, tool-call arguments, audio). A -// function call's name arrives on the `response.output_item.added` event that opens it. -function hasResponsesApiOutput(event: { type: string }): boolean { - const { type, delta, item } = event as { type: string; delta?: unknown; item?: { type?: unknown } }; - if (type === 'response.output_item.added') { - return item?.type === 'function_call'; - } - return type.endsWith('.delta') && typeof delta === 'string' && delta.length > 0; -} - /** * Instruments a stream of OpenAI events, updating the provided span with relevant attributes and * optionally recording output text. This function yields each event from the input stream as it is processed. @@ -258,11 +241,10 @@ export async function* instrumentStream( try { for await (const event of stream) { + recordTimeToFirstChunk(span); if (isChatCompletionChunk(event)) { - if (hasChatCompletionOutput(event)) recordTimeToFirstChunk(span); processChatCompletionChunk(event, state, recordOutputs); } else if (isResponsesApiStreamEvent(event)) { - if (hasResponsesApiOutput(event)) recordTimeToFirstChunk(span); processResponsesApiEvent(event, state, recordOutputs, span); } yield event; diff --git a/packages/server-utils/src/ai/pi-ai/chat.ts b/packages/server-utils/src/ai/pi-ai/chat.ts index a6357ee62fc9..742b90320b65 100644 --- a/packages/server-utils/src/ai/pi-ai/chat.ts +++ b/packages/server-utils/src/ai/pi-ai/chat.ts @@ -123,7 +123,7 @@ function getRequestContentAttributes(context: PiAiContext): Record | undefined): void { const push = events?.push; @@ -131,27 +131,11 @@ export function observePiAiFirstChunk(span: Span, events: Partial return; } events.push = function (this: unknown, event: unknown) { - if (isPiAiOutputEvent(event)) { + // Providers push `start` before the response arrives. + if ((event as { type?: unknown } | undefined)?.type !== 'start') { recordTimeToFirstChunk(span); events.push = push; } return push.call(this, event); }; } - -// `text_start` and `thinking_start` open an empty block, so they are not output. A tool call -// start carries the tool name, which is. -function isPiAiOutputEvent(event: unknown): boolean { - if (!event || typeof event !== 'object') { - return false; - } - const { type, delta } = event as { type?: unknown; delta?: unknown }; - if (type === 'toolcall_start') { - return true; - } - return ( - (type === 'text_delta' || type === 'thinking_delta' || type === 'toolcall_delta') && - typeof delta === 'string' && - delta.length > 0 - ); -} diff --git a/packages/server-utils/src/ai/workers-ai/streaming.ts b/packages/server-utils/src/ai/workers-ai/streaming.ts index ca95a982294d..b0c94dfab714 100644 --- a/packages/server-utils/src/ai/workers-ai/streaming.ts +++ b/packages/server-utils/src/ai/workers-ai/streaming.ts @@ -94,7 +94,6 @@ function processLine( state: StreamResponseState, recordOutputs: boolean, toolCallAccumulator: ToolCallAccumulator, - span: Span, ): void { const trimmed = line.trim(); if (!trimmed.startsWith('data:')) { @@ -113,10 +112,6 @@ function processLine( return; } - if (hasOutput(parsed)) { - recordTimeToFirstChunk(span); - } - if (parsed.usage) { if (typeof parsed.usage.prompt_tokens === 'number') { state.promptTokens = parsed.usage.prompt_tokens; @@ -159,16 +154,6 @@ function processLine( } } -// Native models stream text on `response`; OpenAI-compatible models stream it on `choices[].delta`. -function hasOutput(chunk: WorkersAiStreamChunk): boolean { - return ( - (typeof chunk.response === 'string' && chunk.response.length > 0) || - (Array.isArray(chunk.tool_calls) && chunk.tool_calls.length > 0) || - (Array.isArray(chunk.choices) && - chunk.choices.some(({ delta }) => !!delta && (!!delta.content || !!delta.tool_calls?.length))) - ); -} - /** * Wrap a Workers AI streaming response (a server-sent-events `ReadableStream`) so we can * accumulate the response text and token usage while passing the original bytes through untouched. @@ -230,7 +215,7 @@ export function instrumentWorkersAiStream( // Keep the last (potentially incomplete) line in the buffer unless the stream is done. buffer = isDone ? '' : (lines.pop() ?? ''); for (const line of lines) { - processLine(line, state, recordOutputs, toolCallAccumulator, span); + processLine(line, state, recordOutputs, toolCallAccumulator); } }; @@ -247,6 +232,7 @@ export function instrumentWorkersAiStream( return; } + recordTimeToFirstChunk(span); buffer += decoder.decode(value, { stream: true }); flushBuffer(false); controller.enqueue(value); diff --git a/packages/server-utils/src/integrations/vercel-ai/util.ts b/packages/server-utils/src/integrations/vercel-ai/util.ts index c5ac12b66062..07cbb8b4d82b 100644 --- a/packages/server-utils/src/integrations/vercel-ai/util.ts +++ b/packages/server-utils/src/integrations/vercel-ai/util.ts @@ -41,8 +41,6 @@ interface StreamChunk { delta?: unknown; // v4 text-delta chunks carry the text on `textDelta`; v5+ uses `delta`. textDelta?: unknown; - // v4 tool-call-delta chunks carry argument text on `argsTextDelta`. - argsTextDelta?: unknown; id?: unknown; modelId?: unknown; toolCallId?: unknown; @@ -118,7 +116,8 @@ export function tapModelCallStream( return; } - if (isOutputChunk(value)) { + // The model emits `stream-start` before the response arrives. + if ((value as StreamChunk | undefined)?.type !== 'stream-start') { recordTimeToFirstChunk(span); } text += accumulateChunk(state, value) ?? ''; @@ -138,30 +137,6 @@ export function tapModelCallStream( }); } -// Text, reasoning, tool-call, and file stream parts carry output. `tool-input-start` carries the tool -// name. Metadata, usage, finish, and the other `*-start`/`*-end` parts do not. -function isOutputChunk(chunk: unknown): boolean { - if (!isObjectLike(chunk)) { - return false; - } - const { type, delta, textDelta, argsTextDelta } = chunk as StreamChunk; - switch (type) { - case 'text-delta': - case 'reasoning-delta': - case 'tool-input-delta': - case 'reasoning': - return !!(delta || textDelta); - case 'tool-call-delta': - return !!argsTextDelta; - case 'tool-input-start': - case 'tool-call': - case 'file': - return true; - default: - return false; - } -} - /** * Fold a single streamed chunk into the running aggregate. Returns any text delta so the caller can * accumulate it (kept out of `state` until the end to avoid re-joining on every chunk). diff --git a/packages/server-utils/test/ai/lib/tracing/anthropic-ai-usage.test.ts b/packages/server-utils/test/ai/lib/tracing/anthropic-ai-usage.test.ts index 5d06a19c1e80..b1414ef2c273 100644 --- a/packages/server-utils/test/ai/lib/tracing/anthropic-ai-usage.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/anthropic-ai-usage.test.ts @@ -2,6 +2,7 @@ import { GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_MODEL, GEN_AI_RESPONSE_STREAMING, + GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK, GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, GEN_AI_USAGE_INPUT_TOKENS, @@ -21,6 +22,7 @@ function createMockSpan(): { span: Span; attributes: Record } { let ended = false; const span = { isRecording: () => !ended, + spanContext: () => ({ traceId: '0', spanId: '0', traceFlags: 0 }), setAttribute: (key: string, value: unknown) => { attributes[key] = value; }, @@ -121,7 +123,9 @@ describe.each(['response', 'async iterable', 'message stream'] as const)('Anthro [GEN_AI_USAGE_OUTPUT_TOKENS]: 120, [GEN_AI_USAGE_TOTAL_TOKENS]: totalTokens, ...cacheAttributes, - ...(mode === 'response' ? {} : { [GEN_AI_RESPONSE_STREAMING]: true }), + ...(mode === 'response' + ? {} + : { [GEN_AI_RESPONSE_STREAMING]: true, [GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]: expect.any(Number) }), }); }); }); @@ -305,6 +309,7 @@ describe('instrumentMessageStream', () => { expect(endSpy).toHaveBeenCalledTimes(1); expect(attributes).toEqual({ [GEN_AI_RESPONSE_STREAMING]: true, + [GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]: expect.any(Number), [GEN_AI_RESPONSE_ID]: 'msg_aborted', [GEN_AI_RESPONSE_MODEL]: 'claude-sonnet-4-6', [GEN_AI_USAGE_INPUT_TOKENS]: 42, diff --git a/packages/server-utils/test/ai/lib/tracing/cache-token-usage.test.ts b/packages/server-utils/test/ai/lib/tracing/cache-token-usage.test.ts index 6e88ac2f7421..eca356a176b6 100644 --- a/packages/server-utils/test/ai/lib/tracing/cache-token-usage.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/cache-token-usage.test.ts @@ -2,6 +2,7 @@ import { GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_MODEL, GEN_AI_RESPONSE_STREAMING, + GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK, GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, GEN_AI_USAGE_INPUT_TOKENS, @@ -27,6 +28,7 @@ describe.each(['openai chat', 'openai responses', 'google'] as const)('cache usa const span = { isRecording: () => true, setAttributes: (values: Record) => Object.assign(attributes, values), + spanContext: () => ({ traceId: '0', spanId: '0', traceFlags: 0 }), setAttribute: (key: string, value: unknown) => { attributes[key] = value; }, @@ -99,7 +101,9 @@ describe.each(['openai chat', 'openai responses', 'google'] as const)('cache usa ...(provider === 'google' ? {} : { [GEN_AI_RESPONSE_ID]: 'response_cache_usage', [GEN_AI_RESPONSE_MODEL]: 'gpt-4o' }), - ...(mode === 'response' ? {} : { [GEN_AI_RESPONSE_STREAMING]: true }), + ...(mode === 'response' + ? {} + : { [GEN_AI_RESPONSE_STREAMING]: true, [GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]: expect.any(Number) }), }); }); }); @@ -144,6 +148,7 @@ describe.each(['response', 'stream'] as const)('Google reasoning usage (%s)', mo const span = { isRecording: () => true, setAttributes: (values: Record) => Object.assign(attributes, values), + spanContext: () => ({ traceId: '0', spanId: '0', traceFlags: 0 }), setAttribute: (key: string, value: unknown) => { attributes[key] = value; }, @@ -180,7 +185,9 @@ describe.each(['response', 'stream'] as const)('Google reasoning usage (%s)', mo [GEN_AI_USAGE_OUTPUT_TOKENS]: output, [GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]: thoughts, [GEN_AI_USAGE_TOTAL_TOKENS]: expectedTotal, - ...(mode === 'stream' ? { [GEN_AI_RESPONSE_STREAMING]: true } : {}), + ...(mode === 'stream' + ? { [GEN_AI_RESPONSE_STREAMING]: true, [GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]: expect.any(Number) } + : {}), }); }); }); From 19b1a203d05892e6d7d2846090c59e24ebb07c9f Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:19:34 +0000 Subject: [PATCH 5/5] ref(server-utils): Check the span attribute instead of a WeakSet for the first chunk --- packages/server-utils/src/ai/core/utils.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/server-utils/src/ai/core/utils.ts b/packages/server-utils/src/ai/core/utils.ts index 50bf1701a680..4b9e56e7347b 100644 --- a/packages/server-utils/src/ai/core/utils.ts +++ b/packages/server-utils/src/ai/core/utils.ts @@ -273,18 +273,15 @@ export function markCancelledIfAborted(span: Span, abortSignal: AbortSignal | un } } -const spansWithFirstChunk = new WeakSet(); - /** * Sets `gen_ai.response.time_to_first_chunk` on a streamed span, in seconds from the span start. * Call it for each received chunk. Only the first call per span records a value. */ export function recordTimeToFirstChunk(span: Span): void { - if (spansWithFirstChunk.has(span)) { - return; + const { attributes, start_timestamp } = spanToJSON(span); + if (attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK] === undefined) { + span.setAttribute(GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK, timestampInSeconds() - start_timestamp); } - spansWithFirstChunk.add(span); - span.setAttribute(GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK, timestampInSeconds() - spanToJSON(span).start_timestamp); } /**