Skip to content
Draft
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
GEN_AI_RESPONSE_MODEL,
GEN_AI_RESPONSE_STREAMING,
GEN_AI_RESPONSE_TEXT,
GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK,
GEN_AI_USAGE_INPUT_TOKENS,
GEN_AI_USAGE_OUTPUT_TOKENS,
GEN_AI_USAGE_TOTAL_TOKENS,
Expand Down Expand Up @@ -122,6 +123,7 @@ it('traces a streaming Workers AI text generation request', async ({ signal }) =
type: 'string',
value: 'The capital of France is Paris.',
},
[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]: { value: expect.any(Number), type: 'double' },
[SENTRY_IS_LOCALHOST]: { value: true, type: 'boolean' },
[SENTRY_TRACE_LIFECYCLE]: { value: 'stream', type: 'string' },
[SENTRY_SEGMENT_NAME]: { value: segmentSpan!.name, type: 'string' },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
GEN_AI_RESPONSE_ID,
GEN_AI_RESPONSE_MODEL,
GEN_AI_RESPONSE_STREAMING,
GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK,
GEN_AI_RESPONSE_TEXT,
GEN_AI_RESPONSE_TOOL_CALLS,
GEN_AI_SYSTEM_INSTRUCTIONS,
Expand Down Expand Up @@ -116,6 +117,7 @@ describe('Anthropic integration', () => {
expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307');
expect(streamingSpan!.status).toBe('ok');
expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true);
expect(streamingSpan!.attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]?.value).toEqual(expect.any(Number));
});
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
GEN_AI_RESPONSE_ID,
GEN_AI_RESPONSE_MODEL,
GEN_AI_RESPONSE_STREAMING,
GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK,
GEN_AI_RESPONSE_TEXT,
GEN_AI_RESPONSE_TOOL_CALLS,
GEN_AI_SYSTEM_INSTRUCTIONS,
Expand Down Expand Up @@ -338,6 +339,7 @@ describe('Google GenAI integration', () => {
expect(generateContentSpan!.status).toBe('ok');
expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content');
expect(generateContentSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true);
expect(generateContentSpan!.attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]?.value).toEqual(expect.any(Number));
expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7);
expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.9);
expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
import { DB_OPERATION_NAME, GEN_AI_TOOL_NAME, SENTRY_OP, URL_FULL } from '@sentry/conventions/attributes';
import {
DB_OPERATION_NAME,
GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK,
GEN_AI_TOOL_NAME,
SENTRY_OP,
URL_FULL,
} from '@sentry/conventions/attributes';
import { GEN_AI_CHAT, GEN_AI_EXECUTE_TOOL } from '@sentry/conventions/op';
import { afterAll, expect, describe } from 'vitest';
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner';
Expand Down Expand Up @@ -94,6 +100,7 @@ describe.runIf(supports({ min: 22 }))('Mastra integration', () => {
const cacheGetParentIds = loadSpans.map(span => span.parent_span_id);
const chat = container.items.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_CHAT)!;
expect(chat).toBeDefined();
expect(chat.attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]?.value).toEqual(expect.any(Number));
const chatSpanId = chat.span_id;

const executeTool = container.items.find(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
GEN_AI_RESPONSE_ID,
GEN_AI_RESPONSE_MODEL,
GEN_AI_RESPONSE_STREAMING,
GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK,
GEN_AI_OUTPUT_MESSAGES,
GEN_AI_RESPONSE_TEXT,
GEN_AI_RESPONSE_TOOL_CALLS,
Expand Down Expand Up @@ -70,6 +71,7 @@ describe('Mistral integration', () => {
expect(streamSpan!.name).toBe('chat mistral-large-latest');
expect(streamSpan!.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('chat');
expect(streamSpan!.attributes[GEN_AI_RESPONSE_STREAMING]?.value).toBe(true);
expect(streamSpan!.attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]?.value).toEqual(expect.any(Number));
expect(streamSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(30);

// `chat.stream()` takes no `stream` request field, so the flag has to come from the method.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
GEN_AI_RESPONSE_ID,
GEN_AI_RESPONSE_MODEL,
GEN_AI_RESPONSE_STREAMING,
GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK,
GEN_AI_RESPONSE_TEXT,
GEN_AI_RESPONSE_TOOL_CALLS,
GEN_AI_SYSTEM_INSTRUCTIONS,
Expand Down Expand Up @@ -236,6 +237,9 @@ describe('OpenAI integration', () => {
type: 'boolean',
value: true,
});
expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]?.value).toEqual(
expect.any(Number),
);

const streamingResponsesSpan = spans.find(
span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456',
Expand Down Expand Up @@ -294,6 +298,9 @@ describe('OpenAI integration', () => {
type: 'boolean',
value: true,
});
expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]?.value).toEqual(
expect.any(Number),
);

const streamingErrorSpan = spans.find(
span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
GEN_AI_REQUEST_MODEL,
GEN_AI_RESPONSE_FINISH_REASONS,
GEN_AI_RESPONSE_STREAMING,
GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK,
GEN_AI_SYSTEM_INSTRUCTIONS,
GEN_AI_TOOL_DEFINITIONS,
GEN_AI_USAGE_INPUT_TOKENS,
Expand Down Expand Up @@ -60,6 +61,7 @@ describe.runIf(supports({ min: 22 }))('pi-ai integration', () => {
expect(toolCall!.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('faux');
expect(toolCall!.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('faux-model');
expect(toolCall!.attributes[GEN_AI_RESPONSE_STREAMING]?.value).toBe(true);
expect(toolCall!.attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]?.value).toBeGreaterThan(0);
expect(toolCall!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]?.value).toBe('You are a weather assistant.');
expect(toolCall!.attributes[GEN_AI_TOOL_DEFINITIONS]?.value).toContain('get_weather');
expect(toolCall!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value).toBe('["tool_call"]');
Expand All @@ -75,6 +77,7 @@ describe.runIf(supports({ min: 22 }))('pi-ai integration', () => {
]);

expect(answer!.attributes[GEN_AI_RESPONSE_STREAMING]).toBeUndefined();
expect(answer!.attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]).toBeUndefined();
expect(answer!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value).toBe('["stop"]');
expect(
JSON.parse(String(answer!.attributes[GEN_AI_INPUT_MESSAGES]?.value)).map(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
GEN_AI_PROVIDER_NAME,
GEN_AI_REQUEST_MODEL,
GEN_AI_RESPONSE_FINISH_REASONS,
GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK,
GEN_AI_RESPONSE_MODEL,
GEN_AI_SYSTEM_INSTRUCTIONS,
GEN_AI_TOOL_CALL_ARGUMENTS,
Expand Down Expand Up @@ -1123,6 +1124,9 @@ export function registerVercelAiStreamingTests(version: '6' | '7', vercelAiVersi
expect(generateContent.attributes[GEN_AI_OUTPUT_MESSAGES]?.value).toBe(
'[{"role":"assistant","parts":[{"type":"text","content":"Stream response!"}],"finish_reason":"stop"}]',
);
expect(generateContent.attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK]?.value).toEqual(
expect.any(Number),
);

// The summed usage and output also land on the parent invoke_agent span, whose own
// channel result is otherwise undefined for a stream.
Expand Down
4 changes: 3 additions & 1 deletion packages/server-utils/src/ai/anthropic-ai/streaming.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { captureException, SPAN_STATUS_ERROR } from '@sentry/core';
import type { Span } from '@sentry/core';
import { endStreamSpan, getErrorStatus } from '../core/utils';
import { endStreamSpan, getErrorStatus, recordTimeToFirstChunk } from '../core/utils';
import type { AnthropicAiStreamingEvent } from './types';
import { mapAnthropicErrorToStatusMessage } from './utils';

Expand Down Expand Up @@ -206,6 +206,8 @@ function processEvent(
const isError = isErrorEvent(event, span);
if (isError) return;

recordTimeToFirstChunk(span);

handleMessageMetadata(event, state);

// Tool call events are sent via 3 separate events:
Expand Down
22 changes: 21 additions & 1 deletion packages/server-utils/src/ai/core/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,15 @@
/**
* Shared utils for AI integrations (OpenAI, Anthropic, Verce.AI, etc.)
*/
import { getClient, isObjectLike, isThenable, SPAN_STATUS_ERROR, stringify } from '@sentry/core';
import {
getClient,
isObjectLike,
isThenable,
SPAN_STATUS_ERROR,
spanToJSON,
stringify,
timestampInSeconds,
} from '@sentry/core';
import type { Span, SpanStatus } from '@sentry/core';
import {
GEN_AI_OUTPUT_MESSAGES,
Expand All @@ -11,6 +19,7 @@ import {
GEN_AI_RESPONSE_MODEL,
GEN_AI_RESPONSE_STREAMING,
GEN_AI_RESPONSE_TEXT,
GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK,
GEN_AI_RESPONSE_TOOL_CALLS,
GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS,
GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS,
Expand Down Expand Up @@ -264,6 +273,17 @@ export function markCancelledIfAborted(span: Span, abortSignal: AbortSignal | un
}
}

/**
* Sets `gen_ai.response.time_to_first_chunk` on a streamed span, in seconds from the span start.
* Call it for each received chunk. Only the first call per span records a value.
*/
export function recordTimeToFirstChunk(span: Span): void {
const { attributes, start_timestamp } = spanToJSON(span);
if (attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK] === undefined) {
span.setAttribute(GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK, timestampInSeconds() - start_timestamp);
}
}

/**
* Ends a streaming span by setting all accumulated response attributes and ending the span.
* Shared across OpenAI, Anthropic, and Google GenAI streaming implementations.
Expand Down
3 changes: 2 additions & 1 deletion packages/server-utils/src/ai/google-genai/streaming.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { captureException, SPAN_STATUS_ERROR } from '@sentry/core';
import type { Span } from '@sentry/core';
import { endStreamSpan, getErrorStatus } from '../core/utils';
import { endStreamSpan, getErrorStatus, recordTimeToFirstChunk } from '../core/utils';
import type { GoogleGenAIResponse } from './types';
import type { MessagePart } from './utils';
import { candidatesToMessageParts, setOutputMessagesAttribute } from './utils';
Expand Down Expand Up @@ -111,6 +111,7 @@ function handleCandidateContent(chunk: GoogleGenAIResponse, state: StreamingStat
*/
function processChunk(chunk: GoogleGenAIResponse, state: StreamingState, recordOutputs: boolean, span: Span): void {
if (!chunk || isErrorChunk(chunk, span)) return;
recordTimeToFirstChunk(span);
handleResponseMetadata(chunk, state);
handleCandidateContent(chunk, state, recordOutputs);
}
Expand Down
10 changes: 9 additions & 1 deletion packages/server-utils/src/ai/langchain/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import {
} from '@sentry/conventions/attributes';
import { GEN_AI_CHAT, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op';
import { DEBUG_BUILD } from '../../debug-build';
import { resolveAIRecordingOptions } from '../core/utils';
import { recordTimeToFirstChunk, resolveAIRecordingOptions } from '../core/utils';
import { LANGCHAIN_ORIGIN, MAX_TRACKED_LANGCHAIN_RUNS } from './constants';
import {
addTypeSafeClassifierResponseAttributes,
Expand Down Expand Up @@ -269,6 +269,14 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}):
);
},

// Called for each streamed chunk.
handleLLMNewToken(_token: string, _idx: unknown, runId: string) {
const span = runs.get(runId)?.span;
if (span) {
recordTimeToFirstChunk(span);
}
},

// LLM End Handler - note: handleLLMEnd with capital LLM (used by both LLMs and chat models!)
handleLLMEnd(
output: unknown,
Expand Down
2 changes: 2 additions & 0 deletions packages/server-utils/src/ai/mastra/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ export interface MastraSpanAttributes {
responseId?: string;
finishReason?: string;
streaming?: boolean;
/** When the first streamed chunk arrived. */
completionStartTime?: Date;
usage?: MastraUsageStats;
parameters?: {
temperature?: number;
Expand Down
4 changes: 4 additions & 0 deletions packages/server-utils/src/ai/mastra/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import {
GEN_AI_RESPONSE_ID,
GEN_AI_RESPONSE_MODEL,
GEN_AI_RESPONSE_STREAMING,
GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK,
GEN_AI_RESPONSE_TEXT,
GEN_AI_SYSTEM_INSTRUCTIONS,
GEN_AI_TOOL_CALL_ARGUMENTS,
Expand Down Expand Up @@ -212,6 +213,9 @@ function addModelAttributes(
attributes[GEN_AI_RESPONSE_MODEL] = attrs.responseModel;
attributes[GEN_AI_RESPONSE_ID] = attrs.responseId;
attributes[GEN_AI_RESPONSE_STREAMING] = attrs.streaming;
attributes[GEN_AI_RESPONSE_TIME_TO_FIRST_CHUNK] = attrs.completionStartTime
? (attrs.completionStartTime.getTime() - span.startTime.getTime()) / 1000
: undefined;
attributes[GEN_AI_RESPONSE_FINISH_REASONS] = attrs.finishReason ? JSON.stringify([attrs.finishReason]) : undefined;
attributes[GEN_AI_AGENT_NAME] = span.entityName;

Expand Down
4 changes: 3 additions & 1 deletion packages/server-utils/src/ai/mistral/streaming.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import type { Span } from '@sentry/core';
import { endStreamSpan, getErrorStatus, setOutputMessagesAttribute } from '../core/utils';
import { endStreamSpan, getErrorStatus, recordTimeToFirstChunk, setOutputMessagesAttribute } from '../core/utils';
import type { MistralCompletionChunk, MistralToolCall } from './types';
import { contentToString } from './utils';

Expand Down Expand Up @@ -248,7 +248,9 @@ export function instrumentEventStream(
let settled = false;
let cancelled = false;

// Called for each received event.
const claim = (candidate: StreamConsumer): boolean => {
recordTimeToFirstChunk(span);
consumer ??= candidate;
return consumer === candidate;
};
Expand Down
3 changes: 2 additions & 1 deletion packages/server-utils/src/ai/openai/streaming.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { captureException, SPAN_STATUS_ERROR } from '@sentry/core';
import type { Span } from '@sentry/core';
import { getToolCallKey } from '../core/streaming-tool-calls';
import { endStreamSpan } from '../core/utils';
import { endStreamSpan, recordTimeToFirstChunk } from '../core/utils';
import { RESPONSE_EVENT_TYPES } from './constants';
import type {
ChatCompletionChunk,
Expand Down Expand Up @@ -241,6 +241,7 @@ export async function* instrumentStream<T>(

try {
for await (const event of stream) {
recordTimeToFirstChunk(span);
if (isChatCompletionChunk(event)) {
processChatCompletionChunk(event, state, recordOutputs);
} else if (isResponsesApiStreamEvent(event)) {
Expand Down
23 changes: 22 additions & 1 deletion packages/server-utils/src/ai/pi-ai/chat.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import {
SERVER_PORT,
} from '@sentry/conventions/attributes';
import { GEN_AI_CHAT } from '@sentry/conventions/op';
import { recordTimeToFirstChunk } from '../core/utils';
import { PI_AI_ORIGIN } from './constants';
import type { PiAiContext } from './messages';
import {
Expand All @@ -37,7 +38,7 @@ import {
piAiSystemInstructions,
piAiToolDefinitions,
} from './messages';
import type { PiAssistantMessage, PiModel, PiStreamOptions } from './types';
import type { PiAssistantMessage, PiEventStream, PiModel, PiStreamOptions } from './types';
import { setPiAiUsageAttributes } from './usage';

interface PiAiChatRequest {
Expand Down Expand Up @@ -118,3 +119,23 @@ function getRequestContentAttributes(context: PiAiContext): Record<string, strin
[GEN_AI_TOOL_DEFINITIONS]: tools.length ? stringify(tools) : undefined,
};
}

/**
* Records `gen_ai.response.time_to_first_chunk` for a streamed request. The stream has one consumer,
* so instead of reading it, this wraps `push`, which the provider calls for each event as it decodes
* it. The wrapper is removed after the first chunk.
*/
export function observePiAiFirstChunk(span: Span, events: Partial<PiEventStream> | undefined): void {
const push = events?.push;
if (!events || typeof push !== 'function') {
return;
}
events.push = function (this: unknown, event: unknown) {
// Providers push `start` before the response arrives.
if ((event as { type?: unknown } | undefined)?.type !== 'start') {
recordTimeToFirstChunk(span);
events.push = push;
}
return push.call(this, event);
};
}
1 change: 1 addition & 0 deletions packages/server-utils/src/ai/pi-ai/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,4 +32,5 @@ export interface PiStreamOptions {

export interface PiEventStream {
result(): Promise<PiAssistantMessage>;
push(event: unknown): void;
}
3 changes: 2 additions & 1 deletion packages/server-utils/src/ai/workers-ai/streaming.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import type { Span } from '@sentry/core';
import { getToolCallKey } from '../core/streaming-tool-calls';
import { endStreamSpan, getErrorStatus, type StreamResponseState } from '../core/utils';
import { endStreamSpan, getErrorStatus, recordTimeToFirstChunk, type StreamResponseState } from '../core/utils';
import type { WorkersAiUsage } from './types';
import { setOutputMessagesAttribute } from './utils';

Expand Down Expand Up @@ -232,6 +232,7 @@ export function instrumentWorkersAiStream(
return;
}

recordTimeToFirstChunk(span);
buffer += decoder.decode(value, { stream: true });
flushBuffer(false);
controller.enqueue(value);
Expand Down
3 changes: 2 additions & 1 deletion packages/server-utils/src/integrations/pi-ai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import * as diagnosticsChannel from '../utils/diagnosticsChannel';
import type { IntegrationFn, Span } from '@sentry/core';
import { defineIntegration, isThenable, startInactiveSpan } from '@sentry/core';
import type { GenAiOptions } from '../ai/core/utils';
import { getPiAiChatSpanOptions, setPiAiResponseAttributes } from '../ai/pi-ai/chat';
import { getPiAiChatSpanOptions, observePiAiFirstChunk, setPiAiResponseAttributes } from '../ai/pi-ai/chat';
import { PI_AI_INTEGRATION_NAME } from '../ai/pi-ai/constants';
import type { PiAiContext } from '../ai/pi-ai/messages';
import { isInsidePiAiSpan } from '../ai/pi-ai/providers';
Expand Down Expand Up @@ -66,6 +66,7 @@ function instrumentPiAi(options: PiAiOptions): void {
if (!isThenable(settled)) {
return false;
}
observePiAiFirstChunk(span, events);
settled.then(message => {
setResponse(span, data, message);
end();
Expand Down
Loading
Loading