From 29ad657bf7832ab44750aa639113692095317565 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2026 11:15:09 +0000 Subject: [PATCH 1/3] feat(server-utils): Decide gen_ai content recording per call for Anthropic, Google GenAI and Mistral recordInputs and recordOutputs of anthropicAIIntegration, googleGenAIIntegration, mistralAIIntegration and their instrument*Client wrappers can be functions that decide per call. They get the operation and the request params as passed (for Anthropic tool runner spans: the tool and its input), and run once when the span starts, in the caller's async context. Co-Authored-By: Francesco Gringl-Novy <2411343+mydea@users.noreply.github.com> --- .../server-utils/src/ai/anthropic-ai/index.ts | 27 +-- .../src/ai/anthropic-ai/tool-runner.ts | 8 +- .../server-utils/src/ai/anthropic-ai/types.ts | 19 ++- .../server-utils/src/ai/google-genai/index.ts | 23 ++- .../server-utils/src/ai/google-genai/types.ts | 15 +- packages/server-utils/src/ai/mistral/index.ts | 19 ++- packages/server-utils/src/ai/mistral/types.ts | 12 +- .../src/integrations/anthropic.ts | 44 ++--- .../src/integrations/google-genai.ts | 33 ++-- .../server-utils/src/integrations/mistral.ts | 16 +- .../tracing/anthropic-ai-tool-runner.test.ts | 33 ++++ .../test/ai/lib/tracing/anthropic-ai.test.ts | 59 ++++++- .../test/ai/lib/tracing/google-genai.test.ts | 56 +++++- .../test/ai/lib/tracing/mistral.test.ts | 35 +++- .../gen-ai-call-recording.test.ts | 161 ++++++++++++++++++ 15 files changed, 464 insertions(+), 96 deletions(-) create mode 100644 packages/server-utils/test/integrations/gen-ai-call-recording.test.ts diff --git a/packages/server-utils/src/ai/anthropic-ai/index.ts b/packages/server-utils/src/ai/anthropic-ai/index.ts index 4d987af2a474..6744490e5b4c 100644 --- a/packages/server-utils/src/ai/anthropic-ai/index.ts +++ b/packages/server-utils/src/ai/anthropic-ai/index.ts @@ -18,13 +18,13 @@ import { SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; import { GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../core/gen-ai-attributes'; -import type { InstrumentedMethodEntry } from '../core/utils'; +import type { GenAiOptions, InstrumentedMethodEntry } from '../core/utils'; import { getErrorStatus, getGenAiSpanOp, getRequestSignal, getTokenUsageAttributes, - resolveAIRecordingOptions, + resolveGenAiCallRecording, wrapPromiseWithMethods, } from '../core/utils'; import { ANTHROPIC_AI_ORIGIN, ANTHROPIC_METHOD_REGISTRY } from './constants'; @@ -184,7 +184,7 @@ function handleStreamingRequest( args: T, spanConfig: { name: string; op: string; attributes: Record }, params: Record | undefined, - options: AnthropicAiOptions, + { recordInputs, recordOutputs }: Required, isStreamRequested: boolean, isStreamingMethod: boolean, ): R | Promise { @@ -196,7 +196,7 @@ function handleStreamingRequest( const instrumentedPromise = startSpanManual(spanConfig, (span: Span) => { originalResult = target.apply(invocationThis, args) as Promise; - if (options.recordInputs && params) { + if (recordInputs && params) { addPrivateRequestAttributes(span, params); } @@ -206,7 +206,7 @@ function handleStreamingRequest( return instrumentAsyncIterableStream( result as AsyncIterable, span, - options.recordOutputs ?? false, + recordOutputs, ) as unknown as R; } catch (error) { return handleStreamingError(error, span, abortSignal); @@ -218,7 +218,7 @@ function handleStreamingRequest( } else { return startSpanManual(spanConfig, span => { try { - if (options.recordInputs && params) { + if (recordInputs && params) { addPrivateRequestAttributes(span, params); } // The helper synchronously delegates to `create`; suppress that one internal call so it @@ -226,7 +226,7 @@ function handleStreamingRequest( suppressDelegatedCreate = true; const messageStream = target.apply(invocationThis, args); suppressDelegatedCreate = false; - return instrumentMessageStream(messageStream, span, options.recordOutputs ?? false, abortSignal); + return instrumentMessageStream(messageStream, span, recordOutputs, abortSignal); } catch (error) { suppressDelegatedCreate = false; return handleStreamingError(error, span, abortSignal); @@ -265,10 +265,12 @@ function instrumentMethod( } const operationName = instrumentedMethod.operation || 'unknown'; + const params = typeof args[0] === 'object' && args[0] !== null ? (args[0] as Record) : undefined; + const recording = resolveGenAiCallRecording(options, { operation: 'chat', request: params }); const requestAttributes = extractRequestAttributes( args, operationName, - !!options.recordInputs, + recording.recordInputs, getProviderName(invocationThis), ); const model = requestAttributes[GEN_AI_REQUEST_MODEL] || 'unknown'; @@ -283,7 +285,6 @@ function instrumentMethod( attributes: requestAttributes as Record, }; - const params = typeof args[0] === 'object' ? (args[0] as Record) : undefined; const isStreamRequested = Boolean(params?.stream); if (isStreamRequested || isStreamingMethod) { @@ -293,7 +294,7 @@ function instrumentMethod( args, spanConfig, params, - options, + recording, isStreamRequested, isStreamingMethod, ); @@ -305,13 +306,13 @@ function instrumentMethod( const instrumentedPromise = startSpan(spanConfig, span => { originalResult = target.apply(invocationThis, args) as Promise; - if (options.recordInputs && params) { + if (recording.recordInputs && params) { addPrivateRequestAttributes(span, params); } return originalResult.then( result => { - addResponseAttributes(span, result as AnthropicAiResponse, options.recordOutputs); + addResponseAttributes(span, result as AnthropicAiResponse, recording.recordOutputs); return result; }, error => { @@ -388,5 +389,5 @@ function instrumentClientInPlace(client: T, options: Anthropic * @returns The instrumented client with the same type as the input */ export function instrumentAnthropicAiClient(anthropicAiClient: T, options?: AnthropicAiOptions): T { - return instrumentClientInPlace(anthropicAiClient, resolveAIRecordingOptions(options)); + return instrumentClientInPlace(anthropicAiClient, options ?? {}); } diff --git a/packages/server-utils/src/ai/anthropic-ai/tool-runner.ts b/packages/server-utils/src/ai/anthropic-ai/tool-runner.ts index f373f79c2468..f2a07eeaeaf9 100644 --- a/packages/server-utils/src/ai/anthropic-ai/tool-runner.ts +++ b/packages/server-utils/src/ai/anthropic-ai/tool-runner.ts @@ -29,7 +29,7 @@ import { } from '@sentry/conventions/attributes'; import { GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op'; import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../core/gen-ai-attributes'; -import { getErrorStatus, resolveAIRecordingOptions } from '../core/utils'; +import { getErrorStatus, resolveGenAiCallRecording } from '../core/utils'; import { withCleanScopes } from '../../utils/withCleanScopes'; import { ANTHROPIC_AI_ORIGIN } from './constants'; import type { AnthropicAiOptions } from './types'; @@ -188,8 +188,12 @@ function runWithSpan( args: unknown[], options: AnthropicAiOptions, ): unknown { - const { recordInputs, recordOutputs } = resolveAIRecordingOptions(options); const [input, context] = args as [unknown, RunContext]; + const { recordInputs, recordOutputs } = resolveGenAiCallRecording(options, { + operation: 'execute_tool', + tool, + input, + }); // SDK versions before `toolUse` existed pass only `toolUseBlock`. const callId = (context?.toolUse ?? context?.toolUseBlock)?.id; diff --git a/packages/server-utils/src/ai/anthropic-ai/types.ts b/packages/server-utils/src/ai/anthropic-ai/types.ts index ddee39046d57..36c4b212248d 100644 --- a/packages/server-utils/src/ai/anthropic-ai/types.ts +++ b/packages/server-utils/src/ai/anthropic-ai/types.ts @@ -1,7 +1,22 @@ -import type { GenAiOptions } from '../core/utils'; +import type { GenAiCallOptions } from '../core/utils'; + +/** The call that `recordInputs`/`recordOutputs` decide on: a model call, or a tool call of a tool runner. */ +export type AnthropicAiCallInfo = + | { + operation: 'chat'; + /** The request params, the first argument of the method (such as `messages.create`) as passed. */ + request: Record | undefined; + } + | { + operation: 'execute_tool'; + /** The runnable tool, as passed to the tool runner. */ + tool: { name: string }; + /** The input the model called the tool with. */ + input: unknown; + }; /** Options for the Anthropic AI integration. */ -export type AnthropicAiOptions = GenAiOptions; +export type AnthropicAiOptions = GenAiCallOptions; export type Message = { role: 'user' | 'assistant'; diff --git a/packages/server-utils/src/ai/google-genai/index.ts b/packages/server-utils/src/ai/google-genai/index.ts index d80559565ba4..e63cf1aed668 100644 --- a/packages/server-utils/src/ai/google-genai/index.ts +++ b/packages/server-utils/src/ai/google-genai/index.ts @@ -35,7 +35,7 @@ import { getErrorStatus, getGenAiSpanOp, getTokenUsageAttributes, - resolveAIRecordingOptions, + resolveGenAiCallRecording, } from '../core/utils'; import { GOOGLE_GENAI_METHOD_REGISTRY, GOOGLE_GENAI_SYSTEM_NAME } from './constants'; import { instrumentStream } from './streaming'; @@ -317,14 +317,13 @@ function instrumentMethod( apply(target, _, args: T): R | Promise { const operationName = instrumentedMethod.operation || 'unknown'; const params = args[0] as Record | undefined; + const { recordInputs, recordOutputs } = resolveGenAiCallRecording(options, { + operation: operationName, + request: params, + }); const attributeParams = resolveChatParams(operationName, params, context); const abortSignal = getAbortSignal(attributeParams); - const requestAttributes = extractRequestAttributes( - operationName, - attributeParams, - context, - !!options.recordInputs, - ); + const requestAttributes = extractRequestAttributes(operationName, attributeParams, context, recordInputs); const model = requestAttributes[GEN_AI_REQUEST_MODEL] || 'unknown'; const client = getClient(); // With span streaming, omit the `'unknown'` model sentinel so the name stays low-cardinality. @@ -344,11 +343,11 @@ function instrumentMethod( }, async (span: Span) => { try { - if (options.recordInputs && attributeParams) { + if (recordInputs && attributeParams) { addPrivateRequestAttributes(span, attributeParams, operationName); } const stream = await target.apply(context, args); - return instrumentStream(stream, span, Boolean(options.recordOutputs), abortSignal) as R; + return instrumentStream(stream, span, recordOutputs, abortSignal) as R; } catch (error) { span.setStatus(getErrorStatus(abortSignal)); span.end(); @@ -365,7 +364,7 @@ function instrumentMethod( attributes: requestAttributes, }, (span: Span) => { - if (options.recordInputs && attributeParams) { + if (recordInputs && attributeParams) { addPrivateRequestAttributes(span, attributeParams, operationName); } @@ -377,7 +376,7 @@ function instrumentMethod( result => { // Only add response attributes for content-producing methods, not for embeddings if (!isEmbeddings) { - addResponseAttributes(span, result, options.recordOutputs); + addResponseAttributes(span, result, recordOutputs); } }, ); @@ -476,5 +475,5 @@ function createDeepProxy(target: T, currentPath = '', options: * ``` */ export function instrumentGoogleGenAIClient(client: T, options?: GoogleGenAIOptions): T { - return createDeepProxy(client, '', resolveAIRecordingOptions(options)); + return createDeepProxy(client, '', options ?? {}); } diff --git a/packages/server-utils/src/ai/google-genai/types.ts b/packages/server-utils/src/ai/google-genai/types.ts index c5e0075955ac..444e28a7c8b0 100644 --- a/packages/server-utils/src/ai/google-genai/types.ts +++ b/packages/server-utils/src/ai/google-genai/types.ts @@ -1,7 +1,18 @@ -import type { GenAiOptions } from '../core/utils'; +import type { GenAiCallOptions } from '../core/utils'; + +/** The call that `recordInputs`/`recordOutputs` decide on. */ +export interface GoogleGenAICallInfo { + /** The gen_ai operation of the call: `generate_content`, `embeddings` or `chat`. */ + operation: string; + /** + * The request params, the first argument of the method (such as `models.generateContent` or + * `chat.sendMessage`) as passed. For a chat message this does not include the config of `chats.create()`. + */ + request: Record | undefined; +} /** Options for the Google GenAI integration. */ -export type GoogleGenAIOptions = GenAiOptions; +export type GoogleGenAIOptions = GenAiCallOptions; /** * Google GenAI Content Part diff --git a/packages/server-utils/src/ai/mistral/index.ts b/packages/server-utils/src/ai/mistral/index.ts index 4f6ba7b25f7e..2f3c2d51dfca 100644 --- a/packages/server-utils/src/ai/mistral/index.ts +++ b/packages/server-utils/src/ai/mistral/index.ts @@ -15,7 +15,7 @@ import { buildMethodPath, extractSystemInstructions, getGenAiSpanOp, - resolveAIRecordingOptions, + resolveGenAiCallRecording, wrapPromiseWithMethods, } from '../core/utils'; import { GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../core/gen-ai-attributes'; @@ -110,9 +110,12 @@ function instrumentMethod( return function instrumentedCall(...args: T): Promise { const operationName = instrumentedMethod.operation || 'unknown'; const isStreamRequested = !!instrumentedMethod.streaming; - const requestAttributes = extractRequestAttributes(args, operationName, !!options.recordInputs, isStreamRequested); - const params = args[0] as Record | undefined; + const { recordInputs, recordOutputs } = resolveGenAiCallRecording(options, { + operation: operationName, + request: params, + }); + const requestAttributes = extractRequestAttributes(args, operationName, recordInputs, isStreamRequested); const spanConfig = { name: getSpanName(operationName, requestAttributes), @@ -126,7 +129,7 @@ function instrumentMethod( const instrumentedPromise = startSpanManual(spanConfig, (span: Span) => { originalResult = originalMethod.apply(context, args); - if (options.recordInputs && params) { + if (recordInputs && params) { addRequestAttributes(span, params, operationName); } @@ -134,7 +137,7 @@ function instrumentMethod( result => { // Patched in place so the caller keeps the real `EventStream`; `instrumentEventStream` // takes over ending the span once the stream is drained. - if (!instrumentEventStream(result, span, options.recordOutputs ?? false)) { + if (!instrumentEventStream(result, span, recordOutputs)) { span.end(); } return result; @@ -155,12 +158,12 @@ function instrumentMethod( const instrumentedPromise = startSpan(spanConfig, (span: Span) => { originalResult = originalMethod.apply(context, args); - if (options.recordInputs && params) { + if (recordInputs && params) { addRequestAttributes(span, params, operationName); } return originalResult.then(result => { - addResponseAttributes(span, result, options.recordOutputs); + addResponseAttributes(span, result, recordOutputs); return result; }); }); @@ -202,5 +205,5 @@ function createDeepProxy(target: T, currentPath = '', options: * Can be used across Node.js, Cloudflare Workers, and Vercel Edge. */ export function instrumentMistralAiClient(client: T, options?: MistralOptions): T { - return createDeepProxy(client, '', resolveAIRecordingOptions(options)); + return createDeepProxy(client, '', options ?? {}); } diff --git a/packages/server-utils/src/ai/mistral/types.ts b/packages/server-utils/src/ai/mistral/types.ts index 840830b8e120..d2d600171a94 100644 --- a/packages/server-utils/src/ai/mistral/types.ts +++ b/packages/server-utils/src/ai/mistral/types.ts @@ -1,7 +1,15 @@ -import type { GenAiOptions } from '../core/utils'; +import type { GenAiCallOptions } from '../core/utils'; + +/** The call that `recordInputs`/`recordOutputs` decide on. */ +export interface MistralCallInfo { + /** The gen_ai operation of the call: `chat`, `embeddings` or `invoke_agent`. */ + operation: string; + /** The request params, the first argument of the method (such as `chat.complete`) as passed. */ + request: Record | undefined; +} /** Options for the Mistral integration. */ -export type MistralOptions = GenAiOptions; +export type MistralOptions = GenAiCallOptions; /** * A tool call as it appears in a response/stream delta. During streaming the `function.arguments` diff --git a/packages/server-utils/src/integrations/anthropic.ts b/packages/server-utils/src/integrations/anthropic.ts index 2a9e51d17d22..9677509e22dc 100644 --- a/packages/server-utils/src/integrations/anthropic.ts +++ b/packages/server-utils/src/integrations/anthropic.ts @@ -2,7 +2,7 @@ import { GEN_AI_REQUEST_MODEL, SENTRY_ORIGIN } from '@sentry/conventions/attribu import * as diagnosticsChannel from '../utils/diagnosticsChannel'; import type { IntegrationFn, Span, SpanAttributeValue } from '@sentry/core'; import { defineIntegration, getActiveSpan, getClient, hasSpanStreamingEnabled, startInactiveSpan } from '@sentry/core'; -import { getGenAiSpanOp, getRequestSignal, markCancelledIfAborted, resolveAIRecordingOptions } from '../ai/core/utils'; +import { getGenAiSpanOp, getRequestSignal, markCancelledIfAborted, resolveGenAiCallRecording } from '../ai/core/utils'; import { wrapApiPromiseResponse } from '../ai/core/apiPromise'; import { addPrivateRequestAttributes, addResponseAttributes, extractRequestAttributes } from '../ai/anthropic-ai'; import { ANTHROPIC_AI_ORIGIN } from '../ai/anthropic-ai/constants'; @@ -21,6 +21,9 @@ import { invokeOrchestrionInstrumentation } from '../orchestrion/instrumentation // integration is deduplicated out of the default set. const INTEGRATION_NAME = 'Anthropic_AI' as const; +// The recording decision is made once, when the span starts, in the caller's async context. +type AnthropicChannelContext = OrchestrionChannelContext & { _sentryRecordOutputs?: boolean }; + // `stream` determines how the span is ended const INSTRUMENTED_CHANNELS = [ { channel: CHANNELS.ANTHROPIC_CHAT, operation: 'chat', stream: 'async-iterable' }, @@ -57,32 +60,28 @@ function instrumentAnthropic(options: AnthropicAiOptions): void { disableNativeTracing(); for (const { channel, operation, stream } of INSTRUMENTED_CHANNELS) { bindTracingChannelToSpan( - diagnosticsChannel.tracingChannel(channel), + diagnosticsChannel.tracingChannel(channel), data => createGenAiSpan(data, operation, options, stream), { beforeSpanEnd: (span, data) => { if ('error' in data) { markCancelledIfAborted(span, getRequestSignal(data.arguments?.[1])); } - addResponseAttributes( - span, - data.result as AnthropicAiResponse, - resolveAIRecordingOptions(options).recordOutputs, - ); + addResponseAttributes(span, data.result as AnthropicAiResponse, !!data._sentryRecordOutputs); }, deferSpanEnd: ({ span, data, end }) => - wrapThreadStreamRequest(span, data, end, options) || + wrapThreadStreamRequest(span, data, end) || wrapApiPromiseResponse( data.result, response => { data.result = response; - if (!wrapStreamResult(span, data, stream, options)) { + if (!wrapStreamResult(span, data, stream)) { end(); } }, end, ) || - wrapStreamResult(span, data, stream, options), + wrapStreamResult(span, data, stream), }, ); } @@ -116,8 +115,8 @@ function instrumentAnthropic(options: AnthropicAiOptions): void { * Returning `undefined` opts the payload out so no span is opened. */ function createGenAiSpan( - data: OrchestrionChannelContext, - operation: string, + data: AnthropicChannelContext, + operation: 'chat', options: AnthropicAiOptions, stream: StreamMode, ): Span | undefined { @@ -139,7 +138,8 @@ function createGenAiSpan( const params = typeof args[0] === 'object' && args[0] !== null ? (args[0] as Record) : undefined; - const { recordInputs } = resolveAIRecordingOptions(options); + const { recordInputs, recordOutputs } = resolveGenAiCallRecording(options, { operation, request: params }); + data._sentryRecordOutputs = recordOutputs; const attributes = extractRequestAttributes(args, operation, recordInputs, getProviderName(data.self)); const model = (attributes[GEN_AI_REQUEST_MODEL] as string) || 'unknown'; @@ -222,12 +222,7 @@ const threadStreams = new WeakMap void, - options: AnthropicAiOptions, -): boolean { +function wrapThreadStreamRequest(span: Span, data: AnthropicChannelContext, end: (error?: unknown) => void): boolean { const signal = (data.arguments?.[1] as { signal?: AbortSignal } | undefined)?.signal; const threadStream = signal && threadStreams.get(signal); const request = data.result as { _responseProps?: () => Promise } | undefined; @@ -237,7 +232,7 @@ function wrapThreadStreamRequest( } const { stream, callerSignal } = threadStream; - const { recordOutputs } = resolveAIRecordingOptions(options); + const recordOutputs = !!data._sentryRecordOutputs; // Not `asResponse()`, which on 1.x also ends the SDK's own span when the user turned its tracing on. void request._responseProps().then( () => { @@ -265,13 +260,8 @@ function wrapThreadStreamRequest( * the span when iteration finishes. * - `message-stream`: `instrumentMessageStream` attaches `'message'`/`'error'`/`'end'` listeners that end the span. */ -function wrapStreamResult( - span: Span, - data: OrchestrionChannelContext, - stream: StreamMode, - options: AnthropicAiOptions, -): boolean { - const { recordOutputs } = resolveAIRecordingOptions(options); +function wrapStreamResult(span: Span, data: AnthropicChannelContext, stream: StreamMode): boolean { + const recordOutputs = !!data._sentryRecordOutputs; const result = data.result; if (stream === 'async-iterable' && isAsyncIterable(result)) { diff --git a/packages/server-utils/src/integrations/google-genai.ts b/packages/server-utils/src/integrations/google-genai.ts index 6ffae5f8df56..658690a9795f 100644 --- a/packages/server-utils/src/integrations/google-genai.ts +++ b/packages/server-utils/src/integrations/google-genai.ts @@ -9,7 +9,7 @@ import { spanToJSON, startInactiveSpan, } from '@sentry/core'; -import { getGenAiSpanOp, markCancelledIfAborted, resolveAIRecordingOptions } from '../ai/core/utils'; +import { getGenAiSpanOp, markCancelledIfAborted, resolveGenAiCallRecording } from '../ai/core/utils'; import { addPrivateRequestAttributes, addResponseAttributes, @@ -32,6 +32,9 @@ const INTEGRATION_NAME = 'Google_GenAI' as const; const ORIGIN = 'auto.ai.google_genai'; +// The recording decision is made once, when the span starts, in the caller's async context. +type GoogleGenAIChannelContext = OrchestrionChannelContext & { _sentryRecordOutputs?: boolean }; + // Each instrumented method maps to the gen_ai operation its span reports. const INSTRUMENTED_CHANNELS = [ { channel: CHANNELS.GOOGLE_GENAI_GENERATE_CONTENT, operation: 'generate_content' }, @@ -51,7 +54,7 @@ const _googleGenAIIntegration = ((options: GoogleGenAIOptions = {}) => { function instrumentGoogleGenai(options: GoogleGenAIOptions): void { for (const { channel, operation } of INSTRUMENTED_CHANNELS) { bindTracingChannelToSpan( - diagnosticsChannel.tracingChannel(channel), + diagnosticsChannel.tracingChannel(channel), data => createGenAiSpan(data, operation, options), { beforeSpanEnd: (span, data) => { @@ -65,14 +68,10 @@ function instrumentGoogleGenai(options: GoogleGenAIOptions): void { } // Embeddings responses carry no content attributes. if (operation !== 'embeddings') { - addResponseAttributes( - span, - data.result as GoogleGenAIResponse, - resolveAIRecordingOptions(options).recordOutputs, - ); + addResponseAttributes(span, data.result as GoogleGenAIResponse, !!data._sentryRecordOutputs); } }, - deferSpanEnd: ({ span, data }) => wrapStreamResult(span, data, operation, options), + deferSpanEnd: ({ span, data }) => wrapStreamResult(span, data, operation), }, ); } @@ -83,7 +82,7 @@ function instrumentGoogleGenai(options: GoogleGenAIOptions): void { * Returning `undefined` opts the payload out so no span is opened. */ function createGenAiSpan( - data: OrchestrionChannelContext, + data: GoogleGenAIChannelContext, operation: string, options: GoogleGenAIOptions, ): Span | undefined { @@ -109,9 +108,11 @@ function createGenAiSpan( const args = data.arguments ?? []; // `chat.sendMessage()` carries no config of its own unless the caller passes one, so recover the // one `chats.create()` left on the chat instance that the transform stashed in `data.self`. - const params = resolveChatParams(operation, args[0] as Record | undefined, data.self); + const request = args[0] as Record | undefined; + const params = resolveChatParams(operation, request, data.self); - const { recordInputs } = resolveAIRecordingOptions(options); + const { recordInputs, recordOutputs } = resolveGenAiCallRecording(options, { operation, request }); + data._sentryRecordOutputs = recordOutputs; const attributes = extractRequestAttributes(operation, params, data.self, recordInputs); const model = (attributes[GEN_AI_REQUEST_MODEL] as string) || 'unknown'; @@ -143,24 +144,18 @@ function isAsyncIterable(value: unknown): value is AsyncIterableStream { * For a stream we patch `result[Symbol.asyncIterator]` in place so `instrumentStream` ends the * span when iteration finishes. */ -function wrapStreamResult( - span: Span, - data: OrchestrionChannelContext, - operation: string, - options: GoogleGenAIOptions, -): boolean { +function wrapStreamResult(span: Span, data: GoogleGenAIChannelContext, operation: string): boolean { const result = data.result; if (!isAsyncIterable(result)) { return false; } - const { recordOutputs } = resolveAIRecordingOptions(options); const iterate = result[Symbol.asyncIterator].bind(result); const params = resolveChatParams(operation, data.arguments?.[0] as Record | undefined, data.self); const instrumented = instrumentStream( { [Symbol.asyncIterator]: iterate }, span, - recordOutputs ?? false, + !!data._sentryRecordOutputs, getAbortSignal(params), ); result[Symbol.asyncIterator] = () => instrumented; diff --git a/packages/server-utils/src/integrations/mistral.ts b/packages/server-utils/src/integrations/mistral.ts index 2092a689df77..542f630d1474 100644 --- a/packages/server-utils/src/integrations/mistral.ts +++ b/packages/server-utils/src/integrations/mistral.ts @@ -2,7 +2,7 @@ import { SENTRY_ORIGIN } from '@sentry/conventions/attributes'; import * as diagnosticsChannel from '../utils/diagnosticsChannel'; import type { IntegrationFn, Span, SpanAttributeValue } from '@sentry/core'; import { defineIntegration, startInactiveSpan } from '@sentry/core'; -import { getGenAiSpanOp, resolveAIRecordingOptions } from '../ai/core/utils'; +import { getGenAiSpanOp, resolveGenAiCallRecording } from '../ai/core/utils'; import { addRequestAttributes, extractRequestAttributes } from '../ai/mistral'; import { MISTRAL_INTEGRATION_NAME, MISTRAL_ORIGIN } from '../ai/mistral/constants'; import { instrumentEventStream } from '../ai/mistral/streaming'; @@ -17,6 +17,9 @@ import { bindTracingChannelToSpan } from '../tracing-channel'; const INTEGRATION_NAME = MISTRAL_INTEGRATION_NAME; +// The recording decision is made once, when the span starts, in the caller's async context. +type MistralChannelContext = OrchestrionChannelContext & { _sentryRecordOutputs?: boolean }; + // Each instrumented channel maps to the gen_ai operation its span reports. Streaming methods publish // on their own channel, so the span knows it is a stream before the result exists. const INSTRUMENTED_CHANNELS = [ @@ -39,15 +42,15 @@ const _mistralAIIntegration = ((options: MistralOptions = {}) => { function instrumentMistral(options: MistralOptions): void { for (const { channel, operation, streaming } of INSTRUMENTED_CHANNELS) { bindTracingChannelToSpan( - diagnosticsChannel.tracingChannel(channel), + diagnosticsChannel.tracingChannel(channel), data => createGenAiSpan(data, operation, streaming, options), { beforeSpanEnd: (span, data) => { - addResponseAttributes(span, data.result, resolveAIRecordingOptions(options).recordOutputs); + addResponseAttributes(span, data.result, !!data._sentryRecordOutputs); }, // Streaming: the result is an `EventStream` consumed later, so instrument it and let it end the span. deferSpanEnd: ({ span, data }) => - streaming && instrumentEventStream(data.result, span, resolveAIRecordingOptions(options).recordOutputs), + streaming && instrumentEventStream(data.result, span, !!data._sentryRecordOutputs), }, ); } @@ -58,7 +61,7 @@ function instrumentMistral(options: MistralOptions): void { * Returning `undefined` opts the payload out so no span is opened. */ function createGenAiSpan( - data: OrchestrionChannelContext, + data: MistralChannelContext, operation: string, streaming: boolean, options: MistralOptions, @@ -70,7 +73,8 @@ function createGenAiSpan( const args = data.arguments ?? []; const params = args[0] as Record | undefined; - const { recordInputs } = resolveAIRecordingOptions(options); + const { recordInputs, recordOutputs } = resolveGenAiCallRecording(options, { operation, request: params }); + data._sentryRecordOutputs = recordOutputs; const attributes = extractRequestAttributes(args, operation, recordInputs, streaming); attributes[SENTRY_ORIGIN] = MISTRAL_ORIGIN; diff --git a/packages/server-utils/test/ai/lib/tracing/anthropic-ai-tool-runner.test.ts b/packages/server-utils/test/ai/lib/tracing/anthropic-ai-tool-runner.test.ts index ff685f561b2e..6243a8ee3004 100644 --- a/packages/server-utils/test/ai/lib/tracing/anthropic-ai-tool-runner.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/anthropic-ai-tool-runner.test.ts @@ -31,6 +31,7 @@ import { instrumentAnthropicAiClient } from '../../../../src/ai/anthropic-ai'; import { instrumentSessionToolRunner } from '../../../../src/ai/anthropic-ai/tool-runner'; import { setAsyncLocalStorageAsyncContextStrategy } from '../../../../src/async-context'; import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../src/ai/core/gen-ai-attributes'; +import type { AnthropicAiCallInfo } from '../../../../src/ai/anthropic-ai/types'; import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; type Tool = { name: string; description?: string; input_schema: object; run: (...args: unknown[]) => unknown }; @@ -192,6 +193,38 @@ describe('instrumentAnthropicAiClient tool runner', () => { ]); }); + it("decides recording for each tool call, in the caller's async context", async () => { + const weather = tool('weather', async () => 'sunny'); + const recordInputs = vi.fn( + (call: AnthropicAiCallInfo) => + call.operation === 'execute_tool' && (call.input as { city: string }).city === 'Vienna', + ); + const recordOutputs = () => getCurrentScope().getScopeData().tags.outputs === 'on'; + instrumentAnthropicAiClient(fakeClient(), { recordInputs, recordOutputs }).beta.messages.toolRunner({ + tools: [weather], + }); + + await withScope(scope => { + scope.setTag('outputs', 'off'); + return weather.run({ city: 'Vienna' }, { toolUse: { id: 'toolu_1' } }); + }); + await withScope(scope => { + scope.setTag('outputs', 'on'); + return weather.run({ city: 'Paris' }, { toolUse: { id: 'toolu_2' } }); + }); + + expect(recordInputs).toHaveBeenCalledWith({ operation: 'execute_tool', tool: weather, input: { city: 'Vienna' } }); + expect( + toolSpans().map(({ data }) => ({ + inputs: GEN_AI_TOOL_CALL_ARGUMENTS in data, + outputs: GEN_AI_TOOL_CALL_RESULT in data, + })), + ).toEqual([ + { inputs: true, outputs: false }, + { inputs: false, outputs: true }, + ]); + }); + it('reads the call id from toolUseBlock on SDKs that pass only that', () => { const weather = tool('weather', () => 'sunny'); diff --git a/packages/server-utils/test/ai/lib/tracing/anthropic-ai.test.ts b/packages/server-utils/test/ai/lib/tracing/anthropic-ai.test.ts index 18799c8e871d..31bbf25cd882 100644 --- a/packages/server-utils/test/ai/lib/tracing/anthropic-ai.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/anthropic-ai.test.ts @@ -1,7 +1,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; +import { GEN_AI_INPUT_MESSAGES, GEN_AI_RESPONSE_TEXT } from '@sentry/conventions/attributes'; +import { getCurrentScope, getMainCarrier, setCurrentClient, spanToStaticSpanJSON, withScope } from '@sentry/core'; import type { Span } from '@sentry/core'; import { instrumentAnthropicAiClient } from '../../../../src/ai/anthropic-ai'; +import type { AnthropicAiCallInfo } from '../../../../src/ai/anthropic-ai/types'; import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; describe('instrumentAnthropicAiClient span names', () => { @@ -125,3 +127,58 @@ describe('instrumentAnthropicAiClient provider name', () => { expect(spanToStaticSpanJSON(endedSpans[0]!).data['gen_ai.provider.name']).toBe(expected); }); }); + +describe('instrumentAnthropicAiClient per-call recording', () => { + afterEach(() => { + getMainCarrier().__SENTRY__ = undefined; + }); + + it("decides inputs and outputs separately for each call, in the caller's async context", async () => { + getMainCarrier().__SENTRY__ = undefined; + const client = new TestClient( + getDefaultTestClientOptions({ dsn: 'https://public@dsn.ingest.sentry.io/1337', tracesSampleRate: 1 }), + ); + setCurrentClient(client); + client.init(); + const endedSpans: Span[] = []; + client.on('spanEnd', span => endedSpans.push(span)); + + const recordInputs = vi.fn( + (call: AnthropicAiCallInfo) => + call.operation === 'chat' && (call.request?.metadata as { user_id: string }).user_id === 'public', + ); + const recordOutputs = () => getCurrentScope().getScopeData().tags.outputs === 'on'; + const anthropic = instrumentAnthropicAiClient( + { + messages: { + create: vi.fn().mockResolvedValue({ id: 'msg', model: 'claude', content: [{ type: 'text', text: 'Hi' }] }), + }, + }, + { recordInputs, recordOutputs }, + ); + + const messages = [{ role: 'user', content: 'Hello' }]; + await withScope(scope => { + scope.setTag('outputs', 'off'); + return anthropic.messages.create({ model: 'claude', messages, metadata: { user_id: 'public' } }); + }); + await withScope(scope => { + scope.setTag('outputs', 'on'); + return anthropic.messages.create({ model: 'claude', messages, metadata: { user_id: 'private' } }); + }); + + expect(recordInputs).toHaveBeenCalledWith({ + operation: 'chat', + request: { model: 'claude', messages, metadata: { user_id: 'public' } }, + }); + expect( + endedSpans.map(span => { + const { data } = spanToStaticSpanJSON(span); + return { inputs: GEN_AI_INPUT_MESSAGES in data, outputs: GEN_AI_RESPONSE_TEXT in data }; + }), + ).toEqual([ + { inputs: true, outputs: false }, + { inputs: false, outputs: true }, + ]); + }); +}); diff --git a/packages/server-utils/test/ai/lib/tracing/google-genai.test.ts b/packages/server-utils/test/ai/lib/tracing/google-genai.test.ts index 1001a10ea766..e388f4d9a30c 100644 --- a/packages/server-utils/test/ai/lib/tracing/google-genai.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/google-genai.test.ts @@ -10,12 +10,14 @@ import { GEN_AI_REQUEST_TEMPERATURE, GEN_AI_REQUEST_TOP_K, GEN_AI_REQUEST_TOP_P, + GEN_AI_RESPONSE_TEXT, GEN_AI_SYSTEM_INSTRUCTIONS, GEN_AI_TOOL_DEFINITIONS, } from '@sentry/conventions/attributes'; -import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; +import { getCurrentScope, getMainCarrier, setCurrentClient, spanToStaticSpanJSON, withScope } from '@sentry/core'; import type { Span } from '@sentry/core'; import { instrumentGoogleGenAIClient } from '../../../../src/ai/google-genai'; +import type { GoogleGenAICallInfo } from '../../../../src/ai/google-genai/types'; import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; function setupClient(traceLifecycle: 'static' | 'stream'): Span[] { @@ -314,3 +316,55 @@ describe('instrumentGoogleGenAIClient chat config propagation', () => { expect(data[GEN_AI_REQUEST_MAX_TOKENS]).toBe(150); }); }); + +describe('instrumentGoogleGenAIClient per-call recording', () => { + beforeEach(() => { + getMainCarrier().__SENTRY__ = undefined; + }); + + afterEach(() => { + getMainCarrier().__SENTRY__ = undefined; + }); + + it("decides inputs and outputs separately for each call, in the caller's async context", async () => { + const endedSpans = setupClient('stream'); + const recordInputs = vi.fn( + (call: GoogleGenAICallInfo) => + (call.request?.config as { labels: Record }).labels.privacy === 'public', + ); + const recordOutputs = () => getCurrentScope().getScopeData().tags.outputs === 'on'; + const genai = instrumentGoogleGenAIClient( + { + models: { + generateContent: vi.fn().mockResolvedValue({ candidates: [{ content: { parts: [{ text: 'Hi' }] } }] }), + }, + }, + { recordInputs, recordOutputs }, + ); + + const request = (privacy: string) => ({ + model: 'gemini-2.5-flash', + contents: 'Hello', + config: { labels: { privacy } }, + }); + await withScope(scope => { + scope.setTag('outputs', 'off'); + return genai.models.generateContent(request('public')); + }); + await withScope(scope => { + scope.setTag('outputs', 'on'); + return genai.models.generateContent(request('private')); + }); + + expect(recordInputs).toHaveBeenCalledWith({ operation: 'generate_content', request: request('public') }); + expect( + endedSpans.map(span => { + const { data } = spanToStaticSpanJSON(span); + return { inputs: GEN_AI_INPUT_MESSAGES in data, outputs: GEN_AI_RESPONSE_TEXT in data }; + }), + ).toEqual([ + { inputs: true, outputs: false }, + { inputs: false, outputs: true }, + ]); + }); +}); diff --git a/packages/server-utils/test/ai/lib/tracing/mistral.test.ts b/packages/server-utils/test/ai/lib/tracing/mistral.test.ts index 89ee08ea34cc..af518a45abfd 100644 --- a/packages/server-utils/test/ai/lib/tracing/mistral.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/mistral.test.ts @@ -1,8 +1,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; +import { getCurrentScope, getMainCarrier, setCurrentClient, spanToStaticSpanJSON, withScope } from '@sentry/core'; import type { Span } from '@sentry/core'; import { GEN_AI_AGENT_NAME, + GEN_AI_INPUT_MESSAGES, GEN_AI_OPERATION_NAME, GEN_AI_PROVIDER_NAME, GEN_AI_RESPONSE_STREAMING, @@ -12,6 +13,7 @@ import { GEN_AI_USAGE_TOTAL_TOKENS, } from '@sentry/conventions/attributes'; import { instrumentMistralAiClient } from '../../../../src/ai/mistral'; +import type { MistralCallInfo } from '../../../../src/ai/mistral/types'; import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; const GEN_AI_REQUEST_STREAM = 'gen_ai.request.stream'; @@ -636,4 +638,35 @@ describe('instrumentMistralAiClient', () => { expect(spanToStaticSpanJSON(endedSpans[0]!).status).toBe('internal_error'); }); }); + + it("decides inputs and outputs separately for each call, in the caller's async context", async () => { + const endedSpans = setupClient('stream'); + const recordInputs = vi.fn((call: MistralCallInfo) => call.request?.user === 'public'); + const recordOutputs = () => getCurrentScope().getScopeData().tags.outputs === 'on'; + const client = instrumentMistralAiClient(fakeClient(), { recordInputs, recordOutputs }); + + const messages = [{ role: 'user', content: 'Hello' }]; + await withScope(scope => { + scope.setTag('outputs', 'off'); + return client.chat.complete({ model: 'mistral-small-latest', messages, user: 'public' }); + }); + await withScope(scope => { + scope.setTag('outputs', 'on'); + return client.chat.complete({ model: 'mistral-small-latest', messages, user: 'private' }); + }); + + expect(recordInputs).toHaveBeenCalledWith({ + operation: 'chat', + request: { model: 'mistral-small-latest', messages, user: 'public' }, + }); + expect( + endedSpans.map(span => { + const { data } = spanToStaticSpanJSON(span); + return { inputs: GEN_AI_INPUT_MESSAGES in data, outputs: GEN_AI_RESPONSE_TEXT in data }; + }), + ).toEqual([ + { inputs: true, outputs: false }, + { inputs: false, outputs: true }, + ]); + }); }); diff --git a/packages/server-utils/test/integrations/gen-ai-call-recording.test.ts b/packages/server-utils/test/integrations/gen-ai-call-recording.test.ts new file mode 100644 index 000000000000..3ec6ed898a5c --- /dev/null +++ b/packages/server-utils/test/integrations/gen-ai-call-recording.test.ts @@ -0,0 +1,161 @@ +import { tracingChannel } from 'node:diagnostics_channel'; +import type { Span } from '@sentry/core'; +import { + getCurrentScope, + GLOBAL_OBJ, + setAsyncContextStrategy, + setCurrentClient, + spanToJSON, + withScope, +} from '@sentry/core'; +import { GEN_AI_INPUT_MESSAGES, GEN_AI_RESPONSE_TEXT } from '@sentry/conventions/attributes'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { setAsyncLocalStorageAsyncContextStrategy } from '../../src/async-context'; +import { anthropicAIIntegration } from '../../src/integrations/anthropic'; +import { googleGenAIIntegration } from '../../src/integrations/google-genai'; +import { mistralAIIntegration } from '../../src/integrations/mistral'; +import { CHANNELS } from '../../src/orchestrion/channels'; +import { getDefaultTestClientOptions, TestClient } from '../mocks/client'; + +type CallInfo = { operation: string; request?: Record }; +type Decide = (call: CallInfo) => boolean | undefined; + +const MESSAGES = [{ role: 'user', content: 'Capital of France?' }]; + +const PROVIDERS = [ + { + name: 'anthropicAIIntegration', + channel: CHANNELS.ANTHROPIC_CHAT, + operation: 'chat', + params: (metadata: Record) => ({ model: 'claude-sonnet-4-5', messages: MESSAGES, metadata }), + response: { id: 'msg_1', model: 'claude-sonnet-4-5', content: [{ type: 'text', text: 'Paris.' }] }, + }, + { + name: 'googleGenAIIntegration', + channel: CHANNELS.GOOGLE_GENAI_GENERATE_CONTENT, + operation: 'generate_content', + params: (labels: Record) => ({ + model: 'gemini-2.5-flash', + contents: 'Capital of France?', + config: { labels }, + }), + response: { candidates: [{ content: { role: 'model', parts: [{ text: 'Paris.' }] } }] }, + }, + { + name: 'mistralAIIntegration', + channel: CHANNELS.MISTRAL_CHAT, + operation: 'chat', + params: (metadata: Record) => ({ model: 'mistral-large-latest', messages: MESSAGES, metadata }), + response: { + id: 'cmpl_1', + model: 'mistral-large-latest', + choices: [{ message: { role: 'assistant', content: 'Paris.' }, finishReason: 'stop' }], + }, + }, +]; + +describe('per-call recording options', () => { + let endedSpans: Span[]; + let decideInputs: Decide; + let decideOutputs: Decide; + + function useClient(): void { + const client = new TestClient( + getDefaultTestClientOptions({ dsn: 'https://public@dsn.ingest.sentry.io/1337', tracesSampleRate: 1 }), + ); + setCurrentClient(client); + client.init(); + client.on('spanEnd', span => endedSpans.push(span)); + } + + beforeAll(() => { + setAsyncLocalStorageAsyncContextStrategy(); + vi.stubEnv('ANTHROPIC_OPEN_TELEMETRY', undefined); + GLOBAL_OBJ.__SENTRY_ORCHESTRION__ = { runtime: ['@anthropic-ai/sdk', '@google/genai', '@mistralai/mistralai'] }; + endedSpans = []; + useClient(); + // The channels stay subscribed for the whole file, so each test swaps the decision functions. + const options = { + recordInputs: (call: CallInfo) => decideInputs(call), + recordOutputs: (call: CallInfo) => decideOutputs(call), + }; + for (const integration of [ + anthropicAIIntegration(options), + googleGenAIIntegration(options), + mistralAIIntegration(options), + ]) { + integration.setup!(getCurrentScope().getClient()!); + } + }); + + afterAll(() => { + vi.unstubAllEnvs(); + setAsyncContextStrategy(undefined); + delete GLOBAL_OBJ.__SENTRY_ORCHESTRION__; + }); + + beforeEach(() => { + endedSpans = []; + decideInputs = () => true; + decideOutputs = () => true; + useClient(); + }); + + function call(channel: string, params: Record, response: unknown): Promise { + return tracingChannel(channel).tracePromise(async () => response, { arguments: [params] }); + } + + function recorded(span: Span): { inputs: boolean; outputs: boolean } { + const { attributes } = spanToJSON(span); + return { inputs: GEN_AI_INPUT_MESSAGES in attributes, outputs: GEN_AI_RESPONSE_TEXT in attributes }; + } + + describe.each(PROVIDERS)('$name', ({ channel, operation, params, response }) => { + it('decides for each call from its request', async () => { + const calls: CallInfo[] = []; + decideInputs = info => { + calls.push(info); + return JSON.stringify(info.request).includes('"public"'); + }; + decideOutputs = decideInputs; + + const publicParams = params({ privacy: 'public' }); + await call(channel, publicParams, response); + await call(channel, params({ privacy: 'private' }), response); + + expect(calls[0]).toEqual({ operation, request: publicParams }); + expect(endedSpans.map(recorded)).toEqual([ + { inputs: true, outputs: true }, + { inputs: false, outputs: false }, + ]); + }); + + it("runs the functions in the caller's async context", async () => { + decideInputs = () => getCurrentScope().getScopeData().tags.privacy === 'public'; + decideOutputs = decideInputs; + + await withScope(async scope => { + scope.setTag('privacy', 'public'); + await call(channel, params({}), response); + }); + await withScope(async scope => { + scope.setTag('privacy', 'private'); + await call(channel, params({}), response); + }); + + expect(endedSpans.map(recorded)).toEqual([ + { inputs: true, outputs: true }, + { inputs: false, outputs: false }, + ]); + }); + + it('decides inputs and outputs separately', async () => { + decideInputs = () => false; + decideOutputs = () => true; + + await call(channel, params({}), response); + + expect(endedSpans.map(recorded)).toEqual([{ inputs: false, outputs: true }]); + }); + }); +}); From fe7a0ca5a2afa2297df362aadc8054b09e97076f Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2026 12:53:07 +0000 Subject: [PATCH 2/3] test(node): Cover per-call gen_ai recording for Anthropic, Google GenAI and Mistral with node integration tests Replaces the per-call recording unit tests with one node integration test per integration that runs the real SDK against a mock server: a public call, a private call, and streams whose scope tag flips after they start. The Anthropic test also covers the tool runner's execute_tool spans. Co-Authored-By: Francesco Gringl-Novy <2411343+mydea@users.noreply.github.com> --- .../node-suites/excludes.ts | 9 +- .../node-suites/excludes.ts | 3 + .../instrument-with-recording-functions.mjs | 18 ++ .../anthropic/recording-functions/test.ts | 57 +++++++ .../scenario-recording-functions.mjs | 135 +++++++++++++++ .../instrument-with-recording-functions.mjs | 16 ++ .../google-genai/recording-functions/test.ts | 49 ++++++ .../scenario-recording-functions.mjs | 65 +++++++ .../instrument-with-recording-functions.mjs | 16 ++ .../mistral/recording-functions/test.ts | 50 ++++++ .../mistral/scenario-recording-functions.mjs | 88 ++++++++++ .../tracing/anthropic-ai-tool-runner.test.ts | 33 ---- .../test/ai/lib/tracing/anthropic-ai.test.ts | 59 +------ .../test/ai/lib/tracing/google-genai.test.ts | 56 +----- .../test/ai/lib/tracing/mistral.test.ts | 35 +--- .../gen-ai-call-recording.test.ts | 161 ------------------ 16 files changed, 508 insertions(+), 342 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-recording-functions.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/anthropic/recording-functions/test.ts create mode 100644 dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-recording-functions.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/google-genai/instrument-with-recording-functions.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/google-genai/recording-functions/test.ts create mode 100644 dev-packages/node-integration-tests/suites/tracing/google-genai/scenario-recording-functions.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/mistral/instrument-with-recording-functions.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/mistral/recording-functions/test.ts create mode 100644 dev-packages/node-integration-tests/suites/tracing/mistral/scenario-recording-functions.mjs delete mode 100644 packages/server-utils/test/integrations/gen-ai-call-recording.test.ts diff --git a/dev-packages/bun-integration-tests/node-suites/excludes.ts b/dev-packages/bun-integration-tests/node-suites/excludes.ts index 31d62b28d255..e7ef95296e6e 100644 --- a/dev-packages/bun-integration-tests/node-suites/excludes.ts +++ b/dev-packages/bun-integration-tests/node-suites/excludes.ts @@ -76,6 +76,7 @@ export const NO_AUTO_INSTRUMENTATION = [ 'suites/pino/test.ts', 'suites/tracing/amqplib/test.ts', 'suites/tracing/anthropic/test.ts', + 'suites/tracing/anthropic/recording-functions/test.ts', 'suites/tracing/anthropic/v0/test.ts', 'suites/tracing/anthropic/v0.129/test.ts', 'suites/tracing/anthropic/v1/test.ts', @@ -88,6 +89,7 @@ export const NO_AUTO_INSTRUMENTATION = [ 'suites/tracing/google-genai-v2/test.ts', 'suites/tracing/google-genai-v3/test.ts', 'suites/tracing/google-genai/test.ts', + 'suites/tracing/google-genai/recording-functions/test.ts', 'suites/tracing/groq/test.ts', 'suites/tracing/hapi/test.ts', 'suites/tracing/ioredis-dc/test.ts', @@ -102,6 +104,7 @@ export const NO_AUTO_INSTRUMENTATION = [ 'suites/tracing/mcp-server/**', 'suites/tracing/mistral-v2/test.ts', 'suites/tracing/mistral/test.ts', + 'suites/tracing/mistral/recording-functions/test.ts', 'suites/tracing/mongodb-v4/test.ts', 'suites/tracing/mongodb-v5/test.ts', 'suites/tracing/mongodb-v6/test.ts', @@ -229,7 +232,10 @@ const BUN_BUILD_NO_HTTP_SERVER_INTEGRATION = ['suites/express/multiple-init/test // `@mistralai/mistralai` v3 statically imports its optional peer `@livekit/rtc-node`, so Bun cannot // bundle the scenarios. `mistral-v2` still covers Mistral here. -const BUN_BUILD_UNBUNDLEABLE_DEPENDENCY = ['suites/tracing/mistral/test.ts']; +const BUN_BUILD_UNBUNDLEABLE_DEPENDENCY = [ + 'suites/tracing/mistral/recording-functions/test.ts', + 'suites/tracing/mistral/test.ts', +]; // Some or all tests fail with the bundled scenarios. The cause is not investigated yet. In // `express/tracing` only the request data tests fail: they set `httpIntegration` options, and with @@ -244,6 +250,7 @@ const BUN_BUILD_NOT_TRIAGED = [ 'suites/tracing/google-genai-v2/test.ts', 'suites/tracing/google-genai-v3/test.ts', 'suites/tracing/google-genai/test.ts', + 'suites/tracing/google-genai/recording-functions/test.ts', 'suites/tracing/langchain/v1/**', 'suites/tracing/langgraph/v1/test.ts', 'suites/tracing/mastra/**', diff --git a/dev-packages/deno-integration-tests/node-suites/excludes.ts b/dev-packages/deno-integration-tests/node-suites/excludes.ts index c7e7d700f3a9..4757739940da 100644 --- a/dev-packages/deno-integration-tests/node-suites/excludes.ts +++ b/dev-packages/deno-integration-tests/node-suites/excludes.ts @@ -56,11 +56,13 @@ const REQUIRE_OF_ESM_ONLY_DEPENDENCY = [ // version) and pass on Deno 2.9.0. const NOT_TRIAGED = [ 'suites/tracing/anthropic/test.ts', + 'suites/tracing/anthropic/recording-functions/test.ts', 'suites/tracing/anthropic/v0.129/test.ts', 'suites/tracing/apollo-graphql/**', 'suites/tracing/fastify/test.ts', 'suites/tracing/flue/test.ts', 'suites/tracing/google-genai/test.ts', + 'suites/tracing/google-genai/recording-functions/test.ts', 'suites/tracing/groq/test.ts', 'suites/tracing/http-client-spans/http-strip-query/test.ts', 'suites/tracing/ioredis-dc/test.ts', @@ -69,6 +71,7 @@ const NOT_TRIAGED = [ 'suites/tracing/langgraph/test.ts', 'suites/tracing/mistral-v2/test.ts', 'suites/tracing/mistral/test.ts', + 'suites/tracing/mistral/recording-functions/test.ts', 'suites/tracing/mongodb/test.ts', 'suites/tracing/mongoose-v5/test.ts', 'suites/tracing/mysql/test.ts', diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-recording-functions.mjs new file mode 100644 index 000000000000..1817614f384e --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-recording-functions.mjs @@ -0,0 +1,18 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + integrations: [ + Sentry.anthropicAIIntegration({ + recordInputs: call => + call.operation === 'execute_tool' + ? call.input?.city === 'Vienna' + : !!call.request?.metadata?.user_id?.startsWith('public'), + recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public', + }), + ], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/recording-functions/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/recording-functions/test.ts new file mode 100644 index 000000000000..909e27ff6df7 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/recording-functions/test.ts @@ -0,0 +1,57 @@ +import { afterAll, describe, expect } from 'vitest'; +import { + GEN_AI_INPUT_MESSAGES, + GEN_AI_RESPONSE_ID, + GEN_AI_RESPONSE_TEXT, + GEN_AI_TOOL_CALL_ARGUMENTS, + GEN_AI_TOOL_CALL_RESULT, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import { join } from 'node:path'; +import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; + +describe('Anthropic integration', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + createEsmAndCjsTests( + join(__dirname, '..'), + 'scenario-recording-functions.mjs', + 'instrument-with-recording-functions.mjs', + (createRunner, test) => { + test('decides content recording per call when the options are functions', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic', + ); + const recordedById = Object.fromEntries( + spans.map(span => [ + span.attributes[GEN_AI_RESPONSE_ID]?.value ?? span.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]?.value, + { + inputs: GEN_AI_INPUT_MESSAGES in span.attributes || GEN_AI_TOOL_CALL_ARGUMENTS in span.attributes, + outputs: GEN_AI_RESPONSE_TEXT in span.attributes || GEN_AI_TOOL_CALL_RESULT in span.attributes, + }, + ]), + ); + expect(recordedById).toEqual({ + msg_public: { inputs: true, outputs: true }, + msg_private: { inputs: false, outputs: false }, + 'msg_public-stream': { inputs: true, outputs: false }, + 'msg_public-helper-stream': { inputs: true, outputs: false }, + msg_tool_use: { inputs: false, outputs: true }, + msg_tool_final: { inputs: false, outputs: true }, + toolu_vienna: { inputs: true, outputs: true }, + toolu_paris: { inputs: false, outputs: true }, + }); + }, + }) + .start() + .completed(); + }); + }, + ); +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-recording-functions.mjs new file mode 100644 index 000000000000..b9e42c816f6c --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-recording-functions.mjs @@ -0,0 +1,135 @@ +import Anthropic from '@anthropic-ai/sdk'; +import * as Sentry from '@sentry/node'; +import express from 'express'; + +const MODEL = 'claude-3-haiku-20240307'; + +function message(id, content, stopReason = 'end_turn') { + return { + id, + type: 'message', + role: 'assistant', + model: MODEL, + content, + stop_reason: stopReason, + usage: { input_tokens: 10, output_tokens: 5 }, + }; +} + +function writeStream(res, id) { + res.writeHead(200, { 'Content-Type': 'text/event-stream' }); + const { content, ...start } = message(id, []); + const events = [ + { type: 'message_start', message: { ...start, content } }, + { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, + { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'Paris.' } }, + { type: 'content_block_stop', index: 0 }, + { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 5 } }, + { type: 'message_stop' }, + ]; + for (const event of events) { + res.write(`event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`); + } + res.end(); +} + +function startMockServer() { + const app = express(); + app.use(express.json()); + + app.post('/anthropic/v1/messages', (req, res) => { + const { metadata, stream, messages } = req.body; + + if (metadata.user_id === 'runner') { + const answeredTools = messages.some(m => Array.isArray(m.content) && m.content[0]?.type === 'tool_result'); + res.send( + answeredTools + ? message('msg_tool_final', [{ type: 'text', text: 'Sunny in both.' }]) + : message( + 'msg_tool_use', + [ + { type: 'tool_use', id: 'toolu_vienna', name: 'weather', input: { city: 'Vienna' } }, + { type: 'tool_use', id: 'toolu_paris', name: 'weather', input: { city: 'Paris' } }, + ], + 'tool_use', + ), + ); + return; + } + + const id = `msg_${metadata.user_id}${stream ? '-stream' : ''}`; + if (stream) { + writeStream(res, id); + return; + } + res.send(message(id, [{ type: 'text', text: 'Paris.' }])); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockServer(); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const client = new Anthropic({ + apiKey: 'mock-api-key', + baseURL: `http://localhost:${server.address().port}/anthropic`, + }); + const request = userId => ({ + model: MODEL, + max_tokens: 100, + messages: [{ role: 'user', content: 'What is the capital of France?' }], + metadata: { user_id: userId }, + }); + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'public'); + await client.messages.create(request('public')); + }); + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'private'); + await client.messages.create(request('private')); + + // Changing the tag after a call started must not change its outputs decision. + const stream = await client.messages.create({ ...request('public'), stream: true }); + Sentry.getCurrentScope().setTag('privacy', 'public'); + for await (const event of stream) { + void event; + } + + Sentry.getCurrentScope().setTag('privacy', 'private'); + const helper = client.messages.stream(request('public-helper')); + Sentry.getCurrentScope().setTag('privacy', 'public'); + await helper.finalMessage(); + }); + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'public'); + const runner = client.beta.messages.toolRunner({ + ...request('runner'), + tools: [ + { + name: 'weather', + description: 'Get the weather by city', + input_schema: { type: 'object', properties: { city: { type: 'string' } } }, + run: () => 'sunny', + }, + ], + }); + for await (const msg of runner) { + void msg; + } + }); + }); + + await Sentry.flush(2000); + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai/instrument-with-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/google-genai/instrument-with-recording-functions.mjs new file mode 100644 index 000000000000..d67b7cddf66d --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai/instrument-with-recording-functions.mjs @@ -0,0 +1,16 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + integrations: [ + Sentry.googleGenAIIntegration({ + recordInputs: ({ operation, request }) => + operation === 'generate_content' && !!request?.contents?.startsWith('Public'), + recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public', + }), + ], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai/recording-functions/test.ts b/dev-packages/node-integration-tests/suites/tracing/google-genai/recording-functions/test.ts new file mode 100644 index 000000000000..97c01f59611a --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai/recording-functions/test.ts @@ -0,0 +1,49 @@ +import { afterAll, describe, expect } from 'vitest'; +import { + GEN_AI_INPUT_MESSAGES, + GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_TEXT, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import { join } from 'node:path'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; + +describe('Google GenAI integration', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + createEsmAndCjsTests( + join(__dirname, '..'), + 'scenario-recording-functions.mjs', + 'instrument-with-recording-functions.mjs', + (createRunner, test) => { + test('decides content recording per call when the options are functions', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.google_genai', + ); + const recordedByModel = Object.fromEntries( + spans.map(span => [ + span.attributes[GEN_AI_REQUEST_MODEL]?.value, + { + inputs: GEN_AI_INPUT_MESSAGES in span.attributes, + outputs: GEN_AI_RESPONSE_TEXT in span.attributes, + }, + ]), + ); + expect(recordedByModel).toEqual({ + 'gemini-public': { inputs: true, outputs: true }, + 'gemini-private': { inputs: false, outputs: false }, + 'gemini-public-stream': { inputs: true, outputs: false }, + }); + }, + }) + .start() + .completed(); + }); + }, + ); +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai/scenario-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/google-genai/scenario-recording-functions.mjs new file mode 100644 index 000000000000..3d06b23f6fd2 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai/scenario-recording-functions.mjs @@ -0,0 +1,65 @@ +import { GoogleGenAI } from '@google/genai'; +import * as Sentry from '@sentry/node'; +import express from 'express'; + +const RESPONSE = { + candidates: [{ content: { parts: [{ text: 'Paris.' }], role: 'model' }, finishReason: 'STOP', index: 0 }], + usageMetadata: { promptTokenCount: 8, candidatesTokenCount: 2, totalTokenCount: 10 }, +}; + +function startMockServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1beta/models/:model\\:generateContent', (_req, res) => { + res.send(RESPONSE); + }); + + app.post('/v1beta/models/:model\\:streamGenerateContent', (_req, res) => { + res.setHeader('Content-Type', 'application/json'); + res.write(`data: ${JSON.stringify(RESPONSE)}\n\n`); + res.end(); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockServer(); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const client = new GoogleGenAI({ + apiKey: 'mock-api-key', + httpOptions: { baseUrl: `http://localhost:${server.address().port}` }, + }); + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'public'); + await client.models.generateContent({ model: 'gemini-public', contents: 'Public: capital of France?' }); + }); + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'private'); + await client.models.generateContent({ model: 'gemini-private', contents: 'Private: capital of France?' }); + + // Changing the tag after a call started must not change its outputs decision. + const stream = await client.models.generateContentStream({ + model: 'gemini-public-stream', + contents: 'Public: capital of France?', + }); + Sentry.getCurrentScope().setTag('privacy', 'public'); + for await (const chunk of stream) { + void chunk; + } + }); + }); + + await Sentry.flush(2000); + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/mistral/instrument-with-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/mistral/instrument-with-recording-functions.mjs new file mode 100644 index 000000000000..11b2c29ecd67 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/mistral/instrument-with-recording-functions.mjs @@ -0,0 +1,16 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + integrations: [ + Sentry.mistralAIIntegration({ + recordInputs: ({ operation, request }) => + operation === 'chat' && !!request?.messages?.[0]?.content?.startsWith('Public'), + recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public', + }), + ], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/mistral/recording-functions/test.ts b/dev-packages/node-integration-tests/suites/tracing/mistral/recording-functions/test.ts new file mode 100644 index 000000000000..0316e49651e4 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/mistral/recording-functions/test.ts @@ -0,0 +1,50 @@ +import { afterAll, describe, expect } from 'vitest'; +import { + GEN_AI_INPUT_MESSAGES, + GEN_AI_RESPONSE_ID, + GEN_AI_RESPONSE_TEXT, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import { join } from 'node:path'; +import { cleanupChildProcesses, createEsmTests } from '../../../../utils/runner'; + +// ESM-only, like the rest of the Mistral suite: `@mistralai/mistralai` v2+ ships no CJS build. +describe('Mistral integration', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + createEsmTests( + join(__dirname, '..'), + 'scenario-recording-functions.mjs', + 'instrument-with-recording-functions.mjs', + (createRunner, test) => { + test('decides content recording per call when the options are functions', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mistralai', + ); + const recordedById = Object.fromEntries( + spans.map(span => [ + span.attributes[GEN_AI_RESPONSE_ID]?.value, + { + inputs: GEN_AI_INPUT_MESSAGES in span.attributes, + outputs: GEN_AI_RESPONSE_TEXT in span.attributes, + }, + ]), + ); + expect(recordedById).toEqual({ + 'cmpl-mistral-public': { inputs: true, outputs: true }, + 'cmpl-mistral-private': { inputs: false, outputs: false }, + 'cmpl-mistral-public-stream': { inputs: true, outputs: false }, + }); + }, + }) + .start() + .completed(); + }); + }, + ); +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/mistral/scenario-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/mistral/scenario-recording-functions.mjs new file mode 100644 index 000000000000..1be833a3a76b --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/mistral/scenario-recording-functions.mjs @@ -0,0 +1,88 @@ +import { Mistral } from '@mistralai/mistralai'; +import * as Sentry from '@sentry/node'; +import express from 'express'; + +function startMockServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/chat/completions', (req, res) => { + const { model, stream } = req.body; + const id = `cmpl-${model}`; + + if (!stream) { + res.send({ + id, + object: 'chat.completion', + created: 1677652288, + model, + choices: [{ index: 0, message: { role: 'assistant', content: 'Paris.' }, finish_reason: 'stop' }], + usage: { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 }, + }); + return; + } + + res.setHeader('Content-Type', 'text/event-stream'); + const chunks = [ + { + id, + object: 'chat.completion.chunk', + created: 1677652300, + model, + choices: [{ index: 0, delta: { content: 'Paris.' }, finish_reason: null }], + }, + { + id, + object: 'chat.completion.chunk', + created: 1677652300, + model, + choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], + }, + ]; + for (const chunk of chunks) { + res.write(`data: ${JSON.stringify(chunk)}\n\n`); + } + res.write('data: [DONE]\n\n'); + res.end(); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockServer(); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const client = new Mistral({ + apiKey: 'mock-api-key', + serverURL: `http://localhost:${server.address().port}`, + }); + const messages = privacy => [{ role: 'user', content: `${privacy}: capital of France?` }]; + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'public'); + await client.chat.complete({ model: 'mistral-public', messages: messages('Public') }); + }); + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'private'); + await client.chat.complete({ model: 'mistral-private', messages: messages('Private') }); + + // Changing the tag after a call started must not change its outputs decision. + const stream = await client.chat.stream({ model: 'mistral-public-stream', messages: messages('Public') }); + Sentry.getCurrentScope().setTag('privacy', 'public'); + for await (const event of stream) { + void event; + } + }); + }); + + await Sentry.flush(2000); + server.close(); +} + +run(); diff --git a/packages/server-utils/test/ai/lib/tracing/anthropic-ai-tool-runner.test.ts b/packages/server-utils/test/ai/lib/tracing/anthropic-ai-tool-runner.test.ts index 6243a8ee3004..ff685f561b2e 100644 --- a/packages/server-utils/test/ai/lib/tracing/anthropic-ai-tool-runner.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/anthropic-ai-tool-runner.test.ts @@ -31,7 +31,6 @@ import { instrumentAnthropicAiClient } from '../../../../src/ai/anthropic-ai'; import { instrumentSessionToolRunner } from '../../../../src/ai/anthropic-ai/tool-runner'; import { setAsyncLocalStorageAsyncContextStrategy } from '../../../../src/async-context'; import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../src/ai/core/gen-ai-attributes'; -import type { AnthropicAiCallInfo } from '../../../../src/ai/anthropic-ai/types'; import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; type Tool = { name: string; description?: string; input_schema: object; run: (...args: unknown[]) => unknown }; @@ -193,38 +192,6 @@ describe('instrumentAnthropicAiClient tool runner', () => { ]); }); - it("decides recording for each tool call, in the caller's async context", async () => { - const weather = tool('weather', async () => 'sunny'); - const recordInputs = vi.fn( - (call: AnthropicAiCallInfo) => - call.operation === 'execute_tool' && (call.input as { city: string }).city === 'Vienna', - ); - const recordOutputs = () => getCurrentScope().getScopeData().tags.outputs === 'on'; - instrumentAnthropicAiClient(fakeClient(), { recordInputs, recordOutputs }).beta.messages.toolRunner({ - tools: [weather], - }); - - await withScope(scope => { - scope.setTag('outputs', 'off'); - return weather.run({ city: 'Vienna' }, { toolUse: { id: 'toolu_1' } }); - }); - await withScope(scope => { - scope.setTag('outputs', 'on'); - return weather.run({ city: 'Paris' }, { toolUse: { id: 'toolu_2' } }); - }); - - expect(recordInputs).toHaveBeenCalledWith({ operation: 'execute_tool', tool: weather, input: { city: 'Vienna' } }); - expect( - toolSpans().map(({ data }) => ({ - inputs: GEN_AI_TOOL_CALL_ARGUMENTS in data, - outputs: GEN_AI_TOOL_CALL_RESULT in data, - })), - ).toEqual([ - { inputs: true, outputs: false }, - { inputs: false, outputs: true }, - ]); - }); - it('reads the call id from toolUseBlock on SDKs that pass only that', () => { const weather = tool('weather', () => 'sunny'); diff --git a/packages/server-utils/test/ai/lib/tracing/anthropic-ai.test.ts b/packages/server-utils/test/ai/lib/tracing/anthropic-ai.test.ts index 31bbf25cd882..18799c8e871d 100644 --- a/packages/server-utils/test/ai/lib/tracing/anthropic-ai.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/anthropic-ai.test.ts @@ -1,9 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { GEN_AI_INPUT_MESSAGES, GEN_AI_RESPONSE_TEXT } from '@sentry/conventions/attributes'; -import { getCurrentScope, getMainCarrier, setCurrentClient, spanToStaticSpanJSON, withScope } from '@sentry/core'; +import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; import type { Span } from '@sentry/core'; import { instrumentAnthropicAiClient } from '../../../../src/ai/anthropic-ai'; -import type { AnthropicAiCallInfo } from '../../../../src/ai/anthropic-ai/types'; import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; describe('instrumentAnthropicAiClient span names', () => { @@ -127,58 +125,3 @@ describe('instrumentAnthropicAiClient provider name', () => { expect(spanToStaticSpanJSON(endedSpans[0]!).data['gen_ai.provider.name']).toBe(expected); }); }); - -describe('instrumentAnthropicAiClient per-call recording', () => { - afterEach(() => { - getMainCarrier().__SENTRY__ = undefined; - }); - - it("decides inputs and outputs separately for each call, in the caller's async context", async () => { - getMainCarrier().__SENTRY__ = undefined; - const client = new TestClient( - getDefaultTestClientOptions({ dsn: 'https://public@dsn.ingest.sentry.io/1337', tracesSampleRate: 1 }), - ); - setCurrentClient(client); - client.init(); - const endedSpans: Span[] = []; - client.on('spanEnd', span => endedSpans.push(span)); - - const recordInputs = vi.fn( - (call: AnthropicAiCallInfo) => - call.operation === 'chat' && (call.request?.metadata as { user_id: string }).user_id === 'public', - ); - const recordOutputs = () => getCurrentScope().getScopeData().tags.outputs === 'on'; - const anthropic = instrumentAnthropicAiClient( - { - messages: { - create: vi.fn().mockResolvedValue({ id: 'msg', model: 'claude', content: [{ type: 'text', text: 'Hi' }] }), - }, - }, - { recordInputs, recordOutputs }, - ); - - const messages = [{ role: 'user', content: 'Hello' }]; - await withScope(scope => { - scope.setTag('outputs', 'off'); - return anthropic.messages.create({ model: 'claude', messages, metadata: { user_id: 'public' } }); - }); - await withScope(scope => { - scope.setTag('outputs', 'on'); - return anthropic.messages.create({ model: 'claude', messages, metadata: { user_id: 'private' } }); - }); - - expect(recordInputs).toHaveBeenCalledWith({ - operation: 'chat', - request: { model: 'claude', messages, metadata: { user_id: 'public' } }, - }); - expect( - endedSpans.map(span => { - const { data } = spanToStaticSpanJSON(span); - return { inputs: GEN_AI_INPUT_MESSAGES in data, outputs: GEN_AI_RESPONSE_TEXT in data }; - }), - ).toEqual([ - { inputs: true, outputs: false }, - { inputs: false, outputs: true }, - ]); - }); -}); diff --git a/packages/server-utils/test/ai/lib/tracing/google-genai.test.ts b/packages/server-utils/test/ai/lib/tracing/google-genai.test.ts index e388f4d9a30c..1001a10ea766 100644 --- a/packages/server-utils/test/ai/lib/tracing/google-genai.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/google-genai.test.ts @@ -10,14 +10,12 @@ import { GEN_AI_REQUEST_TEMPERATURE, GEN_AI_REQUEST_TOP_K, GEN_AI_REQUEST_TOP_P, - GEN_AI_RESPONSE_TEXT, GEN_AI_SYSTEM_INSTRUCTIONS, GEN_AI_TOOL_DEFINITIONS, } from '@sentry/conventions/attributes'; -import { getCurrentScope, getMainCarrier, setCurrentClient, spanToStaticSpanJSON, withScope } from '@sentry/core'; +import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; import type { Span } from '@sentry/core'; import { instrumentGoogleGenAIClient } from '../../../../src/ai/google-genai'; -import type { GoogleGenAICallInfo } from '../../../../src/ai/google-genai/types'; import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; function setupClient(traceLifecycle: 'static' | 'stream'): Span[] { @@ -316,55 +314,3 @@ describe('instrumentGoogleGenAIClient chat config propagation', () => { expect(data[GEN_AI_REQUEST_MAX_TOKENS]).toBe(150); }); }); - -describe('instrumentGoogleGenAIClient per-call recording', () => { - beforeEach(() => { - getMainCarrier().__SENTRY__ = undefined; - }); - - afterEach(() => { - getMainCarrier().__SENTRY__ = undefined; - }); - - it("decides inputs and outputs separately for each call, in the caller's async context", async () => { - const endedSpans = setupClient('stream'); - const recordInputs = vi.fn( - (call: GoogleGenAICallInfo) => - (call.request?.config as { labels: Record }).labels.privacy === 'public', - ); - const recordOutputs = () => getCurrentScope().getScopeData().tags.outputs === 'on'; - const genai = instrumentGoogleGenAIClient( - { - models: { - generateContent: vi.fn().mockResolvedValue({ candidates: [{ content: { parts: [{ text: 'Hi' }] } }] }), - }, - }, - { recordInputs, recordOutputs }, - ); - - const request = (privacy: string) => ({ - model: 'gemini-2.5-flash', - contents: 'Hello', - config: { labels: { privacy } }, - }); - await withScope(scope => { - scope.setTag('outputs', 'off'); - return genai.models.generateContent(request('public')); - }); - await withScope(scope => { - scope.setTag('outputs', 'on'); - return genai.models.generateContent(request('private')); - }); - - expect(recordInputs).toHaveBeenCalledWith({ operation: 'generate_content', request: request('public') }); - expect( - endedSpans.map(span => { - const { data } = spanToStaticSpanJSON(span); - return { inputs: GEN_AI_INPUT_MESSAGES in data, outputs: GEN_AI_RESPONSE_TEXT in data }; - }), - ).toEqual([ - { inputs: true, outputs: false }, - { inputs: false, outputs: true }, - ]); - }); -}); diff --git a/packages/server-utils/test/ai/lib/tracing/mistral.test.ts b/packages/server-utils/test/ai/lib/tracing/mistral.test.ts index af518a45abfd..89ee08ea34cc 100644 --- a/packages/server-utils/test/ai/lib/tracing/mistral.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/mistral.test.ts @@ -1,9 +1,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { getCurrentScope, getMainCarrier, setCurrentClient, spanToStaticSpanJSON, withScope } from '@sentry/core'; +import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; import type { Span } from '@sentry/core'; import { GEN_AI_AGENT_NAME, - GEN_AI_INPUT_MESSAGES, GEN_AI_OPERATION_NAME, GEN_AI_PROVIDER_NAME, GEN_AI_RESPONSE_STREAMING, @@ -13,7 +12,6 @@ import { GEN_AI_USAGE_TOTAL_TOKENS, } from '@sentry/conventions/attributes'; import { instrumentMistralAiClient } from '../../../../src/ai/mistral'; -import type { MistralCallInfo } from '../../../../src/ai/mistral/types'; import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; const GEN_AI_REQUEST_STREAM = 'gen_ai.request.stream'; @@ -638,35 +636,4 @@ describe('instrumentMistralAiClient', () => { expect(spanToStaticSpanJSON(endedSpans[0]!).status).toBe('internal_error'); }); }); - - it("decides inputs and outputs separately for each call, in the caller's async context", async () => { - const endedSpans = setupClient('stream'); - const recordInputs = vi.fn((call: MistralCallInfo) => call.request?.user === 'public'); - const recordOutputs = () => getCurrentScope().getScopeData().tags.outputs === 'on'; - const client = instrumentMistralAiClient(fakeClient(), { recordInputs, recordOutputs }); - - const messages = [{ role: 'user', content: 'Hello' }]; - await withScope(scope => { - scope.setTag('outputs', 'off'); - return client.chat.complete({ model: 'mistral-small-latest', messages, user: 'public' }); - }); - await withScope(scope => { - scope.setTag('outputs', 'on'); - return client.chat.complete({ model: 'mistral-small-latest', messages, user: 'private' }); - }); - - expect(recordInputs).toHaveBeenCalledWith({ - operation: 'chat', - request: { model: 'mistral-small-latest', messages, user: 'public' }, - }); - expect( - endedSpans.map(span => { - const { data } = spanToStaticSpanJSON(span); - return { inputs: GEN_AI_INPUT_MESSAGES in data, outputs: GEN_AI_RESPONSE_TEXT in data }; - }), - ).toEqual([ - { inputs: true, outputs: false }, - { inputs: false, outputs: true }, - ]); - }); }); diff --git a/packages/server-utils/test/integrations/gen-ai-call-recording.test.ts b/packages/server-utils/test/integrations/gen-ai-call-recording.test.ts deleted file mode 100644 index 3ec6ed898a5c..000000000000 --- a/packages/server-utils/test/integrations/gen-ai-call-recording.test.ts +++ /dev/null @@ -1,161 +0,0 @@ -import { tracingChannel } from 'node:diagnostics_channel'; -import type { Span } from '@sentry/core'; -import { - getCurrentScope, - GLOBAL_OBJ, - setAsyncContextStrategy, - setCurrentClient, - spanToJSON, - withScope, -} from '@sentry/core'; -import { GEN_AI_INPUT_MESSAGES, GEN_AI_RESPONSE_TEXT } from '@sentry/conventions/attributes'; -import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; -import { setAsyncLocalStorageAsyncContextStrategy } from '../../src/async-context'; -import { anthropicAIIntegration } from '../../src/integrations/anthropic'; -import { googleGenAIIntegration } from '../../src/integrations/google-genai'; -import { mistralAIIntegration } from '../../src/integrations/mistral'; -import { CHANNELS } from '../../src/orchestrion/channels'; -import { getDefaultTestClientOptions, TestClient } from '../mocks/client'; - -type CallInfo = { operation: string; request?: Record }; -type Decide = (call: CallInfo) => boolean | undefined; - -const MESSAGES = [{ role: 'user', content: 'Capital of France?' }]; - -const PROVIDERS = [ - { - name: 'anthropicAIIntegration', - channel: CHANNELS.ANTHROPIC_CHAT, - operation: 'chat', - params: (metadata: Record) => ({ model: 'claude-sonnet-4-5', messages: MESSAGES, metadata }), - response: { id: 'msg_1', model: 'claude-sonnet-4-5', content: [{ type: 'text', text: 'Paris.' }] }, - }, - { - name: 'googleGenAIIntegration', - channel: CHANNELS.GOOGLE_GENAI_GENERATE_CONTENT, - operation: 'generate_content', - params: (labels: Record) => ({ - model: 'gemini-2.5-flash', - contents: 'Capital of France?', - config: { labels }, - }), - response: { candidates: [{ content: { role: 'model', parts: [{ text: 'Paris.' }] } }] }, - }, - { - name: 'mistralAIIntegration', - channel: CHANNELS.MISTRAL_CHAT, - operation: 'chat', - params: (metadata: Record) => ({ model: 'mistral-large-latest', messages: MESSAGES, metadata }), - response: { - id: 'cmpl_1', - model: 'mistral-large-latest', - choices: [{ message: { role: 'assistant', content: 'Paris.' }, finishReason: 'stop' }], - }, - }, -]; - -describe('per-call recording options', () => { - let endedSpans: Span[]; - let decideInputs: Decide; - let decideOutputs: Decide; - - function useClient(): void { - const client = new TestClient( - getDefaultTestClientOptions({ dsn: 'https://public@dsn.ingest.sentry.io/1337', tracesSampleRate: 1 }), - ); - setCurrentClient(client); - client.init(); - client.on('spanEnd', span => endedSpans.push(span)); - } - - beforeAll(() => { - setAsyncLocalStorageAsyncContextStrategy(); - vi.stubEnv('ANTHROPIC_OPEN_TELEMETRY', undefined); - GLOBAL_OBJ.__SENTRY_ORCHESTRION__ = { runtime: ['@anthropic-ai/sdk', '@google/genai', '@mistralai/mistralai'] }; - endedSpans = []; - useClient(); - // The channels stay subscribed for the whole file, so each test swaps the decision functions. - const options = { - recordInputs: (call: CallInfo) => decideInputs(call), - recordOutputs: (call: CallInfo) => decideOutputs(call), - }; - for (const integration of [ - anthropicAIIntegration(options), - googleGenAIIntegration(options), - mistralAIIntegration(options), - ]) { - integration.setup!(getCurrentScope().getClient()!); - } - }); - - afterAll(() => { - vi.unstubAllEnvs(); - setAsyncContextStrategy(undefined); - delete GLOBAL_OBJ.__SENTRY_ORCHESTRION__; - }); - - beforeEach(() => { - endedSpans = []; - decideInputs = () => true; - decideOutputs = () => true; - useClient(); - }); - - function call(channel: string, params: Record, response: unknown): Promise { - return tracingChannel(channel).tracePromise(async () => response, { arguments: [params] }); - } - - function recorded(span: Span): { inputs: boolean; outputs: boolean } { - const { attributes } = spanToJSON(span); - return { inputs: GEN_AI_INPUT_MESSAGES in attributes, outputs: GEN_AI_RESPONSE_TEXT in attributes }; - } - - describe.each(PROVIDERS)('$name', ({ channel, operation, params, response }) => { - it('decides for each call from its request', async () => { - const calls: CallInfo[] = []; - decideInputs = info => { - calls.push(info); - return JSON.stringify(info.request).includes('"public"'); - }; - decideOutputs = decideInputs; - - const publicParams = params({ privacy: 'public' }); - await call(channel, publicParams, response); - await call(channel, params({ privacy: 'private' }), response); - - expect(calls[0]).toEqual({ operation, request: publicParams }); - expect(endedSpans.map(recorded)).toEqual([ - { inputs: true, outputs: true }, - { inputs: false, outputs: false }, - ]); - }); - - it("runs the functions in the caller's async context", async () => { - decideInputs = () => getCurrentScope().getScopeData().tags.privacy === 'public'; - decideOutputs = decideInputs; - - await withScope(async scope => { - scope.setTag('privacy', 'public'); - await call(channel, params({}), response); - }); - await withScope(async scope => { - scope.setTag('privacy', 'private'); - await call(channel, params({}), response); - }); - - expect(endedSpans.map(recorded)).toEqual([ - { inputs: true, outputs: true }, - { inputs: false, outputs: false }, - ]); - }); - - it('decides inputs and outputs separately', async () => { - decideInputs = () => false; - decideOutputs = () => true; - - await call(channel, params({}), response); - - expect(endedSpans.map(recorded)).toEqual([{ inputs: false, outputs: true }]); - }); - }); -}); From 9933911f634c5d2121aa7aed5e60fc0e09c45b55 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2026 12:58:53 +0000 Subject: [PATCH 3/3] test(node): Move per-call recording tests into the existing suite files The Anthropic, Google GenAI and Mistral suites are not split into subdirectories, so the tests belong in each suite's test.ts. Co-Authored-By: Francesco Gringl-Novy <2411343+mydea@users.noreply.github.com> --- .../node-suites/excludes.ts | 9 +-- .../node-suites/excludes.ts | 3 - .../anthropic/recording-functions/test.ts | 57 ------------------- .../suites/tracing/anthropic/test.ts | 46 ++++++++++++++- .../google-genai/recording-functions/test.ts | 49 ---------------- .../suites/tracing/google-genai/test.ts | 32 +++++++++++ .../mistral/recording-functions/test.ts | 50 ---------------- .../suites/tracing/mistral/test.ts | 32 +++++++++++ 8 files changed, 110 insertions(+), 168 deletions(-) delete mode 100644 dev-packages/node-integration-tests/suites/tracing/anthropic/recording-functions/test.ts delete mode 100644 dev-packages/node-integration-tests/suites/tracing/google-genai/recording-functions/test.ts delete mode 100644 dev-packages/node-integration-tests/suites/tracing/mistral/recording-functions/test.ts diff --git a/dev-packages/bun-integration-tests/node-suites/excludes.ts b/dev-packages/bun-integration-tests/node-suites/excludes.ts index e7ef95296e6e..31d62b28d255 100644 --- a/dev-packages/bun-integration-tests/node-suites/excludes.ts +++ b/dev-packages/bun-integration-tests/node-suites/excludes.ts @@ -76,7 +76,6 @@ export const NO_AUTO_INSTRUMENTATION = [ 'suites/pino/test.ts', 'suites/tracing/amqplib/test.ts', 'suites/tracing/anthropic/test.ts', - 'suites/tracing/anthropic/recording-functions/test.ts', 'suites/tracing/anthropic/v0/test.ts', 'suites/tracing/anthropic/v0.129/test.ts', 'suites/tracing/anthropic/v1/test.ts', @@ -89,7 +88,6 @@ export const NO_AUTO_INSTRUMENTATION = [ 'suites/tracing/google-genai-v2/test.ts', 'suites/tracing/google-genai-v3/test.ts', 'suites/tracing/google-genai/test.ts', - 'suites/tracing/google-genai/recording-functions/test.ts', 'suites/tracing/groq/test.ts', 'suites/tracing/hapi/test.ts', 'suites/tracing/ioredis-dc/test.ts', @@ -104,7 +102,6 @@ export const NO_AUTO_INSTRUMENTATION = [ 'suites/tracing/mcp-server/**', 'suites/tracing/mistral-v2/test.ts', 'suites/tracing/mistral/test.ts', - 'suites/tracing/mistral/recording-functions/test.ts', 'suites/tracing/mongodb-v4/test.ts', 'suites/tracing/mongodb-v5/test.ts', 'suites/tracing/mongodb-v6/test.ts', @@ -232,10 +229,7 @@ const BUN_BUILD_NO_HTTP_SERVER_INTEGRATION = ['suites/express/multiple-init/test // `@mistralai/mistralai` v3 statically imports its optional peer `@livekit/rtc-node`, so Bun cannot // bundle the scenarios. `mistral-v2` still covers Mistral here. -const BUN_BUILD_UNBUNDLEABLE_DEPENDENCY = [ - 'suites/tracing/mistral/recording-functions/test.ts', - 'suites/tracing/mistral/test.ts', -]; +const BUN_BUILD_UNBUNDLEABLE_DEPENDENCY = ['suites/tracing/mistral/test.ts']; // Some or all tests fail with the bundled scenarios. The cause is not investigated yet. In // `express/tracing` only the request data tests fail: they set `httpIntegration` options, and with @@ -250,7 +244,6 @@ const BUN_BUILD_NOT_TRIAGED = [ 'suites/tracing/google-genai-v2/test.ts', 'suites/tracing/google-genai-v3/test.ts', 'suites/tracing/google-genai/test.ts', - 'suites/tracing/google-genai/recording-functions/test.ts', 'suites/tracing/langchain/v1/**', 'suites/tracing/langgraph/v1/test.ts', 'suites/tracing/mastra/**', diff --git a/dev-packages/deno-integration-tests/node-suites/excludes.ts b/dev-packages/deno-integration-tests/node-suites/excludes.ts index 4757739940da..c7e7d700f3a9 100644 --- a/dev-packages/deno-integration-tests/node-suites/excludes.ts +++ b/dev-packages/deno-integration-tests/node-suites/excludes.ts @@ -56,13 +56,11 @@ const REQUIRE_OF_ESM_ONLY_DEPENDENCY = [ // version) and pass on Deno 2.9.0. const NOT_TRIAGED = [ 'suites/tracing/anthropic/test.ts', - 'suites/tracing/anthropic/recording-functions/test.ts', 'suites/tracing/anthropic/v0.129/test.ts', 'suites/tracing/apollo-graphql/**', 'suites/tracing/fastify/test.ts', 'suites/tracing/flue/test.ts', 'suites/tracing/google-genai/test.ts', - 'suites/tracing/google-genai/recording-functions/test.ts', 'suites/tracing/groq/test.ts', 'suites/tracing/http-client-spans/http-strip-query/test.ts', 'suites/tracing/ioredis-dc/test.ts', @@ -71,7 +69,6 @@ const NOT_TRIAGED = [ 'suites/tracing/langgraph/test.ts', 'suites/tracing/mistral-v2/test.ts', 'suites/tracing/mistral/test.ts', - 'suites/tracing/mistral/recording-functions/test.ts', 'suites/tracing/mongodb/test.ts', 'suites/tracing/mongoose-v5/test.ts', 'suites/tracing/mysql/test.ts', diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/recording-functions/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/recording-functions/test.ts deleted file mode 100644 index 909e27ff6df7..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/recording-functions/test.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { afterAll, describe, expect } from 'vitest'; -import { - GEN_AI_INPUT_MESSAGES, - GEN_AI_RESPONSE_ID, - GEN_AI_RESPONSE_TEXT, - GEN_AI_TOOL_CALL_ARGUMENTS, - GEN_AI_TOOL_CALL_RESULT, - SENTRY_ORIGIN, -} from '@sentry/conventions/attributes'; -import { join } from 'node:path'; -import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; -import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; - -describe('Anthropic integration', () => { - afterAll(() => { - cleanupChildProcesses(); - }); - - createEsmAndCjsTests( - join(__dirname, '..'), - 'scenario-recording-functions.mjs', - 'instrument-with-recording-functions.mjs', - (createRunner, test) => { - test('decides content recording per call when the options are functions', async () => { - await createRunner() - .expect({ - span: container => { - const spans = container.items.filter( - span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic', - ); - const recordedById = Object.fromEntries( - spans.map(span => [ - span.attributes[GEN_AI_RESPONSE_ID]?.value ?? span.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]?.value, - { - inputs: GEN_AI_INPUT_MESSAGES in span.attributes || GEN_AI_TOOL_CALL_ARGUMENTS in span.attributes, - outputs: GEN_AI_RESPONSE_TEXT in span.attributes || GEN_AI_TOOL_CALL_RESULT in span.attributes, - }, - ]), - ); - expect(recordedById).toEqual({ - msg_public: { inputs: true, outputs: true }, - msg_private: { inputs: false, outputs: false }, - 'msg_public-stream': { inputs: true, outputs: false }, - 'msg_public-helper-stream': { inputs: true, outputs: false }, - msg_tool_use: { inputs: false, outputs: true }, - msg_tool_final: { inputs: false, outputs: true }, - toolu_vienna: { inputs: true, outputs: true }, - toolu_paris: { inputs: false, outputs: true }, - }); - }, - }) - .start() - .completed(); - }); - }, - ); -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts index 54dd27321cd1..c4db3135a003 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts @@ -18,10 +18,15 @@ import { GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + GEN_AI_TOOL_CALL_ARGUMENTS, + GEN_AI_TOOL_CALL_RESULT, SENTRY_OP, SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; -import { GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; +import { + GEN_AI_REQUEST_STREAM_ATTRIBUTE, + GEN_AI_TOOL_CALL_ID_ATTRIBUTE, +} from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; describe('Anthropic integration', () => { @@ -613,4 +618,43 @@ describe('Anthropic integration', () => { .completed(); }); }); + + createEsmAndCjsTests( + __dirname, + 'scenario-recording-functions.mjs', + 'instrument-with-recording-functions.mjs', + (createRunner, test) => { + test('decides content recording per call when the options are functions', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic', + ); + const recordedById = Object.fromEntries( + spans.map(span => [ + span.attributes[GEN_AI_RESPONSE_ID]?.value ?? span.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]?.value, + { + inputs: GEN_AI_INPUT_MESSAGES in span.attributes || GEN_AI_TOOL_CALL_ARGUMENTS in span.attributes, + outputs: GEN_AI_RESPONSE_TEXT in span.attributes || GEN_AI_TOOL_CALL_RESULT in span.attributes, + }, + ]), + ); + expect(recordedById).toEqual({ + msg_public: { inputs: true, outputs: true }, + msg_private: { inputs: false, outputs: false }, + 'msg_public-stream': { inputs: true, outputs: false }, + 'msg_public-helper-stream': { inputs: true, outputs: false }, + msg_tool_use: { inputs: false, outputs: true }, + msg_tool_final: { inputs: false, outputs: true }, + toolu_vienna: { inputs: true, outputs: true }, + toolu_paris: { inputs: false, outputs: true }, + }); + }, + }) + .start() + .completed(); + }); + }, + ); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai/recording-functions/test.ts b/dev-packages/node-integration-tests/suites/tracing/google-genai/recording-functions/test.ts deleted file mode 100644 index 97c01f59611a..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/google-genai/recording-functions/test.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { afterAll, describe, expect } from 'vitest'; -import { - GEN_AI_INPUT_MESSAGES, - GEN_AI_REQUEST_MODEL, - GEN_AI_RESPONSE_TEXT, - SENTRY_ORIGIN, -} from '@sentry/conventions/attributes'; -import { join } from 'node:path'; -import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; - -describe('Google GenAI integration', () => { - afterAll(() => { - cleanupChildProcesses(); - }); - - createEsmAndCjsTests( - join(__dirname, '..'), - 'scenario-recording-functions.mjs', - 'instrument-with-recording-functions.mjs', - (createRunner, test) => { - test('decides content recording per call when the options are functions', async () => { - await createRunner() - .expect({ - span: container => { - const spans = container.items.filter( - span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.google_genai', - ); - const recordedByModel = Object.fromEntries( - spans.map(span => [ - span.attributes[GEN_AI_REQUEST_MODEL]?.value, - { - inputs: GEN_AI_INPUT_MESSAGES in span.attributes, - outputs: GEN_AI_RESPONSE_TEXT in span.attributes, - }, - ]), - ); - expect(recordedByModel).toEqual({ - 'gemini-public': { inputs: true, outputs: true }, - 'gemini-private': { inputs: false, outputs: false }, - 'gemini-public-stream': { inputs: true, outputs: false }, - }); - }, - }) - .start() - .completed(); - }); - }, - ); -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts b/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts index afc273ab86c5..ba43155fe344 100644 --- a/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts @@ -624,4 +624,36 @@ describe('Google GenAI integration', () => { }); }, ); + + createEsmAndCjsTests( + __dirname, + 'scenario-recording-functions.mjs', + 'instrument-with-recording-functions.mjs', + (createRunner, test) => { + test('decides content recording per call when the options are functions', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === EXPECTED_ORIGIN); + const recordedByModel = Object.fromEntries( + spans.map(span => [ + span.attributes[GEN_AI_REQUEST_MODEL]?.value, + { + inputs: GEN_AI_INPUT_MESSAGES in span.attributes, + outputs: GEN_AI_RESPONSE_TEXT in span.attributes, + }, + ]), + ); + expect(recordedByModel).toEqual({ + 'gemini-public': { inputs: true, outputs: true }, + 'gemini-private': { inputs: false, outputs: false }, + 'gemini-public-stream': { inputs: true, outputs: false }, + }); + }, + }) + .start() + .completed(); + }); + }, + ); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/mistral/recording-functions/test.ts b/dev-packages/node-integration-tests/suites/tracing/mistral/recording-functions/test.ts deleted file mode 100644 index 0316e49651e4..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/mistral/recording-functions/test.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { afterAll, describe, expect } from 'vitest'; -import { - GEN_AI_INPUT_MESSAGES, - GEN_AI_RESPONSE_ID, - GEN_AI_RESPONSE_TEXT, - SENTRY_ORIGIN, -} from '@sentry/conventions/attributes'; -import { join } from 'node:path'; -import { cleanupChildProcesses, createEsmTests } from '../../../../utils/runner'; - -// ESM-only, like the rest of the Mistral suite: `@mistralai/mistralai` v2+ ships no CJS build. -describe('Mistral integration', () => { - afterAll(() => { - cleanupChildProcesses(); - }); - - createEsmTests( - join(__dirname, '..'), - 'scenario-recording-functions.mjs', - 'instrument-with-recording-functions.mjs', - (createRunner, test) => { - test('decides content recording per call when the options are functions', async () => { - await createRunner() - .expect({ - span: container => { - const spans = container.items.filter( - span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mistralai', - ); - const recordedById = Object.fromEntries( - spans.map(span => [ - span.attributes[GEN_AI_RESPONSE_ID]?.value, - { - inputs: GEN_AI_INPUT_MESSAGES in span.attributes, - outputs: GEN_AI_RESPONSE_TEXT in span.attributes, - }, - ]), - ); - expect(recordedById).toEqual({ - 'cmpl-mistral-public': { inputs: true, outputs: true }, - 'cmpl-mistral-private': { inputs: false, outputs: false }, - 'cmpl-mistral-public-stream': { inputs: true, outputs: false }, - }); - }, - }) - .start() - .completed(); - }); - }, - ); -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/mistral/test.ts b/dev-packages/node-integration-tests/suites/tracing/mistral/test.ts index 8b5373b22921..f2425430d1ed 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mistral/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/mistral/test.ts @@ -314,4 +314,36 @@ describe('Mistral integration', () => { .completed(); }); }); + + createEsmTests( + __dirname, + 'scenario-recording-functions.mjs', + 'instrument-with-recording-functions.mjs', + (createRunner, test) => { + test('decides content recording per call when the options are functions', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === ORIGIN); + const recordedById = Object.fromEntries( + spans.map(span => [ + span.attributes[GEN_AI_RESPONSE_ID]?.value, + { + inputs: GEN_AI_INPUT_MESSAGES in span.attributes, + outputs: GEN_AI_RESPONSE_TEXT in span.attributes, + }, + ]), + ); + expect(recordedById).toEqual({ + 'cmpl-mistral-public': { inputs: true, outputs: true }, + 'cmpl-mistral-private': { inputs: false, outputs: false }, + 'cmpl-mistral-public-stream': { inputs: true, outputs: false }, + }); + }, + }) + .start() + .completed(); + }); + }, + ); });