From c1a8e6b2bcd4bdeb8e28b3c5ecec104ddc124f77 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2026 11:06:38 +0000 Subject: [PATCH 1/4] feat(server-utils): Decide gen_ai content recording per call for OpenAI recordInputs and recordOutputs of openAIIntegration, instrumentOpenAiClient, groqIntegration and togetherAIIntegration can be functions that decide per call. They get the operation and the request params, and run once when the call starts, in the caller's async context. Co-Authored-By: Francesco Gringl-Novy <2411343+mydea@users.noreply.github.com> --- packages/server-utils/src/ai/core/utils.ts | 57 ++++++- packages/server-utils/src/ai/openai/index.ts | 20 ++- packages/server-utils/src/ai/openai/types.ts | 14 +- .../src/integrations/openai-compatible.ts | 23 +-- .../server-utils/src/integrations/openai.ts | 23 +-- .../test/ai/lib/tracing/ai/utils.test.ts | 60 ++++++- .../test/ai/lib/tracing/openai.test.ts | 16 +- .../test/integrations/openai.test.ts | 158 ++++++++++++++++++ 8 files changed, 336 insertions(+), 35 deletions(-) create mode 100644 packages/server-utils/test/integrations/openai.test.ts diff --git a/packages/server-utils/src/ai/core/utils.ts b/packages/server-utils/src/ai/core/utils.ts index 816c023ab08e..2d3a8e801ab4 100644 --- a/packages/server-utils/src/ai/core/utils.ts +++ b/packages/server-utils/src/ai/core/utils.ts @@ -2,7 +2,7 @@ /** * Shared utils for AI integrations (OpenAI, Anthropic, Verce.AI, etc.) */ -import { getClient, isObjectLike, isThenable, SPAN_STATUS_ERROR, stringify } from '@sentry/core'; +import { debug, getClient, isObjectLike, isThenable, SPAN_STATUS_ERROR, stringify } from '@sentry/core'; import type { Span, SpanStatus } from '@sentry/core'; import { GEN_AI_OUTPUT_MESSAGES, @@ -20,6 +20,7 @@ import { GEN_AI_USAGE_TOTAL_TOKENS, } from '@sentry/conventions/attributes'; import { FUNCTION } from '@sentry/conventions/op'; +import { DEBUG_BUILD } from '../../debug-build'; export interface GenAiOptions { /** @@ -34,6 +35,28 @@ export interface GenAiOptions { recordOutputs?: boolean; } +/** + * Whether to record the content of a gen_ai call: a boolean for every call, or a function that decides for + * each call. The function runs once when the call starts, in the caller's async context, so it can read + * request-scoped state. Returning `undefined` falls back to `dataCollection.genAI`. If it throws, the + * content is not recorded. + */ +export type GenAiRecordingOption = boolean | ((call: TCall) => boolean | undefined); + +/** {@link GenAiOptions} of an integration that can decide per call. `TCall` is what the function gets. */ +export interface GenAiCallOptions { + /** + * Record input messages/prompts on gen_ai spans. Defaults to the global + * `dataCollection.genAI.inputs` setting; an explicit value here takes precedence. + */ + recordInputs?: GenAiRecordingOption; + /** + * Record output text/responses on gen_ai spans. Defaults to the global + * `dataCollection.genAI.outputs` setting; an explicit value here takes precedence. + */ + recordOutputs?: GenAiRecordingOption; +} + /** * A method registry entry describes a single instrumented method: * which gen_ai operation it maps to and whether it is intrinsically streaming. @@ -81,6 +104,38 @@ export function resolveAIRecordingOptions(options?: T): } as T & Required; } +/** + * Resolves the recording options of one call, like {@link resolveAIRecordingOptions}, and calls the + * options that are functions with `call`. Call it once when the call starts, so the functions run in the + * caller's async context. + */ +export function resolveGenAiCallRecording( + options: GenAiCallOptions | undefined, + call: TCall, +): Required { + const genAI = getClient()?.getDataCollectionOptions().genAI; + return { + recordInputs: resolveRecordingOption(options?.recordInputs, call) ?? genAI?.inputs ?? true, + recordOutputs: resolveRecordingOption(options?.recordOutputs, call) ?? genAI?.outputs ?? true, + }; +} + +function resolveRecordingOption( + option: GenAiRecordingOption | undefined, + call: TCall, +): boolean | undefined { + if (typeof option !== 'function') { + return option; + } + try { + const result = option(call); + return result === undefined ? undefined : result === true; + } catch (error) { + DEBUG_BUILD && debug.warn('[gen_ai] `recordInputs`/`recordOutputs` threw, not recording content', error); + return false; + } +} + /** * Build method path from current traversal */ diff --git a/packages/server-utils/src/ai/openai/index.ts b/packages/server-utils/src/ai/openai/index.ts index 06d34be7b412..46c9ac727dc7 100644 --- a/packages/server-utils/src/ai/openai/index.ts +++ b/packages/server-utils/src/ai/openai/index.ts @@ -19,7 +19,7 @@ import { getErrorStatus, getGenAiSpanOp, getRequestSignal, - resolveAIRecordingOptions, + resolveGenAiCallRecording, wrapPromiseWithMethods, } from '../core/utils'; import { OPENAI_METHOD_REGISTRY } from './constants'; @@ -147,10 +147,14 @@ function instrumentMethod( ): (...args: T) => Promise { return function instrumentedCall(...args: T): Promise { const operationName = instrumentedMethod.operation || 'unknown'; - const requestAttributes = extractRequestAttributes(args, operationName, !!options.recordInputs); + const params = args[0] as Record | undefined; + const { recordInputs, recordOutputs } = resolveGenAiCallRecording(options, { + operation: operationName, + request: params, + }); + const requestAttributes = extractRequestAttributes(args, operationName, recordInputs); const model = (requestAttributes[GEN_AI_REQUEST_MODEL] as string) || 'unknown'; - const params = args[0] as Record | undefined; const isStreamRequested = params && typeof params === 'object' && params.stream === true; const abortSignal = getRequestSignal(args[1]); const client = getClient(); @@ -171,7 +175,7 @@ function instrumentMethod( const instrumentedPromise = startSpanManual(spanConfig, (span: Span) => { originalResult = originalMethod.apply(context, args); - if (options.recordInputs && params) { + if (recordInputs && params) { addRequestAttributes(span, params, operationName); } @@ -182,7 +186,7 @@ function instrumentMethod( return instrumentStream( result as OpenAIStream, span, - options.recordOutputs ?? false, + recordOutputs, ) as unknown as R; } catch (error) { span.setStatus(getErrorStatus(abortSignal)); @@ -202,13 +206,13 @@ function instrumentMethod( // Call synchronously to capture the promise originalResult = originalMethod.apply(context, args); - if (options.recordInputs && params) { + if (recordInputs && params) { addRequestAttributes(span, params, operationName); } return originalResult.then( result => { - addResponseAttributes(span, result, options.recordOutputs, operationName); + addResponseAttributes(span, result, recordOutputs, operationName); return result; }, error => { @@ -278,5 +282,5 @@ function createDeepProxy(target: T, currentPath = '', options: * Can be used across Node.js, Cloudflare Workers, and Vercel Edge */ export function instrumentOpenAiClient(client: T, options?: OpenAiOptions): T { - return createDeepProxy(client, '', resolveAIRecordingOptions(options)); + return createDeepProxy(client, '', options ?? {}); } diff --git a/packages/server-utils/src/ai/openai/types.ts b/packages/server-utils/src/ai/openai/types.ts index 43bf03e51288..8590d19b065c 100644 --- a/packages/server-utils/src/ai/openai/types.ts +++ b/packages/server-utils/src/ai/openai/types.ts @@ -1,7 +1,15 @@ -import type { GenAiOptions } from '../core/utils'; +import type { GenAiCallOptions } from '../core/utils'; -/** Options for the OpenAI integration. */ -export type OpenAiOptions = GenAiOptions; +/** The call that `recordInputs`/`recordOutputs` decide on. */ +export interface OpenAiCallInfo { + /** The gen_ai operation of the call, such as `chat`, `embeddings` or `decide`. */ + operation: string; + /** The request params, the first argument of the method (such as `chat.completions.create`) as passed. */ + request: Record | undefined; +} + +/** Options for the OpenAI integration and OpenAI-compatible integrations, such as Groq. */ +export type OpenAiOptions = GenAiCallOptions; /** * @see https://platform.openai.com/docs/api-reference/responses/object diff --git a/packages/server-utils/src/integrations/openai-compatible.ts b/packages/server-utils/src/integrations/openai-compatible.ts index 4e4a0301a8b8..29a504d119c4 100644 --- a/packages/server-utils/src/integrations/openai-compatible.ts +++ b/packages/server-utils/src/integrations/openai-compatible.ts @@ -7,7 +7,7 @@ import { startInactiveSpan, } from '@sentry/core'; import { GEN_AI_PROVIDER_NAME, SENTRY_ORIGIN } from '@sentry/conventions/attributes'; -import { getGenAiSpanOp, getRequestSignal, markCancelledIfAborted, resolveAIRecordingOptions } from '../ai/core/utils'; +import { getGenAiSpanOp, getRequestSignal, markCancelledIfAborted, resolveGenAiCallRecording } from '../ai/core/utils'; import { wrapApiPromiseResponse } from '../ai/core/apiPromise'; import { addRequestAttributes, extractRequestAttributes } from '../ai/openai'; import { instrumentStream } from '../ai/openai/streaming'; @@ -17,6 +17,9 @@ import type { OrchestrionChannelContext } from '../orchestrion/types'; import { invokeOrchestrionInstrumentation } from '../orchestrion/instrumentation'; import { bindTracingChannelToSpan } from '../tracing-channel'; +// The recording decision is made once, when the span starts, in the caller's async context. +type OpenAiChannelContext = OrchestrionChannelContext & { _sentryRecordOutputs?: boolean }; + /** * Describes an OpenAI-compatible provider whose SDK mirrors the openai wire format (Groq, Together, ...), * so the span building, streaming and response parsing can all be reused from `ai/openai`. @@ -52,26 +55,26 @@ export function createOpenAiCompatibleIntegration(channel), + diagnosticsChannel.tracingChannel(channel), data => createGenAiSpan(data, operation, provider, options), { beforeSpanEnd: (span, data) => { if ('error' in data) { markCancelledIfAborted(span, getRequestSignal(data.arguments?.[1])); } - addResponseAttributes(span, data.result, resolveAIRecordingOptions(options).recordOutputs); + addResponseAttributes(span, data.result, !!data._sentryRecordOutputs); }, deferSpanEnd: ({ span, data, end }) => wrapApiPromiseResponse( data.result, response => { data.result = response; - if (!wrapStreamResult(span, data, options)) { + if (!wrapStreamResult(span, data)) { end(); } }, end, - ) || wrapStreamResult(span, data, options), + ) || wrapStreamResult(span, data), }, ); } @@ -92,7 +95,7 @@ export function createOpenAiCompatibleIntegration | undefined; - const { recordInputs } = resolveAIRecordingOptions(options); + const { recordInputs, recordOutputs } = resolveGenAiCallRecording(options, { operation, request: params }); + data._sentryRecordOutputs = recordOutputs; // `extractRequestAttributes` defaults the provider to openai; override it for the concrete provider. const attributes = extractRequestAttributes(args, operation, recordInputs); @@ -144,15 +148,14 @@ function isAsyncIterable(value: unknown): value is AsyncIterableStream { * to hand span-ending ownership to `instrumentStream`; `false` for non-streaming/errored results, which end * via the normal `beforeSpanEnd` path. */ -function wrapStreamResult(span: Span, data: OrchestrionChannelContext, options: OpenAiOptions): boolean { +function wrapStreamResult(span: Span, data: OpenAiChannelContext): boolean { const result = data.result; if (!isAsyncIterable(result)) { return false; } - const { recordOutputs } = resolveAIRecordingOptions(options); const iterate = result[Symbol.asyncIterator].bind(result); - const instrumented = instrumentStream({ [Symbol.asyncIterator]: iterate }, span, recordOutputs ?? false); + const instrumented = instrumentStream({ [Symbol.asyncIterator]: iterate }, span, !!data._sentryRecordOutputs); result[Symbol.asyncIterator] = () => instrumented; return true; diff --git a/packages/server-utils/src/integrations/openai.ts b/packages/server-utils/src/integrations/openai.ts index 3dda9492956e..69d3dc769f1d 100644 --- a/packages/server-utils/src/integrations/openai.ts +++ b/packages/server-utils/src/integrations/openai.ts @@ -2,7 +2,7 @@ import { SENTRY_ORIGIN } from '@sentry/conventions/attributes'; import * as diagnosticsChannel from '../utils/diagnosticsChannel'; import type { IntegrationFn, Span, SpanAttributeValue } from '@sentry/core'; import { defineIntegration, getClient, hasSpanStreamingEnabled, startInactiveSpan } from '@sentry/core'; -import { getGenAiSpanOp, getRequestSignal, markCancelledIfAborted, resolveAIRecordingOptions } from '../ai/core/utils'; +import { getGenAiSpanOp, getRequestSignal, markCancelledIfAborted, resolveGenAiCallRecording } from '../ai/core/utils'; import { addRequestAttributes, extractRequestAttributes } from '../ai/openai'; import { wrapApiPromiseResponse } from '../ai/core/apiPromise'; import { instrumentStream } from '../ai/openai/streaming'; @@ -15,6 +15,9 @@ import { bindTracingChannelToSpan } from '../tracing-channel'; import { openaiModuleNames } from '../orchestrion/config/openai'; import { invokeOrchestrionInstrumentation } from '../orchestrion/instrumentation'; +// The recording decision is made once, when the span starts, in the caller's async context. +type OpenAiChannelContext = OrchestrionChannelContext & { _sentryRecordOutputs?: boolean }; + // Same name as the OTel integration by design, so the OTel 'OpenAI' // integration is deduplicated out of the default set. const INTEGRATION_NAME = 'OpenAI' as const; @@ -40,14 +43,14 @@ const _openAIIntegration = ((options: OpenAiOptions = {}) => { function instrumentOpenai(options: OpenAiOptions): void { for (const { channel, operation } of INSTRUMENTED_CHANNELS) { bindTracingChannelToSpan( - diagnosticsChannel.tracingChannel(channel), + diagnosticsChannel.tracingChannel(channel), data => createGenAiSpan(data, operation, options), { beforeSpanEnd: (span, data) => { if ('error' in data) { markCancelledIfAborted(span, getRequestSignal(data.arguments?.[1])); } - addResponseAttributes(span, data.result, resolveAIRecordingOptions(options).recordOutputs, operation); + addResponseAttributes(span, data.result, !!data._sentryRecordOutputs, operation); }, deferSpanEnd: ({ span, data, end }) => wrapApiPromiseResponse( @@ -55,12 +58,12 @@ function instrumentOpenai(options: OpenAiOptions): void { response => { data.result = response; // stream responses should end only after iteration is completed - if (!wrapStreamResult(span, data, options)) { + if (!wrapStreamResult(span, data)) { end(); } }, end, - ) || wrapStreamResult(span, data, options), + ) || wrapStreamResult(span, data), }, ); } @@ -70,7 +73,7 @@ function instrumentOpenai(options: OpenAiOptions): void { * Build the span for an instrumented `create` call. * Returning `undefined` opts the payload out so no span is opened. */ -function createGenAiSpan(data: OrchestrionChannelContext, operation: string, options: OpenAiOptions): Span | undefined { +function createGenAiSpan(data: OpenAiChannelContext, operation: string, options: OpenAiOptions): Span | undefined { if (shouldSkipAiProviderSpan(INTEGRATION_NAME)) { return undefined; } @@ -78,7 +81,8 @@ function createGenAiSpan(data: OrchestrionChannelContext, operation: string, opt const args = data.arguments ?? []; const params = args[0] as Record | undefined; - const { recordInputs } = resolveAIRecordingOptions(options); + const { recordInputs, recordOutputs } = resolveGenAiCallRecording(options, { operation, request: params }); + data._sentryRecordOutputs = recordOutputs; const attributes = extractRequestAttributes(args, operation, recordInputs); attributes[SENTRY_ORIGIN] = ORIGIN; @@ -114,15 +118,14 @@ function isAsyncIterable(value: unknown): value is AsyncIterableStream { * to hand span-ending ownership to `instrumentStream`; `false` for non-streaming/errored results, which end * via the normal `beforeSpanEnd` path. */ -function wrapStreamResult(span: Span, data: OrchestrionChannelContext, options: OpenAiOptions): boolean { +function wrapStreamResult(span: Span, data: OpenAiChannelContext): boolean { const result = data.result; if (!isAsyncIterable(result)) { return false; } - const { recordOutputs } = resolveAIRecordingOptions(options); const iterate = result[Symbol.asyncIterator].bind(result); - const instrumented = instrumentStream({ [Symbol.asyncIterator]: iterate }, span, recordOutputs ?? false); + const instrumented = instrumentStream({ [Symbol.asyncIterator]: iterate }, span, !!data._sentryRecordOutputs); result[Symbol.asyncIterator] = () => instrumented; return true; diff --git a/packages/server-utils/test/ai/lib/tracing/ai/utils.test.ts b/packages/server-utils/test/ai/lib/tracing/ai/utils.test.ts index 5659fc23d417..0f90916d7f0d 100644 --- a/packages/server-utils/test/ai/lib/tracing/ai/utils.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/ai/utils.test.ts @@ -1,6 +1,10 @@ -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { getMainCarrier, setCurrentClient } from '@sentry/core'; -import { resolveAIRecordingOptions, wrapPromiseWithMethods } from '../../../../../src/ai/core/utils'; +import { + resolveAIRecordingOptions, + resolveGenAiCallRecording, + wrapPromiseWithMethods, +} from '../../../../../src/ai/core/utils'; import { getDefaultTestClientOptions, TestClient } from '../../../../mocks/client'; describe('resolveAIRecordingOptions', () => { @@ -57,6 +61,58 @@ describe('resolveAIRecordingOptions', () => { }); }); +describe('resolveGenAiCallRecording', () => { + beforeEach(() => { + getMainCarrier().__SENTRY__ = undefined; + }); + + afterEach(() => { + getMainCarrier().__SENTRY__ = undefined; + }); + + function setupWithDataCollection(genAI: { inputs?: boolean; outputs?: boolean }): void { + const client = new TestClient(getDefaultTestClientOptions({ tracesSampleRate: 1, dataCollection: { genAI } })); + setCurrentClient(client); + client.init(); + } + + it('resolves booleans like resolveAIRecordingOptions', () => { + setupWithDataCollection({ inputs: true, outputs: true }); + expect(resolveGenAiCallRecording({ recordInputs: false }, 'call')).toEqual({ + recordInputs: false, + recordOutputs: true, + }); + }); + + it('calls function options with the call', () => { + const recordInputs = vi.fn((call: { private: boolean }) => !call.private); + expect(resolveGenAiCallRecording({ recordInputs, recordOutputs: () => false }, { private: true })).toEqual({ + recordInputs: false, + recordOutputs: false, + }); + expect(recordInputs).toHaveBeenCalledWith({ private: true }); + }); + + it('falls back to dataCollection.genAI when a function returns undefined', () => { + setupWithDataCollection({ inputs: false, outputs: true }); + expect(resolveGenAiCallRecording({ recordInputs: () => undefined, recordOutputs: () => undefined }, {})).toEqual({ + recordInputs: false, + recordOutputs: true, + }); + }); + + it('does not record when a function throws', () => { + setupWithDataCollection({ inputs: true, outputs: true }); + const fail = (): boolean => { + throw new Error('boom'); + }; + expect(resolveGenAiCallRecording({ recordInputs: fail, recordOutputs: fail }, {})).toEqual({ + recordInputs: false, + recordOutputs: false, + }); + }); +}); + describe('wrapPromiseWithMethods', () => { /** * Creates a mock APIPromise that mimics the behavior of OpenAI/Anthropic SDK APIPromise. diff --git a/packages/server-utils/test/ai/lib/tracing/openai.test.ts b/packages/server-utils/test/ai/lib/tracing/openai.test.ts index 523d040e8a4c..eb8be1b7b526 100644 --- a/packages/server-utils/test/ai/lib/tracing/openai.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/openai.test.ts @@ -1,9 +1,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { GEN_AI_OUTPUT_MESSAGES } from '@sentry/conventions/attributes'; +import { GEN_AI_INPUT_MESSAGES, GEN_AI_OUTPUT_MESSAGES } from '@sentry/conventions/attributes'; import { GEN_AI_DECIDE } from '@sentry/conventions/op'; import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; import type { Span } from '@sentry/core'; import { instrumentOpenAiClient } from '../../../../src/ai/openai'; +import type { OpenAiCallInfo } from '../../../../src/ai/openai/types'; import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; describe('instrumentOpenAiClient span names', () => { @@ -156,4 +157,17 @@ describe('instrumentOpenAiClient span names', () => { }, ]); }); + + it('decides recording for each call when the options are functions', async () => { + const endedSpans = setupClient('stream'); + const decide = vi.fn((call: OpenAiCallInfo) => call.request?.user === 'public'); + const instrumented = instrumentOpenAiClient(fakeClient(), { recordInputs: decide, recordOutputs: decide }); + + const messages = [{ role: 'user', content: 'Hello' }]; + await instrumented.chat.completions.create({ model: 'gpt-4', messages, user: 'public' }); + await instrumented.chat.completions.create({ model: 'gpt-4', messages, user: 'private' }); + + expect(decide).toHaveBeenCalledWith({ operation: 'chat', request: { model: 'gpt-4', messages, user: 'public' } }); + expect(endedSpans.map(span => GEN_AI_INPUT_MESSAGES in spanToStaticSpanJSON(span).data)).toEqual([true, false]); + }); }); diff --git a/packages/server-utils/test/integrations/openai.test.ts b/packages/server-utils/test/integrations/openai.test.ts new file mode 100644 index 000000000000..67c44bf38ff0 --- /dev/null +++ b/packages/server-utils/test/integrations/openai.test.ts @@ -0,0 +1,158 @@ +import { tracingChannel } from 'node:diagnostics_channel'; +import type { Span } from '@sentry/core'; +import { + getCurrentScope, + GLOBAL_OBJ, + setAsyncContextStrategy, + setCurrentClient, + spanToJSON, + withScope, +} from '@sentry/core'; +import { GEN_AI_INPUT_MESSAGES, GEN_AI_RESPONSE_TEXT } from '@sentry/conventions/attributes'; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { setAsyncLocalStorageAsyncContextStrategy } from '../../src/async-context'; +import type { OpenAiCallInfo } from '../../src/ai/openai/types'; +import { groqIntegration } from '../../src/integrations/groq'; +import { openAIIntegration } from '../../src/integrations/openai'; +import { CHANNELS } from '../../src/orchestrion/channels'; +import { getDefaultTestClientOptions, TestClient } from '../mocks/client'; + +type Decide = (call: OpenAiCallInfo) => boolean | undefined; + +const RESPONSE = { + id: 'chatcmpl-1', + model: 'gpt-4o', + choices: [{ index: 0, message: { role: 'assistant', content: 'Paris.' }, finish_reason: 'stop' }], +}; + +describe('per-call recording options', () => { + let endedSpans: Span[]; + let decideInputs: Decide; + let decideOutputs: Decide; + + function useClient(genAI?: { inputs?: boolean; outputs?: boolean }): void { + const client = new TestClient( + getDefaultTestClientOptions({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + tracesSampleRate: 1, + ...(genAI ? { dataCollection: { genAI } } : {}), + }), + ); + setCurrentClient(client); + client.init(); + client.on('spanEnd', span => endedSpans.push(span)); + } + + beforeAll(() => { + setAsyncLocalStorageAsyncContextStrategy(); + GLOBAL_OBJ.__SENTRY_ORCHESTRION__ = { runtime: ['openai', 'groq-sdk'] }; + endedSpans = []; + useClient(); + // The channels stay subscribed for the whole file, so each test swaps the decision functions. + const options = { + recordInputs: (call: OpenAiCallInfo) => decideInputs(call), + recordOutputs: (call: OpenAiCallInfo) => decideOutputs(call), + }; + for (const integration of [openAIIntegration(options), groqIntegration(options)]) { + integration.setup!(getCurrentScope().getClient()!); + } + }); + + afterAll(() => { + setAsyncContextStrategy(undefined); + delete GLOBAL_OBJ.__SENTRY_ORCHESTRION__; + }); + + beforeEach(() => { + endedSpans = []; + decideInputs = () => true; + decideOutputs = () => true; + useClient(); + }); + + function chat(channel: string, params: Record): Promise { + return tracingChannel(channel).tracePromise(async () => RESPONSE, { arguments: [params] }); + } + + function recorded(span: Span): { inputs: boolean; outputs: boolean } { + const { attributes } = spanToJSON(span); + return { inputs: GEN_AI_INPUT_MESSAGES in attributes, outputs: GEN_AI_RESPONSE_TEXT in attributes }; + } + + const params = (metadata: Record): Record => ({ + model: 'gpt-4o', + messages: [{ role: 'user', content: 'Capital of France?' }], + metadata, + }); + + it.each([ + ['openAIIntegration', CHANNELS.OPENAI_CHAT], + ['groqIntegration', CHANNELS.GROQ_CHAT], + ])('%s decides for each call from its request', async (_name, channel) => { + const calls: OpenAiCallInfo[] = []; + decideInputs = call => { + calls.push(call); + return (call.request?.metadata as Record).privacy === 'public'; + }; + decideOutputs = decideInputs; + + const publicParams = params({ privacy: 'public' }); + await chat(channel, publicParams); + await chat(channel, params({ privacy: 'private' })); + + expect(calls[0]).toEqual({ operation: 'chat', request: publicParams }); + expect(endedSpans.map(recorded)).toEqual([ + { inputs: true, outputs: true }, + { inputs: false, outputs: false }, + ]); + }); + + it("runs the functions in the caller's async context", async () => { + decideInputs = () => getCurrentScope().getScopeData().tags.privacy === 'public'; + decideOutputs = decideInputs; + + await withScope(async scope => { + scope.setTag('privacy', 'public'); + await chat(CHANNELS.OPENAI_CHAT, params({})); + }); + await withScope(async scope => { + scope.setTag('privacy', 'private'); + await chat(CHANNELS.OPENAI_CHAT, params({})); + }); + + expect(endedSpans.map(recorded)).toEqual([ + { inputs: true, outputs: true }, + { inputs: false, outputs: false }, + ]); + }); + + it('decides inputs and outputs separately', async () => { + decideInputs = () => false; + decideOutputs = () => true; + + await chat(CHANNELS.OPENAI_CHAT, params({})); + + expect(endedSpans.map(recorded)).toEqual([{ inputs: false, outputs: true }]); + }); + + it('falls back to dataCollection.genAI when a function returns undefined', async () => { + useClient({ inputs: false, outputs: true }); + decideInputs = () => undefined; + decideOutputs = () => undefined; + + await chat(CHANNELS.OPENAI_CHAT, params({})); + + expect(endedSpans.map(recorded)).toEqual([{ inputs: false, outputs: true }]); + }); + + it('records no content when a function throws, and the call still succeeds', async () => { + decideInputs = () => { + throw new Error('boom'); + }; + decideOutputs = decideInputs; + + await expect(chat(CHANNELS.OPENAI_CHAT, params({}))).resolves.toBe(RESPONSE); + + expect(endedSpans.map(recorded)).toEqual([{ inputs: false, outputs: false }]); + }); +}); From 9b96348a7fda2fadd68b6619f60a694c2655d32a Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2026 12:33:39 +0000 Subject: [PATCH 2/4] test(node): Cover per-call gen_ai recording with a node integration test Replaces the OpenAI unit tests for per-call recording options with one node integration test that runs the real openai client. --- .../instrument-with-recording-functions.mjs | 15 ++ .../openai/scenario-recording-functions.mjs | 86 ++++++++++ .../suites/tracing/openai/test.ts | 33 ++++ .../test/ai/lib/tracing/openai.test.ts | 16 +- .../test/integrations/openai.test.ts | 158 ------------------ 5 files changed, 135 insertions(+), 173 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-recording-functions.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/openai/scenario-recording-functions.mjs delete mode 100644 packages/server-utils/test/integrations/openai.test.ts diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-recording-functions.mjs new file mode 100644 index 000000000000..d8430194b064 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-recording-functions.mjs @@ -0,0 +1,15 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + integrations: [ + Sentry.openAIIntegration({ + recordInputs: ({ operation, request }) => operation === 'chat' && request?.user === 'public', + recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public', + }), + ], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/scenario-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/scenario-recording-functions.mjs new file mode 100644 index 000000000000..aa3c1da8d835 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/scenario-recording-functions.mjs @@ -0,0 +1,86 @@ +import * as Sentry from '@sentry/node'; +import express from 'express'; +import OpenAI from 'openai'; + +function startMockServer() { + const app = express(); + app.use(express.json()); + + app.post('/openai/chat/completions', (req, res) => { + const { model, stream, user } = req.body; + const id = `chatcmpl-${user}${stream ? '-stream' : ''}`; + + if (!stream) { + res.send({ + id, + object: 'chat.completion', + created: 1677652288, + model, + choices: [{ index: 0, message: { role: 'assistant', content: 'Paris.' }, finish_reason: 'stop' }], + }); + return; + } + + res.setHeader('Content-Type', 'text/event-stream'); + const chunks = [ + { + id, + object: 'chat.completion.chunk', + created: 1677652300, + model, + choices: [{ index: 0, delta: { content: 'Paris.' } }], + }, + { + id, + object: 'chat.completion.chunk', + created: 1677652300, + model, + choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], + }, + ]; + for (const chunk of chunks) { + res.write(`data: ${JSON.stringify(chunk)}\n\n`); + } + res.write('data: [DONE]\n\n'); + res.end(); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockServer(); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const client = new OpenAI({ + baseURL: `http://localhost:${server.address().port}/openai`, + apiKey: 'mock-api-key', + }); + const messages = [{ role: 'user', content: 'What is the capital of France?' }]; + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'public'); + await client.chat.completions.create({ model: 'gpt-4o', messages, user: 'public' }); + }); + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'private'); + await client.chat.completions.create({ model: 'gpt-4o', messages, user: 'private' }); + + // Changing the tag after the call started must not change its outputs decision. + const stream = await client.chat.completions.create({ model: 'gpt-4o', messages, user: 'public', stream: true }); + Sentry.getCurrentScope().setTag('privacy', 'public'); + for await (const chunk of stream) { + void chunk; + } + }); + }); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts index 98b9f8755573..dcf81441f453 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts @@ -1747,4 +1747,37 @@ describe('OpenAI integration', () => { .completed(); }); }); + + createEsmAndCjsTests( + __dirname, + 'scenario-recording-functions.mjs', + 'instrument-with-recording-functions.mjs', + (createRunner, test) => { + test('decides content recording per call when the options are functions', async () => { + await createRunner() + .unordered() + .expect({ + span: container => { + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + const recordedById = Object.fromEntries( + spans.map(span => [ + span.attributes[GEN_AI_RESPONSE_ID]?.value, + { + inputs: GEN_AI_INPUT_MESSAGES in span.attributes, + outputs: GEN_AI_RESPONSE_TEXT in span.attributes, + }, + ]), + ); + expect(recordedById).toEqual({ + 'chatcmpl-public': { inputs: true, outputs: true }, + 'chatcmpl-private': { inputs: false, outputs: false }, + 'chatcmpl-public-stream': { inputs: true, outputs: false }, + }); + }, + }) + .start() + .completed(); + }); + }, + ); }); diff --git a/packages/server-utils/test/ai/lib/tracing/openai.test.ts b/packages/server-utils/test/ai/lib/tracing/openai.test.ts index eb8be1b7b526..523d040e8a4c 100644 --- a/packages/server-utils/test/ai/lib/tracing/openai.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/openai.test.ts @@ -1,10 +1,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { GEN_AI_INPUT_MESSAGES, GEN_AI_OUTPUT_MESSAGES } from '@sentry/conventions/attributes'; +import { GEN_AI_OUTPUT_MESSAGES } from '@sentry/conventions/attributes'; import { GEN_AI_DECIDE } from '@sentry/conventions/op'; import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; import type { Span } from '@sentry/core'; import { instrumentOpenAiClient } from '../../../../src/ai/openai'; -import type { OpenAiCallInfo } from '../../../../src/ai/openai/types'; import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; describe('instrumentOpenAiClient span names', () => { @@ -157,17 +156,4 @@ describe('instrumentOpenAiClient span names', () => { }, ]); }); - - it('decides recording for each call when the options are functions', async () => { - const endedSpans = setupClient('stream'); - const decide = vi.fn((call: OpenAiCallInfo) => call.request?.user === 'public'); - const instrumented = instrumentOpenAiClient(fakeClient(), { recordInputs: decide, recordOutputs: decide }); - - const messages = [{ role: 'user', content: 'Hello' }]; - await instrumented.chat.completions.create({ model: 'gpt-4', messages, user: 'public' }); - await instrumented.chat.completions.create({ model: 'gpt-4', messages, user: 'private' }); - - expect(decide).toHaveBeenCalledWith({ operation: 'chat', request: { model: 'gpt-4', messages, user: 'public' } }); - expect(endedSpans.map(span => GEN_AI_INPUT_MESSAGES in spanToStaticSpanJSON(span).data)).toEqual([true, false]); - }); }); diff --git a/packages/server-utils/test/integrations/openai.test.ts b/packages/server-utils/test/integrations/openai.test.ts deleted file mode 100644 index 67c44bf38ff0..000000000000 --- a/packages/server-utils/test/integrations/openai.test.ts +++ /dev/null @@ -1,158 +0,0 @@ -import { tracingChannel } from 'node:diagnostics_channel'; -import type { Span } from '@sentry/core'; -import { - getCurrentScope, - GLOBAL_OBJ, - setAsyncContextStrategy, - setCurrentClient, - spanToJSON, - withScope, -} from '@sentry/core'; -import { GEN_AI_INPUT_MESSAGES, GEN_AI_RESPONSE_TEXT } from '@sentry/conventions/attributes'; -import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; -import { setAsyncLocalStorageAsyncContextStrategy } from '../../src/async-context'; -import type { OpenAiCallInfo } from '../../src/ai/openai/types'; -import { groqIntegration } from '../../src/integrations/groq'; -import { openAIIntegration } from '../../src/integrations/openai'; -import { CHANNELS } from '../../src/orchestrion/channels'; -import { getDefaultTestClientOptions, TestClient } from '../mocks/client'; - -type Decide = (call: OpenAiCallInfo) => boolean | undefined; - -const RESPONSE = { - id: 'chatcmpl-1', - model: 'gpt-4o', - choices: [{ index: 0, message: { role: 'assistant', content: 'Paris.' }, finish_reason: 'stop' }], -}; - -describe('per-call recording options', () => { - let endedSpans: Span[]; - let decideInputs: Decide; - let decideOutputs: Decide; - - function useClient(genAI?: { inputs?: boolean; outputs?: boolean }): void { - const client = new TestClient( - getDefaultTestClientOptions({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - tracesSampleRate: 1, - ...(genAI ? { dataCollection: { genAI } } : {}), - }), - ); - setCurrentClient(client); - client.init(); - client.on('spanEnd', span => endedSpans.push(span)); - } - - beforeAll(() => { - setAsyncLocalStorageAsyncContextStrategy(); - GLOBAL_OBJ.__SENTRY_ORCHESTRION__ = { runtime: ['openai', 'groq-sdk'] }; - endedSpans = []; - useClient(); - // The channels stay subscribed for the whole file, so each test swaps the decision functions. - const options = { - recordInputs: (call: OpenAiCallInfo) => decideInputs(call), - recordOutputs: (call: OpenAiCallInfo) => decideOutputs(call), - }; - for (const integration of [openAIIntegration(options), groqIntegration(options)]) { - integration.setup!(getCurrentScope().getClient()!); - } - }); - - afterAll(() => { - setAsyncContextStrategy(undefined); - delete GLOBAL_OBJ.__SENTRY_ORCHESTRION__; - }); - - beforeEach(() => { - endedSpans = []; - decideInputs = () => true; - decideOutputs = () => true; - useClient(); - }); - - function chat(channel: string, params: Record): Promise { - return tracingChannel(channel).tracePromise(async () => RESPONSE, { arguments: [params] }); - } - - function recorded(span: Span): { inputs: boolean; outputs: boolean } { - const { attributes } = spanToJSON(span); - return { inputs: GEN_AI_INPUT_MESSAGES in attributes, outputs: GEN_AI_RESPONSE_TEXT in attributes }; - } - - const params = (metadata: Record): Record => ({ - model: 'gpt-4o', - messages: [{ role: 'user', content: 'Capital of France?' }], - metadata, - }); - - it.each([ - ['openAIIntegration', CHANNELS.OPENAI_CHAT], - ['groqIntegration', CHANNELS.GROQ_CHAT], - ])('%s decides for each call from its request', async (_name, channel) => { - const calls: OpenAiCallInfo[] = []; - decideInputs = call => { - calls.push(call); - return (call.request?.metadata as Record).privacy === 'public'; - }; - decideOutputs = decideInputs; - - const publicParams = params({ privacy: 'public' }); - await chat(channel, publicParams); - await chat(channel, params({ privacy: 'private' })); - - expect(calls[0]).toEqual({ operation: 'chat', request: publicParams }); - expect(endedSpans.map(recorded)).toEqual([ - { inputs: true, outputs: true }, - { inputs: false, outputs: false }, - ]); - }); - - it("runs the functions in the caller's async context", async () => { - decideInputs = () => getCurrentScope().getScopeData().tags.privacy === 'public'; - decideOutputs = decideInputs; - - await withScope(async scope => { - scope.setTag('privacy', 'public'); - await chat(CHANNELS.OPENAI_CHAT, params({})); - }); - await withScope(async scope => { - scope.setTag('privacy', 'private'); - await chat(CHANNELS.OPENAI_CHAT, params({})); - }); - - expect(endedSpans.map(recorded)).toEqual([ - { inputs: true, outputs: true }, - { inputs: false, outputs: false }, - ]); - }); - - it('decides inputs and outputs separately', async () => { - decideInputs = () => false; - decideOutputs = () => true; - - await chat(CHANNELS.OPENAI_CHAT, params({})); - - expect(endedSpans.map(recorded)).toEqual([{ inputs: false, outputs: true }]); - }); - - it('falls back to dataCollection.genAI when a function returns undefined', async () => { - useClient({ inputs: false, outputs: true }); - decideInputs = () => undefined; - decideOutputs = () => undefined; - - await chat(CHANNELS.OPENAI_CHAT, params({})); - - expect(endedSpans.map(recorded)).toEqual([{ inputs: false, outputs: true }]); - }); - - it('records no content when a function throws, and the call still succeeds', async () => { - decideInputs = () => { - throw new Error('boom'); - }; - decideOutputs = decideInputs; - - await expect(chat(CHANNELS.OPENAI_CHAT, params({}))).resolves.toBe(RESPONSE); - - expect(endedSpans.map(recorded)).toEqual([{ inputs: false, outputs: false }]); - }); -}); From 2dc776d2e24e9e3f75867bfbb7aeae3d88ad4a23 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2026 12:48:22 +0000 Subject: [PATCH 3/4] test(node): Drop unneeded unordered() in per-call recording test --- .../suites/tracing/openai/v5/recording-functions/test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v5/recording-functions/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v5/recording-functions/test.ts index 5e0376cd8c13..f9ee51179f8a 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v5/recording-functions/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v5/recording-functions/test.ts @@ -20,7 +20,6 @@ describe('OpenAI integration', () => { (createRunner, test) => { test('decides content recording per call when the options are functions', async () => { await createRunner() - .unordered() .expect({ span: container => { const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); From aa5682038398d51523229e8b7f0f5307d92487ae Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2026 12:59:04 +0000 Subject: [PATCH 4/4] test(node): Move per-call recording test into the existing OpenAI chat suite Co-Authored-By: Francesco Gringl-Novy <2411343+mydea@users.noreply.github.com> --- .../suites/tracing/openai/v5/chat/test.ts | 32 +++++++++++++ .../openai/v5/recording-functions/test.ts | 47 ------------------- 2 files changed, 32 insertions(+), 47 deletions(-) delete mode 100644 dev-packages/node-integration-tests/suites/tracing/openai/v5/recording-functions/test.ts diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v5/chat/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v5/chat/test.ts index 33f80469f745..8277524097e2 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v5/chat/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v5/chat/test.ts @@ -1133,4 +1133,36 @@ describe('OpenAI integration', () => { .completed(); }); }); + + createEsmAndCjsTests( + join(__dirname, '../..'), + 'scenario-recording-functions.mjs', + 'instrument-with-recording-functions.mjs', + (createRunner, test) => { + test('decides content recording per call when the options are functions', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + const recordedById = Object.fromEntries( + spans.map(span => [ + span.attributes[GEN_AI_RESPONSE_ID]?.value, + { + inputs: GEN_AI_INPUT_MESSAGES in span.attributes, + outputs: GEN_AI_RESPONSE_TEXT in span.attributes, + }, + ]), + ); + expect(recordedById).toEqual({ + 'chatcmpl-public': { inputs: true, outputs: true }, + 'chatcmpl-private': { inputs: false, outputs: false }, + 'chatcmpl-public-stream': { inputs: true, outputs: false }, + }); + }, + }) + .start() + .completed(); + }); + }, + ); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v5/recording-functions/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v5/recording-functions/test.ts deleted file mode 100644 index f9ee51179f8a..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v5/recording-functions/test.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { afterAll, describe, expect } from 'vitest'; -import { - GEN_AI_INPUT_MESSAGES, - GEN_AI_RESPONSE_ID, - GEN_AI_RESPONSE_TEXT, - SENTRY_ORIGIN, -} from '@sentry/conventions/attributes'; -import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../../utils/runner'; -import { join } from 'node:path'; - -describe('OpenAI integration', () => { - afterAll(() => { - cleanupChildProcesses(); - }); - - createEsmAndCjsTests( - join(__dirname, '../..'), - 'scenario-recording-functions.mjs', - 'instrument-with-recording-functions.mjs', - (createRunner, test) => { - test('decides content recording per call when the options are functions', async () => { - await createRunner() - .expect({ - span: container => { - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); - const recordedById = Object.fromEntries( - spans.map(span => [ - span.attributes[GEN_AI_RESPONSE_ID]?.value, - { - inputs: GEN_AI_INPUT_MESSAGES in span.attributes, - outputs: GEN_AI_RESPONSE_TEXT in span.attributes, - }, - ]), - ); - expect(recordedById).toEqual({ - 'chatcmpl-public': { inputs: true, outputs: true }, - 'chatcmpl-private': { inputs: false, outputs: false }, - 'chatcmpl-public-stream': { inputs: true, outputs: false }, - }); - }, - }) - .start() - .completed(); - }); - }, - ); -});