diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-recording-functions.mjs new file mode 100644 index 000000000000..d8430194b064 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-recording-functions.mjs @@ -0,0 +1,15 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + integrations: [ + Sentry.openAIIntegration({ + recordInputs: ({ operation, request }) => operation === 'chat' && request?.user === 'public', + recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public', + }), + ], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/scenario-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/scenario-recording-functions.mjs new file mode 100644 index 000000000000..aa3c1da8d835 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/scenario-recording-functions.mjs @@ -0,0 +1,86 @@ +import * as Sentry from '@sentry/node'; +import express from 'express'; +import OpenAI from 'openai'; + +function startMockServer() { + const app = express(); + app.use(express.json()); + + app.post('/openai/chat/completions', (req, res) => { + const { model, stream, user } = req.body; + const id = `chatcmpl-${user}${stream ? '-stream' : ''}`; + + if (!stream) { + res.send({ + id, + object: 'chat.completion', + created: 1677652288, + model, + choices: [{ index: 0, message: { role: 'assistant', content: 'Paris.' }, finish_reason: 'stop' }], + }); + return; + } + + res.setHeader('Content-Type', 'text/event-stream'); + const chunks = [ + { + id, + object: 'chat.completion.chunk', + created: 1677652300, + model, + choices: [{ index: 0, delta: { content: 'Paris.' } }], + }, + { + id, + object: 'chat.completion.chunk', + created: 1677652300, + model, + choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], + }, + ]; + for (const chunk of chunks) { + res.write(`data: ${JSON.stringify(chunk)}\n\n`); + } + res.write('data: [DONE]\n\n'); + res.end(); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockServer(); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const client = new OpenAI({ + baseURL: `http://localhost:${server.address().port}/openai`, + apiKey: 'mock-api-key', + }); + const messages = [{ role: 'user', content: 'What is the capital of France?' }]; + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'public'); + await client.chat.completions.create({ model: 'gpt-4o', messages, user: 'public' }); + }); + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'private'); + await client.chat.completions.create({ model: 'gpt-4o', messages, user: 'private' }); + + // Changing the tag after the call started must not change its outputs decision. + const stream = await client.chat.completions.create({ model: 'gpt-4o', messages, user: 'public', stream: true }); + Sentry.getCurrentScope().setTag('privacy', 'public'); + for await (const chunk of stream) { + void chunk; + } + }); + }); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v5/chat/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v5/chat/test.ts index 33f80469f745..8277524097e2 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v5/chat/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v5/chat/test.ts @@ -1133,4 +1133,36 @@ describe('OpenAI integration', () => { .completed(); }); }); + + createEsmAndCjsTests( + join(__dirname, '../..'), + 'scenario-recording-functions.mjs', + 'instrument-with-recording-functions.mjs', + (createRunner, test) => { + test('decides content recording per call when the options are functions', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + const recordedById = Object.fromEntries( + spans.map(span => [ + span.attributes[GEN_AI_RESPONSE_ID]?.value, + { + inputs: GEN_AI_INPUT_MESSAGES in span.attributes, + outputs: GEN_AI_RESPONSE_TEXT in span.attributes, + }, + ]), + ); + expect(recordedById).toEqual({ + 'chatcmpl-public': { inputs: true, outputs: true }, + 'chatcmpl-private': { inputs: false, outputs: false }, + 'chatcmpl-public-stream': { inputs: true, outputs: false }, + }); + }, + }) + .start() + .completed(); + }); + }, + ); }); diff --git a/packages/server-utils/src/ai/core/utils.ts b/packages/server-utils/src/ai/core/utils.ts index 816c023ab08e..2d3a8e801ab4 100644 --- a/packages/server-utils/src/ai/core/utils.ts +++ b/packages/server-utils/src/ai/core/utils.ts @@ -2,7 +2,7 @@ /** * Shared utils for AI integrations (OpenAI, Anthropic, Verce.AI, etc.) */ -import { getClient, isObjectLike, isThenable, SPAN_STATUS_ERROR, stringify } from '@sentry/core'; +import { debug, getClient, isObjectLike, isThenable, SPAN_STATUS_ERROR, stringify } from '@sentry/core'; import type { Span, SpanStatus } from '@sentry/core'; import { GEN_AI_OUTPUT_MESSAGES, @@ -20,6 +20,7 @@ import { GEN_AI_USAGE_TOTAL_TOKENS, } from '@sentry/conventions/attributes'; import { FUNCTION } from '@sentry/conventions/op'; +import { DEBUG_BUILD } from '../../debug-build'; export interface GenAiOptions { /** @@ -34,6 +35,28 @@ export interface GenAiOptions { recordOutputs?: boolean; } +/** + * Whether to record the content of a gen_ai call: a boolean for every call, or a function that decides for + * each call. The function runs once when the call starts, in the caller's async context, so it can read + * request-scoped state. Returning `undefined` falls back to `dataCollection.genAI`. If it throws, the + * content is not recorded. + */ +export type GenAiRecordingOption = boolean | ((call: TCall) => boolean | undefined); + +/** {@link GenAiOptions} of an integration that can decide per call. `TCall` is what the function gets. */ +export interface GenAiCallOptions { + /** + * Record input messages/prompts on gen_ai spans. Defaults to the global + * `dataCollection.genAI.inputs` setting; an explicit value here takes precedence. + */ + recordInputs?: GenAiRecordingOption; + /** + * Record output text/responses on gen_ai spans. Defaults to the global + * `dataCollection.genAI.outputs` setting; an explicit value here takes precedence. + */ + recordOutputs?: GenAiRecordingOption; +} + /** * A method registry entry describes a single instrumented method: * which gen_ai operation it maps to and whether it is intrinsically streaming. @@ -81,6 +104,38 @@ export function resolveAIRecordingOptions(options?: T): } as T & Required; } +/** + * Resolves the recording options of one call, like {@link resolveAIRecordingOptions}, and calls the + * options that are functions with `call`. Call it once when the call starts, so the functions run in the + * caller's async context. + */ +export function resolveGenAiCallRecording( + options: GenAiCallOptions | undefined, + call: TCall, +): Required { + const genAI = getClient()?.getDataCollectionOptions().genAI; + return { + recordInputs: resolveRecordingOption(options?.recordInputs, call) ?? genAI?.inputs ?? true, + recordOutputs: resolveRecordingOption(options?.recordOutputs, call) ?? genAI?.outputs ?? true, + }; +} + +function resolveRecordingOption( + option: GenAiRecordingOption | undefined, + call: TCall, +): boolean | undefined { + if (typeof option !== 'function') { + return option; + } + try { + const result = option(call); + return result === undefined ? undefined : result === true; + } catch (error) { + DEBUG_BUILD && debug.warn('[gen_ai] `recordInputs`/`recordOutputs` threw, not recording content', error); + return false; + } +} + /** * Build method path from current traversal */ diff --git a/packages/server-utils/src/ai/openai/index.ts b/packages/server-utils/src/ai/openai/index.ts index 06d34be7b412..46c9ac727dc7 100644 --- a/packages/server-utils/src/ai/openai/index.ts +++ b/packages/server-utils/src/ai/openai/index.ts @@ -19,7 +19,7 @@ import { getErrorStatus, getGenAiSpanOp, getRequestSignal, - resolveAIRecordingOptions, + resolveGenAiCallRecording, wrapPromiseWithMethods, } from '../core/utils'; import { OPENAI_METHOD_REGISTRY } from './constants'; @@ -147,10 +147,14 @@ function instrumentMethod( ): (...args: T) => Promise { return function instrumentedCall(...args: T): Promise { const operationName = instrumentedMethod.operation || 'unknown'; - const requestAttributes = extractRequestAttributes(args, operationName, !!options.recordInputs); + const params = args[0] as Record | undefined; + const { recordInputs, recordOutputs } = resolveGenAiCallRecording(options, { + operation: operationName, + request: params, + }); + const requestAttributes = extractRequestAttributes(args, operationName, recordInputs); const model = (requestAttributes[GEN_AI_REQUEST_MODEL] as string) || 'unknown'; - const params = args[0] as Record | undefined; const isStreamRequested = params && typeof params === 'object' && params.stream === true; const abortSignal = getRequestSignal(args[1]); const client = getClient(); @@ -171,7 +175,7 @@ function instrumentMethod( const instrumentedPromise = startSpanManual(spanConfig, (span: Span) => { originalResult = originalMethod.apply(context, args); - if (options.recordInputs && params) { + if (recordInputs && params) { addRequestAttributes(span, params, operationName); } @@ -182,7 +186,7 @@ function instrumentMethod( return instrumentStream( result as OpenAIStream, span, - options.recordOutputs ?? false, + recordOutputs, ) as unknown as R; } catch (error) { span.setStatus(getErrorStatus(abortSignal)); @@ -202,13 +206,13 @@ function instrumentMethod( // Call synchronously to capture the promise originalResult = originalMethod.apply(context, args); - if (options.recordInputs && params) { + if (recordInputs && params) { addRequestAttributes(span, params, operationName); } return originalResult.then( result => { - addResponseAttributes(span, result, options.recordOutputs, operationName); + addResponseAttributes(span, result, recordOutputs, operationName); return result; }, error => { @@ -278,5 +282,5 @@ function createDeepProxy(target: T, currentPath = '', options: * Can be used across Node.js, Cloudflare Workers, and Vercel Edge */ export function instrumentOpenAiClient(client: T, options?: OpenAiOptions): T { - return createDeepProxy(client, '', resolveAIRecordingOptions(options)); + return createDeepProxy(client, '', options ?? {}); } diff --git a/packages/server-utils/src/ai/openai/types.ts b/packages/server-utils/src/ai/openai/types.ts index 43bf03e51288..8590d19b065c 100644 --- a/packages/server-utils/src/ai/openai/types.ts +++ b/packages/server-utils/src/ai/openai/types.ts @@ -1,7 +1,15 @@ -import type { GenAiOptions } from '../core/utils'; +import type { GenAiCallOptions } from '../core/utils'; -/** Options for the OpenAI integration. */ -export type OpenAiOptions = GenAiOptions; +/** The call that `recordInputs`/`recordOutputs` decide on. */ +export interface OpenAiCallInfo { + /** The gen_ai operation of the call, such as `chat`, `embeddings` or `decide`. */ + operation: string; + /** The request params, the first argument of the method (such as `chat.completions.create`) as passed. */ + request: Record | undefined; +} + +/** Options for the OpenAI integration and OpenAI-compatible integrations, such as Groq. */ +export type OpenAiOptions = GenAiCallOptions; /** * @see https://platform.openai.com/docs/api-reference/responses/object diff --git a/packages/server-utils/src/integrations/openai-compatible.ts b/packages/server-utils/src/integrations/openai-compatible.ts index 4e4a0301a8b8..29a504d119c4 100644 --- a/packages/server-utils/src/integrations/openai-compatible.ts +++ b/packages/server-utils/src/integrations/openai-compatible.ts @@ -7,7 +7,7 @@ import { startInactiveSpan, } from '@sentry/core'; import { GEN_AI_PROVIDER_NAME, SENTRY_ORIGIN } from '@sentry/conventions/attributes'; -import { getGenAiSpanOp, getRequestSignal, markCancelledIfAborted, resolveAIRecordingOptions } from '../ai/core/utils'; +import { getGenAiSpanOp, getRequestSignal, markCancelledIfAborted, resolveGenAiCallRecording } from '../ai/core/utils'; import { wrapApiPromiseResponse } from '../ai/core/apiPromise'; import { addRequestAttributes, extractRequestAttributes } from '../ai/openai'; import { instrumentStream } from '../ai/openai/streaming'; @@ -17,6 +17,9 @@ import type { OrchestrionChannelContext } from '../orchestrion/types'; import { invokeOrchestrionInstrumentation } from '../orchestrion/instrumentation'; import { bindTracingChannelToSpan } from '../tracing-channel'; +// The recording decision is made once, when the span starts, in the caller's async context. +type OpenAiChannelContext = OrchestrionChannelContext & { _sentryRecordOutputs?: boolean }; + /** * Describes an OpenAI-compatible provider whose SDK mirrors the openai wire format (Groq, Together, ...), * so the span building, streaming and response parsing can all be reused from `ai/openai`. @@ -52,26 +55,26 @@ export function createOpenAiCompatibleIntegration(channel), + diagnosticsChannel.tracingChannel(channel), data => createGenAiSpan(data, operation, provider, options), { beforeSpanEnd: (span, data) => { if ('error' in data) { markCancelledIfAborted(span, getRequestSignal(data.arguments?.[1])); } - addResponseAttributes(span, data.result, resolveAIRecordingOptions(options).recordOutputs); + addResponseAttributes(span, data.result, !!data._sentryRecordOutputs); }, deferSpanEnd: ({ span, data, end }) => wrapApiPromiseResponse( data.result, response => { data.result = response; - if (!wrapStreamResult(span, data, options)) { + if (!wrapStreamResult(span, data)) { end(); } }, end, - ) || wrapStreamResult(span, data, options), + ) || wrapStreamResult(span, data), }, ); } @@ -92,7 +95,7 @@ export function createOpenAiCompatibleIntegration | undefined; - const { recordInputs } = resolveAIRecordingOptions(options); + const { recordInputs, recordOutputs } = resolveGenAiCallRecording(options, { operation, request: params }); + data._sentryRecordOutputs = recordOutputs; // `extractRequestAttributes` defaults the provider to openai; override it for the concrete provider. const attributes = extractRequestAttributes(args, operation, recordInputs); @@ -144,15 +148,14 @@ function isAsyncIterable(value: unknown): value is AsyncIterableStream { * to hand span-ending ownership to `instrumentStream`; `false` for non-streaming/errored results, which end * via the normal `beforeSpanEnd` path. */ -function wrapStreamResult(span: Span, data: OrchestrionChannelContext, options: OpenAiOptions): boolean { +function wrapStreamResult(span: Span, data: OpenAiChannelContext): boolean { const result = data.result; if (!isAsyncIterable(result)) { return false; } - const { recordOutputs } = resolveAIRecordingOptions(options); const iterate = result[Symbol.asyncIterator].bind(result); - const instrumented = instrumentStream({ [Symbol.asyncIterator]: iterate }, span, recordOutputs ?? false); + const instrumented = instrumentStream({ [Symbol.asyncIterator]: iterate }, span, !!data._sentryRecordOutputs); result[Symbol.asyncIterator] = () => instrumented; return true; diff --git a/packages/server-utils/src/integrations/openai.ts b/packages/server-utils/src/integrations/openai.ts index 3dda9492956e..69d3dc769f1d 100644 --- a/packages/server-utils/src/integrations/openai.ts +++ b/packages/server-utils/src/integrations/openai.ts @@ -2,7 +2,7 @@ import { SENTRY_ORIGIN } from '@sentry/conventions/attributes'; import * as diagnosticsChannel from '../utils/diagnosticsChannel'; import type { IntegrationFn, Span, SpanAttributeValue } from '@sentry/core'; import { defineIntegration, getClient, hasSpanStreamingEnabled, startInactiveSpan } from '@sentry/core'; -import { getGenAiSpanOp, getRequestSignal, markCancelledIfAborted, resolveAIRecordingOptions } from '../ai/core/utils'; +import { getGenAiSpanOp, getRequestSignal, markCancelledIfAborted, resolveGenAiCallRecording } from '../ai/core/utils'; import { addRequestAttributes, extractRequestAttributes } from '../ai/openai'; import { wrapApiPromiseResponse } from '../ai/core/apiPromise'; import { instrumentStream } from '../ai/openai/streaming'; @@ -15,6 +15,9 @@ import { bindTracingChannelToSpan } from '../tracing-channel'; import { openaiModuleNames } from '../orchestrion/config/openai'; import { invokeOrchestrionInstrumentation } from '../orchestrion/instrumentation'; +// The recording decision is made once, when the span starts, in the caller's async context. +type OpenAiChannelContext = OrchestrionChannelContext & { _sentryRecordOutputs?: boolean }; + // Same name as the OTel integration by design, so the OTel 'OpenAI' // integration is deduplicated out of the default set. const INTEGRATION_NAME = 'OpenAI' as const; @@ -40,14 +43,14 @@ const _openAIIntegration = ((options: OpenAiOptions = {}) => { function instrumentOpenai(options: OpenAiOptions): void { for (const { channel, operation } of INSTRUMENTED_CHANNELS) { bindTracingChannelToSpan( - diagnosticsChannel.tracingChannel(channel), + diagnosticsChannel.tracingChannel(channel), data => createGenAiSpan(data, operation, options), { beforeSpanEnd: (span, data) => { if ('error' in data) { markCancelledIfAborted(span, getRequestSignal(data.arguments?.[1])); } - addResponseAttributes(span, data.result, resolveAIRecordingOptions(options).recordOutputs, operation); + addResponseAttributes(span, data.result, !!data._sentryRecordOutputs, operation); }, deferSpanEnd: ({ span, data, end }) => wrapApiPromiseResponse( @@ -55,12 +58,12 @@ function instrumentOpenai(options: OpenAiOptions): void { response => { data.result = response; // stream responses should end only after iteration is completed - if (!wrapStreamResult(span, data, options)) { + if (!wrapStreamResult(span, data)) { end(); } }, end, - ) || wrapStreamResult(span, data, options), + ) || wrapStreamResult(span, data), }, ); } @@ -70,7 +73,7 @@ function instrumentOpenai(options: OpenAiOptions): void { * Build the span for an instrumented `create` call. * Returning `undefined` opts the payload out so no span is opened. */ -function createGenAiSpan(data: OrchestrionChannelContext, operation: string, options: OpenAiOptions): Span | undefined { +function createGenAiSpan(data: OpenAiChannelContext, operation: string, options: OpenAiOptions): Span | undefined { if (shouldSkipAiProviderSpan(INTEGRATION_NAME)) { return undefined; } @@ -78,7 +81,8 @@ function createGenAiSpan(data: OrchestrionChannelContext, operation: string, opt const args = data.arguments ?? []; const params = args[0] as Record | undefined; - const { recordInputs } = resolveAIRecordingOptions(options); + const { recordInputs, recordOutputs } = resolveGenAiCallRecording(options, { operation, request: params }); + data._sentryRecordOutputs = recordOutputs; const attributes = extractRequestAttributes(args, operation, recordInputs); attributes[SENTRY_ORIGIN] = ORIGIN; @@ -114,15 +118,14 @@ function isAsyncIterable(value: unknown): value is AsyncIterableStream { * to hand span-ending ownership to `instrumentStream`; `false` for non-streaming/errored results, which end * via the normal `beforeSpanEnd` path. */ -function wrapStreamResult(span: Span, data: OrchestrionChannelContext, options: OpenAiOptions): boolean { +function wrapStreamResult(span: Span, data: OpenAiChannelContext): boolean { const result = data.result; if (!isAsyncIterable(result)) { return false; } - const { recordOutputs } = resolveAIRecordingOptions(options); const iterate = result[Symbol.asyncIterator].bind(result); - const instrumented = instrumentStream({ [Symbol.asyncIterator]: iterate }, span, recordOutputs ?? false); + const instrumented = instrumentStream({ [Symbol.asyncIterator]: iterate }, span, !!data._sentryRecordOutputs); result[Symbol.asyncIterator] = () => instrumented; return true; diff --git a/packages/server-utils/test/ai/lib/tracing/ai/utils.test.ts b/packages/server-utils/test/ai/lib/tracing/ai/utils.test.ts index 5659fc23d417..0f90916d7f0d 100644 --- a/packages/server-utils/test/ai/lib/tracing/ai/utils.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/ai/utils.test.ts @@ -1,6 +1,10 @@ -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { getMainCarrier, setCurrentClient } from '@sentry/core'; -import { resolveAIRecordingOptions, wrapPromiseWithMethods } from '../../../../../src/ai/core/utils'; +import { + resolveAIRecordingOptions, + resolveGenAiCallRecording, + wrapPromiseWithMethods, +} from '../../../../../src/ai/core/utils'; import { getDefaultTestClientOptions, TestClient } from '../../../../mocks/client'; describe('resolveAIRecordingOptions', () => { @@ -57,6 +61,58 @@ describe('resolveAIRecordingOptions', () => { }); }); +describe('resolveGenAiCallRecording', () => { + beforeEach(() => { + getMainCarrier().__SENTRY__ = undefined; + }); + + afterEach(() => { + getMainCarrier().__SENTRY__ = undefined; + }); + + function setupWithDataCollection(genAI: { inputs?: boolean; outputs?: boolean }): void { + const client = new TestClient(getDefaultTestClientOptions({ tracesSampleRate: 1, dataCollection: { genAI } })); + setCurrentClient(client); + client.init(); + } + + it('resolves booleans like resolveAIRecordingOptions', () => { + setupWithDataCollection({ inputs: true, outputs: true }); + expect(resolveGenAiCallRecording({ recordInputs: false }, 'call')).toEqual({ + recordInputs: false, + recordOutputs: true, + }); + }); + + it('calls function options with the call', () => { + const recordInputs = vi.fn((call: { private: boolean }) => !call.private); + expect(resolveGenAiCallRecording({ recordInputs, recordOutputs: () => false }, { private: true })).toEqual({ + recordInputs: false, + recordOutputs: false, + }); + expect(recordInputs).toHaveBeenCalledWith({ private: true }); + }); + + it('falls back to dataCollection.genAI when a function returns undefined', () => { + setupWithDataCollection({ inputs: false, outputs: true }); + expect(resolveGenAiCallRecording({ recordInputs: () => undefined, recordOutputs: () => undefined }, {})).toEqual({ + recordInputs: false, + recordOutputs: true, + }); + }); + + it('does not record when a function throws', () => { + setupWithDataCollection({ inputs: true, outputs: true }); + const fail = (): boolean => { + throw new Error('boom'); + }; + expect(resolveGenAiCallRecording({ recordInputs: fail, recordOutputs: fail }, {})).toEqual({ + recordInputs: false, + recordOutputs: false, + }); + }); +}); + describe('wrapPromiseWithMethods', () => { /** * Creates a mock APIPromise that mimics the behavior of OpenAI/Anthropic SDK APIPromise.