Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import * as Sentry from '@sentry/node';
import { loggingTransport } from '@sentry-internal/node-integration-tests';

Sentry.init({
dsn: 'https://public@dsn.ingest.sentry.io/1337',
release: '1.0',
tracesSampleRate: 1.0,
transport: loggingTransport,
integrations: [
Sentry.openAIIntegration({
recordInputs: ({ operation, request }) => operation === 'chat' && request?.user === 'public',
recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public',
}),
],
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
import * as Sentry from '@sentry/node';
import express from 'express';
import OpenAI from 'openai';

function startMockServer() {
const app = express();
app.use(express.json());

app.post('/openai/chat/completions', (req, res) => {
const { model, stream, user } = req.body;
const id = `chatcmpl-${user}${stream ? '-stream' : ''}`;

if (!stream) {
res.send({
id,
object: 'chat.completion',
created: 1677652288,
model,
choices: [{ index: 0, message: { role: 'assistant', content: 'Paris.' }, finish_reason: 'stop' }],
});
return;
}

res.setHeader('Content-Type', 'text/event-stream');
const chunks = [
{
id,
object: 'chat.completion.chunk',
created: 1677652300,
model,
choices: [{ index: 0, delta: { content: 'Paris.' } }],
},
{
id,
object: 'chat.completion.chunk',
created: 1677652300,
model,
choices: [{ index: 0, delta: {}, finish_reason: 'stop' }],
},
];
for (const chunk of chunks) {
res.write(`data: ${JSON.stringify(chunk)}\n\n`);
}
res.write('data: [DONE]\n\n');
res.end();
});

return new Promise(resolve => {
const server = app.listen(0, () => {
resolve(server);
});
});
}

async function run() {
const server = await startMockServer();

await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
const client = new OpenAI({
baseURL: `http://localhost:${server.address().port}/openai`,
apiKey: 'mock-api-key',
});
const messages = [{ role: 'user', content: 'What is the capital of France?' }];

await Sentry.withScope(async scope => {
scope.setTag('privacy', 'public');
await client.chat.completions.create({ model: 'gpt-4o', messages, user: 'public' });
});

await Sentry.withScope(async scope => {
scope.setTag('privacy', 'private');
await client.chat.completions.create({ model: 'gpt-4o', messages, user: 'private' });

// Changing the tag after the call started must not change its outputs decision.
const stream = await client.chat.completions.create({ model: 'gpt-4o', messages, user: 'public', stream: true });
Sentry.getCurrentScope().setTag('privacy', 'public');
for await (const chunk of stream) {
void chunk;
}
});
});

server.close();
}

run();
Original file line number Diff line number Diff line change
Expand Up @@ -1133,4 +1133,36 @@ describe('OpenAI integration', () => {
.completed();
});
});

createEsmAndCjsTests(
join(__dirname, '../..'),
'scenario-recording-functions.mjs',
'instrument-with-recording-functions.mjs',
(createRunner, test) => {
test('decides content recording per call when the options are functions', async () => {
await createRunner()
.expect({
span: container => {
const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai');
const recordedById = Object.fromEntries(
spans.map(span => [
span.attributes[GEN_AI_RESPONSE_ID]?.value,
{
inputs: GEN_AI_INPUT_MESSAGES in span.attributes,
outputs: GEN_AI_RESPONSE_TEXT in span.attributes,
},
]),
);
expect(recordedById).toEqual({
'chatcmpl-public': { inputs: true, outputs: true },
'chatcmpl-private': { inputs: false, outputs: false },
'chatcmpl-public-stream': { inputs: true, outputs: false },
});
},
})
.start()
.completed();
});
},
);
});
57 changes: 56 additions & 1 deletion packages/server-utils/src/ai/core/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
/**
* Shared utils for AI integrations (OpenAI, Anthropic, Verce.AI, etc.)
*/
import { getClient, isObjectLike, isThenable, SPAN_STATUS_ERROR, stringify } from '@sentry/core';
import { debug, getClient, isObjectLike, isThenable, SPAN_STATUS_ERROR, stringify } from '@sentry/core';
import type { Span, SpanStatus } from '@sentry/core';
import {
GEN_AI_OUTPUT_MESSAGES,
Expand All @@ -20,6 +20,7 @@ import {
GEN_AI_USAGE_TOTAL_TOKENS,
} from '@sentry/conventions/attributes';
import { FUNCTION } from '@sentry/conventions/op';
import { DEBUG_BUILD } from '../../debug-build';

export interface GenAiOptions {
/**
Expand All @@ -34,6 +35,28 @@ export interface GenAiOptions {
recordOutputs?: boolean;
}

/**
* Whether to record the content of a gen_ai call: a boolean for every call, or a function that decides for
* each call. The function runs once when the call starts, in the caller's async context, so it can read
* request-scoped state. Returning `undefined` falls back to `dataCollection.genAI`. If it throws, the
* content is not recorded.
*/
export type GenAiRecordingOption<TCall> = boolean | ((call: TCall) => boolean | undefined);

/** {@link GenAiOptions} of an integration that can decide per call. `TCall` is what the function gets. */
export interface GenAiCallOptions<TCall> {
/**
* Record input messages/prompts on gen_ai spans. Defaults to the global
* `dataCollection.genAI.inputs` setting; an explicit value here takes precedence.
*/
recordInputs?: GenAiRecordingOption<TCall>;
/**
* Record output text/responses on gen_ai spans. Defaults to the global
* `dataCollection.genAI.outputs` setting; an explicit value here takes precedence.
*/
recordOutputs?: GenAiRecordingOption<TCall>;
}

/**
* A method registry entry describes a single instrumented method:
* which gen_ai operation it maps to and whether it is intrinsically streaming.
Expand Down Expand Up @@ -81,6 +104,38 @@ export function resolveAIRecordingOptions<T extends GenAiOptions>(options?: T):
} as T & Required<GenAiOptions>;
}

/**
* Resolves the recording options of one call, like {@link resolveAIRecordingOptions}, and calls the
* options that are functions with `call`. Call it once when the call starts, so the functions run in the
* caller's async context.
*/
export function resolveGenAiCallRecording<TCall>(
options: GenAiCallOptions<TCall> | undefined,
call: TCall,
): Required<GenAiOptions> {
const genAI = getClient()?.getDataCollectionOptions().genAI;
return {
recordInputs: resolveRecordingOption(options?.recordInputs, call) ?? genAI?.inputs ?? true,
recordOutputs: resolveRecordingOption(options?.recordOutputs, call) ?? genAI?.outputs ?? true,
};
}

function resolveRecordingOption<TCall>(
option: GenAiRecordingOption<TCall> | undefined,
call: TCall,
): boolean | undefined {
if (typeof option !== 'function') {
return option;
}
try {
const result = option(call);
return result === undefined ? undefined : result === true;
} catch (error) {
DEBUG_BUILD && debug.warn('[gen_ai] `recordInputs`/`recordOutputs` threw, not recording content', error);
return false;
}
}

/**
* Build method path from current traversal
*/
Expand Down
20 changes: 12 additions & 8 deletions packages/server-utils/src/ai/openai/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import {
getErrorStatus,
getGenAiSpanOp,
getRequestSignal,
resolveAIRecordingOptions,
resolveGenAiCallRecording,
wrapPromiseWithMethods,
} from '../core/utils';
import { OPENAI_METHOD_REGISTRY } from './constants';
Expand Down Expand Up @@ -147,10 +147,14 @@ function instrumentMethod<T extends unknown[], R>(
): (...args: T) => Promise<R> {
return function instrumentedCall(...args: T): Promise<R> {
const operationName = instrumentedMethod.operation || 'unknown';
const requestAttributes = extractRequestAttributes(args, operationName, !!options.recordInputs);
const params = args[0] as Record<string, unknown> | undefined;
const { recordInputs, recordOutputs } = resolveGenAiCallRecording(options, {
operation: operationName,
request: params,
});
const requestAttributes = extractRequestAttributes(args, operationName, recordInputs);
const model = (requestAttributes[GEN_AI_REQUEST_MODEL] as string) || 'unknown';

const params = args[0] as Record<string, unknown> | undefined;
const isStreamRequested = params && typeof params === 'object' && params.stream === true;
const abortSignal = getRequestSignal(args[1]);
const client = getClient();
Expand All @@ -171,7 +175,7 @@ function instrumentMethod<T extends unknown[], R>(
const instrumentedPromise = startSpanManual(spanConfig, (span: Span) => {
originalResult = originalMethod.apply(context, args);

if (options.recordInputs && params) {
if (recordInputs && params) {
addRequestAttributes(span, params, operationName);
}

Expand All @@ -182,7 +186,7 @@ function instrumentMethod<T extends unknown[], R>(
return instrumentStream(
result as OpenAIStream<ChatCompletionChunk | ResponseStreamingEvent>,
span,
options.recordOutputs ?? false,
recordOutputs,
) as unknown as R;
} catch (error) {
span.setStatus(getErrorStatus(abortSignal));
Expand All @@ -202,13 +206,13 @@ function instrumentMethod<T extends unknown[], R>(
// Call synchronously to capture the promise
originalResult = originalMethod.apply(context, args);

if (options.recordInputs && params) {
if (recordInputs && params) {
addRequestAttributes(span, params, operationName);
}

return originalResult.then(
result => {
addResponseAttributes(span, result, options.recordOutputs, operationName);
addResponseAttributes(span, result, recordOutputs, operationName);
return result;
},
error => {
Expand Down Expand Up @@ -278,5 +282,5 @@ function createDeepProxy<T extends object>(target: T, currentPath = '', options:
* Can be used across Node.js, Cloudflare Workers, and Vercel Edge
*/
export function instrumentOpenAiClient<T extends object>(client: T, options?: OpenAiOptions): T {
return createDeepProxy(client, '', resolveAIRecordingOptions(options));
return createDeepProxy(client, '', options ?? {});
}
14 changes: 11 additions & 3 deletions packages/server-utils/src/ai/openai/types.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,15 @@
import type { GenAiOptions } from '../core/utils';
import type { GenAiCallOptions } from '../core/utils';

/** Options for the OpenAI integration. */
export type OpenAiOptions = GenAiOptions;
/** The call that `recordInputs`/`recordOutputs` decide on. */
export interface OpenAiCallInfo {
/** The gen_ai operation of the call, such as `chat`, `embeddings` or `decide`. */
operation: string;
/** The request params, the first argument of the method (such as `chat.completions.create`) as passed. */
request: Record<string, unknown> | undefined;
}

/** Options for the OpenAI integration and OpenAI-compatible integrations, such as Groq. */
export type OpenAiOptions = GenAiCallOptions<OpenAiCallInfo>;

/**
* @see https://platform.openai.com/docs/api-reference/responses/object
Expand Down
23 changes: 13 additions & 10 deletions packages/server-utils/src/integrations/openai-compatible.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import {
startInactiveSpan,
} from '@sentry/core';
import { GEN_AI_PROVIDER_NAME, SENTRY_ORIGIN } from '@sentry/conventions/attributes';
import { getGenAiSpanOp, getRequestSignal, markCancelledIfAborted, resolveAIRecordingOptions } from '../ai/core/utils';
import { getGenAiSpanOp, getRequestSignal, markCancelledIfAborted, resolveGenAiCallRecording } from '../ai/core/utils';
import { wrapApiPromiseResponse } from '../ai/core/apiPromise';
import { addRequestAttributes, extractRequestAttributes } from '../ai/openai';
import { instrumentStream } from '../ai/openai/streaming';
Expand All @@ -17,6 +17,9 @@ import type { OrchestrionChannelContext } from '../orchestrion/types';
import { invokeOrchestrionInstrumentation } from '../orchestrion/instrumentation';
import { bindTracingChannelToSpan } from '../tracing-channel';

// The recording decision is made once, when the span starts, in the caller's async context.
type OpenAiChannelContext = OrchestrionChannelContext & { _sentryRecordOutputs?: boolean };

/**
* Describes an OpenAI-compatible provider whose SDK mirrors the openai wire format (Groq, Together, ...),
* so the span building, streaming and response parsing can all be reused from `ai/openai`.
Expand Down Expand Up @@ -52,26 +55,26 @@ export function createOpenAiCompatibleIntegration<T extends OpenAiCompatibleProv
function instrument(options: OpenAiOptions): void {
for (const { channel, operation } of instrumentedChannels) {
bindTracingChannelToSpan(
diagnosticsChannel.tracingChannel<OrchestrionChannelContext>(channel),
diagnosticsChannel.tracingChannel<OpenAiChannelContext>(channel),
data => createGenAiSpan(data, operation, provider, options),
{
beforeSpanEnd: (span, data) => {
if ('error' in data) {
markCancelledIfAborted(span, getRequestSignal(data.arguments?.[1]));
}
addResponseAttributes(span, data.result, resolveAIRecordingOptions(options).recordOutputs);
addResponseAttributes(span, data.result, !!data._sentryRecordOutputs);
},
deferSpanEnd: ({ span, data, end }) =>
wrapApiPromiseResponse(
data.result,
response => {
data.result = response;
if (!wrapStreamResult(span, data, options)) {
if (!wrapStreamResult(span, data)) {
end();
}
},
end,
) || wrapStreamResult(span, data, options),
) || wrapStreamResult(span, data),
},
);
}
Expand All @@ -92,7 +95,7 @@ export function createOpenAiCompatibleIntegration<T extends OpenAiCompatibleProv
* Returning `undefined` opts the payload out so no span is opened.
*/
function createGenAiSpan(
data: OrchestrionChannelContext,
data: OpenAiChannelContext,
operation: string,
provider: OpenAiCompatibleProvider,
options: OpenAiOptions,
Expand All @@ -106,7 +109,8 @@ function createGenAiSpan(
const args = data.arguments ?? [];
const params = args[0] as Record<string, unknown> | undefined;

const { recordInputs } = resolveAIRecordingOptions(options);
const { recordInputs, recordOutputs } = resolveGenAiCallRecording(options, { operation, request: params });
data._sentryRecordOutputs = recordOutputs;

// `extractRequestAttributes` defaults the provider to openai; override it for the concrete provider.
const attributes = extractRequestAttributes(args, operation, recordInputs);
Expand Down Expand Up @@ -144,15 +148,14 @@ function isAsyncIterable(value: unknown): value is AsyncIterableStream {
* to hand span-ending ownership to `instrumentStream`; `false` for non-streaming/errored results, which end
* via the normal `beforeSpanEnd` path.
*/
function wrapStreamResult(span: Span, data: OrchestrionChannelContext, options: OpenAiOptions): boolean {
function wrapStreamResult(span: Span, data: OpenAiChannelContext): boolean {
const result = data.result;
if (!isAsyncIterable(result)) {
return false;
}

const { recordOutputs } = resolveAIRecordingOptions(options);
const iterate = result[Symbol.asyncIterator].bind(result);
const instrumented = instrumentStream({ [Symbol.asyncIterator]: iterate }, span, recordOutputs ?? false);
const instrumented = instrumentStream({ [Symbol.asyncIterator]: iterate }, span, !!data._sentryRecordOutputs);
result[Symbol.asyncIterator] = () => instrumented;

return true;
Expand Down
Loading
Loading