diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-recording-functions.mjs new file mode 100644 index 000000000000..d2b40327cf1b --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-recording-functions.mjs @@ -0,0 +1,16 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + integrations: [ + Sentry.vercelAIIntegration({ + recordInputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public', + recordOutputs: ({ type }) => + type !== 'generateText' && Sentry.getCurrentScope().getScopeData().tags.privacy === 'public', + }), + ], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-recording-functions.mjs new file mode 100644 index 000000000000..ea89814d19d0 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-recording-functions.mjs @@ -0,0 +1,64 @@ +import * as Sentry from '@sentry/node'; +import { generateText, streamText } from 'ai'; +import { MockLanguageModelV3, simulateReadableStream } from 'ai/test'; + +const finishReason = { unified: 'stop', raw: 'stop' }; +const usage = { + inputTokens: { total: 10, noCache: 10, cached: 0 }, + outputTokens: { total: 20, noCache: 20, cached: 0 }, + totalTokens: { total: 30, noCache: 30, cached: 0 }, +}; + +function chat(privacy) { + return Sentry.withScope(async scope => { + scope.setTag('privacy', privacy); + await generateText({ + experimental_telemetry: { isEnabled: true }, + model: new MockLanguageModelV3({ + doGenerate: async () => ({ finishReason, usage, content: [{ type: 'text', text: 'Paris.' }], warnings: [] }), + }), + prompt: `Capital of France? (${privacy})`, + }); + }); +} + +function stream() { + return Sentry.withScope(async scope => { + scope.setTag('privacy', 'public'); + const result = streamText({ + experimental_telemetry: { isEnabled: true }, + model: new MockLanguageModelV3({ + doStream: async () => ({ + stream: simulateReadableStream({ + chunks: [ + { type: 'stream-start', warnings: [] }, + { type: 'text-start', id: '0' }, + { type: 'text-delta', id: '0', delta: 'Stream ' }, + { type: 'text-delta', id: '0', delta: 'response!' }, + { type: 'text-end', id: '0' }, + { type: 'finish', finishReason, usage }, + ], + }), + }), + }), + prompt: 'Stream me a response', + }); + + for await (const part of result.fullStream) { + // Both spans have started once the model streams text, so this must not change what they record. + if (part.type === 'text-delta') { + scope.setTag('privacy', 'private'); + } + } + }); +} + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + await chat('public'); + await chat('private'); + await stream(); + }); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/v7/generation/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/v7/generation/test.ts index 9d93a19482c5..1d201ea25b8b 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/v7/generation/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/v7/generation/test.ts @@ -358,3 +358,47 @@ describe('Vercel AI calls with recording off (v7)', () => { { additionalDependencies: { ai: '^7.0.163' } }, ); }); + +describe('Vercel AI recording functions (v7)', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + createEsmTests( + join(__dirname, '../..'), + 'scenario-recording-functions.mjs', + 'instrument-with-recording-functions.mjs', + (createRunner, test) => { + test('decides content recording per span when the options are functions', async () => { + await createRunner() + .expect({ + span: container => { + const recorded = (operationId: string): { inputs: boolean; outputs: boolean }[] => + container.items + .filter(span => span.attributes['vercel.ai.operationId']?.value === operationId) + .map(span => ({ + inputs: GEN_AI_INPUT_MESSAGES in span.attributes, + outputs: GEN_AI_OUTPUT_MESSAGES in span.attributes, + })); + + // `recordOutputs` skips `generateText` by type; the scope tag decides the rest. + expect(recorded('ai.generateText')).toEqual([ + { inputs: true, outputs: false }, + { inputs: false, outputs: false }, + ]); + expect(recorded('ai.generateText.doGenerate')).toEqual([ + { inputs: true, outputs: true }, + { inputs: false, outputs: false }, + ]); + // The scenario makes the scope private while the stream is read, after both spans started. + expect(recorded('ai.streamText')).toEqual([{ inputs: true, outputs: true }]); + expect(recorded('ai.streamText.doStream')).toEqual([{ inputs: true, outputs: true }]); + }, + }) + .start() + .completed(); + }); + }, + { additionalDependencies: { ai: '^7.0.0' } }, + ); +}); diff --git a/packages/server-utils/src/ai/core/utils.ts b/packages/server-utils/src/ai/core/utils.ts index 2d3a8e801ab4..c4a6ca3eb632 100644 --- a/packages/server-utils/src/ai/core/utils.ts +++ b/packages/server-utils/src/ai/core/utils.ts @@ -120,7 +120,8 @@ export function resolveGenAiCallRecording( }; } -function resolveRecordingOption( +/** Calls `option` with `call` when it is a function; `undefined` means the option does not decide. */ +export function resolveRecordingOption( option: GenAiRecordingOption | undefined, call: TCall, ): boolean | undefined { diff --git a/packages/server-utils/src/integrations/vercel-ai/index.ts b/packages/server-utils/src/integrations/vercel-ai/index.ts index 7d847a76dfe8..fdab27833dfd 100644 --- a/packages/server-utils/src/integrations/vercel-ai/index.ts +++ b/packages/server-utils/src/integrations/vercel-ai/index.ts @@ -1,6 +1,6 @@ import { defineIntegration, waitForTracingChannelBinding, type IntegrationFn } from '@sentry/core'; -import type { GenAiOptions } from '../../ai/core/utils'; -import { subscribeVercelAiTracingChannel } from './vercel-ai-dc-subscriber'; +import type { GenAiCallOptions } from '../../ai/core/utils'; +import { subscribeVercelAiTracingChannel, type VercelAiCallInfo } from './vercel-ai-dc-subscriber'; import { registerVercelAiToolTelemetry } from './vercel-ai-tool-telemetry'; import * as dc from '../../utils/diagnosticsChannel'; import { invokeOrchestrionInstrumentation } from '../../orchestrion/instrumentation'; @@ -8,7 +8,7 @@ import { vercelAiModuleNames } from '../../orchestrion/config/vercel-ai'; import { subscribeVercelAiOrchestrionChannels } from './vercel-ai-orchestrion-subscriber'; /** Options for the Vercel AI integration. */ -export type VercelAiOptions = GenAiOptions; +export type VercelAiOptions = GenAiCallOptions; const _vercelAIIntegration = ((options: VercelAiOptions = {}) => { return { diff --git a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts index d19111cb39be..b81e0d2ec431 100644 --- a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts +++ b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts @@ -55,7 +55,12 @@ import { import type { TracingChannel } from 'node:diagnostics_channel'; import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../ai/core/gen-ai-attributes'; import { isEveGenAiRecordingDefault } from './gen-ai-recording-mode'; -import { getEvaluationOutputMessages, type GenAiOptions } from '../../ai/core/utils'; +import { + type GenAiCallOptions, + type GenAiOptions, + getEvaluationOutputMessages, + resolveRecordingOption, +} from '../../ai/core/utils'; import { getProviderMetadataAttributes, LAST_STEP_ONLY_USAGE_KEYS } from '../../ai/vercel-ai'; import { WORKERS_AI_INTEGRATION_NAME } from '../../ai/workers-ai/constants'; import { bindTracingChannelToSpan } from '../../tracing-channel'; @@ -282,8 +287,21 @@ export interface VercelAiChannelMessage { */ export type VercelAiTracingChannelFactory = (name: string) => TracingChannel; +/** The call that `recordInputs`/`recordOutputs` decide on: one span, with the AI SDK telemetry event as published. */ +export interface VercelAiCallInfo { + /** The channel event type, such as `generateText`, `languageModelCall` or `executeTool`. */ + type: ChannelEventType; + /** The AI SDK telemetry event of the call. */ + event: Record; +} + /** Integration-level recording options, pinned at subscribe time so we never look the integration up per event. */ -export type VercelAiChannelOptions = GenAiOptions; +export type VercelAiChannelOptions = GenAiCallOptions; + +// The integration-level decision of each span, made at span start in the caller's async context. The rest of the +// precedence (eve mode, the per-call flags, `dataCollection.genAI`) is applied when it is needed, because whether +// `ai` withheld the result is only known at span end. +const integrationRecordingBySpan = new WeakMap(); // `ai` >= 7.0.163 publishes no channel `result` for calls with `recordInputs`/`recordOutputs: false`. These stay // here rather than on the shared channel message, which other subscribers can read. @@ -312,14 +330,14 @@ export function subscribeVercelAiTracingChannel( // The helper ends the span; we enrich it from the settled result first (tokens, output messages, // finish reasons, response model/id, provider metadata) and drop the per-operation `callId` maps. beforeSpanEnd: (span, data) => { - enrichSpanOnEnd(span, data, options); + enrichSpanOnEnd(span, data); // The result can hold the provider request and response bodies, and spans outlive the call. fallbackChannelResults.delete(span); clearOperationId(data); }, // A streamed model call resolves before its stream is drained, so we tap the stream, keep the // span open, and end it (via `end`) once the final usage/finish/output chunks arrive. - deferSpanEnd: ({ span, data, end }) => deferStreamedModelCallEnd(span, data, options, end), + deferSpanEnd: ({ span, data, end }) => deferStreamedModelCallEnd(span, data, end), }, ); } @@ -330,12 +348,7 @@ export function subscribeVercelAiTracingChannel( * enrich and end the span once the stream settles. Returns `false` for anything that isn't a streamed * model call so the helper ends the span as usual. */ -function deferStreamedModelCallEnd( - span: Span, - data: VercelAiChannelMessage, - options: VercelAiChannelOptions, - end: (error?: unknown) => void, -): boolean { +function deferStreamedModelCallEnd(span: Span, data: VercelAiChannelMessage, end: (error?: unknown) => void): boolean { const result = fallbackChannelResults.get(span) ?? data.result; if (data.type !== 'languageModelCall' || !isObjectLike(result)) { return false; @@ -346,7 +359,12 @@ function deferStreamedModelCallEnd( } const callId = asString(data.event.callId); - const { recordOutputs } = getRecordingOptions(data.event, options, data.result === undefined); + const invokeAgentSpan = callId ? invokeAgentSpanByCallId.get(callId) : undefined; + const { recordOutputs } = getRecordingOptions( + data.event, + invokeAgentSpan && integrationRecordingBySpan.get(invokeAgentSpan), + data.result === undefined, + ); result.stream = tapModelCallStream( stream, final => { @@ -355,7 +373,9 @@ function deferStreamedModelCallEnd( // streamed and non-streamed spans identical. fallbackChannelResults.set(span, { ...result, ...streamedResultToChannelResult(final) }); end(); - enrichInvokeAgentFromStream(callId, final, recordOutputs); + if (invokeAgentSpan) { + enrichInvokeAgentFromStream(invokeAgentSpan, final, recordOutputs); + } }, error => end(error), ); @@ -394,16 +414,7 @@ export function streamedResultToChannelResult(final: StreamedModelCallResult): R * the enclosing `invoke_agent` span, which has no channel result of its own. Runs before that span ends * because the operation's completion promise settles only after every model-call stream has drained. */ -function enrichInvokeAgentFromStream( - callId: string | undefined, - final: StreamedModelCallResult, - recordOutputs: boolean, -): void { - const span = callId ? invokeAgentSpanByCallId.get(callId) : undefined; - if (!span) { - return; - } - +function enrichInvokeAgentFromStream(span: Span, final: StreamedModelCallResult, recordOutputs: boolean): void { const usage = isObjectLike(final.usage) ? final.usage : undefined; if (usage) { const input = tokenCount(usage.inputTokens) ?? tokenCount(usage.promptTokens) ?? tokenCount(usage.tokens); @@ -452,7 +463,22 @@ export function createSpanFromMessage( return undefined; } - const { recordInputs } = getRecordingOptions(event, channelOptions); + const integrationRecording: GenAiOptions = { + recordInputs: resolveRecordingOption(channelOptions.recordInputs, { type, event }), + recordOutputs: resolveRecordingOption(channelOptions.recordOutputs, { type, event }), + }; + const span = startSpanFromEvent(type, event, getRecordingOptions(event, integrationRecording).recordInputs); + if (span) { + integrationRecordingBySpan.set(span, integrationRecording); + } + return span; +} + +function startSpanFromEvent( + type: ChannelEventType, + event: Record, + recordInputs: boolean, +): Span | undefined { const provider = asString(event.provider); const modelId = asString(event.modelId); const callId = asString(event.callId); @@ -663,18 +689,18 @@ function buildToolSpan( * Everything here is guarded: when a field is missing or the shape differs across `ai` versions, * we simply don't set the attribute rather than emit a malformed span. */ -export function enrichSpanOnEnd( - span: Span, - data: VercelAiChannelMessage, - channelOptions: VercelAiChannelOptions, -): void { +export function enrichSpanOnEnd(span: Span, data: VercelAiChannelMessage): void { const { type } = data; const result = fallbackChannelResults.get(span) ?? data.result; if (!isObjectLike(result)) { return; } - const { recordOutputs } = getRecordingOptions(data.event, channelOptions, data.result === undefined); + const { recordOutputs } = getRecordingOptions( + data.event, + integrationRecordingBySpan.get(span), + data.result === undefined, + ); if (type === 'executeTool') { // From V5 on, tool errors are not rejected (so the `error` channel verb never fires) — they @@ -918,7 +944,7 @@ export function captureToolError(span: Span, data: VercelAiChannelMessage, error function getRecordingOptions( event: Record, - channelOptions: VercelAiChannelOptions, + integrationRecording: GenAiOptions = {}, channelWithheldResult = false, ): { recordInputs: boolean; @@ -931,8 +957,8 @@ function getRecordingOptions( const eveMode = client && !channelWithheldResult ? isEveGenAiRecordingDefault(client) : false; return { - recordInputs: resolveRecording(channelOptions.recordInputs, event.recordInputs, genAI?.inputs, eveMode), - recordOutputs: resolveRecording(channelOptions.recordOutputs, event.recordOutputs, genAI?.outputs, eveMode), + recordInputs: resolveRecording(integrationRecording.recordInputs, event.recordInputs, genAI?.inputs, eveMode), + recordOutputs: resolveRecording(integrationRecording.recordOutputs, event.recordOutputs, genAI?.outputs, eveMode), }; } diff --git a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-orchestrion-subscriber.ts b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-orchestrion-subscriber.ts index 84fb1ac96c70..f85b90d73cf3 100644 --- a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-orchestrion-subscriber.ts +++ b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-orchestrion-subscriber.ts @@ -304,7 +304,7 @@ function bindOperation( // v6's `executeToolCall` returns the tool result/error object directly, whereas the shared core // (matching v7) expects it nested under `output`; wrap it so tool-error detection works. message.result = message.type === 'executeTool' ? { output: data.result } : data.result; - enrichSpanOnEnd(span, message, options); + enrichSpanOnEnd(span, message); } // A `streamText`/`streamObject` model call runs after this (synchronously-returning) operation's // span has already ended, so its `callId` entry must outlive the operation — it's cleared once the @@ -318,7 +318,7 @@ function bindOperation( // before the stream drains — losing the aggregate usage/output. Defer the end and await the result's // completion promises (`totalUsage`/`text`/…, which resolve on drain), mirroring how v7's channel // defers the operation span on the SDK's total-usage promise. - deferSpanEnd: ({ span, data, end }) => deferStreamTextOperationEnd(span, data, end, options), + deferSpanEnd: ({ span, data, end }) => deferStreamTextOperationEnd(span, data, end), }, ); } @@ -337,7 +337,6 @@ function deferStreamTextOperationEnd( span: Span, data: TracingChannelPayloadWithSpan, end: (error?: unknown) => void, - options: VercelAiChannelOptions, ): boolean { const message = messages.get(data); const type = message?.type; @@ -371,7 +370,7 @@ function deferStreamTextOperationEnd( // The model call itself succeeded, so keep its usage on the span; `beforeSpanEnd` skips // enrichment once `end(error)` marks the payload as errored. message.result = { usage, toolCalls, finishReason, response }; - enrichSpanOnEnd(span, message, options); + enrichSpanOnEnd(span, message); end(objectOutcome.error); return; } @@ -568,7 +567,7 @@ function patchModelMethod( value.stream, final => { message.result = { ...value, ...streamedResultToChannelResult(final) }; - enrichSpanOnEnd(span, message, options); + enrichSpanOnEnd(span, message); span.end(); clearStreamCallId(); }, @@ -587,7 +586,7 @@ function patchModelMethod( // `doGenerate` (and any non-stream result) settles with the full result; end here, start/end // bracket the call to match the channel timing. message.result = value; - enrichSpanOnEnd(span, message, options); + enrichSpanOnEnd(span, message); span.end(); clearStreamCallId(); @@ -681,7 +680,7 @@ function patchToolExecute( return result.then(value => { // The shared core (matching v6/v7) expects the tool result nested under `output`. message.result = { output: value }; - enrichSpanOnEnd(span, message, options); + enrichSpanOnEnd(span, message); span.end(); return value; }, failSpan); diff --git a/packages/server-utils/test/integrations/vercel-ai/cache-tokens.test.ts b/packages/server-utils/test/integrations/vercel-ai/cache-tokens.test.ts index 54f249be0591..30cf07a6cffa 100644 --- a/packages/server-utils/test/integrations/vercel-ai/cache-tokens.test.ts +++ b/packages/server-utils/test/integrations/vercel-ai/cache-tokens.test.ts @@ -48,7 +48,7 @@ describe('Vercel AI SDK cache tokens', () => { const message = { type, event: {}, result } as Parameters[0]; const span = createSpanFromMessage(message, {} as Parameters[1]); span!.setAttributes(existingAttributes); - enrichSpanOnEnd(span!, message, {} as Parameters[2]); + enrichSpanOnEnd(span!, message); span?.end(); return spanToStaticSpanJSON(endedSpans[0]!).data ?? {}; } diff --git a/packages/server-utils/test/integrations/vercel-ai/conversation-id.test.ts b/packages/server-utils/test/integrations/vercel-ai/conversation-id.test.ts index 07b5a35da9f5..854b4b9713a6 100644 --- a/packages/server-utils/test/integrations/vercel-ai/conversation-id.test.ts +++ b/packages/server-utils/test/integrations/vercel-ai/conversation-id.test.ts @@ -47,7 +47,7 @@ describe('Vercel AI SDK conversation id', () => { function run(message: Message): Record { const span = createSpanFromMessage(message, channelOptions)!; - enrichSpanOnEnd(span, message, {} as Parameters[2]); + enrichSpanOnEnd(span, message); span.end(); return spanToStaticSpanJSON(endedSpans[endedSpans.length - 1]!).data ?? {}; } diff --git a/packages/server-utils/test/integrations/vercel-ai/decide.test.ts b/packages/server-utils/test/integrations/vercel-ai/decide.test.ts index 8a225cab123b..8a2c60f83f62 100644 --- a/packages/server-utils/test/integrations/vercel-ai/decide.test.ts +++ b/packages/server-utils/test/integrations/vercel-ai/decide.test.ts @@ -44,7 +44,7 @@ describe('Vercel AI decide spans', () => { } satisfies Message; const span = createSpanFromMessage(message, {} as Parameters[1])!; - enrichSpanOnEnd(span, message, {} as Parameters[2]); + enrichSpanOnEnd(span, message); span.end(); return JSON.parse(spanToStaticSpanJSON(endedSpans[0]!).data?.[GEN_AI_OUTPUT_MESSAGES] as string)[0].answers; @@ -67,7 +67,7 @@ describe('Vercel AI decide spans', () => { const span = createSpanFromMessage(message, {} as Parameters[1])!; expect(span).toBeDefined(); - enrichSpanOnEnd(span, message, {} as Parameters[2]); + enrichSpanOnEnd(span, message); span.end(); const json = spanToStaticSpanJSON(endedSpans[0]!); diff --git a/packages/server-utils/test/integrations/vercel-ai/eve-recording.test.ts b/packages/server-utils/test/integrations/vercel-ai/eve-recording.test.ts index 826a2e1c6f26..590a028df1c3 100644 --- a/packages/server-utils/test/integrations/vercel-ai/eve-recording.test.ts +++ b/packages/server-utils/test/integrations/vercel-ai/eve-recording.test.ts @@ -99,7 +99,7 @@ describe('Vercel AI recording under eve', () => { }, } as Parameters[0]; const span = createSpanFromMessage(message, {})!; - enrichSpanOnEnd(span, message, {}); + enrichSpanOnEnd(span, message); expect(spanToStaticSpanJSON(span).data?.[GEN_AI_OUTPUT_MESSAGES]).toContain('It is sunny in Paris.'); }); diff --git a/packages/server-utils/test/integrations/vercel-ai/gemini-reasoning-tokens.test.ts b/packages/server-utils/test/integrations/vercel-ai/gemini-reasoning-tokens.test.ts index 51040686b90e..5869ba590603 100644 --- a/packages/server-utils/test/integrations/vercel-ai/gemini-reasoning-tokens.test.ts +++ b/packages/server-utils/test/integrations/vercel-ai/gemini-reasoning-tokens.test.ts @@ -48,7 +48,7 @@ describe('Vercel AI Gemini reasoning tokens', () => { const endedSpans = setupClient(); const message = { type, event: {}, result } as Parameters[0]; const span = createSpanFromMessage(message, {} as Parameters[1]); - enrichSpanOnEnd(span!, message, {} as Parameters[2]); + enrichSpanOnEnd(span!, message); span?.end(); return spanToStaticSpanJSON(endedSpans[0]!).data ?? {}; }