Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
import * as Sentry from '@sentry/node';
import { loggingTransport } from '@sentry-internal/node-integration-tests';

Sentry.init({
dsn: 'https://public@dsn.ingest.sentry.io/1337',
release: '1.0',
tracesSampleRate: 1.0,
transport: loggingTransport,
integrations: [
Sentry.vercelAIIntegration({
recordInputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public',
recordOutputs: ({ type }) =>
type !== 'generateText' && Sentry.getCurrentScope().getScopeData().tags.privacy === 'public',
}),
],
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
import * as Sentry from '@sentry/node';
import { generateText, streamText } from 'ai';
import { MockLanguageModelV3, simulateReadableStream } from 'ai/test';

const finishReason = { unified: 'stop', raw: 'stop' };
const usage = {
inputTokens: { total: 10, noCache: 10, cached: 0 },
outputTokens: { total: 20, noCache: 20, cached: 0 },
totalTokens: { total: 30, noCache: 30, cached: 0 },
};

function chat(privacy) {
return Sentry.withScope(async scope => {
scope.setTag('privacy', privacy);
await generateText({
experimental_telemetry: { isEnabled: true },
model: new MockLanguageModelV3({
doGenerate: async () => ({ finishReason, usage, content: [{ type: 'text', text: 'Paris.' }], warnings: [] }),
}),
prompt: `Capital of France? (${privacy})`,
});
});
}

function stream() {
return Sentry.withScope(async scope => {
scope.setTag('privacy', 'public');
const result = streamText({
experimental_telemetry: { isEnabled: true },
model: new MockLanguageModelV3({
doStream: async () => ({
stream: simulateReadableStream({
chunks: [
{ type: 'stream-start', warnings: [] },
{ type: 'text-start', id: '0' },
{ type: 'text-delta', id: '0', delta: 'Stream ' },
{ type: 'text-delta', id: '0', delta: 'response!' },
{ type: 'text-end', id: '0' },
{ type: 'finish', finishReason, usage },
],
}),
}),
}),
prompt: 'Stream me a response',
});

for await (const part of result.fullStream) {
// Both spans have started once the model streams text, so this must not change what they record.
if (part.type === 'text-delta') {
scope.setTag('privacy', 'private');
}
}
});
}

async function run() {
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
await chat('public');
await chat('private');
await stream();
});
}

run();
Original file line number Diff line number Diff line change
Expand Up @@ -358,3 +358,47 @@ describe('Vercel AI calls with recording off (v7)', () => {
{ additionalDependencies: { ai: '^7.0.163' } },
);
});

describe('Vercel AI recording functions (v7)', () => {
afterAll(() => {
cleanupChildProcesses();
});

createEsmTests(
join(__dirname, '../..'),
'scenario-recording-functions.mjs',
'instrument-with-recording-functions.mjs',
(createRunner, test) => {
test('decides content recording per span when the options are functions', async () => {
await createRunner()
.expect({
span: container => {
const recorded = (operationId: string): { inputs: boolean; outputs: boolean }[] =>
container.items
.filter(span => span.attributes['vercel.ai.operationId']?.value === operationId)
.map(span => ({
inputs: GEN_AI_INPUT_MESSAGES in span.attributes,
outputs: GEN_AI_OUTPUT_MESSAGES in span.attributes,
}));

// `recordOutputs` skips `generateText` by type; the scope tag decides the rest.
expect(recorded('ai.generateText')).toEqual([
{ inputs: true, outputs: false },
{ inputs: false, outputs: false },
]);
expect(recorded('ai.generateText.doGenerate')).toEqual([
{ inputs: true, outputs: true },
{ inputs: false, outputs: false },
]);
// The scenario makes the scope private while the stream is read, after both spans started.
expect(recorded('ai.streamText')).toEqual([{ inputs: true, outputs: true }]);
expect(recorded('ai.streamText.doStream')).toEqual([{ inputs: true, outputs: true }]);
},
})
.start()
.completed();
});
},
{ additionalDependencies: { ai: '^7.0.0' } },
);
});
3 changes: 2 additions & 1 deletion packages/server-utils/src/ai/core/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,8 @@ export function resolveGenAiCallRecording<TCall>(
};
}

function resolveRecordingOption<TCall>(
/** Calls `option` with `call` when it is a function; `undefined` means the option does not decide. */
export function resolveRecordingOption<TCall>(
option: GenAiRecordingOption<TCall> | undefined,
call: TCall,
): boolean | undefined {
Expand Down
6 changes: 3 additions & 3 deletions packages/server-utils/src/integrations/vercel-ai/index.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
import { defineIntegration, waitForTracingChannelBinding, type IntegrationFn } from '@sentry/core';
import type { GenAiOptions } from '../../ai/core/utils';
import { subscribeVercelAiTracingChannel } from './vercel-ai-dc-subscriber';
import type { GenAiCallOptions } from '../../ai/core/utils';
import { subscribeVercelAiTracingChannel, type VercelAiCallInfo } from './vercel-ai-dc-subscriber';
import { registerVercelAiToolTelemetry } from './vercel-ai-tool-telemetry';
import * as dc from '../../utils/diagnosticsChannel';
import { invokeOrchestrionInstrumentation } from '../../orchestrion/instrumentation';
import { vercelAiModuleNames } from '../../orchestrion/config/vercel-ai';
import { subscribeVercelAiOrchestrionChannels } from './vercel-ai-orchestrion-subscriber';

/** Options for the Vercel AI integration. */
export type VercelAiOptions = GenAiOptions;
export type VercelAiOptions = GenAiCallOptions<VercelAiCallInfo>;

const _vercelAIIntegration = ((options: VercelAiOptions = {}) => {
return {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,12 @@ import {
import type { TracingChannel } from 'node:diagnostics_channel';
import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../ai/core/gen-ai-attributes';
import { isEveGenAiRecordingDefault } from './gen-ai-recording-mode';
import { getEvaluationOutputMessages, type GenAiOptions } from '../../ai/core/utils';
import {
type GenAiCallOptions,
type GenAiOptions,
getEvaluationOutputMessages,
resolveRecordingOption,
} from '../../ai/core/utils';
import { getProviderMetadataAttributes, LAST_STEP_ONLY_USAGE_KEYS } from '../../ai/vercel-ai';
import { WORKERS_AI_INTEGRATION_NAME } from '../../ai/workers-ai/constants';
import { bindTracingChannelToSpan } from '../../tracing-channel';
Expand Down Expand Up @@ -282,8 +287,21 @@ export interface VercelAiChannelMessage {
*/
export type VercelAiTracingChannelFactory = <T extends object>(name: string) => TracingChannel<T, T>;

/** The call that `recordInputs`/`recordOutputs` decide on: one span, with the AI SDK telemetry event as published. */
export interface VercelAiCallInfo {
/** The channel event type, such as `generateText`, `languageModelCall` or `executeTool`. */
type: ChannelEventType;
/** The AI SDK telemetry event of the call. */
event: Record<string, unknown>;
}

/** Integration-level recording options, pinned at subscribe time so we never look the integration up per event. */
export type VercelAiChannelOptions = GenAiOptions;
export type VercelAiChannelOptions = GenAiCallOptions<VercelAiCallInfo>;

// The integration-level decision of each span, made at span start in the caller's async context. The rest of the
// precedence (eve mode, the per-call flags, `dataCollection.genAI`) is applied when it is needed, because whether
// `ai` withheld the result is only known at span end.
const integrationRecordingBySpan = new WeakMap<Span, GenAiOptions>();

// `ai` >= 7.0.163 publishes no channel `result` for calls with `recordInputs`/`recordOutputs: false`. These stay
// here rather than on the shared channel message, which other subscribers can read.
Expand Down Expand Up @@ -312,14 +330,14 @@ export function subscribeVercelAiTracingChannel(
// The helper ends the span; we enrich it from the settled result first (tokens, output messages,
// finish reasons, response model/id, provider metadata) and drop the per-operation `callId` maps.
beforeSpanEnd: (span, data) => {
enrichSpanOnEnd(span, data, options);
enrichSpanOnEnd(span, data);
// The result can hold the provider request and response bodies, and spans outlive the call.
fallbackChannelResults.delete(span);
clearOperationId(data);
},
// A streamed model call resolves before its stream is drained, so we tap the stream, keep the
// span open, and end it (via `end`) once the final usage/finish/output chunks arrive.
deferSpanEnd: ({ span, data, end }) => deferStreamedModelCallEnd(span, data, options, end),
deferSpanEnd: ({ span, data, end }) => deferStreamedModelCallEnd(span, data, end),
},
);
}
Expand All @@ -330,12 +348,7 @@ export function subscribeVercelAiTracingChannel(
* enrich and end the span once the stream settles. Returns `false` for anything that isn't a streamed
* model call so the helper ends the span as usual.
*/
function deferStreamedModelCallEnd(
span: Span,
data: VercelAiChannelMessage,
options: VercelAiChannelOptions,
end: (error?: unknown) => void,
): boolean {
function deferStreamedModelCallEnd(span: Span, data: VercelAiChannelMessage, end: (error?: unknown) => void): boolean {
const result = fallbackChannelResults.get(span) ?? data.result;
if (data.type !== 'languageModelCall' || !isObjectLike(result)) {
return false;
Expand All @@ -346,7 +359,12 @@ function deferStreamedModelCallEnd(
}

const callId = asString(data.event.callId);
const { recordOutputs } = getRecordingOptions(data.event, options, data.result === undefined);
const invokeAgentSpan = callId ? invokeAgentSpanByCallId.get(callId) : undefined;
const { recordOutputs } = getRecordingOptions(
data.event,
invokeAgentSpan && integrationRecordingBySpan.get(invokeAgentSpan),
data.result === undefined,
);
result.stream = tapModelCallStream(
stream,
final => {
Expand All @@ -355,7 +373,9 @@ function deferStreamedModelCallEnd(
// streamed and non-streamed spans identical.
fallbackChannelResults.set(span, { ...result, ...streamedResultToChannelResult(final) });
end();
enrichInvokeAgentFromStream(callId, final, recordOutputs);
if (invokeAgentSpan) {
enrichInvokeAgentFromStream(invokeAgentSpan, final, recordOutputs);
}
},
error => end(error),
);
Expand Down Expand Up @@ -394,16 +414,7 @@ export function streamedResultToChannelResult(final: StreamedModelCallResult): R
* the enclosing `invoke_agent` span, which has no channel result of its own. Runs before that span ends
* because the operation's completion promise settles only after every model-call stream has drained.
*/
function enrichInvokeAgentFromStream(
callId: string | undefined,
final: StreamedModelCallResult,
recordOutputs: boolean,
): void {
const span = callId ? invokeAgentSpanByCallId.get(callId) : undefined;
if (!span) {
return;
}

function enrichInvokeAgentFromStream(span: Span, final: StreamedModelCallResult, recordOutputs: boolean): void {
const usage = isObjectLike(final.usage) ? final.usage : undefined;
if (usage) {
const input = tokenCount(usage.inputTokens) ?? tokenCount(usage.promptTokens) ?? tokenCount(usage.tokens);
Expand Down Expand Up @@ -452,7 +463,22 @@ export function createSpanFromMessage(
return undefined;
}

const { recordInputs } = getRecordingOptions(event, channelOptions);
const integrationRecording: GenAiOptions = {
recordInputs: resolveRecordingOption(channelOptions.recordInputs, { type, event }),
recordOutputs: resolveRecordingOption(channelOptions.recordOutputs, { type, event }),
};
const span = startSpanFromEvent(type, event, getRecordingOptions(event, integrationRecording).recordInputs);
if (span) {
integrationRecordingBySpan.set(span, integrationRecording);
}
return span;
}

function startSpanFromEvent(
type: ChannelEventType,
event: Record<string, unknown>,
recordInputs: boolean,
): Span | undefined {
const provider = asString(event.provider);
const modelId = asString(event.modelId);
const callId = asString(event.callId);
Expand Down Expand Up @@ -663,18 +689,18 @@ function buildToolSpan(
* Everything here is guarded: when a field is missing or the shape differs across `ai` versions,
* we simply don't set the attribute rather than emit a malformed span.
*/
export function enrichSpanOnEnd(
span: Span,
data: VercelAiChannelMessage,
channelOptions: VercelAiChannelOptions,
): void {
export function enrichSpanOnEnd(span: Span, data: VercelAiChannelMessage): void {
const { type } = data;
const result = fallbackChannelResults.get(span) ?? data.result;
if (!isObjectLike(result)) {
return;
}

const { recordOutputs } = getRecordingOptions(data.event, channelOptions, data.result === undefined);
const { recordOutputs } = getRecordingOptions(
data.event,
integrationRecordingBySpan.get(span),
data.result === undefined,
);

if (type === 'executeTool') {
// From V5 on, tool errors are not rejected (so the `error` channel verb never fires) — they
Expand Down Expand Up @@ -918,7 +944,7 @@ export function captureToolError(span: Span, data: VercelAiChannelMessage, error

function getRecordingOptions(
event: Record<string, unknown>,
channelOptions: VercelAiChannelOptions,
integrationRecording: GenAiOptions = {},
channelWithheldResult = false,
): {
recordInputs: boolean;
Expand All @@ -931,8 +957,8 @@ function getRecordingOptions(
const eveMode = client && !channelWithheldResult ? isEveGenAiRecordingDefault(client) : false;

return {
recordInputs: resolveRecording(channelOptions.recordInputs, event.recordInputs, genAI?.inputs, eveMode),
recordOutputs: resolveRecording(channelOptions.recordOutputs, event.recordOutputs, genAI?.outputs, eveMode),
recordInputs: resolveRecording(integrationRecording.recordInputs, event.recordInputs, genAI?.inputs, eveMode),
recordOutputs: resolveRecording(integrationRecording.recordOutputs, event.recordOutputs, genAI?.outputs, eveMode),
};
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,7 @@ function bindOperation(
// v6's `executeToolCall` returns the tool result/error object directly, whereas the shared core
// (matching v7) expects it nested under `output`; wrap it so tool-error detection works.
message.result = message.type === 'executeTool' ? { output: data.result } : data.result;
enrichSpanOnEnd(span, message, options);
enrichSpanOnEnd(span, message);
}
// A `streamText`/`streamObject` model call runs after this (synchronously-returning) operation's
// span has already ended, so its `callId` entry must outlive the operation — it's cleared once the
Expand All @@ -318,7 +318,7 @@ function bindOperation(
// before the stream drains — losing the aggregate usage/output. Defer the end and await the result's
// completion promises (`totalUsage`/`text`/…, which resolve on drain), mirroring how v7's channel
// defers the operation span on the SDK's total-usage promise.
deferSpanEnd: ({ span, data, end }) => deferStreamTextOperationEnd(span, data, end, options),
deferSpanEnd: ({ span, data, end }) => deferStreamTextOperationEnd(span, data, end),
},
);
}
Expand All @@ -337,7 +337,6 @@ function deferStreamTextOperationEnd(
span: Span,
data: TracingChannelPayloadWithSpan<OrchestrionChannelContext>,
end: (error?: unknown) => void,
options: VercelAiChannelOptions,
): boolean {
const message = messages.get(data);
const type = message?.type;
Expand Down Expand Up @@ -371,7 +370,7 @@ function deferStreamTextOperationEnd(
// The model call itself succeeded, so keep its usage on the span; `beforeSpanEnd` skips
// enrichment once `end(error)` marks the payload as errored.
message.result = { usage, toolCalls, finishReason, response };
enrichSpanOnEnd(span, message, options);
enrichSpanOnEnd(span, message);
end(objectOutcome.error);
return;
}
Expand Down Expand Up @@ -568,7 +567,7 @@ function patchModelMethod(
value.stream,
final => {
message.result = { ...value, ...streamedResultToChannelResult(final) };
enrichSpanOnEnd(span, message, options);
enrichSpanOnEnd(span, message);
span.end();
clearStreamCallId();
},
Expand All @@ -587,7 +586,7 @@ function patchModelMethod(
// `doGenerate` (and any non-stream result) settles with the full result; end here, start/end
// bracket the call to match the channel timing.
message.result = value;
enrichSpanOnEnd(span, message, options);
enrichSpanOnEnd(span, message);
span.end();
clearStreamCallId();

Expand Down Expand Up @@ -681,7 +680,7 @@ function patchToolExecute(
return result.then(value => {
// The shared core (matching v6/v7) expects the tool result nested under `output`.
message.result = { output: value };
enrichSpanOnEnd(span, message, options);
enrichSpanOnEnd(span, message);
span.end();
return value;
}, failSpan);
Expand Down
Loading
Loading