diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts index 75945701e92f..e6ec70fa35ff 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts @@ -492,7 +492,7 @@ describe('Vercel AI integration (v4)', () => { }); createEsmAndCjsTests(__dirname, 'scenario-conversation-id.mjs', 'instrument.mjs', (createRunner, test) => { - test('does not overwrite conversation id set via Sentry.setConversationId with responseId from provider metadata', async () => { + test('keeps the conversation id set via Sentry.setConversationId and ignores the provider responseId', async () => { await createRunner() .expect({ transaction: { transaction: 'main' } }) .expect({ diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-openai-conversation.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-openai-conversation.mjs new file mode 100644 index 000000000000..fd724db00276 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-openai-conversation.mjs @@ -0,0 +1,75 @@ +import * as Sentry from '@sentry/node'; +import { generateText, tool } from 'ai'; +import { MockLanguageModelV3 } from 'ai/test'; +import { z } from 'zod'; + +const usage = { + inputTokens: { total: 10, noCache: 10, cached: 0 }, + outputTokens: { total: 5, noCache: 5, cached: 0 }, + totalTokens: { total: 15, noCache: 15, cached: 0 }, +}; + +const textModel = new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'stop', raw: 'stop' }, + usage, + content: [{ type: 'text', text: 'Hello!' }], + warnings: [], + // A per-response id: present on every turn, never the conversation id. + providerMetadata: { openai: { responseId: 'resp_turn' } }, + }), +}); + +const toolCallModel = new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, + usage, + content: [{ type: 'tool-call', toolCallId: 'tc-1', toolName: 'echo', input: JSON.stringify({ text: 'hi' }) }], + warnings: [], + }), +}); + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + // A turn of an OpenAI Conversations API conversation. + await generateText({ + experimental_telemetry: { isEnabled: true }, + model: textModel, + prompt: 'First turn', + providerOptions: { openai: { conversation: 'conv_abc123' } }, + }); + + // The Azure Responses API carries the same option under the `azure` key; this turn also runs a tool. + await generateText({ + experimental_telemetry: { isEnabled: true }, + model: toolCallModel, + prompt: 'Second turn', + providerOptions: { azure: { conversation: 'conv_azure' } }, + tools: { + echo: tool({ + inputSchema: z.object({ text: z.string() }), + execute: async ({ text }) => text, + }), + }, + }); + + // Chaining on the previous response names a response, not a thread, so no conversation id. + await generateText({ + experimental_telemetry: { isEnabled: true }, + model: textModel, + prompt: 'Chained turn', + providerOptions: { openai: { previousResponseId: 'resp_turn' } }, + }); + + // An id set through the SDK API wins over the provider option. Last, since it stays on the scope. + Sentry.setConversationId('conv-from-api'); + await generateText({ + experimental_telemetry: { isEnabled: true }, + model: textModel, + prompt: 'API turn', + providerOptions: { openai: { conversation: 'conv_ignored' } }, + }); + }); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts index f8a26a410833..b55c3889ffee 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -767,7 +767,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe 'scenario-provider-metadata.mjs', 'instrument.mjs', (createRunner, test) => { - test('derives provider-metadata token breakdown, conversation id and system instructions', async () => { + test('derives provider-metadata token breakdown and system instructions', async () => { await createRunner() .expect({ transaction: { transaction: 'main' } }) .expect({ @@ -781,12 +781,13 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe )!; expect(generateContent).toBeDefined(); - // Cache/reasoning token breakdown and conversation id are derived from the model's - // `providerMetadata` — by the OTel processor on v6 and by the channel subscriber on v7, - // both via the shared `getProviderMetadataAttributes` helper, so the shape is identical. + // Cache/reasoning token breakdown is derived from the model's `providerMetadata` — by the + // OTel processor on v6 and by the channel subscriber on v7, both via the shared + // `getProviderMetadataAttributes` helper, so the shape is identical. expect(generateContent.attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]?.value).toBe(5); expect(generateContent.attributes[GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]?.value).toBe(7); - expect(generateContent.attributes[GEN_AI_CONVERSATION_ID]?.value).toBe('resp_abc123'); + // The per-response `responseId` is not a conversation id and must not be recorded as one. + expect(generateContent.attributes[GEN_AI_CONVERSATION_ID]).toBeUndefined(); const invokeAgent = container.items.find( span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', @@ -813,6 +814,66 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, ); + createEsmTests( + __dirname, + 'scenario-openai-conversation.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('derives gen_ai.conversation.id from the OpenAI `conversation` provider option', async () => { + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + const genAiSpans = container.items.filter(s => + String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.'), + ); + const conversationIdOf = (span: (typeof genAiSpans)[number]) => + span.attributes[GEN_AI_CONVERSATION_ID]?.value; + const invokeAgentSpans = genAiSpans.filter( + s => s.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', + ); + expect(invokeAgentSpans).toHaveLength(4); + const [firstTurn, secondTurn, chainedTurn, apiTurn] = invokeAgentSpans.sort( + (a, b) => a.start_timestamp - b.start_timestamp, + ); + + // `providerOptions.openai.conversation` is the Conversations API id: the same on every turn. + expect(conversationIdOf(firstTurn!)).toBe('conv_abc123'); + // The Azure Responses API uses the `azure` key for the same option. + expect(conversationIdOf(secondTurn!)).toBe('conv_azure'); + // `previousResponseId` names a response rather than a thread, and the response's own + // `responseId` is recorded as `gen_ai.response.id` only. + expect(conversationIdOf(chainedTurn!)).toBeUndefined(); + // `Sentry.setConversationId()` beats the provider option. + expect(conversationIdOf(apiTurn!)).toBe('conv-from-api'); + + // Model-call and tool spans carry their operation's id, even though their start events + // do not carry `providerOptions`. + const modelCallSpans = genAiSpans.filter( + s => s.attributes['sentry.op']?.value === 'gen_ai.generate_content', + ); + expect(modelCallSpans.map(conversationIdOf).sort()).toEqual([ + 'conv-from-api', + 'conv_abc123', + 'conv_azure', + undefined, + ]); + const toolSpan = genAiSpans.find(s => s.attributes['sentry.op']?.value === 'gen_ai.execute_tool')!; + expect(toolSpan).toBeDefined(); + expect(conversationIdOf(toolSpan)).toBe('conv_azure'); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + ai: vercelAiVersion, + }, + }, + ); + createEsmTests( __dirname, 'scenario-cache-tokens.mjs', diff --git a/packages/server-utils/src/ai/vercel-ai/index.ts b/packages/server-utils/src/ai/vercel-ai/index.ts index 76a355fdc588..3296eb18a73f 100644 --- a/packages/server-utils/src/ai/vercel-ai/index.ts +++ b/packages/server-utils/src/ai/vercel-ai/index.ts @@ -1,6 +1,5 @@ import type { SpanAttributeValue } from '@sentry/core'; import { - GEN_AI_CONVERSATION_ID, GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, @@ -10,8 +9,8 @@ import { import type { OpenAiProviderMetadata, ProviderMetadata } from './vercel-ai-attributes'; /** - * Derive the `gen_ai.usage.*` cache/reasoning/prediction token attributes and `gen_ai.conversation.id` - * from an AI SDK `providerMetadata` object. + * Derive the `gen_ai.usage.*` cache/reasoning/prediction token attributes from an AI SDK + * `providerMetadata` object. * * Used by the `ai` >= 7 tracing-channel subscriber, which receives `providerMetadata` as an object on * the channel result. Pass the already-parsed object; unknown/empty input yields `{}`. @@ -39,7 +38,6 @@ export function getProviderMetadataAttributes(providerMetadata: unknown): Record 'gen_ai.usage.output_tokens.prediction_rejected', openaiMetadata.rejectedPredictionTokens, ); - setAttributeIfDefined(attributes, GEN_AI_CONVERSATION_ID, openaiMetadata.responseId); } if (metadata.anthropic) { diff --git a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts index 78b4763f0240..526b3b4fb379 100644 --- a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts +++ b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts @@ -37,6 +37,7 @@ import type { Span, SpanAttributes } from '@sentry/core'; import { _INTERNAL_skipAiProviderWrapping, captureException, + getActiveSpan, getClient, isObjectLike, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, @@ -142,6 +143,25 @@ export function clearOperationCallId(callId: string): void { invokeAgentSpanByCallId.delete(callId); } +/** + * The OpenAI Conversations API id from `providerOptions.openai.conversation` (or `azure`): the one + * provider-level value that is the same on every turn. A `Sentry.setConversationId()` value still wins, + * since `conversationIdIntegration` writes it on `spanStart`, after these start attributes. + */ +function getOpenAiConversationId(providerOptions: unknown): string | undefined { + if (!isObjectLike(providerOptions)) { + return undefined; + } + const openaiOptions = providerOptions.openai ?? providerOptions.azure; + return isObjectLike(openaiOptions) ? asString(openaiOptions.conversation) : undefined; +} + +/** The `gen_ai.conversation.id` already on the active span, which for a child event is its operation span. */ +function getActiveSpanConversationId(): string | undefined { + const active = getActiveSpan(); + return active ? asString(spanToJSON(active).attributes[GEN_AI_CONVERSATION_ID]) : undefined; +} + /** * `providerMetadata` is last-step only; drop derived usage on spans that report an aggregate. * @@ -417,11 +437,16 @@ export function createSpanFromMessage( recordToolDescriptions(callId, event.tools); } + // Only an operation's start event carries `providerOptions`; its model-call and tool events start + // while the operation span is active, so they inherit the id from it. + const conversationId = getOpenAiConversationId(event.providerOptions) ?? getActiveSpanConversationId(); + const baseAttributes: Record = { [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: ORIGIN, ...(provider ? { [GEN_AI_PROVIDER_NAME]: provider, [VERCEL_AI_MODEL_PROVIDER_ATTRIBUTE]: provider } : {}), ...(modelId ? { [GEN_AI_REQUEST_MODEL]: modelId } : {}), ...(maxRetries !== undefined ? { [VERCEL_AI_SETTINGS_MAX_RETRIES_ATTRIBUTE]: maxRetries } : {}), + ...(conversationId ? { [GEN_AI_CONVERSATION_ID]: conversationId } : {}), }; switch (type) { @@ -437,7 +462,7 @@ export function createSpanFromMessage( return buildModelCallSpan(event, baseAttributes, recordInputs, callId, modelId); case 'executeTool': - return buildToolSpan(event, recordInputs); + return buildToolSpan(event, recordInputs, conversationId); case 'embed': case 'embedMany': { // `embed` carries a single `value`; `embedMany` a `values` array — both map to the embeddings input. @@ -523,7 +548,11 @@ function buildModelCallSpan( }); } -function buildToolSpan(event: Record, recordInputs: boolean): Span { +function buildToolSpan( + event: Record, + recordInputs: boolean, + conversationId: string | undefined, +): Span { const toolCall = isObjectLike(event.toolCall) ? event.toolCall : {}; const toolName = asString(toolCall.toolName); const toolCallId = asString(event.toolCallId) ?? asString(toolCall.toolCallId); @@ -538,6 +567,7 @@ function buildToolSpan(event: Record, recordInputs: boolean): S ...(toolCallId ? { [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: toolCallId } : {}), ...(description ? { [GEN_AI_TOOL_DESCRIPTION]: description } : {}), ...(recordInputs && toolInput !== undefined ? { [GEN_AI_TOOL_CALL_ARGUMENTS]: stringify(toolInput) } : {}), + ...(conversationId ? { [GEN_AI_CONVERSATION_ID]: conversationId } : {}), }); } @@ -609,17 +639,11 @@ export function enrichSpanOnEnd( span.setAttribute(GEN_AI_RESPONSE_MODEL, responseModel); } - // Provider-specific cache/reasoning/prediction token breakdowns and `gen_ai.conversation.id`. - // The channel exposes `providerMetadata` as an object (the OTel path parses it from a string); - // both share `getProviderMetadataAttributes` so the emitted shape is identical. + // Provider-specific cache/reasoning/prediction token breakdowns. The channel exposes `providerMetadata` + // as an object (the OTel path parses it from a string); both share `getProviderMetadataAttributes` so + // the emitted shape is identical. const providerMetadata = (result as { providerMetadata?: unknown }).providerMetadata; const providerAttributes = getProviderMetadataAttributes(providerMetadata); - // Don't overwrite a conversation id already set on span start (e.g. by `conversationIdIntegration` - // from a user-set scope value); the provider-derived id is only a fallback. Matches the OTel path. - if (GEN_AI_CONVERSATION_ID in providerAttributes && spanToJSON(span).attributes[GEN_AI_CONVERSATION_ID]) { - // oxlint-disable-next-line typescript/no-dynamic-delete - delete providerAttributes[GEN_AI_CONVERSATION_ID]; - } dropLastStepOnlyUsage(providerAttributes, type); span.setAttributes(providerAttributes); diff --git a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-orchestrion-subscriber.ts b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-orchestrion-subscriber.ts index ea64ae6aacd9..e8d52aa517f1 100644 --- a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-orchestrion-subscriber.ts +++ b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-orchestrion-subscriber.ts @@ -669,6 +669,9 @@ function buildTextMessage(type: 'generateText' | 'streamText' | 'generateObject' // Normalize to the message-array shape the shared core (and v7's channel) expects: a bare string // `prompt` becomes a single user message, matching the SDK's own normalization. messages: normalizePromptMessages(options), + // v7's native start event carries `providerOptions`; the shared core reads the OpenAI + // Conversations API id from it. + providerOptions: options.providerOptions, ...recording(telemetry), }, }); diff --git a/packages/server-utils/test/integrations/vercel-ai/conversation-id.test.ts b/packages/server-utils/test/integrations/vercel-ai/conversation-id.test.ts new file mode 100644 index 000000000000..fb5d1e347cd5 --- /dev/null +++ b/packages/server-utils/test/integrations/vercel-ai/conversation-id.test.ts @@ -0,0 +1,119 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { GEN_AI_CONVERSATION_ID, GEN_AI_RESPONSE_ID } from '@sentry/conventions/attributes'; +import { + conversationIdIntegration, + getMainCarrier, + setConversationId, + setCurrentClient, + spanToStaticSpanJSON, + withActiveSpan, +} from '@sentry/core'; +import type { Span } from '@sentry/core'; +import { getProviderMetadataAttributes } from '../../../src/ai/vercel-ai'; +import { + clearOperationCallId, + createSpanFromMessage, + enrichSpanOnEnd, +} from '../../../src/integrations/vercel-ai/vercel-ai-dc-subscriber'; +import { getDefaultTestClientOptions, TestClient } from '../../mocks/client'; + +type Message = Parameters[0]; +const channelOptions = {} as Parameters[1]; + +describe('Vercel AI SDK conversation id', () => { + let endedSpans: Span[]; + + beforeEach(() => { + getMainCarrier().__SENTRY__ = undefined; + // The bare test client has no default integrations; the Node SDK registers this one itself. + const client = new TestClient( + getDefaultTestClientOptions({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + tracesSampleRate: 1, + integrations: [conversationIdIntegration()], + }), + ); + setCurrentClient(client); + client.init(); + endedSpans = []; + client.on('spanEnd', span => endedSpans.push(span)); + }); + + afterEach(() => { + setConversationId(undefined); + clearOperationCallId('call-1'); + getMainCarrier().__SENTRY__ = undefined; + }); + + function run(message: Message): Record { + const span = createSpanFromMessage(message, channelOptions)!; + enrichSpanOnEnd(span, message, {} as Parameters[2]); + span.end(); + return spanToStaticSpanJSON(endedSpans[endedSpans.length - 1]!).data ?? {}; + } + + it('reads the OpenAI Conversations API id from the operation `providerOptions`', () => { + const data = run({ + type: 'generateText', + event: { callId: 'call-1', providerOptions: { openai: { conversation: 'conv_abc' } } }, + }); + expect(data[GEN_AI_CONVERSATION_ID]).toBe('conv_abc'); + }); + + it('reads the id from the `azure` provider options', () => { + const data = run({ + type: 'generateText', + event: { callId: 'call-1', providerOptions: { azure: { conversation: 'conv_azure' } } }, + }); + expect(data[GEN_AI_CONVERSATION_ID]).toBe('conv_azure'); + }); + + it('passes the operation id on to the model-call and tool spans that start under it', () => { + const operationMessage: Message = { + type: 'generateText', + event: { callId: 'call-1', providerOptions: { openai: { conversation: 'conv_abc' } } }, + }; + const operationSpan = createSpanFromMessage(operationMessage, channelOptions)!; + // Child events start while the operation span is the active span (the channel binds it as context). + const [modelCall, toolCall] = withActiveSpan(operationSpan, () => [ + run({ type: 'languageModelCall', event: { callId: 'call-1', modelId: 'gpt-4' } }), + run({ + type: 'executeTool', + event: { callId: 'call-1', toolCall: { toolName: 'echo', toolCallId: 'tc-1', input: {} } }, + }), + ]); + operationSpan.end(); + + expect(spanToStaticSpanJSON(operationSpan).data?.[GEN_AI_CONVERSATION_ID]).toBe('conv_abc'); + expect(modelCall![GEN_AI_CONVERSATION_ID]).toBe('conv_abc'); + expect(toolCall![GEN_AI_CONVERSATION_ID]).toBe('conv_abc'); + }); + + it('sets nothing for `previousResponseId` chaining, which names a response rather than a thread', () => { + const data = run({ + type: 'generateText', + event: { callId: 'call-1', providerOptions: { openai: { previousResponseId: 'resp_1' } } }, + }); + expect(data[GEN_AI_CONVERSATION_ID]).toBeUndefined(); + }); + + it('does not record the provider `responseId` as the conversation id', () => { + const data = run({ + type: 'languageModelCall', + event: { callId: 'call-1', modelId: 'gpt-4' }, + result: { response: { id: 'resp_1' }, providerMetadata: { openai: { responseId: 'resp_1' } } }, + }); + expect(data[GEN_AI_RESPONSE_ID]).toBe('resp_1'); + expect(data[GEN_AI_CONVERSATION_ID]).toBeUndefined(); + expect(getProviderMetadataAttributes({ openai: { responseId: 'resp_1' } })).toEqual({}); + }); + + it('lets an id set via `Sentry.setConversationId()` win over the provider option', () => { + setConversationId('conv-from-scope'); + const data = run({ + type: 'generateText', + event: { callId: 'call-1', providerOptions: { openai: { conversation: 'conv_abc' } } }, + }); + expect(data[GEN_AI_CONVERSATION_ID]).toBe('conv-from-scope'); + }); +});