diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json index 2dbf1013a228..bdf2c25ca0a7 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json @@ -1,6 +1,6 @@ { "name": "gen-ai-libraries", - "description": "Real gen_ai spans for every instrumented AI library (OpenAI, Anthropic, Mistral, Together, Vercel AI), each driven through OpenRouter with a chat query and a tool call, on Node and on Cloudflare", + "description": "Real gen_ai spans for every instrumented AI library (OpenAI, Anthropic, Mistral, Together, Vercel AI, TanStack AI), each driven through OpenRouter with a chat query and a tool call, on Node and on Cloudflare", "version": "1.0.0", "private": true, "type": "module", @@ -21,6 +21,9 @@ "@openrouter/ai-sdk-provider": "~3.0.0", "@sentry/cloudflare": "file:../../packed/sentry-cloudflare-packed.tgz", "@sentry/node": "file:../../packed/sentry-node-packed.tgz", + "@tanstack/ai": "^1.0.0", + "@tanstack/ai-openai": "^1.0.0", + "@tanstack/ai-openrouter": "^1.0.0", "ai": "~7.0.97", "express": "^4.21.2", "openai": "5.18.1", diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/pnpm-workspace.yaml b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/pnpm-workspace.yaml index fd29a82085e0..6f72f0ab7167 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/pnpm-workspace.yaml +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/pnpm-workspace.yaml @@ -1,4 +1,5 @@ allowBuilds: + '@openrouter/sdk': false esbuild: true workerd: true minimumReleaseAge: 0 diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts index 265b8122f0df..e7cca78ad3fb 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts @@ -2,8 +2,10 @@ // `@sentry/cloudflare/vite` bundler plugin (build-time channel injection) and run on workerd. import * as Sentry from '@sentry/cloudflare'; import { libraries } from './libraries'; +import { tanstackAiActivities } from './tanstack-ai-activities'; const byId = new Map(libraries.map(library => [library.id, library])); +const tanstackAiActivityById = new Map(tanstackAiActivities.map(activity => [activity.id, activity])); export default Sentry.withSentry( (env: Env) => ({ @@ -17,8 +19,14 @@ export default Sentry.withSentry( const url = new URL(request.url); const [, id, action] = url.pathname.split('/'); const library = id ? byId.get(id) : undefined; + const run = + library && (action === 'chat' || action === 'tools') + ? library[action] + : id === 'tanstack-ai' && action + ? tanstackAiActivityById.get(action)?.run + : undefined; - if (!library || (action !== 'chat' && action !== 'tools')) { + if (!run) { return new Response('Not found', { status: 404 }); } @@ -29,7 +37,7 @@ export default Sentry.withSentry( try { const spanName = action === 'tools' ? 'ai-tool-workflow' : 'ai-workflow'; - const result = await Sentry.startSpan({ name: spanName, op: 'function' }, () => library[action](apiKey)); + const result = await Sentry.startSpan({ name: spanName, op: 'function' }, () => run(apiKey)); return Response.json({ result }); } catch (error) { return Response.json({ message: (error as Error).message }, { status: 500 }); diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.node.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.node.ts index 726924dbd69d..17a3ebbdae3a 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.node.ts +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.node.ts @@ -2,6 +2,7 @@ import * as Sentry from '@sentry/node'; import express from 'express'; import { libraries } from './libraries'; +import { tanstackAiActivities } from './tanstack-ai-activities'; const apiKey = process.env.E2E_OPENROUTER_API_KEY; if (!apiKey) { @@ -35,6 +36,17 @@ for (const library of libraries) { }); } +for (const activity of tanstackAiActivities) { + app.get(`/tanstack-ai/${activity.id}`, async (_req, res, next) => { + try { + const result = await Sentry.startSpan({ name: 'ai-workflow', op: 'function' }, () => activity.run(apiKey)); + res.send({ result }); + } catch (error) { + next(error); + } + }); +} + Sentry.setupExpressErrorHandler(app); app.use((error: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => { diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts index 1b1490e7e9b4..abcc4ae7adf9 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts @@ -19,6 +19,8 @@ import Anthropic from '@anthropic-ai/sdk'; import { createOpenRouter } from '@openrouter/ai-sdk-provider'; import { Mistral } from '@mistralai/mistralai'; +import { chat, toolDefinition } from '@tanstack/ai'; +import { createOpenRouterText } from '@tanstack/ai-openrouter'; import OpenAI from 'openai'; import Together from 'together-ai'; import { generateText, tool } from 'ai'; @@ -54,9 +56,9 @@ export interface Library { id: string; /** The op of the model-call span; asserted by the tests. */ op: 'gen_ai.chat' | 'gen_ai.generate_content'; - /** `gen_ai.provider.name` for the direct-SDK libraries; unset for the Vercel AI SDK. */ + /** `gen_ai.provider.name` of the model-call span; unset for the Vercel AI SDK. */ provider?: string; - /** `sentry.origin` for the direct-SDK libraries; unset for the Vercel AI SDK. */ + /** `sentry.origin` of the model-call span; unset for the Vercel AI SDK. */ origin?: string; chat: (apiKey: string) => Promise; tools: (apiKey: string) => Promise; @@ -216,4 +218,49 @@ export const libraries: Library[] = [ return result.toolCalls ?? []; }, }, + + { + id: 'tanstack-ai', + // TanStack AI runs an agent loop like the Vercel AI SDK: an `invoke_agent` span with one `chat` span + // per model call, and an `execute_tool` span for each tool it runs. + op: 'gen_ai.chat', + provider: 'openrouter', + origin: 'auto.ai.tanstack_ai', + chat: async apiKey => { + const { text } = await chat({ + adapter: createOpenRouterText(MODEL, apiKey), + threadId: crypto.randomUUID(), + systemPrompts: [SYSTEM], + messages: [{ role: 'user', content: CHAT_PROMPT }], + modelOptions: { temperature: 0 }, + stream: false, + }); + return text; + }, + tools: async apiKey => { + const getWeather = toolDefinition({ + name: 'get_weather', + description: 'Get the current weather for a city.', + inputSchema: z.object({ city: z.string().describe('The city name') }), + }).server(async ({ city }) => `It is sunny in ${city}.`); + + const { text } = await chat({ + adapter: createOpenRouterText(MODEL, apiKey), + threadId: crypto.randomUUID(), + messages: [{ role: 'user', content: WEATHER_PROMPT }], + tools: [getWeather], + // A `toolChoice` option would apply to every model call of the loop, so the model could never + // answer. Forcing only the first call keeps the turn at one tool call and one answer. + middleware: [ + { + name: 'force-first-tool-call', + onConfig: ctx => + ctx.phase === 'beforeModel' && ctx.iteration === 0 ? { toolChoice: 'required' } : undefined, + }, + ], + stream: false, + }); + return text; + }, + }, ]; diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/tanstack-ai-activities.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/tanstack-ai-activities.ts new file mode 100644 index 000000000000..f9c02b796705 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/tanstack-ai-activities.ts @@ -0,0 +1,52 @@ +// The TanStack AI activities other than `chat()`, run against OpenRouter like the libraries in +// `libraries.ts`. The embedding adapter calls the OpenAI SDK, whose Sentry integration TanStack AI skips +// only during its own call, so the `openai` tests that run later still get their spans. +import { boolean, choice, decide, embed, rerank } from '@tanstack/ai'; +import { createOpenaiEmbedding } from '@tanstack/ai-openai'; +import { createOpenRouterDecider, createOpenRouterRerank } from '@tanstack/ai-openrouter'; + +export interface TanstackAiActivity { + id: string; + run: (apiKey: string) => Promise; +} + +export const tanstackAiActivities: TanstackAiActivity[] = [ + { + id: 'embed', + run: async apiKey => { + const { embeddings } = await embed({ + // OpenRouter names the model with its vendor, which the model type of the adapter does not allow. + adapter: createOpenaiEmbedding('openai/text-embedding-3-small' as 'text-embedding-3-small', apiKey, { + baseURL: 'https://openrouter.ai/api/v1', + }), + input: ['a red guitar', 'a blue drum kit'], + }); + return embeddings.length; + }, + }, + { + id: 'decide', + run: async apiKey => { + const result = await decide({ + adapter: createOpenRouterDecider('~typesafe/jev-latest', apiKey), + state: 'I was charged twice for my subscription.', + questions: { + queue: choice({ instructions: 'Which team handles this ticket?', options: { billing: null, tech: null } }), + refund: boolean({ instructions: 'Does the customer ask for a refund?' }), + }, + }); + return { queue: result.queue.value, refund: result.refund.value }; + }, + }, + { + id: 'rerank', + run: async apiKey => { + const { ranking } = await rerank({ + adapter: createOpenRouterRerank('cohere/rerank-v3.5', apiKey), + query: 'A laptop for travel', + documents: ['A heavy gaming desktop', 'A lightweight ultrabook with all-day battery'], + }); + return ranking.map(entry => entry.index); + }, + }, +]; diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/tanstack-ai-activities.test.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/tanstack-ai-activities.test.ts new file mode 100644 index 000000000000..fe7594c69352 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/tanstack-ai-activities.test.ts @@ -0,0 +1,114 @@ +import { expect, test } from '@playwright/test'; +import { collectStreamedSpans, getSpanOp } from '@sentry-internal/test-utils'; +import { APP } from './constants'; +import { attr, describeTree } from './utils'; + +// The span of an activity is active while the activity runs, so the provider request nests under it. +// Each test waits for the segment span, which ends last, so that all spans of the trace are there. + +test('tanstack-ai: an embed() call emits a gen_ai.embeddings span, and no span of the OpenAI integration', async ({ + baseURL, +}) => { + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => + spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.embeddings') && spansOfTrace.some(span => span.is_segment), + ); + + const response = await fetch(`${baseURL}/tanstack-ai/embed`); + expect(response.status).toBe(200); + + const spans = await spansPromise; + const tree = describeTree(spans); + const workflow = spans.find(span => span.name === 'ai-workflow'); + const embeddings = spans.filter(span => getSpanOp(span) === 'gen_ai.embeddings'); + const providerRequests = spans.filter( + span => getSpanOp(span) === 'http.client' && span.parent_span_id === embeddings[0]?.span_id, + ); + + expect(embeddings, tree).toHaveLength(1); + expect(embeddings[0]!.parent_span_id, tree).toBe(workflow?.span_id); + expect(embeddings[0]!.name).toBe('embeddings openai/text-embedding-3-small'); + expect(embeddings[0]!.status).toBe('ok'); + expect(attr(embeddings[0]!, 'sentry.origin')).toBe('auto.ai.tanstack_ai'); + expect(attr(embeddings[0]!, 'gen_ai.operation.name')).toBe('embeddings'); + expect(attr(embeddings[0]!, 'gen_ai.provider.name')).toBe('openai'); + expect(attr(embeddings[0]!, 'gen_ai.request.model')).toBe('openai/text-embedding-3-small'); + expect(typeof attr(embeddings[0]!, 'gen_ai.response.model')).toBe('string'); + expect(attr(embeddings[0]!, 'gen_ai.usage.input_tokens')).toBeGreaterThan(0); + expect(attr(embeddings[0]!, 'gen_ai.embeddings.input')).toBe('["a red guitar","a blue drum kit"]'); + + expect(providerRequests, tree).toHaveLength(1); + expect(String(attr(providerRequests[0]!, 'server.address') ?? providerRequests[0]!.name)).toContain('openrouter'); +}); + +test('tanstack-ai: a decide() call emits a gen_ai.decide span', async ({ baseURL }) => { + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => + spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.decide') && spansOfTrace.some(span => span.is_segment), + ); + + const response = await fetch(`${baseURL}/tanstack-ai/decide`); + expect(response.status).toBe(200); + + const spans = await spansPromise; + const tree = describeTree(spans); + const workflow = spans.find(span => span.name === 'ai-workflow'); + const decideSpan = spans.find(span => getSpanOp(span) === 'gen_ai.decide'); + const providerRequests = spans.filter( + span => getSpanOp(span) === 'http.client' && span.parent_span_id === decideSpan?.span_id, + ); + + expect(decideSpan, tree).toBeDefined(); + expect(decideSpan!.parent_span_id, tree).toBe(workflow?.span_id); + expect(decideSpan!.name).toBe('decide ~typesafe/jev-latest'); + expect(decideSpan!.status).toBe('ok'); + expect(attr(decideSpan!, 'sentry.origin')).toBe('auto.ai.tanstack_ai'); + expect(attr(decideSpan!, 'gen_ai.operation.name')).toBe('decide'); + expect(attr(decideSpan!, 'gen_ai.provider.name')).toBe('openrouter'); + expect(attr(decideSpan!, 'gen_ai.request.model')).toBe('~typesafe/jev-latest'); + expect(typeof attr(decideSpan!, 'gen_ai.response.model')).toBe('string'); + expect(typeof attr(decideSpan!, 'gen_ai.response.id')).toBe('string'); + expect(attr(decideSpan!, 'gen_ai.usage.input_tokens')).toBeGreaterThan(0); + expect(attr(decideSpan!, 'gen_ai.usage.output_tokens')).toBeGreaterThan(0); + expect(attr(decideSpan!, 'gen_ai.cost.total_tokens')).toBeGreaterThan(0); + expect(attr(decideSpan!, 'gen_ai.input.messages')).toContain('I was charged twice for my subscription.'); + expect(attr(decideSpan!, 'gen_ai.output.messages')).toContain('"queue":{"type":"choice"'); + expect(attr(decideSpan!, 'gen_ai.output.messages')).toContain('"refund":{"type":"boolean"'); + + expect(providerRequests, tree).toHaveLength(1); + expect(String(attr(providerRequests[0]!, 'server.address') ?? providerRequests[0]!.name)).toContain('openrouter'); +}); + +// The rerank models of OpenRouter come from Cohere, and the zero data retention guardrail of the account +// of the test key blocks them, so OpenRouter refuses the request. Thus the test checks only what the span +// records when OpenRouter answers and when it refuses. +test('tanstack-ai: a rerank() call emits a gen_ai.rerank span', async ({ baseURL }) => { + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => + spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.rerank') && spansOfTrace.some(span => span.is_segment), + ); + + await fetch(`${baseURL}/tanstack-ai/rerank`); + + const spans = await spansPromise; + const tree = describeTree(spans); + const workflow = spans.find(span => span.name === 'ai-workflow'); + const rerankSpan = spans.find(span => getSpanOp(span) === 'gen_ai.rerank'); + const providerRequests = spans.filter( + span => getSpanOp(span) === 'http.client' && span.parent_span_id === rerankSpan?.span_id, + ); + + expect(rerankSpan, tree).toBeDefined(); + expect(rerankSpan!.parent_span_id, tree).toBe(workflow?.span_id); + expect(rerankSpan!.name).toBe('rerank cohere/rerank-v3.5'); + expect(attr(rerankSpan!, 'sentry.origin')).toBe('auto.ai.tanstack_ai'); + expect(attr(rerankSpan!, 'gen_ai.operation.name')).toBe('rerank'); + expect(attr(rerankSpan!, 'gen_ai.provider.name')).toBe('openrouter'); + expect(attr(rerankSpan!, 'gen_ai.request.model')).toBe('cohere/rerank-v3.5'); + + expect(providerRequests, tree).toHaveLength(1); + expect(String(attr(providerRequests[0]!, 'server.address') ?? providerRequests[0]!.name)).toContain('openrouter'); +}); diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/tools.test.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/tools.test.ts index d2ff269ca0a7..3036fe40f456 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/tools.test.ts +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/tools.test.ts @@ -5,7 +5,7 @@ import { attr, describeTree, hasRecordedToolCalls, isExecuteToolSpan, isModelCal // The direct-SDK libraries record the model's tool call on the chat span's `gen_ai.response.tool_calls` // attribute, which only exists when the model actually returned tool calls. -const DIRECT_SDK_LIBRARIES = LIBRARIES.filter(library => library.provider); +const DIRECT_SDK_LIBRARIES = LIBRARIES.filter(library => !library.executesTools); for (const library of DIRECT_SDK_LIBRARIES) { test(`${library.id}: the model's tool call is recorded on the ${library.op} span`, async ({ baseURL }) => { @@ -28,20 +28,76 @@ for (const library of DIRECT_SDK_LIBRARIES) { }); } -// The Vercel AI SDK executes the tool and emits a dedicated `gen_ai.execute_tool` span instead. -test('vercel-ai: the tool call is captured as a gen_ai.execute_tool span', async ({ baseURL }) => { +// The libraries that run an agent loop execute the tool and emit a dedicated `gen_ai.execute_tool` span. +for (const library of LIBRARIES.filter(library => library.executesTools)) { + test(`${library.id}: the tool call is captured as a gen_ai.execute_tool span`, async ({ baseURL }) => { + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => spansOfTrace.some(isExecuteToolSpan) && spansOfTrace.some(span => isModelCallSpan(span, library)), + ); + + const response = await fetch(`${baseURL}/${library.id}/tools`); + expect(response.status).toBe(200); + + const spans = await spansPromise; + const toolSpan = spans.find(isExecuteToolSpan); + + expect(toolSpan, `expected a gen_ai.execute_tool span in:\n${describeTree(spans)}`).toBeDefined(); + expect(attr(toolSpan!, 'gen_ai.tool.name')).toBe('get_weather'); + }); +} + +// A real provider request goes over HTTP, which a fake adapter cannot show. TanStack AI wraps that fetch +// so the request nests under the `chat` span of its model call. +test('tanstack-ai: the agent loop nests its model calls, tool call and provider requests', async ({ baseURL }) => { + const library = LIBRARIES.find(library => library.id === 'tanstack-ai')!; const spansPromise = collectStreamedSpans( APP, spansOfTrace => - spansOfTrace.some(isExecuteToolSpan) && spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.generate_content'), + spansOfTrace.some(isExecuteToolSpan) && + spansOfTrace.filter(span => isModelCallSpan(span, library)).length >= 2 && + spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.invoke_agent'), ); - const response = await fetch(`${baseURL}/vercel-ai/tools`); + const response = await fetch(`${baseURL}/tanstack-ai/tools`); expect(response.status).toBe(200); const spans = await spansPromise; + const tree = describeTree(spans); + const agent = spans.find(span => getSpanOp(span) === 'gen_ai.invoke_agent'); + const chats = spans.filter(span => isModelCallSpan(span, library)); const toolSpan = spans.find(isExecuteToolSpan); - expect(toolSpan, `expected a gen_ai.execute_tool span in:\n${describeTree(spans)}`).toBeDefined(); - expect(attr(toolSpan!, 'gen_ai.tool.name')).toBe('get_weather'); + expect(agent, tree).toBeDefined(); + expect(attr(agent!, 'gen_ai.provider.name')).toBe('openrouter'); + expect(chats, tree).toHaveLength(2); + expect( + chats.every(span => span.parent_span_id === agent!.span_id), + tree, + ).toBe(true); + expect(toolSpan!.parent_span_id, tree).toBe(agent!.span_id); + + // The first call returns the tool call, the second answers with the tool result. + const [first, second] = [...chats].sort((a, b) => a.start_timestamp - b.start_timestamp); + expect(attr(first!, 'gen_ai.response.finish_reasons')).toBe('["tool_calls"]'); + expect(attr(first!, 'gen_ai.output.messages')).toContain('get_weather'); + expect(attr(second!, 'gen_ai.response.finish_reasons')).toBe('["stop"]'); + expect(typeof attr(second!, 'gen_ai.response.id')).toBe('string'); + + // The agent span sums the usage of both model calls. OpenRouter reports the cost itself, also for a BYOK + // key, where its own fee is 0. + expect(attr(agent!, 'gen_ai.usage.input_tokens')).toBe( + (attr(first!, 'gen_ai.usage.input_tokens') as number) + (attr(second!, 'gen_ai.usage.input_tokens') as number), + ); + expect(attr(agent!, 'gen_ai.cost.total_tokens')).toBeGreaterThan(0); + + const providerRequests = spans.filter( + span => + getSpanOp(span) === 'http.client' && String(attr(span, 'server.address') ?? span.name).includes('openrouter'), + ); + expect(providerRequests, tree).toHaveLength(2); + expect( + providerRequests.every(span => chats.some(chat => chat.span_id === span.parent_span_id)), + tree, + ).toBe(true); }); diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/utils.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/utils.ts index 4861a01c5334..51baab904857 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/utils.ts +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/utils.ts @@ -8,11 +8,14 @@ export interface LibraryUnderTest { /** The op of the model-call span: `gen_ai.chat` for the direct SDKs, `gen_ai.generate_content` for * the Vercel AI SDK. */ op: string; - /** `gen_ai.provider.name` for the direct-SDK libraries; unset for the Vercel AI SDK, whose provider - * name comes from the model id and is not asserted. */ + /** `gen_ai.provider.name` of the model-call span; unset for the Vercel AI SDK, whose provider name + * comes from the model id and is not asserted. */ provider?: string; - /** `sentry.origin` for the direct-SDK libraries; unset for the Vercel AI SDK. */ + /** `sentry.origin` of the model-call span; unset for the Vercel AI SDK. */ origin?: string; + /** The library runs the tool itself and records a `gen_ai.execute_tool` span for it, instead of only + * returning the model's tool call to the caller. */ + executesTools?: boolean; } export const LIBRARIES: LibraryUnderTest[] = [ @@ -20,24 +23,27 @@ export const LIBRARIES: LibraryUnderTest[] = [ { id: 'together', op: 'gen_ai.chat', provider: 'together_ai', origin: 'auto.ai.together_ai' }, { id: 'mistral', op: 'gen_ai.chat', provider: 'mistralai', origin: 'auto.ai.mistralai' }, { id: 'anthropic', op: 'gen_ai.chat', provider: 'anthropic', origin: 'auto.ai.anthropic' }, - { id: 'vercel-ai', op: 'gen_ai.generate_content' }, + { id: 'vercel-ai', op: 'gen_ai.generate_content', executesTools: true }, + { id: 'tanstack-ai', op: 'gen_ai.chat', provider: 'openrouter', origin: 'auto.ai.tanstack_ai', executesTools: true }, ]; export const attr = (span: SerializedStreamedSpan, key: string): unknown => span.attributes?.[key]?.value; /** - * The model-call span for a library. The four direct SDKs all report `gen_ai.chat`, so the provider - * name is what tells them apart — without it, a leftover span from another library's request could be + * The model-call span for a library. Most libraries report `gen_ai.chat`, so the provider name and origin + * are what tell them apart. Without them, a leftover span from another library's request could be * mistaken for this one's. */ export const isModelCallSpan = (span: SerializedStreamedSpan, library: LibraryUnderTest): boolean => - getSpanOp(span) === library.op && (!library.provider || attr(span, 'gen_ai.provider.name') === library.provider); + getSpanOp(span) === library.op && + (!library.provider || attr(span, 'gen_ai.provider.name') === library.provider) && + (!library.origin || attr(span, 'sentry.origin') === library.origin); /** A span that recorded the model returning tool calls on a chat-completions request (direct SDKs). */ export const hasRecordedToolCalls = (span: SerializedStreamedSpan): boolean => typeof attr(span, 'gen_ai.response.tool_calls') === 'string'; -/** The dedicated tool-execution span the Vercel AI SDK emits. */ +/** The dedicated tool-execution span the libraries that run tools emit. */ export const isExecuteToolSpan = (span: SerializedStreamedSpan): boolean => getSpanOp(span) === 'gen_ai.execute_tool'; /** Whether a trace shows any evidence of a tool call, used to tell a chat request from a tools one. */ diff --git a/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/scenario.mjs b/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/scenario.mjs index c1705ff0d403..a8238cdbdb5c 100644 --- a/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/scenario.mjs +++ b/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/scenario.mjs @@ -1,4 +1,5 @@ import * as Sentry from '@sentry/aws-serverless'; +import { _INTERNAL_withAiProviderWrappingSkipped } from '@sentry/core'; import { BedrockRuntimeClient, ConverseCommand, InvokeModelCommand } from '@aws-sdk/client-bedrock-runtime'; // Force the HTTP/1 handler so `nock` can intercept the request instead of hitting real AWS. import { NodeHttpHandler } from '@smithy/node-http-handler'; @@ -76,8 +77,16 @@ async function invokeModel() { async function run() { await Sentry.startSpan({ name: 'Test Transaction' }, async () => { - await converse(); - await invokeModel(); + if (process.env.SKIP_BEDROCK) { + // What an AI framework integration does around the Bedrock calls of its adapters, which it records itself. + await _INTERNAL_withAiProviderWrappingSkipped(['AwsBedrock'], async () => { + await converse(); + await invokeModel(); + }); + } else { + await converse(); + await invokeModel(); + } }); } diff --git a/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/test.ts b/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/test.ts index a7bc374499b4..0709b7f647b7 100644 --- a/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/test.ts +++ b/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/test.ts @@ -65,6 +65,26 @@ describe('awsIntegration - Bedrock', () => { test('auto-instruments Bedrock Converse and InvokeModel', { timeout: 90_000 }, async () => { await createTestRunner().ignore('event').expect({ span: assertBedrockSpans }).start().completed(); }); + + test( + 'records no Bedrock spans while an AI framework integration skips Bedrock', + { timeout: 90_000 }, + async () => { + await createTestRunner() + .withEnv({ SKIP_BEDROCK: '1' }) + .ignore('event') + .expect({ + span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'Test Transaction')).toBeDefined(); + expect( + container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.aws.aws_sdk'), + ).toEqual([]); + }, + }) + .start() + .completed(); + }, + ); }, { additionalDependencies: { '@aws-sdk/client-bedrock-runtime': '^3.1046.0' } }, ); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/instrument-no-auto.mjs b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/instrument-no-auto.mjs new file mode 100644 index 000000000000..0800cae7780b --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/instrument-no-auto.mjs @@ -0,0 +1,11 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +// Without the integration nothing injects the middleware, so only an explicit one records spans. +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + integrations: integrations => integrations.filter(integration => integration.name !== 'TanStackAI'), +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/instrument.mjs new file mode 100644 index 000000000000..46a27dd03b74 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/instrument.mjs @@ -0,0 +1,9 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-decide.mjs b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-decide.mjs new file mode 100644 index 000000000000..58ca3db3930e --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-decide.mjs @@ -0,0 +1,66 @@ +import express from 'express'; +import * as Sentry from '@sentry/node'; +import { boolean, choice, decide } from '@tanstack/ai'; +import { createOpenaiDecider } from '@tanstack/ai-openai'; +import OpenAI from 'openai'; + +// Inlined OpenAI decisions mock: the runner copies only this file into its temp dir. It refuses the question +// about a refund when the input asks it to. +function startMockProvider() { + const app = express(); + app.use(express.json()); + app.post('/v1/decisions', (req, res) => { + res.json({ + model: 'gpt-6-luna-2026-09-01', + answers: [ + { + type: 'choice', + name: 'queue', + choice: 'billing', + probabilities: [ + { value: 'billing', probability: 0.9 }, + { value: 'tech', probability: 0.1 }, + ], + confidence: 0.85, + }, + req.body.input.includes('refuse') + ? { type: 'refusal', name: 'refund', refusal: 'I cannot answer this.' } + : { type: 'predicate', name: 'refund', probability: 0.8 }, + ], + usage: { input_tokens: 30, output_tokens: 4, total_tokens: 34 }, + }); + }); + return new Promise(resolve => { + const server = app.listen(0, () => resolve(server)); + }); +} + +const server = await startMockProvider(); +const baseURL = `http://localhost:${server.address().port}/v1`; +const adapter = createOpenaiDecider('gpt-6-luna', 'test-key', { baseURL }); +const questions = { + queue: choice({ instructions: 'Which team handles this ticket?', options: { billing: null, tech: null } }), + refund: boolean({ instructions: 'Does the customer ask for a refund?' }), +}; + +await Sentry.startSpan({ name: 'main' }, async () => { + await decide({ adapter, state: 'I was charged twice for my subscription.', questions }); + + await Sentry.startSpan({ name: 'refused' }, async () => { + try { + await decide({ adapter, state: 'Please refuse to answer.', questions }); + } catch { + // The model refused the question about a refund. + } + }); + + await Sentry.startSpan({ name: 'direct' }, () => + new OpenAI({ apiKey: 'test-key', baseURL }).decisions.create({ + model: 'gpt-6-luna', + input: 'I was charged twice for my subscription.', + questions: [{ type: 'predicate', name: 'refund', instructions: 'Does the customer ask for a refund?' }], + }), + ); +}); + +server.close(); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-embed.mjs b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-embed.mjs new file mode 100644 index 000000000000..ade74dbbf6f8 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-embed.mjs @@ -0,0 +1,39 @@ +import express from 'express'; +import * as Sentry from '@sentry/node'; +import { embed } from '@tanstack/ai'; +import { createOpenaiEmbedding } from '@tanstack/ai-openai'; +import OpenAI from 'openai'; + +// Inlined OpenAI-compatible mock: the runner copies only this file into its temp dir. +function startMockProvider() { + const app = express(); + app.use(express.json()); + app.post('/v1/embeddings', (req, res) => { + res.json({ + object: 'list', + data: req.body.input.map((_text, index) => ({ object: 'embedding', embedding: [0.1, 0.2, 0.3], index })), + model: 'text-embedding-3-small', + usage: { prompt_tokens: 8, total_tokens: 8 }, + }); + }); + return new Promise(resolve => { + const server = app.listen(0, () => resolve(server)); + }); +} + +const server = await startMockProvider(); +const baseURL = `http://localhost:${server.address().port}/v1`; +const adapter = createOpenaiEmbedding('text-embedding-3-small', 'test-key', { baseURL }); + +await Sentry.startSpan({ name: 'main' }, async () => { + await embed({ adapter, input: ['a red guitar', 'a blue drum kit'] }); + + await Sentry.startSpan({ name: 'direct' }, async () => { + await new OpenAI({ apiKey: 'test-key', baseURL }).embeddings.create({ + model: 'text-embedding-3-small', + input: ['a green piano'], + }); + }); +}); + +server.close(); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-interrupt.mjs b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-interrupt.mjs new file mode 100644 index 000000000000..a573bea99af7 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-interrupt.mjs @@ -0,0 +1,28 @@ +import * as Sentry from '@sentry/node'; +import { chat, defineInterrupt } from '@tanstack/ai'; +import { fakeText } from '@tanstack/ai/testing'; + +// A middleware pauses the run at the interrupt boundary that `INTERRUPT` names: `beforeModel` or `afterModel`. +const confirm = defineInterrupt({ id: 'confirm' }); +const pauseRun = { + name: 'confirm', + onInterruptBoundary: ctx => + ctx.phase === process.env.INTERRUPT + ? { interrupts: [confirm.interrupt({ key: 'confirm-1', reason: 'review', message: 'Please confirm.' })] } + : undefined, +}; + +const adapter = fakeText({ model: 'fake-model' }); +adapter.setResponses([{ text: 'It is 21 degrees in Vienna.' }]); + +await Sentry.startSpan({ name: 'main' }, async () => { + for await (const _chunk of chat({ + adapter, + threadId: 'thread-1', + messages: [{ role: 'user', content: 'What is the weather in Vienna?' }], + interrupts: [confirm], + middleware: [pauseRun], + })) { + // drain + } +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-openai.mjs b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-openai.mjs new file mode 100644 index 000000000000..d9dc55fa9a59 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-openai.mjs @@ -0,0 +1,61 @@ +import express from 'express'; +import * as Sentry from '@sentry/node'; +import { chat } from '@tanstack/ai'; +import { createOpenaiChatCompletions } from '@tanstack/ai-openai'; +import OpenAI from 'openai'; + +// Inlined OpenAI-compatible streaming mock: the runner copies only this file into its temp dir. +function startMockProvider() { + const app = express(); + app.use(express.json()); + app.post('/v1/chat/completions', (req, res) => { + const base = { id: 'chatcmpl-1', object: 'chat.completion.chunk', created: 1, model: 'gpt-4o-2024-08-06' }; + res.setHeader('content-type', 'text/event-stream'); + const send = chunk => res.write(`data: ${JSON.stringify({ ...base, ...chunk })}\n\n`); + send({ choices: [{ index: 0, delta: { role: 'assistant', content: 'Hello!' }, finish_reason: null }] }); + send({ choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] }); + send({ + choices: [], + usage: { + prompt_tokens: 20, + completion_tokens: 5, + total_tokens: 25, + prompt_tokens_details: { cached_tokens: 8 }, + completion_tokens_details: { reasoning_tokens: 2 }, + }, + }); + res.end('data: [DONE]\n\n'); + }); + return new Promise(resolve => { + const server = app.listen(0, () => resolve(server)); + }); +} + +const server = await startMockProvider(); +const baseURL = `http://localhost:${server.address().port}/v1`; +const adapter = createOpenaiChatCompletions('gpt-4o', 'test-key', { baseURL }); +const messages = [{ role: 'user', content: 'Hi' }]; + +await Sentry.startSpan({ name: 'main' }, async () => { + if (process.env.STREAM) { + // The adapter calls the SDK while the app reads the stream, after `chat()` has returned. + for await (const _chunk of chat({ adapter, threadId: 'thread-1', messages, stream: true })) { + // drain + } + } else { + await chat({ adapter, threadId: 'thread-1', messages, stream: false }); + } + + await Sentry.startSpan({ name: 'direct' }, async () => { + const stream = await new OpenAI({ apiKey: 'test-key', baseURL }).chat.completions.create({ + model: 'gpt-4o', + messages, + stream: true, + }); + for await (const _chunk of stream) { + // drain + } + }); +}); + +server.close(); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-provider-error.mjs b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-provider-error.mjs new file mode 100644 index 000000000000..9c691d6618e6 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-provider-error.mjs @@ -0,0 +1,31 @@ +import express from 'express'; +import * as Sentry from '@sentry/node'; +import { chat } from '@tanstack/ai'; +import { createOpenaiChatCompletions } from '@tanstack/ai-openai'; + +// Inlined OpenAI mock that rejects the key: the runner copies only this file into its temp dir. +function startMockProvider() { + const app = express(); + app.post('/v1/chat/completions', (_req, res) => { + res.status(401).json({ error: { message: 'Incorrect API key provided', type: 'invalid_request_error' } }); + }); + return new Promise(resolve => { + const server = app.listen(0, () => resolve(server)); + }); +} + +const server = await startMockProvider(); +const adapter = createOpenaiChatCompletions('gpt-4o', 'test-key', { + baseURL: `http://localhost:${server.address().port}/v1`, +}); + +await Sentry.startSpan({ name: 'main' }, async () => { + try { + // Without streaming, `chat()` rejects on the provider error and reads no further chunks. + await chat({ adapter, threadId: 'thread-1', messages: [{ role: 'user', content: 'Hi' }], stream: false }); + } catch { + // The 401 of the provider. + } +}); + +server.close(); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-rerank.mjs b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-rerank.mjs new file mode 100644 index 000000000000..5a4c2d9864e8 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-rerank.mjs @@ -0,0 +1,48 @@ +import express from 'express'; +import * as Sentry from '@sentry/node'; +import { rerank } from '@tanstack/ai'; +import { createOpenRouterRerank } from '@tanstack/ai-openrouter'; + +// Inlined OpenRouter rerank mock: the runner copies only this file into its temp dir. +function startMockProvider() { + const app = express(); + app.use(express.json()); + app.post('/rerank', (req, res) => { + const results = req.body.documents + .map((text, index) => ({ index, relevance_score: 1 - index / 10, document: { text } })) + .reverse(); + res.json({ + id: 'gen-rerank-1', + model: 'cohere/rerank-v3.5', + // An index that is out of range makes `rerank()` throw after the provider answered. + results: req.body.query.includes('out of range') + ? [...results, { index: results.length, relevance_score: 0, document: { text: '' } }] + : results, + usage: { total_tokens: 12, search_units: 1, cost: 0.002 }, + }); + }); + return new Promise(resolve => { + const server = app.listen(0, () => resolve(server)); + }); +} + +const server = await startMockProvider(); +const adapter = createOpenRouterRerank('cohere/rerank-v3.5', 'test-key', { + serverURL: `http://localhost:${server.address().port}`, +}); + +const documents = ['A heavy gaming desktop', 'A lightweight ultrabook with all-day battery']; + +await Sentry.startSpan({ name: 'main' }, async () => { + await rerank({ adapter, query: 'A laptop for travel', documents }); + + await Sentry.startSpan({ name: 'failed' }, async () => { + try { + await rerank({ adapter, query: 'Answer out of range', documents }); + } catch { + // `rerank()` throws on the index that is out of range. + } + }); +}); + +server.close(); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-retry.mjs b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-retry.mjs new file mode 100644 index 000000000000..927f56115b07 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-retry.mjs @@ -0,0 +1,19 @@ +import * as Sentry from '@sentry/node'; +import { chat } from '@tanstack/ai'; +import { fakeText } from '@tanstack/ai/testing'; + +// A middleware calls the model again after the first call fails. +const retry = { name: 'retry', onModelError: ctx => ctx.retry() }; + +const adapter = fakeText({ model: 'fake-model' }); +adapter.setResponses([{ error: 'Overloaded' }, { text: 'It is 21 degrees in Vienna.' }]); + +await Sentry.startSpan({ name: 'main' }, async () => { + await chat({ + adapter, + threadId: 'thread-1', + messages: [{ role: 'user', content: 'What is the weather in Vienna?' }], + middleware: [retry], + stream: false, + }); +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario.mjs new file mode 100644 index 000000000000..a5d899bd9d21 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario.mjs @@ -0,0 +1,49 @@ +import * as Sentry from '@sentry/node'; +import { chat, toolDefinition } from '@tanstack/ai'; +import { fakeText } from '@tanstack/ai/testing'; + +// `MIDDLEWARE=manual` passes the middleware explicitly, and `MIDDLEWARE=transform-args` passes a middleware that +// changes the arguments of each tool call; otherwise the call carries none. Except with `manual`, it relies on the +// integration injecting the middleware. +const middleware = { + manual: [Sentry.createTanstackAiMiddleware()], + 'transform-args': [ + { + name: 'transform-args', + onBeforeToolCall: (_ctx, { args }) => ({ type: 'transformArgs', args: { ...args, unit: 'celsius' } }), + }, + ], +}[process.env.MIDDLEWARE]; + +const getWeather = toolDefinition({ + name: 'get_weather', + description: 'Get the current weather for a city.', + inputSchema: { type: 'object', properties: { city: { type: 'string' } } }, +}).server(async ({ city }) => Sentry.startSpan({ name: 'weather-lookup' }, () => ({ city, temperature: 21 }))); + +// The fake estimates prompt caching per thread, so the second model call reports cached tokens. +const adapter = fakeText({ model: 'fake-model', cache: true }); +adapter.setResponses([ + { text: 'Let me check.', toolCalls: [{ name: 'get_weather', input: { city: 'Vienna' } }] }, + { text: 'It is 21 degrees in Vienna.' }, +]); + +const options = { + adapter, + threadId: 'thread-1', + systemPrompts: ['You are a helpful assistant.'], + messages: [{ role: 'user', content: 'What is the weather in Vienna?' }], + tools: [getWeather], + middleware, +}; + +await Sentry.startSpan({ name: 'main' }, async () => { + if (process.env.STREAM) { + // The run only advances while the consumer pulls chunks, as when streaming to a response. + for await (const _chunk of chat({ ...options, stream: true })) { + // drain + } + } else { + await chat({ ...options, stream: false }); + } +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts new file mode 100644 index 000000000000..0f0d6bb0163e --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts @@ -0,0 +1,592 @@ +import { + GEN_AI_CONVERSATION_ID, + GEN_AI_COST_TOTAL_TOKENS, + GEN_AI_EMBEDDINGS_INPUT, + GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_FINISH_REASONS, + GEN_AI_RESPONSE_ID, + GEN_AI_RESPONSE_MODEL, + GEN_AI_SYSTEM_INSTRUCTIONS, + GEN_AI_TOOL_CALL_ARGUMENTS, + GEN_AI_TOOL_CALL_RESULT, + GEN_AI_TOOL_DEFINITIONS, + GEN_AI_TOOL_NAME, + GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_REASONING_OUTPUT_TOKENS, + GEN_AI_USAGE_TOTAL_TOKENS, +} from '@sentry/conventions/attributes'; +import type { SerializedStreamedSpan } from '@sentry/core'; +import { afterAll, describe, expect } from 'vitest'; +import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; + +const TANSTACK_AI_DEPENDENCIES = { + additionalDependencies: { + '@tanstack/ai': '1.0.0', + '@tanstack/ai-openai': '1.0.0', + '@tanstack/ai-openrouter': '1.0.0', + // In the range of `@tanstack/ai-openai`, so that the scenarios and the adapter share one copy. + openai: '7.32.0', + }, +}; + +const ORIGIN = 'auto.ai.tanstack_ai'; + +type Spans = SerializedStreamedSpan[]; + +function attr(span: SerializedStreamedSpan | undefined, key: string): unknown { + return span?.attributes?.[key]?.value; +} + +function expectAgentTree(spans: Spans): void { + const main = spans.find(span => span.name === 'main')!; + expect(main.is_segment).toBe(true); + + // Counted: an explicit middleware plus the injected one would double every span. + expect(spans.filter(span => attr(span, 'sentry.origin') === ORIGIN)).toHaveLength(4); + + const agent = spans.find(span => span.name === 'invoke_agent')!; + expect(attr(agent, 'sentry.op')).toBe('gen_ai.invoke_agent'); + expect(attr(agent, GEN_AI_PROVIDER_NAME)).toBe('fake'); + expect(attr(agent, GEN_AI_REQUEST_MODEL)).toBe('fake-model'); + expect(attr(agent, GEN_AI_CONVERSATION_ID)).toBe('thread-1'); + expect(agent.parent_span_id).toBe(main.span_id); + + const chats = spans + .filter(span => span.name === 'chat fake-model') + .sort((a, b) => a.start_timestamp - b.start_timestamp); + expect(chats).toHaveLength(2); + for (const chat of chats) { + expect(attr(chat, 'sentry.op')).toBe('gen_ai.chat'); + expect(attr(chat, GEN_AI_RESPONSE_MODEL)).toBe('fake-model'); + expect(chat.parent_span_id).toBe(agent.span_id); + } + expect(chats.map(chat => attr(chat, GEN_AI_RESPONSE_FINISH_REASONS))).toEqual(['["tool_calls"]', '["stop"]']); + expect(attr(agent, GEN_AI_RESPONSE_MODEL)).toBe('fake-model'); + + // The fake caches per thread: the first call writes the whole prompt, the second reads the + // repeated prefix back. + const [first, second] = chats.map(chat => ({ + input: attr(chat, GEN_AI_USAGE_INPUT_TOKENS) as number, + output: attr(chat, GEN_AI_USAGE_OUTPUT_TOKENS) as number, + cacheRead: attr(chat, GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS) as number, + })); + expect(first!.input).toBeGreaterThan(0); + expect(attr(chats[0], GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS)).toBe(first!.input); + expect(second!.cacheRead).toBeGreaterThan(0); + // The agent span sums both model calls. + expect(attr(agent, GEN_AI_USAGE_INPUT_TOKENS)).toBe(first!.input + second!.input); + expect(attr(agent, GEN_AI_USAGE_OUTPUT_TOKENS)).toBe(first!.output + second!.output); + expect(attr(agent, GEN_AI_USAGE_TOTAL_TOKENS)).toBe(first!.input + first!.output + second!.input + second!.output); + expect(attr(agent, GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS)).toBe(second!.cacheRead); + + const tool = spans.find(span => span.name === 'execute_tool get_weather')!; + expect(tool.status).toBe('ok'); + // Each model call ends before the tool that it asked for runs. + expect(chats[0]!.end_timestamp).toBeLessThanOrEqual(tool.start_timestamp); + expect(tool.end_timestamp).toBeLessThanOrEqual(chats[1]!.start_timestamp); + expect(attr(tool, 'sentry.op')).toBe('gen_ai.execute_tool'); + expect(attr(tool, GEN_AI_TOOL_NAME)).toBe('get_weather'); + expect(tool.parent_span_id).toBe(agent.span_id); + + // Work inside the tool parents under the tool span, not beside it. + const lookup = spans.find(span => span.name === 'weather-lookup')!; + expect(lookup.parent_span_id).toBe(tool.span_id); +} + +describe('TanStack AI integration', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + // `@tanstack/ai` is ESM-only. + if (mode === 'cjs') { + return; + } + + test('injects the middleware into chat() calls that carry none', async () => { + await createRunner() + .expect({ + span: container => { + expectAgentTree(container.items); + + const chats = container.items + .filter(span => span.name === 'chat fake-model') + .sort((a, b) => a.start_timestamp - b.start_timestamp); + expect(JSON.parse(String(attr(chats[0], GEN_AI_INPUT_MESSAGES)))).toEqual([ + { role: 'user', parts: [{ type: 'text', content: 'What is the weather in Vienna?' }] }, + ]); + expect(attr(chats[0], GEN_AI_SYSTEM_INSTRUCTIONS)).toBe('You are a helpful assistant.'); + expect(JSON.parse(String(attr(chats[0], GEN_AI_TOOL_DEFINITIONS)))[0]).toMatchObject({ + name: 'get_weather', + description: 'Get the current weather for a city.', + }); + expect(JSON.parse(String(attr(chats[0], GEN_AI_OUTPUT_MESSAGES)))).toEqual([ + { + role: 'assistant', + parts: [ + { type: 'text', content: 'Let me check.' }, + { type: 'tool_call', id: expect.any(String), name: 'get_weather', arguments: '{"city":"Vienna"}' }, + ], + finish_reason: 'tool_calls', + }, + ]); + const toolCallId = JSON.parse(String(attr(chats[0], GEN_AI_OUTPUT_MESSAGES)))[0].parts[1].id; + const secondInput = JSON.parse(String(attr(chats[1], GEN_AI_INPUT_MESSAGES))); + expect(secondInput.map((message: { role: string }) => message.role)).toEqual([ + 'user', + 'assistant', + 'tool', + ]); + + // The agent span has the input of the run and the answer of its last model call. + const agent = container.items.find(span => span.name === 'invoke_agent'); + expect(attr(agent, GEN_AI_INPUT_MESSAGES)).toBe(attr(chats[0], GEN_AI_INPUT_MESSAGES)); + expect(attr(agent, GEN_AI_SYSTEM_INSTRUCTIONS)).toBe('You are a helpful assistant.'); + expect(JSON.parse(String(attr(agent, GEN_AI_OUTPUT_MESSAGES)))).toEqual([ + { + role: 'assistant', + parts: [{ type: 'text', content: 'It is 21 degrees in Vienna.' }], + finish_reason: 'stop', + }, + ]); + + const tool = container.items.find(span => span.name === 'execute_tool get_weather'); + expect(attr(tool, GEN_AI_TOOL_CALL_ID_ATTRIBUTE)).toBe(toolCallId); + expect(attr(tool, GEN_AI_TOOL_CALL_ARGUMENTS)).toBe('{"city":"Vienna"}'); + expect(attr(tool, GEN_AI_TOOL_CALL_RESULT)).toBe('{"city":"Vienna","temperature":21}'); + }, + }) + .start() + .completed(); + }); + + test('records the same spans when the app consumes the stream', async () => { + await createRunner() + .withEnv({ STREAM: '1' }) + .expect({ span: container => expectAgentTree(container.items) }) + .start() + .completed(); + }); + + test('does not add a second middleware when the app passes one', async () => { + await createRunner() + .withEnv({ MIDDLEWARE: 'manual' }) + .expect({ span: container => expectAgentTree(container.items) }) + .start() + .completed(); + }); + + test('records the tool span when an earlier middleware changes the tool arguments', async () => { + await createRunner() + .withEnv({ MIDDLEWARE: 'transform-args' }) + .expect({ + span: container => { + expectAgentTree(container.items); + const tool = container.items.find(span => span.name === 'execute_tool get_weather'); + expect(attr(tool, GEN_AI_TOOL_CALL_ARGUMENTS)).toBe('{"city":"Vienna","unit":"celsius"}'); + }, + }) + .start() + .completed(); + }); + }, + TANSTACK_AI_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument-no-auto.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + test('records the same spans from an explicit middleware alone', async () => { + await createRunner() + .withEnv({ MIDDLEWARE: 'manual' }) + .expect({ span: container => expectAgentTree(container.items) }) + .start() + .completed(); + }); + + test('records nothing without the integration or an explicit middleware', async () => { + await createRunner() + .expect({ + span: container => { + expect(container.items.find(span => span.name === 'main')).toBeDefined(); + expect(container.items.filter(span => attr(span, 'sentry.origin') === ORIGIN)).toHaveLength(0); + }, + }) + .start() + .completed(); + }); + }, + TANSTACK_AI_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-openai.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + test('reports the provider call once, with the HTTP request under the chat span', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items; + const direct = spans.find(span => span.name === 'direct')!; + const chats = spans.filter(span => attr(span, 'sentry.op') === 'gen_ai.chat'); + // The OpenAI integration is skipped for TanStack's model call, but not for the direct call after it. + expect(chats.map(span => attr(span, 'sentry.origin')).sort()).toEqual(['auto.ai.openai', ORIGIN]); + const chat = chats.find(span => attr(span, 'sentry.origin') === ORIGIN)!; + const directChat = chats.find(span => attr(span, 'sentry.origin') === 'auto.ai.openai')!; + expect(directChat.parent_span_id).toBe(direct.span_id); + // TanStack reports the adapter's name, which is per API, not the vendor. + expect(attr(chat, GEN_AI_PROVIDER_NAME)).toBe('openai-chat'); + expect(attr(chat, GEN_AI_REQUEST_MODEL)).toBe('gpt-4o'); + expect(attr(chat, GEN_AI_RESPONSE_MODEL)).toBe('gpt-4o-2024-08-06'); + expect(attr(chat, GEN_AI_RESPONSE_ID)).toBe('chatcmpl-1'); + const agent = spans.find(span => attr(span, 'sentry.op') === 'gen_ai.invoke_agent'); + expect(attr(agent, GEN_AI_RESPONSE_ID)).toBe('chatcmpl-1'); + expect(attr(chat, GEN_AI_USAGE_INPUT_TOKENS)).toBe(20); + expect(attr(chat, GEN_AI_USAGE_OUTPUT_TOKENS)).toBe(5); + expect(attr(chat, GEN_AI_USAGE_TOTAL_TOKENS)).toBe(25); + expect(attr(chat, GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS)).toBe(8); + expect(attr(chat, GEN_AI_USAGE_REASONING_OUTPUT_TOKENS)).toBe(2); + + const requests = spans.filter(span => attr(span, 'sentry.op') === 'http.client'); + expect(requests.map(span => span.parent_span_id).sort()).toEqual( + [chat.span_id, directChat.span_id].sort(), + ); + }, + }) + .start() + .completed(); + }); + + test('skips the OpenAI integration also while the app reads the stream', async () => { + await createRunner() + .withEnv({ STREAM: '1' }) + .expect({ + span: container => { + const spans = container.items; + const direct = spans.find(span => span.name === 'direct')!; + const chats = spans.filter(span => attr(span, 'sentry.op') === 'gen_ai.chat'); + expect(chats.map(span => attr(span, 'sentry.origin')).sort()).toEqual(['auto.ai.openai', ORIGIN]); + const directChat = chats.find(span => attr(span, 'sentry.origin') === 'auto.ai.openai')!; + expect(directChat.parent_span_id).toBe(direct.span_id); + }, + }) + .start() + .completed(); + }); + }, + TANSTACK_AI_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-interrupt.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + // TanStack calls no terminal hook for a run that pauses at an interrupt before or after a model call. + test('ends the spans of a run that pauses at an interrupt before the model call', async () => { + await createRunner() + .withEnv({ INTERRUPT: 'beforeModel' }) + .expect({ + span: container => { + const agent = container.items.find(span => span.name === 'invoke_agent'); + const chat = container.items.find(span => span.name === 'chat fake-model'); + expect(agent?.status).toBe('ok'); + expect(chat?.parent_span_id).toBe(agent?.span_id); + }, + }) + .start() + .completed(); + }); + + test('ends the spans of a run that pauses at an interrupt after the model call', async () => { + await createRunner() + .withEnv({ INTERRUPT: 'afterModel' }) + .expect({ + span: container => { + const agent = container.items.find(span => span.name === 'invoke_agent'); + const chat = container.items.find(span => span.name === 'chat fake-model'); + expect(agent?.status).toBe('ok'); + expect(chat?.parent_span_id).toBe(agent?.span_id); + expect(attr(chat, GEN_AI_RESPONSE_FINISH_REASONS)).toBe('["stop"]'); + }, + }) + .start() + .completed(); + }); + }, + TANSTACK_AI_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-retry.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + test('ends a model call that a middleware retried with an error status', async () => { + await createRunner() + .expect({ + span: container => { + const agent = container.items.find(span => span.name === 'invoke_agent'); + const chats = container.items + .filter(span => span.name === 'chat fake-model') + .sort((a, b) => a.start_timestamp - b.start_timestamp); + expect(chats.map(chat => chat.status)).toEqual(['error', 'ok']); + expect(chats.map(chat => chat.parent_span_id)).toEqual([agent?.span_id, agent?.span_id]); + expect(agent?.status).toBe('ok'); + }, + }) + .start() + .completed(); + }); + }, + TANSTACK_AI_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-provider-error.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + test('ends the spans with an error status when the provider call fails', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items; + const agent = spans.find(span => attr(span, 'sentry.op') === 'gen_ai.invoke_agent'); + const chat = spans.find(span => attr(span, 'sentry.op') === 'gen_ai.chat'); + expect(agent?.status).toBe('error'); + expect(chat?.status).toBe('error'); + expect(chat?.parent_span_id).toBe(agent?.span_id); + }, + }) + .start() + .completed(); + }); + }, + TANSTACK_AI_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-embed.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + test('records an embeddings span, with the provider request under it', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items; + const main = spans.find(span => span.name === 'main')!; + const direct = spans.find(span => span.name === 'direct')!; + const embeddings = spans.filter(span => attr(span, 'sentry.op') === 'gen_ai.embeddings'); + // The OpenAI integration is skipped for TanStack's call, but not for the direct call after it. + expect(embeddings.map(span => attr(span, 'sentry.origin')).sort()).toEqual(['auto.ai.openai', ORIGIN]); + const embedding = embeddings.find(span => attr(span, 'sentry.origin') === ORIGIN)!; + const directEmbedding = embeddings.find(span => attr(span, 'sentry.origin') === 'auto.ai.openai')!; + expect(directEmbedding.parent_span_id).toBe(direct.span_id); + expect(embedding.name).toBe('embeddings text-embedding-3-small'); + expect(embedding.parent_span_id).toBe(main.span_id); + expect(embedding.status).toBe('ok'); + expect(attr(embedding, 'sentry.origin')).toBe(ORIGIN); + expect(attr(embedding, GEN_AI_OPERATION_NAME)).toBe('embeddings'); + expect(attr(embedding, GEN_AI_PROVIDER_NAME)).toBe('openai'); + expect(attr(embedding, GEN_AI_REQUEST_MODEL)).toBe('text-embedding-3-small'); + expect(attr(embedding, GEN_AI_RESPONSE_MODEL)).toBe('text-embedding-3-small'); + expect(attr(embedding, GEN_AI_EMBEDDINGS_INPUT)).toBe('["a red guitar","a blue drum kit"]'); + expect(attr(embedding, GEN_AI_USAGE_INPUT_TOKENS)).toBe(8); + expect(attr(embedding, GEN_AI_USAGE_TOTAL_TOKENS)).toBe(8); + expect(attr(embedding, GEN_AI_USAGE_OUTPUT_TOKENS)).toBeUndefined(); + + const requests = spans.filter(span => attr(span, 'sentry.op') === 'http.client'); + expect(requests.map(span => span.parent_span_id).sort()).toEqual( + [embedding.span_id, directEmbedding.span_id].sort(), + ); + }, + }) + .start() + .completed(); + }); + }, + TANSTACK_AI_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-rerank.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + test('records rerank spans with the usage, with the provider request under them', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items; + const main = spans.find(span => span.name === 'main')!; + const failed = spans.find(span => span.name === 'failed')!; + const reranks = spans.filter(span => attr(span, 'sentry.op') === 'gen_ai.rerank'); + expect(reranks).toHaveLength(2); + const rerank = reranks.find(span => span.parent_span_id === main.span_id)!; + expect(rerank.name).toBe('rerank cohere/rerank-v3.5'); + expect(rerank.status).toBe('ok'); + expect(attr(rerank, 'sentry.origin')).toBe(ORIGIN); + expect(attr(rerank, GEN_AI_OPERATION_NAME)).toBe('rerank'); + expect(attr(rerank, GEN_AI_PROVIDER_NAME)).toBe('openrouter'); + expect(attr(rerank, GEN_AI_REQUEST_MODEL)).toBe('cohere/rerank-v3.5'); + // The adapter reports no model, and makes the id itself when the response has none. + expect(attr(rerank, GEN_AI_RESPONSE_MODEL)).toBeUndefined(); + expect(attr(rerank, GEN_AI_RESPONSE_ID)).toBeUndefined(); + expect(attr(rerank, GEN_AI_USAGE_TOTAL_TOKENS)).toBe(12); + expect(attr(rerank, GEN_AI_USAGE_INPUT_TOKENS)).toBeUndefined(); + expect(attr(rerank, GEN_AI_COST_TOTAL_TOKENS)).toBe(0.002); + + // `rerank()` throws after the provider answered, and the provider bills the call. + const failedRerank = reranks.find(span => span.parent_span_id === failed.span_id)!; + expect(failedRerank.status).toBe('error'); + expect(attr(failedRerank, GEN_AI_USAGE_TOTAL_TOKENS)).toBe(12); + expect(attr(failedRerank, GEN_AI_COST_TOTAL_TOKENS)).toBe(0.002); + + const requests = spans.filter(span => attr(span, 'sentry.op') === 'http.client'); + expect(requests.map(span => span.parent_span_id).sort()).toEqual( + [rerank.span_id, failedRerank.span_id].sort(), + ); + }, + }) + .start() + .completed(); + }); + }, + TANSTACK_AI_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-decide.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + test('records decide spans with the questions, the answers and the usage, with the provider request under them', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items; + const main = spans.find(span => span.name === 'main')!; + const refused = spans.find(span => span.name === 'refused')!; + const direct = spans.find(span => span.name === 'direct')!; + const decisions = spans.filter(span => attr(span, 'sentry.op') === 'gen_ai.decide'); + // The OpenAI integration is skipped for TanStack's calls, but not for the direct call after them. + expect(decisions.map(span => attr(span, 'sentry.origin')).sort()).toEqual([ + 'auto.ai.openai', + ORIGIN, + ORIGIN, + ]); + const decision = decisions.find(span => span.parent_span_id === main.span_id)!; + const refusal = decisions.find(span => span.parent_span_id === refused.span_id)!; + const directDecision = decisions.find(span => span.parent_span_id === direct.span_id)!; + expect(attr(directDecision, 'sentry.origin')).toBe('auto.ai.openai'); + expect(decision.name).toBe('decide gpt-6-luna'); + expect(decision.status).toBe('ok'); + expect(attr(decision, 'sentry.origin')).toBe(ORIGIN); + expect(attr(decision, GEN_AI_OPERATION_NAME)).toBe('decide'); + expect(attr(decision, GEN_AI_PROVIDER_NAME)).toBe('openai'); + expect(attr(decision, GEN_AI_REQUEST_MODEL)).toBe('gpt-6-luna'); + expect(attr(decision, GEN_AI_RESPONSE_MODEL)).toBe('gpt-6-luna-2026-09-01'); + expect(attr(decision, GEN_AI_USAGE_INPUT_TOKENS)).toBe(30); + expect(attr(decision, GEN_AI_USAGE_OUTPUT_TOKENS)).toBe(4); + expect(attr(decision, GEN_AI_USAGE_TOTAL_TOKENS)).toBe(34); + expect(JSON.parse(String(attr(decision, GEN_AI_INPUT_MESSAGES)))).toEqual([ + { + type: 'evaluation', + state: 'I was charged twice for my subscription.', + questions: { + queue: { + type: 'choice', + instructions: 'Which team handles this ticket?', + criteria: { billing: null, tech: null }, + }, + refund: { type: 'noul', instructions: 'Does the customer ask for a refund?' }, + }, + }, + ]); + expect(JSON.parse(String(attr(decision, GEN_AI_OUTPUT_MESSAGES)))).toEqual([ + { + type: 'evaluation', + answers: { + queue: { + type: 'choice', + value: 'billing', + probability: 0.9, + confidence: 0.85, + probabilities: { billing: 0.9, tech: 0.1 }, + }, + refund: { type: 'boolean', value: true, probability: 0.8 }, + }, + }, + ]); + + // A refused call fails, but is still billed. + expect(refusal.status).toBe('error'); + expect(attr(refusal, 'sentry.origin')).toBe(ORIGIN); + expect(attr(refusal, GEN_AI_RESPONSE_MODEL)).toBe('gpt-6-luna-2026-09-01'); + expect(attr(refusal, GEN_AI_USAGE_INPUT_TOKENS)).toBe(30); + expect(attr(refusal, GEN_AI_USAGE_OUTPUT_TOKENS)).toBe(4); + + const requests = spans.filter(span => attr(span, 'sentry.op') === 'http.client'); + expect(requests.map(span => span.parent_span_id).sort()).toEqual( + [decision.span_id, refusal.span_id, directDecision.span_id].sort(), + ); + }, + }) + .start() + .completed(); + }); + }, + TANSTACK_AI_DEPENDENCIES, + ); +}); diff --git a/packages/astro/src/index.server.ts b/packages/astro/src/index.server.ts index 80158e311e49..6b458f67d643 100644 --- a/packages/astro/src/index.server.ts +++ b/packages/astro/src/index.server.ts @@ -99,6 +99,8 @@ export { groqIntegration, togetherAIIntegration, typesafeIntegration, + tanstackAiIntegration, + createTanstackAiMiddleware, langChainIntegration, langGraphIntegration, createFlueInstrumentation, diff --git a/packages/aws-serverless/src/index.ts b/packages/aws-serverless/src/index.ts index fd07e76a3fba..6cf94dfbf052 100644 --- a/packages/aws-serverless/src/index.ts +++ b/packages/aws-serverless/src/index.ts @@ -64,6 +64,8 @@ export { groqIntegration, togetherAIIntegration, typesafeIntegration, + tanstackAiIntegration, + createTanstackAiMiddleware, langChainIntegration, langGraphIntegration, mastraIntegration, diff --git a/packages/bun/src/index.ts b/packages/bun/src/index.ts index 4e91068d21f8..98d2d7aeebe4 100644 --- a/packages/bun/src/index.ts +++ b/packages/bun/src/index.ts @@ -86,6 +86,8 @@ export { groqIntegration, togetherAIIntegration, typesafeIntegration, + tanstackAiIntegration, + createTanstackAiMiddleware, langChainIntegration, langGraphIntegration, mastraIntegration, diff --git a/packages/cloudflare/src/index.ts b/packages/cloudflare/src/index.ts index 88ce8deaac01..2ddf149598b2 100644 --- a/packages/cloudflare/src/index.ts +++ b/packages/cloudflare/src/index.ts @@ -108,7 +108,7 @@ export { spanStreamingIntegration, } from '@sentry/core'; export { trpcMiddleware, wrapMcpServerWithSentry } from '@sentry/core/server'; -export { createFlueInstrumentation, instrumentPostgresJsSql } from '@sentry/server-utils'; +export { createFlueInstrumentation, createTanstackAiMiddleware, instrumentPostgresJsSql } from '@sentry/server-utils'; export type { FlueOptions } from '@sentry/server-utils'; export { withSentry } from './withSentry'; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 1eb1d0ff4d72..12e45cc0b4a7 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -77,6 +77,7 @@ export { } from './integration'; export { _INTERNAL_skipAiProviderWrapping, + _INTERNAL_withAiProviderWrappingSkipped, _INTERNAL_shouldSkipAiProviderWrapping, _INTERNAL_clearAiProviderSkips, } from './utils/ai/providerSkip'; diff --git a/packages/core/src/utils/ai/providerSkip.ts b/packages/core/src/utils/ai/providerSkip.ts index 0b7ca2a5c3bc..94529eb3a5c1 100644 --- a/packages/core/src/utils/ai/providerSkip.ts +++ b/packages/core/src/utils/ai/providerSkip.ts @@ -1,3 +1,4 @@ +import { getCurrentScope, withScope } from '../../currentScopes'; import { DEBUG_BUILD } from '../../debug-build'; import { debug } from '../debug-logger'; @@ -9,6 +10,9 @@ import { debug } from '../debug-logger'; */ const SKIPPED_AI_PROVIDERS = new Set(); +/** Scope metadata key of the modules that `_INTERNAL_withAiProviderWrappingSkipped` skips, as `{ [module]: true }`. */ +const SCOPED_SKIPPED_AI_PROVIDERS_KEY = '__SENTRY_SKIPPED_AI_PROVIDERS__'; + /** * Mark AI provider modules to skip instrumentation wrapping. * @@ -31,6 +35,25 @@ export function _INTERNAL_skipAiProviderWrapping(modules: string[]): void { }); } +/** + * Run `callback` with AI provider modules skipped, for the calls that it makes and the async work that + * it starts. A call to the same provider outside of `callback` is still instrumented. + * + * @internal + * @param modules - Array of npm module names to skip (e.g., '@anthropic-ai/sdk', 'openai') + */ +export function _INTERNAL_withAiProviderWrappingSkipped(modules: string[], callback: () => T): T { + return withScope(scope => { + const skipped: Record = {}; + for (const module of modules) { + skipped[module] = true; + } + // `setSDKProcessingMetadata` merges one level deep, so a nested call adds to the modules of the outer one. + scope.setSDKProcessingMetadata({ [SCOPED_SKIPPED_AI_PROVIDERS_KEY]: skipped }); + return callback(); + }); +} + /** * Check if an AI provider module should skip instrumentation wrapping. * @@ -47,7 +70,13 @@ export function _INTERNAL_skipAiProviderWrapping(modules: string[]): void { * ``` */ export function _INTERNAL_shouldSkipAiProviderWrapping(module: string): boolean { - return SKIPPED_AI_PROVIDERS.has(module); + if (SKIPPED_AI_PROVIDERS.has(module)) { + return true; + } + const scoped = getCurrentScope().getScopeData().sdkProcessingMetadata[SCOPED_SKIPPED_AI_PROVIDERS_KEY] as + | Record + | undefined; + return scoped?.[module] === true; } /** diff --git a/packages/core/test/lib/utils/ai/providerSkip.test.ts b/packages/core/test/lib/utils/ai/providerSkip.test.ts index de39778a2b5e..b072a73f7501 100644 --- a/packages/core/test/lib/utils/ai/providerSkip.test.ts +++ b/packages/core/test/lib/utils/ai/providerSkip.test.ts @@ -3,6 +3,7 @@ import { _INTERNAL_clearAiProviderSkips, _INTERNAL_shouldSkipAiProviderWrapping, _INTERNAL_skipAiProviderWrapping, + _INTERNAL_withAiProviderWrappingSkipped, } from '../../../../src/index'; const OPENAI_INTEGRATION_NAME = 'OpenAI'; @@ -36,6 +37,40 @@ describe('AI Provider Skip', () => { }); }); + describe('_INTERNAL_withAiProviderWrappingSkipped', () => { + it('skips the providers only inside the callback', () => { + const skippedInside = _INTERNAL_withAiProviderWrappingSkipped([OPENAI_INTEGRATION_NAME], () => [ + _INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME), + _INTERNAL_shouldSkipAiProviderWrapping(ANTHROPIC_AI_INTEGRATION_NAME), + ]); + + expect(skippedInside).toEqual([true, false]); + expect(_INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME)).toBe(false); + }); + + it('keeps the skip in the async work that the callback starts', async () => { + const skippedAfterAwait = await _INTERNAL_withAiProviderWrappingSkipped([OPENAI_INTEGRATION_NAME], async () => { + await Promise.resolve(); + return _INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME); + }); + + expect(skippedAfterAwait).toBe(true); + expect(_INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME)).toBe(false); + }); + + it('adds the providers of a nested call to the providers of the outer call', () => { + const skippedInside = _INTERNAL_withAiProviderWrappingSkipped([OPENAI_INTEGRATION_NAME], () => + _INTERNAL_withAiProviderWrappingSkipped([ANTHROPIC_AI_INTEGRATION_NAME], () => [ + _INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME), + _INTERNAL_shouldSkipAiProviderWrapping(ANTHROPIC_AI_INTEGRATION_NAME), + _INTERNAL_shouldSkipAiProviderWrapping(GOOGLE_GENAI_INTEGRATION_NAME), + ]), + ); + + expect(skippedInside).toEqual([true, true, false]); + }); + }); + describe('_INTERNAL_shouldSkipAiProviderWrapping', () => { it('returns false for unmarked providers', () => { expect(_INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME)).toBe(false); diff --git a/packages/deno/src/index.ts b/packages/deno/src/index.ts index f55fe14de14f..8033186eaa8c 100644 --- a/packages/deno/src/index.ts +++ b/packages/deno/src/index.ts @@ -180,6 +180,8 @@ export { groqIntegration, togetherAIIntegration, typesafeIntegration, + tanstackAiIntegration, + createTanstackAiMiddleware, postgresIntegration, postgresJsIntegration, neonIntegration, diff --git a/packages/elysia/src/index.ts b/packages/elysia/src/index.ts index 76c135d6f3f4..c557e3eecbd6 100644 --- a/packages/elysia/src/index.ts +++ b/packages/elysia/src/index.ts @@ -65,6 +65,8 @@ export { groqIntegration, togetherAIIntegration, typesafeIntegration, + tanstackAiIntegration, + createTanstackAiMiddleware, langChainIntegration, langGraphIntegration, createFlueInstrumentation, diff --git a/packages/google-cloud-serverless/src/index.ts b/packages/google-cloud-serverless/src/index.ts index b90e609bbfc1..ce5f6dee1995 100644 --- a/packages/google-cloud-serverless/src/index.ts +++ b/packages/google-cloud-serverless/src/index.ts @@ -64,6 +64,8 @@ export { groqIntegration, togetherAIIntegration, typesafeIntegration, + tanstackAiIntegration, + createTanstackAiMiddleware, langChainIntegration, langGraphIntegration, mastraIntegration, diff --git a/packages/node/src/index.ts b/packages/node/src/index.ts index 130194f72cf5..399162c62373 100644 --- a/packages/node/src/index.ts +++ b/packages/node/src/index.ts @@ -37,6 +37,8 @@ export { openAIIntegration, togetherAIIntegration, typesafeIntegration, + tanstackAiIntegration, + createTanstackAiMiddleware, postgresIntegration, postgresJsIntegration, neonIntegration, diff --git a/packages/server-utils/src/ai/index.ts b/packages/server-utils/src/ai/index.ts index 3fc0f4bd5fc9..b5dd51f0ef57 100644 --- a/packages/server-utils/src/ai/index.ts +++ b/packages/server-utils/src/ai/index.ts @@ -15,3 +15,5 @@ export { instrumentStateGraph, instrumentStateGraphCompile, instrumentCreateReac export { SentryMastraExporter } from './mastra'; export { createFlueInstrumentation } from './flue'; export type { FlueOptions } from './flue'; +export { createTanstackAiMiddleware } from './tanstack-ai'; +export type { TanstackAiOptions } from './tanstack-ai'; diff --git a/packages/server-utils/src/ai/tanstack-ai/constants.ts b/packages/server-utils/src/ai/tanstack-ai/constants.ts new file mode 100644 index 000000000000..5d57d070f58c --- /dev/null +++ b/packages/server-utils/src/ai/tanstack-ai/constants.ts @@ -0,0 +1,9 @@ +export const TANSTACK_AI_INTEGRATION_NAME = 'TanStackAI' as const; + +export const TANSTACK_AI_ORIGIN = 'auto.ai.tanstack_ai'; + +/** + * Marks the middleware so auto-injection can tell whether the app already passed one to `chat()`. + * `Symbol.for` so two copies of the SDK in one process still recognise each other's middleware. + */ +export const TANSTACK_AI_MIDDLEWARE_BRAND = Symbol.for('sentry.tanstack-ai.middleware'); diff --git a/packages/server-utils/src/ai/tanstack-ai/generation.ts b/packages/server-utils/src/ai/tanstack-ai/generation.ts new file mode 100644 index 000000000000..7319447abf2e --- /dev/null +++ b/packages/server-utils/src/ai/tanstack-ai/generation.ts @@ -0,0 +1,169 @@ +import type { Span, SpanAttributes } from '@sentry/core'; +import { isObjectLike, SPAN_STATUS_ERROR, startInactiveSpan, stringify } from '@sentry/core'; +import { + GEN_AI_EMBEDDINGS_INPUT, + GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_ID, + GEN_AI_RESPONSE_MODEL, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import type { OrchestrionChannelContext } from '../../orchestrion/types'; +import type { GenAiOptions } from '../core/utils'; +import { + getEvaluationInputMessages, + getEvaluationOutputMessages, + getGenAiSpanOp, + resolveAIRecordingOptions, +} from '../core/utils'; +import { TANSTACK_AI_ORIGIN } from './constants'; +import { usageAttributes } from './utils'; + +/** + * A `@tanstack/ai` activity other than `chat()`, traced through its orchestrion channel. Its middleware + * gets neither the input of the call nor the result, and cannot make a span active during the provider + * request. + */ +export interface TanstackAiGenerationActivity { + /** The `gen_ai.operation.name` of the span. */ + operation: string; + /** The name of the adapter method that sends the provider request. */ + adapterMethod: string; + /** Attributes from the options of the call. */ + requestAttributes?: (options: Record) => SpanAttributes; + /** + * Attributes from the value that the adapter method resolves to. They are recorded also when TanStack AI throws + * after the provider answered, e.g. for a rerank index out of range, because the provider bills that call too. + */ + responseAttributes: (adapterResult: Record) => SpanAttributes; + /** Attributes from the value that the call resolves to, recorded only when outputs are recorded. */ + outputAttributes?: (result: Record) => SpanAttributes; +} + +export const TANSTACK_AI_EMBED: TanstackAiGenerationActivity = { + operation: 'embeddings', + adapterMethod: 'createEmbeddings', + requestAttributes: ({ input }) => (input !== undefined ? { [GEN_AI_EMBEDDINGS_INPUT]: embeddingsInput(input) } : {}), + // The adapters make `id` themselves, so it is not the provider's response id. An embedding has no output + // tokens, but the OpenAI adapter reports 0. + responseAttributes: ({ model, usage }) => + responseMetadataAttributes({ + model, + usage: isObjectLike(usage) ? { ...usage, completionTokens: undefined } : usage, + }), +}; + +export const TANSTACK_AI_RERANK: TanstackAiGenerationActivity = { + operation: 'rerank', + adapterMethod: 'rerank', + // Rerank models bill search units. The OpenRouter adapter reports 0 input and output tokens, so only the + // total and the cost are real, and it makes `id` itself when the response has none. + responseAttributes: ({ model, usage }) => + responseMetadataAttributes({ + model, + usage: isObjectLike(usage) ? { ...usage, promptTokens: undefined, completionTokens: undefined } : usage, + }), +}; + +export const TANSTACK_AI_DECIDE: TanstackAiGenerationActivity = { + operation: 'decide', + adapterMethod: 'evaluate', + requestAttributes: ({ state, questions }) => ({ + [GEN_AI_INPUT_MESSAGES]: getEvaluationInputMessages({ state, questions }), + }), + responseAttributes: responseMetadataAttributes, + // `decide()` resolves to the answers keyed by question name, plus `meta` with the model, the usage and the id. + outputAttributes: ({ meta: _meta, ...answers }) => ({ + [GEN_AI_OUTPUT_MESSAGES]: getEvaluationOutputMessages(answers), + }), +}; + +/** Start the span of an activity call, or return `undefined` for options that are not a call's options. */ +export function startGenerationSpan( + activity: TanstackAiGenerationActivity, + callOptions: unknown, + options: GenAiOptions, +): Span | undefined { + if (!isObjectLike(callOptions) || !isObjectLike(callOptions.adapter)) { + return undefined; + } + // As in `chat()`, `provider` is the vendor of an adapter whose `name` also names the wire API. + const { name, provider = name, model } = callOptions.adapter; + if (typeof provider !== 'string' || typeof model !== 'string') { + return undefined; + } + + const { recordInputs } = resolveAIRecordingOptions(options); + return startInactiveSpan({ + name: `${activity.operation} ${model}`, + op: getGenAiSpanOp(activity.operation), + attributes: { + [SENTRY_ORIGIN]: TANSTACK_AI_ORIGIN, + [GEN_AI_OPERATION_NAME]: activity.operation, + [GEN_AI_PROVIDER_NAME]: provider, + [GEN_AI_REQUEST_MODEL]: model, + ...(recordInputs ? activity.requestAttributes?.(callOptions) : {}), + }, + }); +} + +export function setGenerationResponseAttributes( + span: Span, + activity: TanstackAiGenerationActivity, + adapterResult: unknown, +): void { + if (isObjectLike(adapterResult)) { + span.setAttributes(activity.responseAttributes(adapterResult)); + } +} + +/** Record the value that a call resolved to, or what the error that it threw tells. */ +export function setGenerationEndAttributes( + span: Span, + activity: TanstackAiGenerationActivity, + call: OrchestrionChannelContext, + options: GenAiOptions, +): void { + if ('result' in call) { + if (activity.outputAttributes && isObjectLike(call.result) && resolveAIRecordingOptions(options).recordOutputs) { + span.setAttributes(activity.outputAttributes(call.result)); + } + return; + } + if (isObjectLike(call.error) && call.error.name === 'EvaluateError') { + // A refused `decide()` is still billed. TanStack AI puts the usage and the model on the error. + setGenerationResponseAttributes(span, activity, call.error); + } + // As for `chat()`, a call that the app aborted counts as cancelled. + const [callOptions] = call.arguments; + if (isObjectLike(callOptions) && (callOptions.abortSignal as AbortSignal | undefined)?.aborted) { + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'cancelled' }); + } +} + +function responseMetadataAttributes({ id, model, usage }: Record): SpanAttributes { + return { + ...(typeof id === 'string' ? { [GEN_AI_RESPONSE_ID]: id } : {}), + ...(typeof model === 'string' ? { [GEN_AI_RESPONSE_MODEL]: model } : {}), + ...(isObjectLike(usage) ? usageAttributes(usage) : {}), + }; +} + +/** An input item is a string, a content part, or an array of content parts. */ +function embeddingsInput(input: unknown): string | undefined { + if (typeof input === 'string') { + return input; + } + // Media parts carry inline bytes or provider handles; keep only their kind. + const withoutMedia = (part: unknown): unknown => + isObjectLike(part) && typeof part.type === 'string' + ? part.type === 'text' + ? part.content + : { type: part.type } + : part; + const items = Array.isArray(input) ? input : [input]; + return stringify(items.map(item => (Array.isArray(item) ? item.map(withoutMedia) : withoutMedia(item)))); +} diff --git a/packages/server-utils/src/ai/tanstack-ai/index.ts b/packages/server-utils/src/ai/tanstack-ai/index.ts new file mode 100644 index 000000000000..95c2359790d1 --- /dev/null +++ b/packages/server-utils/src/ai/tanstack-ai/index.ts @@ -0,0 +1,299 @@ +import { captureException, SPAN_STATUS_ERROR, startInactiveSpan, stringify, withActiveSpan } from '@sentry/core'; +import { + GEN_AI_AGENT_NAME, + GEN_AI_CONVERSATION_ID, + GEN_AI_OPERATION_NAME, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_FINISH_REASONS, + GEN_AI_RESPONSE_ID, + GEN_AI_RESPONSE_MODEL, + GEN_AI_RESPONSE_STREAMING, + GEN_AI_TOOL_CALL_ARGUMENTS, + GEN_AI_TOOL_CALL_RESULT, + GEN_AI_TOOL_DESCRIPTION, + GEN_AI_TOOL_NAME, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../core/gen-ai-attributes'; +import type { GenAiOptions } from '../core/utils'; +import { getGenAiSpanOp, resolveAIRecordingOptions } from '../core/utils'; +import { TANSTACK_AI_MIDDLEWARE_BRAND, TANSTACK_AI_ORIGIN } from './constants'; +import type { TanstackAiChatMiddleware, TanstackAiFetch, TanstackAiMiddlewareContext, TanstackAiTool } from './types'; +import type { RunState } from './utils'; +import { + addUsage, + endChatSpan, + endSpan, + getProviderName, + inputAttributes, + setOutputMessages, + startChatSpan, + usageAttributes, +} from './utils'; + +export type TanstackAiOptions = GenAiOptions; + +/** + * Create a `@tanstack/ai` chat middleware that records `invoke_agent`, `chat` and `execute_tool` + * spans for a `chat()` call. + * + * ```js + * chat({ adapter, messages, threadId, middleware: [Sentry.createTanstackAiMiddleware()] }); + * ``` + */ +export function createTanstackAiMiddleware(options: TanstackAiOptions = {}): TanstackAiChatMiddleware { + // Keyed by the middleware context, which TanStack creates once per `chat()` call. + const runs = new WeakMap(); + + const endRun = (ctx: TanstackAiMiddlewareContext, errorStatus?: string): void => { + const run = runs.get(ctx); + if (!run) { + return; + } + runs.delete(ctx); + + for (const span of run.toolSpans.values()) { + endSpan(span, errorStatus); + } + endChatSpan(run, errorStatus); + run.agentSpan.setAttributes(usageAttributes(run.usage)); + endSpan(run.agentSpan, errorStatus); + }; + + return { + name: 'sentry', + [TANSTACK_AI_MIDDLEWARE_BRAND]: true, + + onConfig(ctx, config) { + // TanStack runs the `init` phase once per `chat()` call, before `onStart` and before any tool can run, so the + // run starts here and its tools get the span of their call from the start. + if (ctx.phase === 'init' && ctx.activity === 'chat') { + const { recordInputs, recordOutputs } = resolveAIRecordingOptions(options); + const agentName = ctx.subagentName; + runs.set(ctx, { + agentSpan: startInactiveSpan({ + name: agentName ? `invoke_agent ${agentName}` : 'invoke_agent', + op: getGenAiSpanOp('invoke_agent'), + attributes: { + [SENTRY_ORIGIN]: TANSTACK_AI_ORIGIN, + [GEN_AI_OPERATION_NAME]: 'invoke_agent', + [GEN_AI_PROVIDER_NAME]: getProviderName(ctx), + [GEN_AI_REQUEST_MODEL]: ctx.model, + [GEN_AI_CONVERSATION_ID]: ctx.threadId, + [GEN_AI_RESPONSE_STREAMING]: ctx.streaming, + ...(agentName ? { [GEN_AI_AGENT_NAME]: agentName } : {}), + ...(recordInputs ? inputAttributes(config.messages, config.systemPrompts) : {}), + }, + }), + recordInputs, + recordOutputs, + outputText: '', + outputToolCalls: [], + toolCallNames: new Map(), + usage: {}, + toolSpans: new Map(), + wrappedTools: new WeakMap(), + }); + } + + const run = runs.get(ctx); + if (!run) { + return undefined; + } + const tools = config.tools.some(tool => run.wrappedTools.get(tool) !== tool) + ? config.tools.map(tool => wrapTool(run, tool)) + : undefined; + // `beforeModel` opens each agent-loop model call; `structuredOutput` the separate final call + // made for `outputSchema`. + if (ctx.phase !== 'beforeModel' && ctx.phase !== 'structuredOutput') { + return tools ? { tools } : undefined; + } + + // A model call that completes ends its span at the `afterModel` boundary. A span that is still open here + // belongs to a call that failed and that a middleware retried with `ctx.retry()`. + endChatSpan(run, 'internal_error'); + const chatSpan = startChatSpan(ctx, config, run); + run.chatSpan = chatSpan; + + return { + ...(tools ? { tools } : {}), + // Chains inside any wrapper set before us, so the provider request (and the HTTP span the + // SDK records for it) runs with the `chat` span active. + wrapFetch: + (next: TanstackAiFetch): TanstackAiFetch => + (input, init) => + withActiveSpan(chatSpan, () => next(input, init)), + }; + }, + + onChunk(ctx, chunk) { + const run = runs.get(ctx); + if (!run) { + return; + } + if (chunk.type === 'RUN_FINISHED' || chunk.type === 'RUN_ERROR') { + // TanStack sets the call metadata of the `structuredOutput` call only after `onConfig`. + const provider = getProviderName(ctx); + run.chatSpan?.setAttribute(GEN_AI_PROVIDER_NAME, provider); + run.agentSpan.setAttribute(GEN_AI_PROVIDER_NAME, provider); + } + // A provider error ends the run with this chunk. `chat({ stream: false })` throws on it and stops + // reading, so TanStack calls no terminal hook. + if (chunk.type === 'RUN_ERROR') { + endRun(ctx, ctx.signal?.aborted ? 'cancelled' : 'internal_error'); + return; + } + // A run that pauses at an interrupt ends with this chunk. For an interrupt at `beforeModel` or + // `afterModel`, TanStack calls no terminal hook. + if (chunk.type === 'RUN_FINISHED' && chunk.outcome?.type === 'interrupt') { + endRun(ctx); + return; + } + const span = run.chatSpan; + if (!span) { + return; + } + + if (chunk.type === 'TEXT_MESSAGE_CONTENT' && run.recordOutputs && chunk.delta) { + run.outputText += chunk.delta; + } else if (chunk.type === 'TOOL_CALL_START' && chunk.toolCallId && chunk.toolCallName) { + run.toolCallNames.set(chunk.toolCallId, chunk.toolCallName); + } else if (chunk.type === 'TOOL_CALL_END' && run.recordOutputs && chunk.toolCallId) { + run.outputToolCalls.push({ + type: 'tool_call', + id: chunk.toolCallId, + name: run.toolCallNames.get(chunk.toolCallId), + arguments: stringify(chunk.input ?? {}), + }); + } else if (chunk.type === 'RUN_FINISHED') { + // An adapter can report these only in the chunk's `metadata.tanstack`. TanStack copies them + // into `callMetadata` before middleware sees the chunk. + const responseModel = chunk.model ?? ctx.callMetadata?.model; + const responseId = chunk.responseId ?? ctx.callMetadata?.responseId; + if (chunk.finishReason) { + span.setAttribute(GEN_AI_RESPONSE_FINISH_REASONS, stringify([chunk.finishReason])); + } + // The agent span keeps the response of the last model call, which is the answer of the run. + for (const target of [span, run.agentSpan]) { + if (responseModel) { + target.setAttribute(GEN_AI_RESPONSE_MODEL, responseModel); + } + if (responseId) { + target.setAttribute(GEN_AI_RESPONSE_ID, responseId); + } + setOutputMessages(target, run, chunk.finishReason ?? undefined); + } + } + }, + + // Runs after each model call, before the tools that it asked for, so the `chat` span does not cover them. + onInterruptBoundary(ctx) { + const run = runs.get(ctx); + if (run && ctx.phase === 'afterModel') { + endChatSpan(run); + } + }, + + onUsage(ctx, usage) { + const run = runs.get(ctx); + if (!run) { + return; + } + run.chatSpan?.setAttributes(usageAttributes(usage)); + run.usage = addUsage(run.usage, usage); + }, + + // Guards run after every `onBeforeToolCall` decision and after the input check, so `args` are the + // arguments the tool gets. A `transformArgs` decision of an earlier middleware ends the `onBeforeToolCall` + // chain, but not the guards. + onGuardToolCall(ctx, { toolName, toolCallId, args, tool }) { + const run = runs.get(ctx); + // A client tool runs in the browser, not here, so there is nothing to time. + if (!run || (tool && typeof tool.execute !== 'function')) { + return; + } + + run.toolSpans.set( + toolCallId, + startInactiveSpan({ + name: `execute_tool ${toolName}`, + op: getGenAiSpanOp('execute_tool'), + parentSpan: run.agentSpan, + attributes: { + [SENTRY_ORIGIN]: TANSTACK_AI_ORIGIN, + [GEN_AI_OPERATION_NAME]: 'execute_tool', + [GEN_AI_TOOL_NAME]: toolName, + [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: toolCallId, + [GEN_AI_CONVERSATION_ID]: ctx.threadId, + ...(tool?.description ? { [GEN_AI_TOOL_DESCRIPTION]: tool.description } : {}), + ...(run.recordInputs ? { [GEN_AI_TOOL_CALL_ARGUMENTS]: stringify(args) } : {}), + }, + }), + ); + }, + + onAfterToolCall(ctx, { toolCallId, ok, result, error }) { + const run = runs.get(ctx); + const span = run?.toolSpans.get(toolCallId); + if (!run || !span) { + return; + } + run.toolSpans.delete(toolCallId); + + if (run.recordOutputs && ok && result !== undefined) { + span.setAttribute(GEN_AI_TOOL_CALL_RESULT, typeof result === 'string' ? result : stringify(result)); + } + if (!ok && ctx.signal?.aborted) { + // TanStack fails each tool that has not started yet with an `Operation aborted` error. + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'cancelled' }); + } else if (!ok) { + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'internal_error' }); + // TanStack hands a tool failure back to the model as its result, so it never reaches the + // app's error handlers. + withActiveSpan(span, () => { + captureException(error, { mechanism: { type: TANSTACK_AI_ORIGIN, handled: true } }); + }); + } + span.end(); + }, + + onFinish(ctx) { + endRun(ctx); + }, + + // As in the other AI integrations, an aborted call counts as failed. + onAbort(ctx) { + endRun(ctx, 'cancelled'); + }, + + onError(ctx) { + endRun(ctx, 'internal_error'); + }, + }; +} + +// TanStack runs `execute` outside any of the middleware hooks, so wrapping it is the only way to make the +// tool span active while the tool runs and have the tool's own spans nest under it. +function wrapTool(run: RunState, tool: TanstackAiTool): TanstackAiTool { + const known = run.wrappedTools.get(tool); + if (known) { + return known; + } + const { execute } = tool; + const wrapped: TanstackAiTool = + typeof execute === 'function' + ? { + ...tool, + execute: (args, context) => { + const span = context?.toolCallId ? run.toolSpans.get(context.toolCallId) : undefined; + return span + ? withActiveSpan(span, () => execute.call(tool, args, context)) + : execute.call(tool, args, context); + }, + } + : tool; + run.wrappedTools.set(tool, wrapped); + run.wrappedTools.set(wrapped, wrapped); + return wrapped; +} diff --git a/packages/server-utils/src/ai/tanstack-ai/types.ts b/packages/server-utils/src/ai/tanstack-ai/types.ts new file mode 100644 index 000000000000..aae06d2e50d1 --- /dev/null +++ b/packages/server-utils/src/ai/tanstack-ai/types.ts @@ -0,0 +1,118 @@ +/** + * Structural types for the subset of the `@tanstack/ai` chat middleware contract we consume. + * + * Declared locally rather than imported: `@tanstack/ai` is ESM-only and not a dependency of this + * package. Mirrors `ChatMiddleware`, `ChatMiddlewareContext` and `ChatMiddlewareConfig` as of + * `@tanstack/ai` 1.x. + */ + +import type { TANSTACK_AI_MIDDLEWARE_BRAND } from './constants'; + +export type TanstackAiFetch = (input: unknown, init?: unknown) => Promise; + +/** Mirrors `TokenUsage` from `@tanstack/ai-event-client`. */ +export interface TanstackAiUsage { + promptTokens?: number; + completionTokens?: number; + totalTokens?: number; + cost?: number; + /** `upstreamCost` is the provider's charge that a gateway passes through, set by OpenRouter for BYOK keys. */ + costDetails?: { upstreamCost?: number }; + promptTokensDetails?: { cachedTokens?: number; cacheWriteTokens?: number }; + completionTokensDetails?: { reasoningTokens?: number }; +} + +export interface TanstackAiTool { + name: string; + description?: string; + inputSchema?: unknown; + execute?: (args: unknown, context?: { toolCallId?: string }) => unknown; +} + +/** Mirrors the `TanStackMessageMetadata` fields of the current model call. */ +export interface TanstackAiCallMetadata { + model?: string; + responseId?: string; + /** `provider` is the adapter's `provider ?? name`, e.g. `openrouter` for the `openrouter-responses` adapter. */ + source?: { provider?: string; api?: string }; +} + +export interface TanstackAiMiddlewareContext { + activity: string; + phase: string; + /** The adapter's `name`, e.g. `openai-chat`. */ + provider: string; + model: string; + threadId: string; + streaming: boolean; + subagentName?: string; + callMetadata?: TanstackAiCallMetadata; + /** The signal of the `abortController` passed to `chat()`. */ + signal?: AbortSignal; + /** The conversation history of the run, as passed to `chat()`. */ + messages: ReadonlyArray; + systemPrompts: Array; +} + +/** Mirrors `ModelMessage`. */ +export interface TanstackAiModelMessage { + role: 'user' | 'assistant' | 'tool'; + content: string | null | Array<{ type: string; content?: string }>; + toolCalls?: Array<{ id: string; function?: { name: string; arguments: string } }>; + toolCallId?: string; +} + +export interface TanstackAiMiddlewareConfig { + messages: TanstackAiModelMessage[]; + /** The messages sent to the provider, when a middleware sets them apart from `messages`. */ + providerMessages?: TanstackAiModelMessage[]; + systemPrompts: Array; + tools: TanstackAiTool[]; + modelOptions?: Record; + wrapFetch?: (next: TanstackAiFetch) => TanstackAiFetch; +} + +/** A stream chunk: an AG-UI event discriminated by `type`. */ +export interface TanstackAiChunk { + type: string; + delta?: string; + model?: string; + responseId?: string; + finishReason?: string | null; + toolCallId?: string; + toolCallName?: string; + input?: unknown; + /** `{ type: 'interrupt' }` on the `RUN_FINISHED` chunk of a run that pauses at an interrupt. */ + outcome?: { type: string }; +} + +export interface TanstackAiToolCallHookContext { + toolName: string; + toolCallId: string; + args: unknown; + tool?: TanstackAiTool; +} + +export interface TanstackAiAfterToolCallInfo { + toolCallId: string; + ok: boolean; + result?: unknown; + error?: unknown; +} + +export interface TanstackAiChatMiddleware { + name: string; + [TANSTACK_AI_MIDDLEWARE_BRAND]: true; + onConfig( + ctx: TanstackAiMiddlewareContext, + config: TanstackAiMiddlewareConfig, + ): Partial | undefined; + onChunk(ctx: TanstackAiMiddlewareContext, chunk: TanstackAiChunk): void; + onInterruptBoundary(ctx: TanstackAiMiddlewareContext): void; + onUsage(ctx: TanstackAiMiddlewareContext, usage: TanstackAiUsage): void; + onGuardToolCall(ctx: TanstackAiMiddlewareContext, hookCtx: TanstackAiToolCallHookContext): void; + onAfterToolCall(ctx: TanstackAiMiddlewareContext, info: TanstackAiAfterToolCallInfo): void; + onFinish(ctx: TanstackAiMiddlewareContext): void; + onAbort(ctx: TanstackAiMiddlewareContext): void; + onError(ctx: TanstackAiMiddlewareContext, info: { error: unknown }): void; +} diff --git a/packages/server-utils/src/ai/tanstack-ai/utils.ts b/packages/server-utils/src/ai/tanstack-ai/utils.ts new file mode 100644 index 000000000000..7e11b9381f66 --- /dev/null +++ b/packages/server-utils/src/ai/tanstack-ai/utils.ts @@ -0,0 +1,314 @@ +import type { Span } from '@sentry/core'; +import { + _INTERNAL_withAiProviderWrappingSkipped, + isObjectLike, + isThenable, + SPAN_STATUS_ERROR, + startInactiveSpan, + stringify, +} from '@sentry/core'; +import { + GEN_AI_CONVERSATION_ID, + GEN_AI_COST_TOTAL_TOKENS, + GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MAX_TOKENS, + GEN_AI_REQUEST_MODEL, + GEN_AI_REQUEST_TEMPERATURE, + GEN_AI_REQUEST_TOP_P, + GEN_AI_RESPONSE_STREAMING, + GEN_AI_SYSTEM_INSTRUCTIONS, + GEN_AI_TOOL_DEFINITIONS, + GEN_AI_USAGE_REASONING_OUTPUT_TOKENS, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import { AWS_BEDROCK_INTEGRATION_NAME } from '../../integrations/aws-sdk/constants'; +import { ANTHROPIC_AI_INTEGRATION_NAME } from '../anthropic-ai/constants'; +import { getGenAiSpanOp, getTokenUsageAttributes } from '../core/utils'; +import { GOOGLE_GENAI_INTEGRATION_NAME } from '../google-genai/constants'; +import { MISTRAL_INTEGRATION_NAME } from '../mistral/constants'; +import { OPENAI_INTEGRATION_NAME } from '../openai/constants'; +import { WORKERS_AI_INTEGRATION_NAME } from '../workers-ai/constants'; +import { TANSTACK_AI_ORIGIN } from './constants'; +import type { + TanstackAiMiddlewareConfig, + TanstackAiMiddlewareContext, + TanstackAiModelMessage, + TanstackAiTool, + TanstackAiUsage, +} from './types'; + +/** What the middleware tracks for one `chat()` call. */ +export interface RunState { + agentSpan: Span; + chatSpan?: Span; + recordInputs: boolean; + recordOutputs: boolean; + outputText: string; + outputToolCalls: Array>; + // `TOOL_CALL_END` carries the parsed input but not the name, which only `TOOL_CALL_START` has. + toolCallNames: Map; + usage: TanstackAiUsage; + /** The open `execute_tool` spans of the run, keyed by tool call id. */ + toolSpans: Map; + /** Each tool of the run mapped to the copy whose `execute` runs with the tool span active. */ + wrappedTools: WeakMap; +} + +// The Sentry instrumentations of the vendor SDKs and bindings that adapters call. Each one would report the +// model call of an adapter a second time beside our span. +const PROVIDER_INTEGRATIONS = [ + OPENAI_INTEGRATION_NAME, + ANTHROPIC_AI_INTEGRATION_NAME, + GOOGLE_GENAI_INTEGRATION_NAME, + MISTRAL_INTEGRATION_NAME, + // The Cloudflare adapter calls `env.AI.run()`, which `@sentry/cloudflare` instruments. + WORKERS_AI_INTEGRATION_NAME, + // The Bedrock adapters call `@aws-sdk/client-bedrock-runtime`, which `awsIntegration` instruments. + AWS_BEDROCK_INTEGRATION_NAME, +]; + +type AdapterMethod = (...args: unknown[]) => unknown; + +/** + * The adapter, with the Sentry integrations of the vendor SDKs skipped during its calls, as our spans record + * them. A direct call to the same SDK elsewhere in the app still gets its span. + * + * `onResult.callback` gets the value that the adapter method named `onResult.method` resolves to. + */ +export function skipProviderIntegrations( + adapter: T, + onResult?: { method: string; callback: (result: unknown) => void }, +): T { + if (!isObjectLike(adapter)) { + return adapter; + } + const skipped = (callback: () => R): R => _INTERNAL_withAiProviderWrappingSkipped(PROVIDER_INTEGRATIONS, callback); + // So that each read of a method returns the same function. + const wrappedMethods = new WeakMap(); + + return new Proxy(adapter, { + get(target, property) { + const value: unknown = Reflect.get(target, property); + const descriptor = Reflect.getOwnPropertyDescriptor(target, property); + // A Proxy must return a read-only, non-configurable property (e.g. of a frozen adapter) as it is. + if ( + typeof value !== 'function' || + property === 'constructor' || + (descriptor && !descriptor.configurable && !descriptor.writable) + ) { + return value; + } + const known = wrappedMethods.get(value as AdapterMethod); + if (known) { + return known; + } + const callback = property === onResult?.method ? onResult.callback : undefined; + const wrapped: AdapterMethod = (...args) => { + const result: unknown = skipped(() => value.apply(target, args)); + // A text adapter returns an async generator, whose body runs on each `next()` in the async + // context of the caller, which for a streamed `chat()` is the app's code. + if (isAsyncIterable(result)) { + return skipDuringIteration(result, skipped); + } + return callback && isThenable(result) + ? result.then(resolved => { + callback(resolved); + return resolved; + }) + : result; + }; + wrappedMethods.set(value as AdapterMethod, wrapped); + return wrapped; + }, + }); +} + +function isAsyncIterable(value: unknown): value is AsyncIterable { + return isObjectLike(value) && typeof value[Symbol.asyncIterator] === 'function'; +} + +function skipDuringIteration( + iterable: AsyncIterable, + skipped: (callback: () => R) => R, +): AsyncIterableIterator { + const iterator = iterable[Symbol.asyncIterator](); + return { + next: (...args) => skipped(() => iterator.next(...args)), + return: value => skipped(() => iterator.return?.(value) ?? Promise.resolve({ done: true, value })), + throw: error => skipped(() => iterator.throw?.(error) ?? Promise.reject(error)), + [Symbol.asyncIterator]() { + return this; + }, + }; +} + +/** + * `ctx.provider` is the adapter name, which for some adapters also names the wire API (e.g. + * `openrouter-responses`). The call metadata carries the adapter's provider identity, but only once the + * first model call has started. + */ +export function getProviderName(ctx: TanstackAiMiddlewareContext): string { + return ctx.callMetadata?.source?.provider ?? ctx.provider; +} + +export function startChatSpan( + ctx: TanstackAiMiddlewareContext, + config: TanstackAiMiddlewareConfig, + run: RunState, +): Span { + const attributes: Record = { + [SENTRY_ORIGIN]: TANSTACK_AI_ORIGIN, + [GEN_AI_OPERATION_NAME]: 'chat', + [GEN_AI_PROVIDER_NAME]: getProviderName(ctx), + [GEN_AI_REQUEST_MODEL]: ctx.model, + [GEN_AI_CONVERSATION_ID]: ctx.threadId, + [GEN_AI_RESPONSE_STREAMING]: ctx.streaming, + ...requestAttributes(config.modelOptions), + }; + + // The `structuredOutput` call sends no tools, although its config lists them. + if (run.recordInputs && config.tools.length && ctx.phase !== 'structuredOutput') { + attributes[GEN_AI_TOOL_DEFINITIONS] = stringify( + config.tools.map(({ name, description, inputSchema }) => ({ name, description, inputSchema })), + ); + } + if (run.recordInputs) { + // A middleware can send the provider other messages than the history of the run. + Object.assign(attributes, inputAttributes(config.providerMessages ?? config.messages, config.systemPrompts)); + } + + run.outputText = ''; + run.outputToolCalls = []; + run.toolCallNames.clear(); + return startInactiveSpan({ + name: `chat ${ctx.model}`, + op: getGenAiSpanOp('chat'), + parentSpan: run.agentSpan, + attributes, + }); +} + +export function inputAttributes( + messages: ReadonlyArray, + systemPrompts: TanstackAiMiddlewareConfig['systemPrompts'], +): Record { + return { + [GEN_AI_INPUT_MESSAGES]: stringify(messages.map(toGenAiMessage)), + [GEN_AI_SYSTEM_INSTRUCTIONS]: systemPrompts.length + ? systemPrompts.map(prompt => (typeof prompt === 'string' ? prompt : prompt.content)).join('\n') + : undefined, + }; +} + +export function endChatSpan(run: RunState, errorStatus?: string): void { + const span = run.chatSpan; + run.chatSpan = undefined; + if (span) { + endSpan(span, errorStatus); + } +} + +export function endSpan(span: Span, errorStatus?: string): void { + if (errorStatus) { + span.setStatus({ code: SPAN_STATUS_ERROR, message: errorStatus }); + } + span.end(); +} + +export function setOutputMessages(span: Span, run: RunState, finishReason: string | undefined): void { + const parts = [...(run.outputText ? [{ type: 'text', content: run.outputText }] : []), ...run.outputToolCalls]; + if (parts.length) { + span.setAttribute(GEN_AI_OUTPUT_MESSAGES, stringify([{ role: 'assistant', parts, finish_reason: finishReason }])); + } +} + +/** TanStack's `ModelMessage` in the gen_ai conventions shape (`{ role, parts }`). */ +function toGenAiMessage(message: TanstackAiModelMessage): Record { + const parts: Array> = []; + const { content } = message; + + if (message.role === 'tool') { + parts.push({ type: 'tool_call_response', id: message.toolCallId, result: content }); + } else if (typeof content === 'string') { + parts.push({ type: 'text', content }); + } else if (Array.isArray(content)) { + for (const part of content) { + // Media parts carry inline bytes or provider handles; keep only their kind. + parts.push(part.type === 'text' ? { type: 'text', content: part.content } : { type: part.type }); + } + } + for (const toolCall of message.toolCalls ?? []) { + parts.push({ + type: 'tool_call', + id: toolCall.id, + name: toolCall.function?.name, + arguments: toolCall.function?.arguments, + }); + } + return { role: message.role, parts }; +} + +/** Sampling options live in the provider-native `modelOptions`, so accept the common spellings. */ +function requestAttributes(modelOptions: Record = {}): Record { + const pick = (...keys: string[]): number | undefined => { + for (const key of keys) { + const value = modelOptions[key]; + if (typeof value === 'number') { + return value; + } + } + return undefined; + }; + return { + [GEN_AI_REQUEST_TEMPERATURE]: pick('temperature'), + [GEN_AI_REQUEST_TOP_P]: pick('top_p', 'topP'), + [GEN_AI_REQUEST_MAX_TOKENS]: pick('max_tokens', 'maxTokens', 'max_output_tokens', 'maxOutputTokens'), + }; +} + +export function usageAttributes(usage: TanstackAiUsage): Record { + const attributes = getTokenUsageAttributes( + usage.promptTokens, + usage.completionTokens, + usage.promptTokensDetails?.cacheWriteTokens, + usage.promptTokensDetails?.cachedTokens, + usage.totalTokens, + ); + if (typeof usage.completionTokensDetails?.reasoningTokens === 'number') { + attributes[GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] = usage.completionTokensDetails.reasoningTokens; + } + // The provider's own figure (e.g. OpenRouter) includes cache discounts and gateway markup that a + // price table cannot reproduce. With a BYOK key, OpenRouter reports only its fee as `cost` and the + // provider's charge as `upstreamCost`. Relay keeps any cost it gets, so a 0 would replace the cost it + // computes from the token counts. + const cost = (usage.cost ?? 0) + (usage.costDetails?.upstreamCost ?? 0); + if (cost > 0) { + attributes[GEN_AI_COST_TOTAL_TOKENS] = cost; + } + return attributes; +} + +export function addUsage(total: TanstackAiUsage, usage: TanstackAiUsage): TanstackAiUsage { + const add = (a: number | undefined, b: number | undefined): number | undefined => + a === undefined && b === undefined ? undefined : (a ?? 0) + (b ?? 0); + return { + promptTokens: add(total.promptTokens, usage.promptTokens), + completionTokens: add(total.completionTokens, usage.completionTokens), + totalTokens: add(total.totalTokens, usage.totalTokens), + cost: add(total.cost, usage.cost), + costDetails: { upstreamCost: add(total.costDetails?.upstreamCost, usage.costDetails?.upstreamCost) }, + promptTokensDetails: { + cachedTokens: add(total.promptTokensDetails?.cachedTokens, usage.promptTokensDetails?.cachedTokens), + cacheWriteTokens: add(total.promptTokensDetails?.cacheWriteTokens, usage.promptTokensDetails?.cacheWriteTokens), + }, + completionTokensDetails: { + reasoningTokens: add( + total.completionTokensDetails?.reasoningTokens, + usage.completionTokensDetails?.reasoningTokens, + ), + }, + }; +} diff --git a/packages/server-utils/src/index.ts b/packages/server-utils/src/index.ts index dacf7ff16b7f..d73aafc5899a 100644 --- a/packages/server-utils/src/index.ts +++ b/packages/server-utils/src/index.ts @@ -63,6 +63,9 @@ export { mistralAIIntegration } from './integrations/mistral'; export { groqIntegration } from './integrations/groq'; export { togetherAIIntegration } from './integrations/together-ai'; export { typesafeIntegration } from './integrations/typesafe'; +export { tanstackAiIntegration } from './integrations/tanstack-ai'; +export { createTanstackAiMiddleware } from './ai/tanstack-ai'; +export type { TanstackAiOptions } from './ai/tanstack-ai'; export { mysqlIntegration } from './integrations/mysql'; export { mysql2Integration } from './integrations/mysql2'; export { openAIIntegration } from './integrations/openai'; diff --git a/packages/server-utils/src/integrations/aws-sdk/constants.ts b/packages/server-utils/src/integrations/aws-sdk/constants.ts index ae27a6e50315..27165dd153d3 100644 --- a/packages/server-utils/src/integrations/aws-sdk/constants.ts +++ b/packages/server-utils/src/integrations/aws-sdk/constants.ts @@ -17,3 +17,11 @@ export const MESSAGING_DESTINATION_KIND_VALUE_TOPIC = 'topic'; export const GEN_AI_OPERATION_NAME_VALUE_CHAT = 'chat'; export const GEN_AI_OPERATION_NAME_VALUE_GENERATE_CONTENT = 'generate_content'; export const GEN_AI_SYSTEM_VALUE_AWS_BEDROCK = 'aws.bedrock'; + +/** + * The key of Bedrock in the AI provider skip registry. An AI framework integration that records the Bedrock calls + * of its adapters itself skips it, so `awsIntegration` records no span for these calls. + * + * @see `_INTERNAL_withAiProviderWrappingSkipped` + */ +export const AWS_BEDROCK_INTEGRATION_NAME = 'AwsBedrock' as const; diff --git a/packages/server-utils/src/integrations/aws-sdk/index.ts b/packages/server-utils/src/integrations/aws-sdk/index.ts index 95b38b41eece..26f035726123 100644 --- a/packages/server-utils/src/integrations/aws-sdk/index.ts +++ b/packages/server-utils/src/integrations/aws-sdk/index.ts @@ -1,6 +1,6 @@ import * as diagnosticsChannel from '../../utils/diagnosticsChannel'; import type { IntegrationFn, Span } from '@sentry/core'; -import { defineIntegration, startInactiveSpan } from '@sentry/core'; +import { _INTERNAL_shouldSkipAiProviderWrapping, defineIntegration, startInactiveSpan } from '@sentry/core'; import { _AWS_REQUEST_ID as AWS_REQUEST_ID, AWS_REQUEST_EXTENDED_ID, @@ -16,7 +16,7 @@ import { awsSdkModuleNames } from '../../orchestrion/config/aws-sdk'; import { invokeOrchestrionInstrumentation } from '../../orchestrion/instrumentation'; import type { TracingChannelLifeCycleOptions } from '../../tracing-channel'; import { bindTracingChannelToSpan, safeChannelCallback } from '../../tracing-channel'; -import { AWS_SDK_ORIGIN } from './constants'; +import { AWS_BEDROCK_INTEGRATION_NAME, AWS_SDK_ORIGIN } from './constants'; import { ServicesExtensions } from './services'; import type { NormalizedRequest, NormalizedResponse, RequestMetadata } from './types'; import { extractAttributesFromNormalizedRequest, normalizeV3Request, removeSuffixFromStringIfExists } from './utils'; @@ -101,6 +101,12 @@ function instrumentAwsSdk(servicesExtensions: ServicesExtensions): void { command.input = {}; } const normalizedRequest = normalizeV3Request(serviceName, commandName, command.input, undefined); + if ( + normalizedRequest.serviceName === 'BedrockRuntime' && + _INTERNAL_shouldSkipAiProviderWrapping(AWS_BEDROCK_INTEGRATION_NAME) + ) { + return undefined; + } const requestMetadata = servicesExtensions.requestPreSpanHook(normalizedRequest); const span = startInactiveSpan({ diff --git a/packages/server-utils/src/integrations/index.ts b/packages/server-utils/src/integrations/index.ts index 9643f95250ff..9b2f08243e43 100644 --- a/packages/server-utils/src/integrations/index.ts +++ b/packages/server-utils/src/integrations/index.ts @@ -26,6 +26,7 @@ import { mistralAIIntegration } from './mistral'; import { groqIntegration } from './groq'; import { togetherAIIntegration } from './together-ai'; import { typesafeIntegration } from './typesafe'; +import { tanstackAiIntegration } from './tanstack-ai'; import { postgresJsIntegration } from './postgres-js'; import { neonIntegration } from './neon'; import { firebaseIntegration } from './firebase'; @@ -63,6 +64,7 @@ export function getTracingIntegrations(): Integration[] { langGraphIntegration(), mastraIntegration(), piDurableIntegration(), + tanstackAiIntegration(), vercelAIIntegration(), openAIIntegration(), anthropicAIIntegration(), diff --git a/packages/server-utils/src/integrations/tanstack-ai.ts b/packages/server-utils/src/integrations/tanstack-ai.ts new file mode 100644 index 000000000000..8686095d5274 --- /dev/null +++ b/packages/server-utils/src/integrations/tanstack-ai.ts @@ -0,0 +1,99 @@ +import * as diagnosticsChannel from '../utils/diagnosticsChannel'; +import type { IntegrationFn } from '@sentry/core'; +import { defineIntegration, isObjectLike } from '@sentry/core'; +import type { TanstackAiOptions } from '../ai/tanstack-ai'; +import { createTanstackAiMiddleware } from '../ai/tanstack-ai'; +import { TANSTACK_AI_INTEGRATION_NAME, TANSTACK_AI_MIDDLEWARE_BRAND } from '../ai/tanstack-ai/constants'; +import type { TanstackAiGenerationActivity } from '../ai/tanstack-ai/generation'; +import { + setGenerationEndAttributes, + setGenerationResponseAttributes, + startGenerationSpan, + TANSTACK_AI_DECIDE, + TANSTACK_AI_EMBED, + TANSTACK_AI_RERANK, +} from '../ai/tanstack-ai/generation'; +import { skipProviderIntegrations } from '../ai/tanstack-ai/utils'; +import { CHANNELS } from '../orchestrion/channels'; +import { tanstackAiModuleNames } from '../orchestrion/config/tanstack-ai'; +import { invokeOrchestrionInstrumentation } from '../orchestrion/instrumentation'; +import type { OrchestrionChannelContext } from '../orchestrion/types'; +import { bindTracingChannelToSpan, safeChannelCallback } from '../tracing-channel'; + +const GENERATION_ACTIVITIES: Array<[channel: string, activity: TanstackAiGenerationActivity]> = [ + [CHANNELS.TANSTACK_AI_EMBED, TANSTACK_AI_EMBED], + [CHANNELS.TANSTACK_AI_RERANK, TANSTACK_AI_RERANK], + [CHANNELS.TANSTACK_AI_DECIDE, TANSTACK_AI_DECIDE], +]; + +const _tanstackAiIntegration = ((options: TanstackAiOptions = {}) => { + return { + name: TANSTACK_AI_INTEGRATION_NAME, + setup(client) { + // Injecting the middleware opens no span; the middleware parents its spans explicitly. + invokeOrchestrionInstrumentation(client, tanstackAiModuleNames, injectMiddleware, [options], { + requiresTracingChannelBinding: false, + }); + // The other activities open their own spans via `bindTracingChannelToSpan`, which needs the binding. + invokeOrchestrionInstrumentation(client, tanstackAiModuleNames, instrumentGenerationActivities, [options]); + }, + }; +}) satisfies IntegrationFn; + +/** + * Add our middleware to every `chat()` call that does not already carry one, and skip the integrations of + * the vendor SDKs during the calls of the adapter. + */ +function injectMiddleware(options: TanstackAiOptions): void { + const middleware = createTanstackAiMiddleware(options); + + diagnosticsChannel.tracingChannel(CHANNELS.TANSTACK_AI_CHAT).start.subscribe(message => { + safeChannelCallback(() => { + const data = message as OrchestrionChannelContext; + const chatOptions = data.arguments[0]; + if (!isObjectLike(chatOptions)) { + return; + } + const existing = chatOptions.middleware; + // `chat()` throws on a non-array itself; leave that error to it. + if (existing !== undefined && !Array.isArray(existing)) { + return; + } + const hasOurMiddleware = existing?.some(entry => isObjectLike(entry) && entry[TANSTACK_AI_MIDDLEWARE_BRAND]); + // A copy, so options the app reuses across calls do not collect one middleware per call. + data.arguments[0] = { + ...chatOptions, + adapter: skipProviderIntegrations(chatOptions.adapter), + ...(hasOurMiddleware ? {} : { middleware: [...(existing ?? []), middleware] }), + }; + }); + }); +} + +/** Record a span for each call of the activities other than `chat()`, active while the call runs. */ +function instrumentGenerationActivities(options: TanstackAiOptions): void { + for (const [channel, activity] of GENERATION_ACTIVITIES) { + bindTracingChannelToSpan( + diagnosticsChannel.tracingChannel(channel), + data => { + const callOptions = data.arguments[0]; + const span = startGenerationSpan(activity, callOptions, options); + // Only a call that our span records skips the integrations of the vendor SDKs. + if (span && isObjectLike(callOptions)) { + data.arguments[0] = { + ...callOptions, + adapter: skipProviderIntegrations(callOptions.adapter, { + method: activity.adapterMethod, + callback: adapterResult => + safeChannelCallback(() => setGenerationResponseAttributes(span, activity, adapterResult)), + }), + }; + } + return span; + }, + { beforeSpanEnd: (span, data) => setGenerationEndAttributes(span, activity, data, options) }, + ); + } +} + +export const tanstackAiIntegration = defineIntegration(_tanstackAiIntegration); diff --git a/packages/server-utils/src/orchestrion/channels.ts b/packages/server-utils/src/orchestrion/channels.ts index b5919b9eb071..8371669d9831 100644 --- a/packages/server-utils/src/orchestrion/channels.ts +++ b/packages/server-utils/src/orchestrion/channels.ts @@ -35,6 +35,7 @@ import { redisChannels } from './config/redis'; import { remixChannels } from './config/remix'; import { tediousChannels } from './config/tedious'; import { togetherAiChannels } from './config/together-ai'; +import { tanstackAiChannels } from './config/tanstack-ai'; import { typesafeChannels } from './config/typesafe'; import { vercelAiChannels } from './config/vercel-ai'; @@ -92,6 +93,7 @@ export const CHANNELS = { ...remixChannels, ...tediousChannels, ...togetherAiChannels, + ...tanstackAiChannels, ...typesafeChannels, ...vercelAiChannels, } as const; diff --git a/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts b/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts index 46cd69d31bf9..c75b700f8262 100644 --- a/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts +++ b/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts @@ -35,6 +35,7 @@ export const CHANNEL_INTEGRATION_DEFINITIONS = [ { exportName: 'groqIntegration', modules: ['groq-sdk'] }, { exportName: 'togetherAIIntegration', modules: ['together-ai'] }, { exportName: 'typesafeIntegration', modules: ['@typesafe-ai/sdk'] }, + { exportName: 'tanstackAiIntegration', modules: ['@tanstack/ai'] }, { exportName: 'vercelAIIntegration', modules: ['ai'] }, { exportName: 'langChainIntegration', diff --git a/packages/server-utils/src/orchestrion/config/index.ts b/packages/server-utils/src/orchestrion/config/index.ts index e60810315420..aa6901bcf14e 100644 --- a/packages/server-utils/src/orchestrion/config/index.ts +++ b/packages/server-utils/src/orchestrion/config/index.ts @@ -42,6 +42,7 @@ import { remixConfig } from './remix'; import { remixV3Config } from './remix-v3'; import { tediousConfig } from './tedious'; import { togetherAiConfig } from './together-ai'; +import { tanstackAiConfig } from './tanstack-ai'; import { typesafeConfig } from './typesafe'; import { vercelAiConfig } from './vercel-ai'; // Kept sorted alphabetically by module so concurrent additions insert at different @@ -101,6 +102,7 @@ export const SENTRY_INSTRUMENTATIONS: InstrumentationConfig[] = [ ...remixV3Config, ...tediousConfig, ...togetherAiConfig, + ...tanstackAiConfig, ...typesafeConfig, ...vercelAiConfig, ]; diff --git a/packages/server-utils/src/orchestrion/config/tanstack-ai.ts b/packages/server-utils/src/orchestrion/config/tanstack-ai.ts new file mode 100644 index 000000000000..482055e46f0b --- /dev/null +++ b/packages/server-utils/src/orchestrion/config/tanstack-ai.ts @@ -0,0 +1,40 @@ +import type { InstrumentationConfig } from '../apmTypes'; +import { getModuleNames } from './module-names'; + +// `chat()` takes its middleware per call and `@tanstack/ai` has no global registry, so the only way +// in without user code is to rewrite the options of each call. `chat` is a plain function declaration +// in a stable, unhashed file (`dist/esm/activities/chat/index.js` in 1.0.0). It returns a stream or a +// promise depending on `stream`, so it is wrapped `Sync`: we only need the `start` event. +export const tanstackAiConfig = [ + { + channelName: 'chat', + module: { name: '@tanstack/ai', versionRange: '>=1.0.0 <2.0.0', filePath: 'dist/esm/activities/chat/index.js' }, + functionQuery: { functionName: 'chat', kind: 'Sync' }, + }, + // The other activities take observe-only middleware that never sees the input or the result, so we + // trace the call itself. + { + channelName: 'embed', + module: { name: '@tanstack/ai', versionRange: '>=1.0.0 <2.0.0', filePath: 'dist/esm/activities/embed/index.js' }, + functionQuery: { functionName: 'embed', kind: 'Async' }, + }, + { + channelName: 'rerank', + module: { name: '@tanstack/ai', versionRange: '>=1.0.0 <2.0.0', filePath: 'dist/esm/activities/rerank/index.js' }, + functionQuery: { functionName: 'rerank', kind: 'Async' }, + }, + { + channelName: 'decide', + module: { name: '@tanstack/ai', versionRange: '>=1.0.0 <2.0.0', filePath: 'dist/esm/activities/evaluate/index.js' }, + functionQuery: { functionName: 'decide', kind: 'Async' }, + }, +] satisfies InstrumentationConfig[]; + +export const tanstackAiModuleNames = getModuleNames(tanstackAiConfig); + +export const tanstackAiChannels = { + TANSTACK_AI_CHAT: 'orchestrion:@tanstack/ai:chat', + TANSTACK_AI_EMBED: 'orchestrion:@tanstack/ai:embed', + TANSTACK_AI_RERANK: 'orchestrion:@tanstack/ai:rerank', + TANSTACK_AI_DECIDE: 'orchestrion:@tanstack/ai:decide', +} as const; diff --git a/packages/server-utils/test/ai/lib/tracing/tanstack-ai-generation.test.ts b/packages/server-utils/test/ai/lib/tracing/tanstack-ai-generation.test.ts new file mode 100644 index 000000000000..3ce4944b3ca2 --- /dev/null +++ b/packages/server-utils/test/ai/lib/tracing/tanstack-ai-generation.test.ts @@ -0,0 +1,232 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; +import { + setGenerationEndAttributes, + setGenerationResponseAttributes, + startGenerationSpan, + TANSTACK_AI_DECIDE, + TANSTACK_AI_EMBED, + TANSTACK_AI_RERANK, +} from '../../../../src/ai/tanstack-ai/generation'; +import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; + +describe('TanStack AI activities', () => { + beforeEach(() => { + getMainCarrier().__SENTRY__ = undefined; + const client = new TestClient( + getDefaultTestClientOptions({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + tracesSampleRate: 1, + traceLifecycle: 'stream', + }), + ); + setCurrentClient(client); + client.init(); + }); + + afterEach(() => { + getMainCarrier().__SENTRY__ = undefined; + }); + + it('opens no span for options without an adapter', () => { + expect(startGenerationSpan(TANSTACK_AI_EMBED, { input: 'a red guitar' }, {})).toBeUndefined(); + }); + + it('reports the provider of an adapter whose name also names the API', () => { + const span = startGenerationSpan( + TANSTACK_AI_EMBED, + { + adapter: { name: 'bedrock-embed', provider: 'amazon-bedrock', model: 'amazon.titan-embed-text-v2:0' }, + input: 'a', + }, + {}, + ); + + expect(spanToStaticSpanJSON(span!).data['gen_ai.provider.name']).toBe('amazon-bedrock'); + }); + + it('marks a call that the app aborted as cancelled', () => { + const abortController = new AbortController(); + abortController.abort(); + const callOptions = { + adapter: { name: 'openai', model: 'text-embedding-3-small' }, + input: 'a', + abortSignal: abortController.signal, + }; + const span = startGenerationSpan(TANSTACK_AI_EMBED, callOptions, {})!; + const error = Object.assign(new Error('Request was aborted.'), { name: 'APIUserAbortError' }); + setGenerationEndAttributes(span, TANSTACK_AI_EMBED, { arguments: [callOptions], error }, {}); + + expect(spanToStaticSpanJSON(span).status).toBe('cancelled'); + }); + + describe('embed', () => { + it('records the input without media', () => { + const span = startGenerationSpan( + TANSTACK_AI_EMBED, + { + adapter: { name: 'openai', model: 'text-embedding-3-small' }, + input: [ + 'a red guitar', + [ + { type: 'text', content: 'product photo' }, + { type: 'image', source: { type: 'data', value: 'iVBORw0KGgo', mimeType: 'image/png' } }, + ], + ], + }, + {}, + ); + + expect(spanToStaticSpanJSON(span!)).toMatchObject({ + description: 'embeddings text-embedding-3-small', + op: 'gen_ai.embeddings', + data: { + 'sentry.origin': 'auto.ai.tanstack_ai', + 'gen_ai.operation.name': 'embeddings', + 'gen_ai.provider.name': 'openai', + 'gen_ai.request.model': 'text-embedding-3-small', + 'gen_ai.embeddings.input': '["a red guitar",["product photo",{"type":"image"}]]', + }, + }); + }); + + it('keeps a single string input as it is', () => { + const span = startGenerationSpan( + TANSTACK_AI_EMBED, + { adapter: { name: 'ollama', model: 'nomic-embed-text' }, input: 'a red guitar' }, + {}, + ); + + expect(spanToStaticSpanJSON(span!).data['gen_ai.embeddings.input']).toBe('a red guitar'); + }); + + it('leaves out the input when inputs are not recorded', () => { + const span = startGenerationSpan( + TANSTACK_AI_EMBED, + { adapter: { name: 'openai', model: 'text-embedding-3-small' }, input: 'a red guitar' }, + { recordInputs: false }, + ); + + expect(spanToStaticSpanJSON(span!).data).not.toHaveProperty('gen_ai.embeddings.input'); + }); + + it('records the response model, usage and provider cost, but not the id and the output tokens that the adapter made', () => { + const span = startGenerationSpan(TANSTACK_AI_EMBED, { adapter: { name: 'openai', model: 'm' }, input: 'a' }, {})!; + setGenerationResponseAttributes(span, TANSTACK_AI_EMBED, { + id: 'openai-123', + model: 'text-embedding-3-small', + embeddings: [{ vector: [0.1], index: 0 }], + usage: { promptTokens: 8, completionTokens: 0, totalTokens: 8, cost: 0.0001 }, + }); + + const { data } = spanToStaticSpanJSON(span); + expect(data).toMatchObject({ + 'gen_ai.response.model': 'text-embedding-3-small', + 'gen_ai.usage.input_tokens': 8, + 'gen_ai.usage.total_tokens': 8, + 'gen_ai.cost.total_tokens': 0.0001, + }); + expect(data).not.toHaveProperty('gen_ai.response.id'); + expect(data).not.toHaveProperty('gen_ai.usage.output_tokens'); + }); + }); + + describe('rerank', () => { + it('records the total tokens and provider cost, but not the id and the input and output tokens that the adapter made', () => { + const span = startGenerationSpan( + TANSTACK_AI_RERANK, + { adapter: { name: 'openrouter', model: 'cohere/rerank-v3.5' }, query: 'travel laptop', documents: ['a', 'b'] }, + {}, + )!; + setGenerationResponseAttributes(span, TANSTACK_AI_RERANK, { + id: 'rerank-1792850464143-k3j9x2a', + ranking: [{ index: 1, score: 0.9 }], + usage: { promptTokens: 0, completionTokens: 0, totalTokens: 12, cost: 0.002 }, + }); + + expect(spanToStaticSpanJSON(span)).toMatchObject({ + description: 'rerank cohere/rerank-v3.5', + op: 'gen_ai.rerank', + data: { + 'gen_ai.operation.name': 'rerank', + 'gen_ai.provider.name': 'openrouter', + 'gen_ai.request.model': 'cohere/rerank-v3.5', + 'gen_ai.usage.total_tokens': 12, + 'gen_ai.cost.total_tokens': 0.002, + }, + }); + const { data } = spanToStaticSpanJSON(span); + expect(data).not.toHaveProperty('gen_ai.response.id'); + expect(data).not.toHaveProperty('gen_ai.usage.input_tokens'); + expect(data).not.toHaveProperty('gen_ai.usage.output_tokens'); + }); + }); + + describe('decide', () => { + const callOptions = { + adapter: { name: 'openai', model: 'gpt-6-luna' }, + state: 'I was charged twice.', + questions: { refund: { type: 'noul', instructions: 'Does the customer ask for a refund?' } }, + }; + const adapterResult = { + id: 'decision-1', + model: 'gpt-6-luna-2026-09-01', + answers: { refund: { type: 'noul', noul: 0.8 } }, + usage: { promptTokens: 30, completionTokens: 4, totalTokens: 34 }, + }; + const result = { + refund: { type: 'boolean', value: true, probability: 0.8 }, + meta: { id: adapterResult.id, model: adapterResult.model, usage: adapterResult.usage }, + }; + + it('records the questions and answers', () => { + const span = startGenerationSpan(TANSTACK_AI_DECIDE, callOptions, {})!; + setGenerationResponseAttributes(span, TANSTACK_AI_DECIDE, adapterResult); + setGenerationEndAttributes(span, TANSTACK_AI_DECIDE, { arguments: [callOptions], result }, {}); + + expect(spanToStaticSpanJSON(span)).toMatchObject({ + description: 'decide gpt-6-luna', + op: 'gen_ai.decide', + data: { + 'gen_ai.operation.name': 'decide', + 'gen_ai.input.messages': + '[{"type":"evaluation","state":"I was charged twice.","questions":{"refund":{"type":"noul","instructions":"Does the customer ask for a refund?"}}}]', + 'gen_ai.output.messages': + '[{"type":"evaluation","answers":{"refund":{"type":"boolean","value":true,"probability":0.8}}}]', + 'gen_ai.response.id': 'decision-1', + 'gen_ai.response.model': 'gpt-6-luna-2026-09-01', + 'gen_ai.usage.input_tokens': 30, + 'gen_ai.usage.output_tokens': 4, + }, + }); + }); + + it('leaves out the questions and answers when inputs and outputs are not recorded', () => { + const options = { recordInputs: false, recordOutputs: false }; + const span = startGenerationSpan(TANSTACK_AI_DECIDE, callOptions, options)!; + setGenerationResponseAttributes(span, TANSTACK_AI_DECIDE, adapterResult); + setGenerationEndAttributes(span, TANSTACK_AI_DECIDE, { arguments: [callOptions], result }, options); + + const { data } = spanToStaticSpanJSON(span); + expect(data).not.toHaveProperty('gen_ai.input.messages'); + expect(data).not.toHaveProperty('gen_ai.output.messages'); + expect(data['gen_ai.response.model']).toBe('gpt-6-luna-2026-09-01'); + }); + + it('records the model and usage of a refused call, which the provider bills', () => { + const span = startGenerationSpan(TANSTACK_AI_DECIDE, callOptions, {})!; + const error = Object.assign(new Error('The model refused to answer "refund".'), { + name: 'EvaluateError', + model: 'gpt-6-luna-2026-09-01', + usage: { promptTokens: 30, completionTokens: 4, totalTokens: 34 }, + }); + setGenerationEndAttributes(span, TANSTACK_AI_DECIDE, { arguments: [callOptions], error }, {}); + + expect(spanToStaticSpanJSON(span).data).toMatchObject({ + 'gen_ai.response.model': 'gpt-6-luna-2026-09-01', + 'gen_ai.usage.input_tokens': 30, + 'gen_ai.usage.output_tokens': 4, + }); + }); + }); +}); diff --git a/packages/server-utils/test/ai/lib/tracing/tanstack-ai.test.ts b/packages/server-utils/test/ai/lib/tracing/tanstack-ai.test.ts new file mode 100644 index 000000000000..cb4502351626 --- /dev/null +++ b/packages/server-utils/test/ai/lib/tracing/tanstack-ai.test.ts @@ -0,0 +1,514 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Span } from '@sentry/core'; +import { + _INTERNAL_clearAiProviderSkips, + _INTERNAL_shouldSkipAiProviderWrapping, + captureMessage, + getActiveSpan, + getMainCarrier, + setCurrentClient, + spanToStaticSpanJSON, +} from '@sentry/core'; +import { ANTHROPIC_AI_INTEGRATION_NAME } from '../../../../src/ai/anthropic-ai/constants'; +import { OPENAI_INTEGRATION_NAME } from '../../../../src/ai/openai/constants'; +import { createTanstackAiMiddleware } from '../../../../src/ai/tanstack-ai'; +import type { TanstackAiMiddlewareConfig, TanstackAiMiddlewareContext } from '../../../../src/ai/tanstack-ai/types'; +import { skipProviderIntegrations } from '../../../../src/ai/tanstack-ai/utils'; +import { WORKERS_AI_INTEGRATION_NAME } from '../../../../src/ai/workers-ai/constants'; +import { AWS_BEDROCK_INTEGRATION_NAME } from '../../../../src/integrations/aws-sdk/constants'; +import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; + +// TanStack creates one context per `chat()` call and the middleware keys its state by it, so every test +// spreads its own copy. +const CTX: TanstackAiMiddlewareContext = { + activity: 'chat', + phase: 'init', + provider: 'openrouter-responses', + model: 'openai/gpt-4o-mini', + threadId: 'thread-1', + streaming: true, + callMetadata: {}, + messages: [], + systemPrompts: [], +}; + +const CONFIG: TanstackAiMiddlewareConfig = { + messages: [{ role: 'user', content: 'What is the weather in Vienna?' }], + systemPrompts: [], + tools: [], +}; + +describe('createTanstackAiMiddleware', () => { + let endedSpans: Span[]; + let client: TestClient; + + beforeEach(() => { + _INTERNAL_clearAiProviderSkips(); + getMainCarrier().__SENTRY__ = undefined; + client = new TestClient( + getDefaultTestClientOptions({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + tracesSampleRate: 1, + traceLifecycle: 'stream', + }), + ); + setCurrentClient(client); + client.init(); + + endedSpans = []; + client.on('spanEnd', span => endedSpans.push(span)); + }); + + afterEach(() => { + _INTERNAL_clearAiProviderSkips(); + getMainCarrier().__SENTRY__ = undefined; + }); + + it('reports the provider and response fields from the call metadata', () => { + const middleware = createTanstackAiMiddleware(); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, CONFIG); + // TanStack sets the call metadata when the model call starts, after `onStart`. + ctx.phase = 'beforeModel'; + ctx.callMetadata = { source: { provider: 'openrouter', api: 'responses' } }; + middleware.onConfig(ctx, CONFIG); + ctx.callMetadata = { ...ctx.callMetadata, model: 'openai/gpt-4o-mini-2024-07-18', responseId: 'gen-123' }; + middleware.onChunk(ctx, { type: 'RUN_FINISHED', finishReason: 'stop' }); + middleware.onFinish(ctx); + + const chat = endedSpans.map(span => spanToStaticSpanJSON(span)).find(span => span.op === 'gen_ai.chat'); + const agent = endedSpans.map(span => spanToStaticSpanJSON(span)).find(span => span.op === 'gen_ai.invoke_agent'); + expect(chat?.data).toMatchObject({ + 'gen_ai.provider.name': 'openrouter', + 'gen_ai.response.model': 'openai/gpt-4o-mini-2024-07-18', + 'gen_ai.response.id': 'gen-123', + 'gen_ai.response.finish_reasons': '["stop"]', + }); + expect(agent?.data['gen_ai.provider.name']).toBe('openrouter'); + }); + + it('records the input of the run and the answer of the last model call on the agent span', () => { + const middleware = createTanstackAiMiddleware({ recordInputs: true, recordOutputs: true }); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, { ...CONFIG, systemPrompts: ['Be brief.'] }); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + middleware.onChunk(ctx, { type: 'TOOL_CALL_START', toolCallId: 'call-1', toolCallName: 'get_weather' }); + middleware.onChunk(ctx, { type: 'TOOL_CALL_END', toolCallId: 'call-1', input: { city: 'Vienna' } }); + middleware.onChunk(ctx, { type: 'RUN_FINISHED', finishReason: 'tool_calls', responseId: 'gen-1' }); + ctx.phase = 'afterModel'; + middleware.onInterruptBoundary(ctx); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + middleware.onChunk(ctx, { type: 'TEXT_MESSAGE_CONTENT', delta: 'Sunny' }); + middleware.onChunk(ctx, { type: 'RUN_FINISHED', finishReason: 'stop', responseId: 'gen-2', model: 'gpt-4o-mini' }); + middleware.onFinish(ctx); + + const agent = endedSpans.map(span => spanToStaticSpanJSON(span)).find(span => span.op === 'gen_ai.invoke_agent'); + expect(agent?.data).toMatchObject({ + 'gen_ai.input.messages': '[{"role":"user","parts":[{"type":"text","content":"What is the weather in Vienna?"}]}]', + 'gen_ai.system_instructions': 'Be brief.', + 'gen_ai.response.id': 'gen-2', + 'gen_ai.response.model': 'gpt-4o-mini', + 'gen_ai.output.messages': + '[{"role":"assistant","parts":[{"type":"text","content":"Sunny"}],"finish_reason":"stop"}]', + }); + // The finish reason belongs to a single model call. + expect(agent?.data).not.toHaveProperty('gen_ai.response.finish_reasons'); + }); + + it('sums the usage and provider cost of every model call on the agent span', () => { + const middleware = createTanstackAiMiddleware(); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + middleware.onUsage(ctx, { promptTokens: 10, completionTokens: 2, totalTokens: 12, cost: 0.001 }); + ctx.phase = 'afterModel'; + middleware.onInterruptBoundary(ctx); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + // OpenRouter with a BYOK key: its fee is 0, and the provider's charge is the upstream cost. + middleware.onUsage(ctx, { + promptTokens: 20, + completionTokens: 5, + totalTokens: 25, + cost: 0, + costDetails: { upstreamCost: 0.002 }, + promptTokensDetails: { cachedTokens: 8 }, + }); + middleware.onFinish(ctx); + + const agent = endedSpans.map(span => spanToStaticSpanJSON(span)).find(span => span.op === 'gen_ai.invoke_agent'); + expect(agent?.data).toMatchObject({ + 'gen_ai.usage.input_tokens': 30, + 'gen_ai.usage.output_tokens': 7, + 'gen_ai.usage.total_tokens': 37, + 'gen_ai.usage.cache_read.input_tokens': 8, + 'gen_ai.cost.total_tokens': 0.003, + }); + expect(endedSpans.filter(span => spanToStaticSpanJSON(span).op === 'gen_ai.chat')).toHaveLength(2); + }); + + it('leaves out a cost of 0, so Relay computes the cost from the token counts', () => { + const middleware = createTanstackAiMiddleware(); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + middleware.onUsage(ctx, { promptTokens: 10, completionTokens: 2, totalTokens: 12, cost: 0 }); + middleware.onFinish(ctx); + + const spans = endedSpans.map(span => spanToStaticSpanJSON(span)); + expect(spans).toHaveLength(2); + expect(spans.every(span => span.data['gen_ai.usage.input_tokens'] === 10)).toBe(true); + expect(spans.some(span => 'gen_ai.cost.total_tokens' in span.data)).toBe(false); + }); + + it('marks a failed tool call and captures its error, which TanStack hands to the model', async () => { + const middleware = createTanstackAiMiddleware(); + const ctx = { ...CTX }; + const tool = { name: 'get_weather', execute: () => undefined }; + + middleware.onConfig(ctx, CONFIG); + middleware.onGuardToolCall(ctx, { toolName: 'get_weather', toolCallId: 'call-1', args: { city: 'Vienna' }, tool }); + middleware.onAfterToolCall(ctx, { toolCallId: 'call-1', ok: false, error: new TypeError('kaboom') }); + middleware.onFinish(ctx); + await client.flush(); + + const toolSpan = endedSpans.map(span => spanToStaticSpanJSON(span)).find(span => span.op === 'gen_ai.execute_tool'); + expect(toolSpan?.status).toBe('internal_error'); + expect(toolSpan?.data).toMatchObject({ + 'gen_ai.tool.call.id': 'call-1', + 'gen_ai.tool.call.arguments': '{"city":"Vienna"}', + }); + const exception = client.event?.exception?.values?.[0]; + expect(exception?.value).toBe('kaboom'); + expect(exception?.mechanism).toEqual({ type: 'auto.ai.tanstack_ai', handled: true }); + }); + + it('records no tool span for a client tool, which runs in the browser', () => { + const middleware = createTanstackAiMiddleware(); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, CONFIG); + middleware.onGuardToolCall(ctx, { + toolName: 'pick_color', + toolCallId: 'call-1', + args: {}, + tool: { name: 'pick_color' }, + }); + middleware.onFinish(ctx); + + expect(endedSpans.map(span => spanToStaticSpanJSON(span).op)).toEqual(['gen_ai.invoke_agent']); + }); + + it('ends every open span with an error status when the run fails', () => { + const middleware = createTanstackAiMiddleware(); + const ctx = { ...CTX }; + const tool = { name: 'get_weather', execute: () => undefined }; + + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + middleware.onGuardToolCall(ctx, { toolName: 'get_weather', toolCallId: 'call-1', args: {}, tool }); + middleware.onError(ctx, { error: new Error('provider down') }); + + expect(endedSpans.map(span => [spanToStaticSpanJSON(span).op, spanToStaticSpanJSON(span).status])).toEqual([ + ['gen_ai.execute_tool', 'internal_error'], + ['gen_ai.chat', 'internal_error'], + ['gen_ai.invoke_agent', 'internal_error'], + ]); + }); + + it('ends the run with an error status when the provider error arrives as a chunk', () => { + const middleware = createTanstackAiMiddleware(); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + // `chat({ stream: false })` throws on this chunk and stops reading, so no terminal hook follows. + middleware.onChunk(ctx, { type: 'RUN_ERROR' }); + + expect(endedSpans.map(span => [spanToStaticSpanJSON(span).op, spanToStaticSpanJSON(span).status])).toEqual([ + ['gen_ai.chat', 'internal_error'], + ['gen_ai.invoke_agent', 'internal_error'], + ]); + }); + + it('marks an aborted run as cancelled', () => { + const middleware = createTanstackAiMiddleware(); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + middleware.onAbort(ctx); + + expect(endedSpans.map(span => spanToStaticSpanJSON(span).status)).toEqual(['cancelled', 'cancelled']); + }); + + it('marks the provider error that ends an aborted run as cancelled', () => { + const middleware = createTanstackAiMiddleware(); + const abortController = new AbortController(); + const ctx = { ...CTX, signal: abortController.signal }; + + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + abortController.abort(); + // TanStack yields the held error of a rate-limited call when an abort ends the wait for the retry. + middleware.onChunk(ctx, { type: 'RUN_ERROR' }); + + expect(endedSpans.map(span => spanToStaticSpanJSON(span).status)).toEqual(['cancelled', 'cancelled']); + }); + + it('marks a tool call that fails because the run was aborted as cancelled, and captures no error', async () => { + const middleware = createTanstackAiMiddleware(); + const abortController = new AbortController(); + const ctx = { ...CTX, signal: abortController.signal }; + const tool = { name: 'get_weather', execute: () => undefined }; + const sendEvent = vi.spyOn(client, 'sendEvent'); + + middleware.onConfig(ctx, CONFIG); + middleware.onGuardToolCall(ctx, { toolName: 'get_weather', toolCallId: 'call-1', args: {}, tool }); + abortController.abort(); + middleware.onAfterToolCall(ctx, { toolCallId: 'call-1', ok: false, error: new Error('Operation aborted') }); + middleware.onAbort(ctx); + // Sentinel: once this event is sent, an event for the tool error would have been sent before it. + captureMessage('sentinel'); + await client.flush(); + + const toolSpan = endedSpans.map(span => spanToStaticSpanJSON(span)).find(span => span.op === 'gen_ai.execute_tool'); + expect(toolSpan?.status).toBe('cancelled'); + expect(sendEvent).toHaveBeenCalledTimes(1); + expect(sendEvent).toHaveBeenCalledWith(expect.objectContaining({ message: 'sentinel' }), expect.anything()); + }); + + it('ends a model call that a middleware retried with an error status', () => { + const middleware = createTanstackAiMiddleware(); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + // The `RUN_ERROR` of a call that `ctx.retry()` retries never reaches `onChunk`. TanStack runs `onConfig` again. + middleware.onConfig(ctx, CONFIG); + middleware.onChunk(ctx, { type: 'RUN_FINISHED', finishReason: 'stop' }); + ctx.phase = 'afterModel'; + middleware.onInterruptBoundary(ctx); + middleware.onFinish(ctx); + + expect(endedSpans.map(span => [spanToStaticSpanJSON(span).op, spanToStaticSpanJSON(span).status])).toEqual([ + ['gen_ai.chat', 'internal_error'], + ['gen_ai.chat', 'ok'], + ['gen_ai.invoke_agent', 'ok'], + ]); + }); + + it('ends the run when it pauses at an interrupt, for which TanStack calls no terminal hook', () => { + const middleware = createTanstackAiMiddleware(); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + middleware.onChunk(ctx, { type: 'RUN_FINISHED', finishReason: 'stop', outcome: { type: 'interrupt' } }); + + expect(endedSpans.map(span => [spanToStaticSpanJSON(span).op, spanToStaticSpanJSON(span).status])).toEqual([ + ['gen_ai.chat', 'ok'], + ['gen_ai.invoke_agent', 'ok'], + ]); + }); + + it('ends the chat span after the model call, before the tools run', () => { + const middleware = createTanstackAiMiddleware(); + const ctx = { ...CTX }; + const tool = { name: 'get_weather', execute: () => undefined }; + + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'afterModel'; + middleware.onInterruptBoundary(ctx); + middleware.onGuardToolCall(ctx, { toolName: 'get_weather', toolCallId: 'call-1', args: {}, tool }); + middleware.onAfterToolCall(ctx, { toolCallId: 'call-1', ok: true, result: 'sunny' }); + middleware.onFinish(ctx); + + expect(endedSpans.map(span => spanToStaticSpanJSON(span).op)).toEqual([ + 'gen_ai.chat', + 'gen_ai.execute_tool', + 'gen_ai.invoke_agent', + ]); + }); + + it('keeps the tool spans of two runs apart when their tool calls have the same id', () => { + const middleware = createTanstackAiMiddleware(); + const first = { ...CTX, threadId: 'thread-1' }; + const second = { ...CTX, threadId: 'thread-2' }; + const tool = { name: 'get_weather', execute: () => undefined }; + + middleware.onConfig(first, CONFIG); + middleware.onConfig(second, CONFIG); + middleware.onGuardToolCall(first, { toolName: 'get_weather', toolCallId: 'get_weather_1', args: {}, tool }); + middleware.onGuardToolCall(second, { toolName: 'get_weather', toolCallId: 'get_weather_1', args: {}, tool }); + middleware.onAfterToolCall(first, { toolCallId: 'get_weather_1', ok: true, result: 'sunny' }); + middleware.onAfterToolCall(second, { toolCallId: 'get_weather_1', ok: true, result: 'rainy' }); + + const tools = endedSpans.map(span => spanToStaticSpanJSON(span)).filter(span => span.op === 'gen_ai.execute_tool'); + expect(tools.map(span => [span.data['gen_ai.conversation.id'], span.data['gen_ai.tool.call.result']])).toEqual([ + ['thread-1', 'sunny'], + ['thread-2', 'rainy'], + ]); + }); + + it('records the messages that a middleware sends to the provider instead of the history', () => { + const middleware = createTanstackAiMiddleware({ recordInputs: true }); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, { ...CONFIG, providerMessages: [{ role: 'user', content: 'Weather in Vienna?' }] }); + middleware.onFinish(ctx); + + const chat = endedSpans.map(span => spanToStaticSpanJSON(span)).find(span => span.op === 'gen_ai.chat'); + expect(chat?.data['gen_ai.input.messages']).toBe( + '[{"role":"user","parts":[{"type":"text","content":"Weather in Vienna?"}]}]', + ); + }); + + it('records the provider of the structured output call, and no tools, which that call does not send', () => { + const middleware = createTanstackAiMiddleware({ recordInputs: true }); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, CONFIG); + // Without tools, the structured output call is the only model call, and TanStack sets its call metadata after + // `onConfig`. + ctx.phase = 'structuredOutput'; + middleware.onConfig(ctx, { ...CONFIG, tools: [{ name: 'get_weather', description: 'Get the weather.' }] }); + ctx.callMetadata = { source: { provider: 'openrouter', api: 'responses' } }; + middleware.onChunk(ctx, { type: 'RUN_FINISHED', finishReason: 'stop' }); + middleware.onFinish(ctx); + + const spans = endedSpans.map(span => spanToStaticSpanJSON(span)); + expect(spans.map(span => span.data['gen_ai.provider.name'])).toEqual(['openrouter', 'openrouter']); + expect(spans.find(span => span.op === 'gen_ai.chat')?.data).not.toHaveProperty('gen_ai.tool.definitions'); + }); + + it('leaves out the tool definitions when inputs are not recorded', () => { + const middleware = createTanstackAiMiddleware({ recordInputs: false }); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, { ...CONFIG, tools: [{ name: 'get_weather', description: 'Get the weather.' }] }); + middleware.onFinish(ctx); + + const chat = endedSpans.map(span => spanToStaticSpanJSON(span)).find(span => span.op === 'gen_ai.chat'); + expect(chat?.data).not.toHaveProperty('gen_ai.tool.definitions'); + }); + + it('runs each tool with the tool span of its own run active, also when two runs have the same tool call id', async () => { + const middleware = createTanstackAiMiddleware(); + const getWeather = { + name: 'get_weather', + execute: () => spanToStaticSpanJSON(getActiveSpan()!).data['gen_ai.conversation.id'], + }; + const first = { ...CTX, threadId: 'thread-1' }; + const second = { ...CTX, threadId: 'thread-2' }; + + const [firstTool] = middleware.onConfig(first, { ...CONFIG, tools: [getWeather] })!.tools!; + const [secondTool] = middleware.onConfig(second, { ...CONFIG, tools: [getWeather] })!.tools!; + middleware.onGuardToolCall(first, { + toolName: 'get_weather', + toolCallId: 'get_weather_1', + args: {}, + tool: firstTool, + }); + middleware.onGuardToolCall(second, { + toolName: 'get_weather', + toolCallId: 'get_weather_1', + args: {}, + tool: secondTool, + }); + + expect([ + await firstTool!.execute!({}, { toolCallId: 'get_weather_1' }), + await secondTool!.execute!({}, { toolCallId: 'get_weather_1' }), + ]).toEqual(['thread-1', 'thread-2']); + }); +}); + +describe('skipProviderIntegrations', () => { + it('skips every provider integration during a call, also after an await, but not after the call', async () => { + // `@tanstack/openai-base` lets an adapter have a name of its own. + const adapter = skipProviderIntegrations({ + name: 'deepseek', + async structuredOutput() { + await Promise.resolve(); + return [ + _INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME), + _INTERNAL_shouldSkipAiProviderWrapping(ANTHROPIC_AI_INTEGRATION_NAME), + _INTERNAL_shouldSkipAiProviderWrapping(WORKERS_AI_INTEGRATION_NAME), + _INTERNAL_shouldSkipAiProviderWrapping(AWS_BEDROCK_INTEGRATION_NAME), + ]; + }, + }); + + expect(await adapter.structuredOutput()).toEqual([true, true, true, true]); + expect(_INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME)).toBe(false); + }); + + it('skips the integrations in the body of a returned async generator, but not in the code that reads it', async () => { + const adapter = skipProviderIntegrations({ + name: 'openai-chat', + async *chatStream() { + yield _INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME); + await Promise.resolve(); + yield _INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME); + }, + }); + + const skipped: boolean[] = []; + for await (const skippedInAdapter of adapter.chatStream()) { + skipped.push(skippedInAdapter, _INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME)); + } + + expect(skipped).toEqual([true, false, true, false]); + }); + + it('calls the methods on the adapter itself, so that they can read its private fields', async () => { + class Adapter { + public readonly name = 'openai-chat'; + readonly #model = 'gpt-4o'; + + public async structuredOutput(): Promise { + return this.#model; + } + } + + const adapter = skipProviderIntegrations(new Adapter()); + + expect(await adapter.structuredOutput()).toBe('gpt-4o'); + }); + + it('returns the same function for each read of a method, and the constructor and frozen methods as they are', () => { + class Adapter { + public async structuredOutput(): Promise { + return 'ok'; + } + } + const frozen = Object.freeze({ chatStream: () => 'frozen' }); + + const adapter = skipProviderIntegrations(new Adapter()); + + expect(adapter.structuredOutput).toBe(adapter.structuredOutput); + expect(adapter.constructor).toBe(Adapter); + expect(skipProviderIntegrations(frozen).chatStream).toBe(frozen.chatStream); + }); +}); diff --git a/packages/vercel-edge/src/index.ts b/packages/vercel-edge/src/index.ts index d1f92c146338..91949ae846f8 100644 --- a/packages/vercel-edge/src/index.ts +++ b/packages/vercel-edge/src/index.ts @@ -115,6 +115,7 @@ export { instrumentAnthropicAiClient, instrumentGoogleGenAIClient, createLangChainCallbackHandler, + createTanstackAiMiddleware, instrumentLangChainEmbeddings, instrumentStateGraph, instrumentCreateReactAgent,