From 316613b73b5080a0441e7ca3e62ac0d1fd371aab Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 14:55:05 +0000 Subject: [PATCH 1/6] feat(server-utils): Add TanStack AI instrumentation (POC) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Record invoke_agent, chat and execute_tool spans for @tanstack/ai chat() calls through its chat middleware API, and inject that middleware automatically via an orchestrion channel on chat(). Refs #18690 Co-Authored-By: Jan VamPeer Stöcklmair --- .../tanstack-ai/instrument-no-auto.mjs | 11 + .../suites/tracing/tanstack-ai/instrument.mjs | 9 + .../tracing/tanstack-ai/scenario-openai.mjs | 42 +++ .../suites/tracing/tanstack-ai/scenario.mjs | 40 +++ .../suites/tracing/tanstack-ai/test.ts | 240 ++++++++++++++++ packages/node/src/index.ts | 2 + .../src/ai/tanstack-ai/constants.ts | 9 + .../server-utils/src/ai/tanstack-ai/index.ts | 263 ++++++++++++++++++ .../server-utils/src/ai/tanstack-ai/types.ts | 96 +++++++ .../server-utils/src/ai/tanstack-ai/utils.ts | 184 ++++++++++++ packages/server-utils/src/index.ts | 3 + .../server-utils/src/integrations/index.ts | 2 + .../src/integrations/tanstack-ai.ts | 50 ++++ .../server-utils/src/orchestrion/channels.ts | 2 + .../config/channel-integration-definitions.ts | 1 + .../src/orchestrion/config/index.ts | 2 + .../src/orchestrion/config/tanstack-ai.ts | 20 ++ 17 files changed, 976 insertions(+) create mode 100644 dev-packages/node-integration-tests/suites/tracing/tanstack-ai/instrument-no-auto.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/tanstack-ai/instrument.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-openai.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts create mode 100644 packages/server-utils/src/ai/tanstack-ai/constants.ts create mode 100644 packages/server-utils/src/ai/tanstack-ai/index.ts create mode 100644 packages/server-utils/src/ai/tanstack-ai/types.ts create mode 100644 packages/server-utils/src/ai/tanstack-ai/utils.ts create mode 100644 packages/server-utils/src/integrations/tanstack-ai.ts create mode 100644 packages/server-utils/src/orchestrion/config/tanstack-ai.ts diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/instrument-no-auto.mjs b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/instrument-no-auto.mjs new file mode 100644 index 000000000000..0800cae7780b --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/instrument-no-auto.mjs @@ -0,0 +1,11 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +// Without the integration nothing injects the middleware, so only an explicit one records spans. +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + integrations: integrations => integrations.filter(integration => integration.name !== 'TanStackAI'), +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/instrument.mjs new file mode 100644 index 000000000000..46a27dd03b74 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/instrument.mjs @@ -0,0 +1,9 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-openai.mjs b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-openai.mjs new file mode 100644 index 000000000000..d7d459dc034e --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-openai.mjs @@ -0,0 +1,42 @@ +import express from 'express'; +import * as Sentry from '@sentry/node'; +import { chat } from '@tanstack/ai'; +import { createOpenaiChatCompletions } from '@tanstack/ai-openai'; + +// Inlined OpenAI-compatible streaming mock: the runner copies only this file into its temp dir. +function startMockProvider() { + const app = express(); + app.use(express.json()); + app.post('/v1/chat/completions', (req, res) => { + const base = { id: 'chatcmpl-1', object: 'chat.completion.chunk', created: 1, model: 'gpt-4o-2024-08-06' }; + res.setHeader('content-type', 'text/event-stream'); + const send = chunk => res.write(`data: ${JSON.stringify({ ...base, ...chunk })}\n\n`); + send({ choices: [{ index: 0, delta: { role: 'assistant', content: 'Hello!' }, finish_reason: null }] }); + send({ choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] }); + send({ + choices: [], + usage: { + prompt_tokens: 20, + completion_tokens: 5, + total_tokens: 25, + prompt_tokens_details: { cached_tokens: 8 }, + completion_tokens_details: { reasoning_tokens: 2 }, + }, + }); + res.end('data: [DONE]\n\n'); + }); + return new Promise(resolve => { + const server = app.listen(0, () => resolve(server)); + }); +} + +const server = await startMockProvider(); +const adapter = createOpenaiChatCompletions('gpt-4o', 'test-key', { + baseURL: `http://localhost:${server.address().port}/v1`, +}); + +await Sentry.startSpan({ name: 'main' }, async () => { + await chat({ adapter, threadId: 'thread-1', messages: [{ role: 'user', content: 'Hi' }], stream: false }); +}); + +server.close(); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario.mjs new file mode 100644 index 000000000000..3fc4afeb3fbf --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario.mjs @@ -0,0 +1,40 @@ +import * as Sentry from '@sentry/node'; +import { chat, toolDefinition } from '@tanstack/ai'; +import { fakeText } from '@tanstack/ai/testing'; + +// `MIDDLEWARE=manual` passes the middleware explicitly; otherwise the call carries none and relies +// on the integration injecting it. +const middleware = process.env.MIDDLEWARE === 'manual' ? [Sentry.createTanstackAiMiddleware()] : undefined; + +const getWeather = toolDefinition({ + name: 'get_weather', + description: 'Get the current weather for a city.', + inputSchema: { type: 'object', properties: { city: { type: 'string' } } }, +}).server(async ({ city }) => Sentry.startSpan({ name: 'weather-lookup' }, () => ({ city, temperature: 21 }))); + +// The fake estimates prompt caching per thread, so the second model call reports cached tokens. +const adapter = fakeText({ model: 'fake-model', cache: true }); +adapter.setResponses([ + { text: 'Let me check.', toolCalls: [{ name: 'get_weather', input: { city: 'Vienna' } }] }, + { text: 'It is 21 degrees in Vienna.' }, +]); + +const options = { + adapter, + threadId: 'thread-1', + systemPrompts: ['You are a helpful assistant.'], + messages: [{ role: 'user', content: 'What is the weather in Vienna?' }], + tools: [getWeather], + middleware, +}; + +await Sentry.startSpan({ name: 'main' }, async () => { + if (process.env.STREAM) { + // The run only advances while the consumer pulls chunks, as when streaming to a response. + for await (const _chunk of chat({ ...options, stream: true })) { + // drain + } + } else { + await chat({ ...options, stream: false }); + } +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts new file mode 100644 index 000000000000..94230c76587e --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts @@ -0,0 +1,240 @@ +import { + GEN_AI_CONVERSATION_ID, + GEN_AI_INPUT_MESSAGES, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_FINISH_REASONS, + GEN_AI_RESPONSE_MODEL, + GEN_AI_SYSTEM_INSTRUCTIONS, + GEN_AI_TOOL_CALL_ARGUMENTS, + GEN_AI_TOOL_CALL_RESULT, + GEN_AI_TOOL_DEFINITIONS, + GEN_AI_TOOL_NAME, + GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_REASONING_OUTPUT_TOKENS, + GEN_AI_USAGE_TOTAL_TOKENS, +} from '@sentry/conventions/attributes'; +import type { SerializedStreamedSpan } from '@sentry/core'; +import { afterAll, describe, expect } from 'vitest'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; + +const TANSTACK_AI_DEPENDENCIES = { + additionalDependencies: { + '@tanstack/ai': '1.0.0', + '@tanstack/ai-openai': '1.0.0', + }, +}; + +const ORIGIN = 'auto.ai.tanstack_ai'; + +type Spans = SerializedStreamedSpan[]; + +function attr(span: SerializedStreamedSpan | undefined, key: string): unknown { + return span?.attributes?.[key]?.value; +} + +function expectAgentTree(spans: Spans): void { + const main = spans.find(span => span.name === 'main')!; + expect(main.is_segment).toBe(true); + + // Counted: an explicit middleware plus the injected one would double every span. + expect(spans.filter(span => attr(span, 'sentry.origin') === ORIGIN)).toHaveLength(4); + + const agent = spans.find(span => span.name === 'invoke_agent')!; + expect(attr(agent, 'sentry.op')).toBe('gen_ai.invoke_agent'); + expect(attr(agent, GEN_AI_PROVIDER_NAME)).toBe('fake'); + expect(attr(agent, GEN_AI_REQUEST_MODEL)).toBe('fake-model'); + expect(attr(agent, GEN_AI_CONVERSATION_ID)).toBe('thread-1'); + expect(agent.parent_span_id).toBe(main.span_id); + + const chats = spans + .filter(span => span.name === 'chat fake-model') + .sort((a, b) => a.start_timestamp - b.start_timestamp); + expect(chats).toHaveLength(2); + for (const chat of chats) { + expect(attr(chat, 'sentry.op')).toBe('gen_ai.chat'); + expect(attr(chat, GEN_AI_RESPONSE_MODEL)).toBe('fake-model'); + expect(chat.parent_span_id).toBe(agent.span_id); + } + expect(chats.map(chat => attr(chat, GEN_AI_RESPONSE_FINISH_REASONS))).toEqual(['["tool_calls"]', '["stop"]']); + + // The fake caches per thread: the first call writes the whole prompt, the second reads the + // repeated prefix back. + const [first, second] = chats.map(chat => ({ + input: attr(chat, GEN_AI_USAGE_INPUT_TOKENS) as number, + output: attr(chat, GEN_AI_USAGE_OUTPUT_TOKENS) as number, + cacheRead: attr(chat, GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS) as number, + })); + expect(first!.input).toBeGreaterThan(0); + expect(attr(chats[0], GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS)).toBe(first!.input); + expect(second!.cacheRead).toBeGreaterThan(0); + // The agent span sums both model calls. + expect(attr(agent, GEN_AI_USAGE_INPUT_TOKENS)).toBe(first!.input + second!.input); + expect(attr(agent, GEN_AI_USAGE_OUTPUT_TOKENS)).toBe(first!.output + second!.output); + expect(attr(agent, GEN_AI_USAGE_TOTAL_TOKENS)).toBe(first!.input + first!.output + second!.input + second!.output); + expect(attr(agent, GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS)).toBe(second!.cacheRead); + + const tool = spans.find(span => span.name === 'execute_tool get_weather')!; + expect(tool.status).toBe('ok'); + expect(attr(tool, 'sentry.op')).toBe('gen_ai.execute_tool'); + expect(attr(tool, GEN_AI_TOOL_NAME)).toBe('get_weather'); + expect(tool.parent_span_id).toBe(agent.span_id); + + // Work inside the tool parents under the tool span, not beside it. + const lookup = spans.find(span => span.name === 'weather-lookup')!; + expect(lookup.parent_span_id).toBe(tool.span_id); +} + +describe('TanStack AI integration', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + // `@tanstack/ai` is ESM-only. + if (mode === 'cjs') { + return; + } + + test('injects the middleware into chat() calls that carry none', async () => { + await createRunner() + .expect({ + span: container => { + expectAgentTree(container.items); + + const chats = container.items + .filter(span => span.name === 'chat fake-model') + .sort((a, b) => a.start_timestamp - b.start_timestamp); + expect(JSON.parse(String(attr(chats[0], GEN_AI_INPUT_MESSAGES)))).toEqual([ + { role: 'user', parts: [{ type: 'text', content: 'What is the weather in Vienna?' }] }, + ]); + expect(attr(chats[0], GEN_AI_SYSTEM_INSTRUCTIONS)).toBe('You are a helpful assistant.'); + expect(JSON.parse(String(attr(chats[0], GEN_AI_TOOL_DEFINITIONS)))[0]).toMatchObject({ + name: 'get_weather', + description: 'Get the current weather for a city.', + }); + expect(JSON.parse(String(attr(chats[0], GEN_AI_OUTPUT_MESSAGES)))).toEqual([ + { + role: 'assistant', + parts: [ + { type: 'text', content: 'Let me check.' }, + { type: 'tool_call', id: expect.any(String), name: 'get_weather', arguments: '{"city":"Vienna"}' }, + ], + finish_reason: 'tool_calls', + }, + ]); + const secondInput = JSON.parse(String(attr(chats[1], GEN_AI_INPUT_MESSAGES))); + expect(secondInput.map((message: { role: string }) => message.role)).toEqual([ + 'user', + 'assistant', + 'tool', + ]); + + const tool = container.items.find(span => span.name === 'execute_tool get_weather'); + expect(attr(tool, GEN_AI_TOOL_CALL_ARGUMENTS)).toBe('{"city":"Vienna"}'); + expect(attr(tool, GEN_AI_TOOL_CALL_RESULT)).toBe('{"city":"Vienna","temperature":21}'); + }, + }) + .start() + .completed(); + }); + + test('records the same spans when the app consumes the stream', async () => { + await createRunner() + .withEnv({ STREAM: '1' }) + .expect({ span: container => expectAgentTree(container.items) }) + .start() + .completed(); + }); + + test('does not add a second middleware when the app passes one', async () => { + await createRunner() + .withEnv({ MIDDLEWARE: 'manual' }) + .expect({ span: container => expectAgentTree(container.items) }) + .start() + .completed(); + }); + }, + TANSTACK_AI_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument-no-auto.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + test('records the same spans from an explicit middleware alone', async () => { + await createRunner() + .withEnv({ MIDDLEWARE: 'manual' }) + .expect({ span: container => expectAgentTree(container.items) }) + .start() + .completed(); + }); + + test('records nothing without the integration or an explicit middleware', async () => { + await createRunner() + .expect({ + span: container => { + expect(container.items.find(span => span.name === 'main')).toBeDefined(); + expect(container.items.filter(span => attr(span, 'sentry.origin') === ORIGIN)).toHaveLength(0); + }, + }) + .start() + .completed(); + }); + }, + TANSTACK_AI_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-openai.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + test('reports the provider call once, with the HTTP request under the chat span', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items; + const chats = spans.filter(span => attr(span, 'sentry.op') === 'gen_ai.chat'); + // Only ours: the OpenAI SDK integration is skipped for TanStack's model calls. + expect(chats).toHaveLength(1); + const chat = chats[0]!; + expect(attr(chat, 'sentry.origin')).toBe(ORIGIN); + // TanStack reports the adapter's name, which is per API, not the vendor. + expect(attr(chat, GEN_AI_PROVIDER_NAME)).toBe('openai-chat'); + expect(attr(chat, GEN_AI_REQUEST_MODEL)).toBe('gpt-4o'); + expect(attr(chat, GEN_AI_RESPONSE_MODEL)).toBe('gpt-4o-2024-08-06'); + expect(attr(chat, GEN_AI_USAGE_INPUT_TOKENS)).toBe(20); + expect(attr(chat, GEN_AI_USAGE_OUTPUT_TOKENS)).toBe(5); + expect(attr(chat, GEN_AI_USAGE_TOTAL_TOKENS)).toBe(25); + expect(attr(chat, GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS)).toBe(8); + expect(attr(chat, GEN_AI_USAGE_REASONING_OUTPUT_TOKENS)).toBe(2); + + const http = spans.find(span => attr(span, 'sentry.op') === 'http.client'); + expect(http?.parent_span_id).toBe(chat.span_id); + }, + }) + .start() + .completed(); + }); + }, + TANSTACK_AI_DEPENDENCIES, + ); +}); diff --git a/packages/node/src/index.ts b/packages/node/src/index.ts index 130194f72cf5..399162c62373 100644 --- a/packages/node/src/index.ts +++ b/packages/node/src/index.ts @@ -37,6 +37,8 @@ export { openAIIntegration, togetherAIIntegration, typesafeIntegration, + tanstackAiIntegration, + createTanstackAiMiddleware, postgresIntegration, postgresJsIntegration, neonIntegration, diff --git a/packages/server-utils/src/ai/tanstack-ai/constants.ts b/packages/server-utils/src/ai/tanstack-ai/constants.ts new file mode 100644 index 000000000000..5d57d070f58c --- /dev/null +++ b/packages/server-utils/src/ai/tanstack-ai/constants.ts @@ -0,0 +1,9 @@ +export const TANSTACK_AI_INTEGRATION_NAME = 'TanStackAI' as const; + +export const TANSTACK_AI_ORIGIN = 'auto.ai.tanstack_ai'; + +/** + * Marks the middleware so auto-injection can tell whether the app already passed one to `chat()`. + * `Symbol.for` so two copies of the SDK in one process still recognise each other's middleware. + */ +export const TANSTACK_AI_MIDDLEWARE_BRAND = Symbol.for('sentry.tanstack-ai.middleware'); diff --git a/packages/server-utils/src/ai/tanstack-ai/index.ts b/packages/server-utils/src/ai/tanstack-ai/index.ts new file mode 100644 index 000000000000..52cc1643ef18 --- /dev/null +++ b/packages/server-utils/src/ai/tanstack-ai/index.ts @@ -0,0 +1,263 @@ +import type { Span } from '@sentry/core'; +import { + _INTERNAL_shouldSkipAiProviderWrapping, + _INTERNAL_skipAiProviderWrapping, + captureException, + SPAN_STATUS_ERROR, + startInactiveSpan, + stringify, + withActiveSpan, +} from '@sentry/core'; +import { + GEN_AI_AGENT_NAME, + GEN_AI_CONVERSATION_ID, + GEN_AI_OPERATION_NAME, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_FINISH_REASONS, + GEN_AI_RESPONSE_MODEL, + GEN_AI_RESPONSE_STREAMING, + GEN_AI_TOOL_CALL_ARGUMENTS, + GEN_AI_TOOL_CALL_RESULT, + GEN_AI_TOOL_DESCRIPTION, + GEN_AI_TOOL_NAME, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import { ANTHROPIC_AI_INTEGRATION_NAME } from '../anthropic-ai/constants'; +import type { GenAiOptions } from '../core/utils'; +import { getGenAiSpanOp, resolveAIRecordingOptions } from '../core/utils'; +import { GOOGLE_GENAI_INTEGRATION_NAME } from '../google-genai/constants'; +import { OPENAI_INTEGRATION_NAME } from '../openai/constants'; +import { TANSTACK_AI_MIDDLEWARE_BRAND, TANSTACK_AI_ORIGIN } from './constants'; +import type { TanstackAiChatMiddleware, TanstackAiFetch, TanstackAiMiddlewareContext, TanstackAiTool } from './types'; +import type { RunState } from './utils'; +import { addUsage, endChatSpan, setOutputMessages, startChatSpan, usageAttributes } from './utils'; + +export type TanstackAiOptions = GenAiOptions; + +// TanStack's provider adapters call the vendor SDKs that Sentry instruments too, which would report +// each model call a second time beside our `chat` span. +const PROVIDER_INTEGRATIONS = [OPENAI_INTEGRATION_NAME, ANTHROPIC_AI_INTEGRATION_NAME, GOOGLE_GENAI_INTEGRATION_NAME]; + +/** + * Create a `@tanstack/ai` chat middleware that records `invoke_agent`, `chat` and `execute_tool` + * spans for a `chat()` call. + * + * ```js + * chat({ adapter, messages, threadId, middleware: [Sentry.createTanstackAiMiddleware()] }); + * ``` + */ +export function createTanstackAiMiddleware(options: TanstackAiOptions = {}): TanstackAiChatMiddleware { + // Keyed by the middleware context, which TanStack creates once per `chat()` call. + const runs = new WeakMap(); + // Tool call ids are unique per call, so one map serves every run this middleware sees. + const toolSpans = new Map(); + const wrappedTools = new WeakMap(); + + // TanStack runs `execute` outside any of the middleware hooks, so wrapping it is the only way to + // make the tool span active while the tool runs and have the tool's own spans nest under it. + const wrapTool = (tool: TanstackAiTool): TanstackAiTool => { + const known = wrappedTools.get(tool); + if (known) { + return known; + } + const { execute } = tool; + if (typeof execute !== 'function') { + wrappedTools.set(tool, tool); + return tool; + } + const wrapped: TanstackAiTool = { + ...tool, + execute: (args, context) => { + const span = context?.toolCallId ? toolSpans.get(context.toolCallId) : undefined; + return span ? withActiveSpan(span, () => execute.call(tool, args, context)) : execute.call(tool, args, context); + }, + }; + wrappedTools.set(tool, wrapped); + wrappedTools.set(wrapped, wrapped); + return wrapped; + }; + + const endRun = (ctx: TanstackAiMiddlewareContext, error?: unknown): void => { + const run = runs.get(ctx); + if (!run) { + return; + } + runs.delete(ctx); + + for (const id of run.toolCallIds) { + toolSpans.get(id)?.end(); + toolSpans.delete(id); + } + endChatSpan(run, error); + + run.agentSpan.setAttributes(usageAttributes(run.usage)); + if (error !== undefined) { + run.agentSpan.setStatus({ code: SPAN_STATUS_ERROR, message: 'internal_error' }); + } + run.agentSpan.end(); + }; + + return { + name: 'sentry', + [TANSTACK_AI_MIDDLEWARE_BRAND]: true, + + onStart(ctx) { + if (ctx.activity !== 'chat') { + return; + } + if (!PROVIDER_INTEGRATIONS.every(name => _INTERNAL_shouldSkipAiProviderWrapping(name))) { + _INTERNAL_skipAiProviderWrapping(PROVIDER_INTEGRATIONS); + } + + const { recordInputs, recordOutputs } = resolveAIRecordingOptions(options); + const agentName = ctx.subagentName; + runs.set(ctx, { + agentSpan: startInactiveSpan({ + name: agentName ? `invoke_agent ${agentName}` : 'invoke_agent', + op: getGenAiSpanOp('invoke_agent'), + attributes: { + [SENTRY_ORIGIN]: TANSTACK_AI_ORIGIN, + [GEN_AI_OPERATION_NAME]: 'invoke_agent', + [GEN_AI_PROVIDER_NAME]: ctx.provider, + [GEN_AI_REQUEST_MODEL]: ctx.model, + [GEN_AI_CONVERSATION_ID]: ctx.threadId, + [GEN_AI_RESPONSE_STREAMING]: ctx.streaming, + ...(agentName ? { [GEN_AI_AGENT_NAME]: agentName } : {}), + }, + }), + recordInputs, + recordOutputs, + outputText: '', + outputToolCalls: [], + toolCallNames: new Map(), + usage: {}, + toolCallIds: new Set(), + }); + }, + + onConfig(ctx, config) { + const tools = config.tools.some(tool => wrappedTools.get(tool) !== tool) ? config.tools.map(wrapTool) : undefined; + + const run = runs.get(ctx); + // `beforeModel` opens each agent-loop model call; `structuredOutput` the separate final call + // made for `outputSchema`. `init` fires once before `onStart`, with no run yet. + if (!run || (ctx.phase !== 'beforeModel' && ctx.phase !== 'structuredOutput')) { + return tools ? { tools } : undefined; + } + + endChatSpan(run); + const chatSpan = startChatSpan(ctx, config, run); + run.chatSpan = chatSpan; + + return { + ...(tools ? { tools } : {}), + // Chains inside any wrapper set before us, so the provider request (and the HTTP span the + // SDK records for it) runs with the `chat` span active. + wrapFetch: + (next: TanstackAiFetch): TanstackAiFetch => + (input, init) => + withActiveSpan(chatSpan, () => next(input, init)), + }; + }, + + onChunk(ctx, chunk) { + const run = runs.get(ctx); + const span = run?.chatSpan; + if (!run || !span) { + return; + } + + if (chunk.type === 'TEXT_MESSAGE_CONTENT' && run.recordOutputs && chunk.delta) { + run.outputText += chunk.delta; + } else if (chunk.type === 'TOOL_CALL_START' && chunk.toolCallId && chunk.toolCallName) { + run.toolCallNames.set(chunk.toolCallId, chunk.toolCallName); + } else if (chunk.type === 'TOOL_CALL_END' && run.recordOutputs && chunk.toolCallId) { + run.outputToolCalls.push({ + type: 'tool_call', + id: chunk.toolCallId, + name: run.toolCallNames.get(chunk.toolCallId), + arguments: stringify(chunk.input ?? {}), + }); + } else if (chunk.type === 'RUN_FINISHED') { + if (chunk.model) { + span.setAttribute(GEN_AI_RESPONSE_MODEL, chunk.model); + } + if (chunk.finishReason) { + span.setAttribute(GEN_AI_RESPONSE_FINISH_REASONS, stringify([chunk.finishReason])); + } + setOutputMessages(span, run, chunk.finishReason ?? undefined); + } + }, + + onUsage(ctx, usage) { + const run = runs.get(ctx); + if (!run) { + return; + } + run.chatSpan?.setAttributes(usageAttributes(usage)); + run.usage = addUsage(run.usage, usage); + }, + + onBeforeToolCall(ctx, { toolName, toolCallId, args, tool }) { + const run = runs.get(ctx); + // A client tool runs in the browser, not here, so there is nothing to time. + if (!run || (tool && typeof tool.execute !== 'function')) { + return; + } + + run.toolCallIds.add(toolCallId); + toolSpans.set( + toolCallId, + startInactiveSpan({ + name: `execute_tool ${toolName}`, + op: getGenAiSpanOp('execute_tool'), + parentSpan: run.agentSpan, + attributes: { + [SENTRY_ORIGIN]: TANSTACK_AI_ORIGIN, + [GEN_AI_OPERATION_NAME]: 'execute_tool', + [GEN_AI_TOOL_NAME]: toolName, + [GEN_AI_CONVERSATION_ID]: ctx.threadId, + ...(tool?.description ? { [GEN_AI_TOOL_DESCRIPTION]: tool.description } : {}), + ...(run.recordInputs ? { [GEN_AI_TOOL_CALL_ARGUMENTS]: stringify(args) } : {}), + }, + }), + ); + }, + + onAfterToolCall(ctx, { toolCallId, ok, result, error }) { + const span = toolSpans.get(toolCallId); + if (!span) { + return; + } + toolSpans.delete(toolCallId); + const run = runs.get(ctx); + run?.toolCallIds.delete(toolCallId); + + if (run?.recordOutputs && ok && result !== undefined) { + span.setAttribute(GEN_AI_TOOL_CALL_RESULT, typeof result === 'string' ? result : stringify(result)); + } + if (!ok) { + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'internal_error' }); + // TanStack hands a tool failure back to the model as its result, so it never reaches the + // app's error handlers. + withActiveSpan(span, () => { + captureException(error, { mechanism: { type: TANSTACK_AI_ORIGIN, handled: true } }); + }); + } + span.end(); + }, + + onFinish(ctx) { + endRun(ctx); + }, + + onAbort(ctx) { + endRun(ctx); + }, + + onError(ctx, { error }) { + endRun(ctx, error); + }, + }; +} diff --git a/packages/server-utils/src/ai/tanstack-ai/types.ts b/packages/server-utils/src/ai/tanstack-ai/types.ts new file mode 100644 index 000000000000..c50688c4660b --- /dev/null +++ b/packages/server-utils/src/ai/tanstack-ai/types.ts @@ -0,0 +1,96 @@ +/** + * Structural types for the subset of the `@tanstack/ai` chat middleware contract we consume. + * + * Declared locally rather than imported: `@tanstack/ai` is ESM-only and not a dependency of this + * package. Mirrors `ChatMiddleware`, `ChatMiddlewareContext` and `ChatMiddlewareConfig` as of + * `@tanstack/ai` 1.x. + */ + +import type { TANSTACK_AI_MIDDLEWARE_BRAND } from './constants'; + +export type TanstackAiFetch = (input: unknown, init?: unknown) => Promise; + +/** Mirrors `TokenUsage` from `@tanstack/ai-event-client`. */ +export interface TanstackAiUsage { + promptTokens?: number; + completionTokens?: number; + totalTokens?: number; + cost?: number; + promptTokensDetails?: { cachedTokens?: number; cacheWriteTokens?: number }; + completionTokensDetails?: { reasoningTokens?: number }; +} + +export interface TanstackAiTool { + name: string; + description?: string; + inputSchema?: unknown; + execute?: (args: unknown, context?: { toolCallId?: string }) => unknown; +} + +export interface TanstackAiMiddlewareContext { + activity: string; + phase: string; + provider: string; + model: string; + threadId: string; + streaming: boolean; + subagentName?: string; +} + +/** Mirrors `ModelMessage`. */ +export interface TanstackAiModelMessage { + role: 'user' | 'assistant' | 'tool'; + content: string | null | Array<{ type: string; content?: string }>; + toolCalls?: Array<{ id: string; function?: { name: string; arguments: string } }>; + toolCallId?: string; +} + +export interface TanstackAiMiddlewareConfig { + messages: TanstackAiModelMessage[]; + systemPrompts: Array; + tools: TanstackAiTool[]; + modelOptions?: Record; + wrapFetch?: (next: TanstackAiFetch) => TanstackAiFetch; +} + +/** A stream chunk: an AG-UI event discriminated by `type`. */ +export interface TanstackAiChunk { + type: string; + delta?: string; + model?: string; + finishReason?: string | null; + toolCallId?: string; + toolCallName?: string; + input?: unknown; +} + +export interface TanstackAiToolCallHookContext { + toolName: string; + toolCallId: string; + args: unknown; + tool?: TanstackAiTool; +} + +export interface TanstackAiAfterToolCallInfo { + toolCallId: string; + ok: boolean; + result?: unknown; + error?: unknown; +} + +export interface TanstackAiChatMiddleware { + name: string; + [TANSTACK_AI_MIDDLEWARE_BRAND]: true; + onConfig( + ctx: TanstackAiMiddlewareContext, + config: TanstackAiMiddlewareConfig, + ): Partial | undefined; + onStart(ctx: TanstackAiMiddlewareContext): void; + onChunk(ctx: TanstackAiMiddlewareContext, chunk: TanstackAiChunk): void; + onUsage(ctx: TanstackAiMiddlewareContext, usage: TanstackAiUsage): void; + onBeforeToolCall(ctx: TanstackAiMiddlewareContext, hookCtx: TanstackAiToolCallHookContext): void; + onAfterToolCall(ctx: TanstackAiMiddlewareContext, info: TanstackAiAfterToolCallInfo): void; + onFinish(ctx: TanstackAiMiddlewareContext): void; + onAbort(ctx: TanstackAiMiddlewareContext): void; + onError(ctx: TanstackAiMiddlewareContext, info: { error: unknown }): void; +} diff --git a/packages/server-utils/src/ai/tanstack-ai/utils.ts b/packages/server-utils/src/ai/tanstack-ai/utils.ts new file mode 100644 index 000000000000..f9438624cf57 --- /dev/null +++ b/packages/server-utils/src/ai/tanstack-ai/utils.ts @@ -0,0 +1,184 @@ +import type { Span } from '@sentry/core'; +import { SPAN_STATUS_ERROR, startInactiveSpan, stringify } from '@sentry/core'; +import { + GEN_AI_CONVERSATION_ID, + GEN_AI_COST_TOTAL_TOKENS, + GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MAX_TOKENS, + GEN_AI_REQUEST_MODEL, + GEN_AI_REQUEST_TEMPERATURE, + GEN_AI_REQUEST_TOP_P, + GEN_AI_RESPONSE_STREAMING, + GEN_AI_SYSTEM_INSTRUCTIONS, + GEN_AI_TOOL_DEFINITIONS, + GEN_AI_USAGE_REASONING_OUTPUT_TOKENS, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import { getGenAiSpanOp, getTokenUsageAttributes } from '../core/utils'; +import { TANSTACK_AI_ORIGIN } from './constants'; +import type { + TanstackAiMiddlewareConfig, + TanstackAiMiddlewareContext, + TanstackAiModelMessage, + TanstackAiUsage, +} from './types'; + +/** What the middleware tracks for one `chat()` call. */ +export interface RunState { + agentSpan: Span; + chatSpan?: Span; + recordInputs: boolean; + recordOutputs: boolean; + outputText: string; + outputToolCalls: Array>; + // `TOOL_CALL_END` carries the parsed input but not the name, which only `TOOL_CALL_START` has. + toolCallNames: Map; + usage: TanstackAiUsage; + toolCallIds: Set; +} + +export function startChatSpan( + ctx: TanstackAiMiddlewareContext, + config: TanstackAiMiddlewareConfig, + run: RunState, +): Span { + const attributes: Record = { + [SENTRY_ORIGIN]: TANSTACK_AI_ORIGIN, + [GEN_AI_OPERATION_NAME]: 'chat', + [GEN_AI_PROVIDER_NAME]: ctx.provider, + [GEN_AI_REQUEST_MODEL]: ctx.model, + [GEN_AI_CONVERSATION_ID]: ctx.threadId, + [GEN_AI_RESPONSE_STREAMING]: ctx.streaming, + ...requestAttributes(config.modelOptions), + }; + + if (config.tools.length) { + attributes[GEN_AI_TOOL_DEFINITIONS] = stringify( + config.tools.map(({ name, description, inputSchema }) => ({ name, description, inputSchema })), + ); + } + if (run.recordInputs) { + attributes[GEN_AI_INPUT_MESSAGES] = stringify(config.messages.map(toGenAiMessage)); + if (config.systemPrompts.length) { + attributes[GEN_AI_SYSTEM_INSTRUCTIONS] = config.systemPrompts + .map(prompt => (typeof prompt === 'string' ? prompt : prompt.content)) + .join('\n'); + } + } + + run.outputText = ''; + run.outputToolCalls = []; + run.toolCallNames.clear(); + return startInactiveSpan({ + name: `chat ${ctx.model}`, + op: getGenAiSpanOp('chat'), + parentSpan: run.agentSpan, + attributes, + }); +} + +export function endChatSpan(run: RunState, error?: unknown): void { + const span = run.chatSpan; + if (!span) { + return; + } + run.chatSpan = undefined; + if (error !== undefined) { + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'internal_error' }); + } + span.end(); +} + +export function setOutputMessages(span: Span, run: RunState, finishReason: string | undefined): void { + const parts = [...(run.outputText ? [{ type: 'text', content: run.outputText }] : []), ...run.outputToolCalls]; + if (parts.length) { + span.setAttribute(GEN_AI_OUTPUT_MESSAGES, stringify([{ role: 'assistant', parts, finish_reason: finishReason }])); + } +} + +/** TanStack's `ModelMessage` in the gen_ai conventions shape (`{ role, parts }`). */ +function toGenAiMessage(message: TanstackAiModelMessage): Record { + const parts: Array> = []; + const { content } = message; + + if (message.role === 'tool') { + parts.push({ type: 'tool_call_response', id: message.toolCallId, result: content }); + } else if (typeof content === 'string') { + parts.push({ type: 'text', content }); + } else if (Array.isArray(content)) { + for (const part of content) { + // Media parts carry inline bytes or provider handles; keep only their kind. + parts.push(part.type === 'text' ? { type: 'text', content: part.content } : { type: part.type }); + } + } + for (const toolCall of message.toolCalls ?? []) { + parts.push({ + type: 'tool_call', + id: toolCall.id, + name: toolCall.function?.name, + arguments: toolCall.function?.arguments, + }); + } + return { role: message.role, parts }; +} + +/** Sampling options live in the provider-native `modelOptions`, so accept the common spellings. */ +function requestAttributes(modelOptions: Record = {}): Record { + const pick = (...keys: string[]): number | undefined => { + for (const key of keys) { + const value = modelOptions[key]; + if (typeof value === 'number') { + return value; + } + } + return undefined; + }; + return { + [GEN_AI_REQUEST_TEMPERATURE]: pick('temperature'), + [GEN_AI_REQUEST_TOP_P]: pick('top_p', 'topP'), + [GEN_AI_REQUEST_MAX_TOKENS]: pick('max_tokens', 'maxTokens', 'max_output_tokens', 'maxOutputTokens'), + }; +} + +export function usageAttributes(usage: TanstackAiUsage): Record { + const attributes = getTokenUsageAttributes( + usage.promptTokens, + usage.completionTokens, + usage.promptTokensDetails?.cacheWriteTokens, + usage.promptTokensDetails?.cachedTokens, + usage.totalTokens, + ); + if (typeof usage.completionTokensDetails?.reasoningTokens === 'number') { + attributes[GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] = usage.completionTokensDetails.reasoningTokens; + } + // The provider's own figure (e.g. OpenRouter) includes cache discounts and gateway markup that a + // price table cannot reproduce. + if (typeof usage.cost === 'number') { + attributes[GEN_AI_COST_TOTAL_TOKENS] = usage.cost; + } + return attributes; +} + +export function addUsage(total: TanstackAiUsage, usage: TanstackAiUsage): TanstackAiUsage { + const add = (a: number | undefined, b: number | undefined): number | undefined => + a === undefined && b === undefined ? undefined : (a ?? 0) + (b ?? 0); + return { + promptTokens: add(total.promptTokens, usage.promptTokens), + completionTokens: add(total.completionTokens, usage.completionTokens), + totalTokens: add(total.totalTokens, usage.totalTokens), + cost: add(total.cost, usage.cost), + promptTokensDetails: { + cachedTokens: add(total.promptTokensDetails?.cachedTokens, usage.promptTokensDetails?.cachedTokens), + cacheWriteTokens: add(total.promptTokensDetails?.cacheWriteTokens, usage.promptTokensDetails?.cacheWriteTokens), + }, + completionTokensDetails: { + reasoningTokens: add( + total.completionTokensDetails?.reasoningTokens, + usage.completionTokensDetails?.reasoningTokens, + ), + }, + }; +} diff --git a/packages/server-utils/src/index.ts b/packages/server-utils/src/index.ts index dacf7ff16b7f..d73aafc5899a 100644 --- a/packages/server-utils/src/index.ts +++ b/packages/server-utils/src/index.ts @@ -63,6 +63,9 @@ export { mistralAIIntegration } from './integrations/mistral'; export { groqIntegration } from './integrations/groq'; export { togetherAIIntegration } from './integrations/together-ai'; export { typesafeIntegration } from './integrations/typesafe'; +export { tanstackAiIntegration } from './integrations/tanstack-ai'; +export { createTanstackAiMiddleware } from './ai/tanstack-ai'; +export type { TanstackAiOptions } from './ai/tanstack-ai'; export { mysqlIntegration } from './integrations/mysql'; export { mysql2Integration } from './integrations/mysql2'; export { openAIIntegration } from './integrations/openai'; diff --git a/packages/server-utils/src/integrations/index.ts b/packages/server-utils/src/integrations/index.ts index 9643f95250ff..9b2f08243e43 100644 --- a/packages/server-utils/src/integrations/index.ts +++ b/packages/server-utils/src/integrations/index.ts @@ -26,6 +26,7 @@ import { mistralAIIntegration } from './mistral'; import { groqIntegration } from './groq'; import { togetherAIIntegration } from './together-ai'; import { typesafeIntegration } from './typesafe'; +import { tanstackAiIntegration } from './tanstack-ai'; import { postgresJsIntegration } from './postgres-js'; import { neonIntegration } from './neon'; import { firebaseIntegration } from './firebase'; @@ -63,6 +64,7 @@ export function getTracingIntegrations(): Integration[] { langGraphIntegration(), mastraIntegration(), piDurableIntegration(), + tanstackAiIntegration(), vercelAIIntegration(), openAIIntegration(), anthropicAIIntegration(), diff --git a/packages/server-utils/src/integrations/tanstack-ai.ts b/packages/server-utils/src/integrations/tanstack-ai.ts new file mode 100644 index 000000000000..da38f1f50f74 --- /dev/null +++ b/packages/server-utils/src/integrations/tanstack-ai.ts @@ -0,0 +1,50 @@ +import * as diagnosticsChannel from '../utils/diagnosticsChannel'; +import type { IntegrationFn } from '@sentry/core'; +import { defineIntegration, isObjectLike } from '@sentry/core'; +import type { TanstackAiOptions } from '../ai/tanstack-ai'; +import { createTanstackAiMiddleware } from '../ai/tanstack-ai'; +import { TANSTACK_AI_INTEGRATION_NAME, TANSTACK_AI_MIDDLEWARE_BRAND } from '../ai/tanstack-ai/constants'; +import { CHANNELS } from '../orchestrion/channels'; +import { tanstackAiModuleNames } from '../orchestrion/config/tanstack-ai'; +import { invokeOrchestrionInstrumentation } from '../orchestrion/instrumentation'; +import type { OrchestrionChannelContext } from '../orchestrion/types'; +import { safeChannelCallback } from '../tracing-channel'; + +const _tanstackAiIntegration = ((options: TanstackAiOptions = {}) => { + return { + name: TANSTACK_AI_INTEGRATION_NAME, + setup(client) { + // Injecting the middleware opens no span; the middleware parents its spans explicitly. + invokeOrchestrionInstrumentation(client, tanstackAiModuleNames, injectMiddleware, [options], { + requiresTracingChannelBinding: false, + }); + }, + }; +}) satisfies IntegrationFn; + +/** Add our middleware to every `chat()` call that does not already carry one. */ +function injectMiddleware(options: TanstackAiOptions): void { + const middleware = createTanstackAiMiddleware(options); + + diagnosticsChannel.tracingChannel(CHANNELS.TANSTACK_AI_CHAT).start.subscribe(message => { + safeChannelCallback(() => { + const data = message as OrchestrionChannelContext; + const chatOptions = data.arguments[0]; + if (!isObjectLike(chatOptions)) { + return; + } + const existing = chatOptions.middleware; + // `chat()` throws on a non-array itself; leave that error to it. + if (existing !== undefined && !Array.isArray(existing)) { + return; + } + if (existing?.some(entry => isObjectLike(entry) && entry[TANSTACK_AI_MIDDLEWARE_BRAND])) { + return; + } + // A copy, so options the app reuses across calls do not collect one middleware per call. + data.arguments[0] = { ...chatOptions, middleware: [...(existing ?? []), middleware] }; + }); + }); +} + +export const tanstackAiIntegration = defineIntegration(_tanstackAiIntegration); diff --git a/packages/server-utils/src/orchestrion/channels.ts b/packages/server-utils/src/orchestrion/channels.ts index b5919b9eb071..8371669d9831 100644 --- a/packages/server-utils/src/orchestrion/channels.ts +++ b/packages/server-utils/src/orchestrion/channels.ts @@ -35,6 +35,7 @@ import { redisChannels } from './config/redis'; import { remixChannels } from './config/remix'; import { tediousChannels } from './config/tedious'; import { togetherAiChannels } from './config/together-ai'; +import { tanstackAiChannels } from './config/tanstack-ai'; import { typesafeChannels } from './config/typesafe'; import { vercelAiChannels } from './config/vercel-ai'; @@ -92,6 +93,7 @@ export const CHANNELS = { ...remixChannels, ...tediousChannels, ...togetherAiChannels, + ...tanstackAiChannels, ...typesafeChannels, ...vercelAiChannels, } as const; diff --git a/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts b/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts index 46cd69d31bf9..c75b700f8262 100644 --- a/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts +++ b/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts @@ -35,6 +35,7 @@ export const CHANNEL_INTEGRATION_DEFINITIONS = [ { exportName: 'groqIntegration', modules: ['groq-sdk'] }, { exportName: 'togetherAIIntegration', modules: ['together-ai'] }, { exportName: 'typesafeIntegration', modules: ['@typesafe-ai/sdk'] }, + { exportName: 'tanstackAiIntegration', modules: ['@tanstack/ai'] }, { exportName: 'vercelAIIntegration', modules: ['ai'] }, { exportName: 'langChainIntegration', diff --git a/packages/server-utils/src/orchestrion/config/index.ts b/packages/server-utils/src/orchestrion/config/index.ts index e60810315420..aa6901bcf14e 100644 --- a/packages/server-utils/src/orchestrion/config/index.ts +++ b/packages/server-utils/src/orchestrion/config/index.ts @@ -42,6 +42,7 @@ import { remixConfig } from './remix'; import { remixV3Config } from './remix-v3'; import { tediousConfig } from './tedious'; import { togetherAiConfig } from './together-ai'; +import { tanstackAiConfig } from './tanstack-ai'; import { typesafeConfig } from './typesafe'; import { vercelAiConfig } from './vercel-ai'; // Kept sorted alphabetically by module so concurrent additions insert at different @@ -101,6 +102,7 @@ export const SENTRY_INSTRUMENTATIONS: InstrumentationConfig[] = [ ...remixV3Config, ...tediousConfig, ...togetherAiConfig, + ...tanstackAiConfig, ...typesafeConfig, ...vercelAiConfig, ]; diff --git a/packages/server-utils/src/orchestrion/config/tanstack-ai.ts b/packages/server-utils/src/orchestrion/config/tanstack-ai.ts new file mode 100644 index 000000000000..fda0f7eda09f --- /dev/null +++ b/packages/server-utils/src/orchestrion/config/tanstack-ai.ts @@ -0,0 +1,20 @@ +import type { InstrumentationConfig } from '../apmTypes'; +import { getModuleNames } from './module-names'; + +// `chat()` takes its middleware per call and `@tanstack/ai` has no global registry, so the only way +// in without user code is to rewrite the options of each call. `chat` is a plain function declaration +// in a stable, unhashed file (`dist/esm/activities/chat/index.js` in 1.0.0). It returns a stream or a +// promise depending on `stream`, so it is wrapped `Sync`: we only need the `start` event. +export const tanstackAiConfig = [ + { + channelName: 'chat', + module: { name: '@tanstack/ai', versionRange: '>=1.0.0 <2.0.0', filePath: 'dist/esm/activities/chat/index.js' }, + functionQuery: { functionName: 'chat', kind: 'Sync' }, + }, +] satisfies InstrumentationConfig[]; + +export const tanstackAiModuleNames = getModuleNames(tanstackAiConfig); + +export const tanstackAiChannels = { + TANSTACK_AI_CHAT: 'orchestrion:@tanstack/ai:chat', +} as const; From 42a7338b5eca5eeb55f92d01a7948dea8964bc0a Mon Sep 17 00:00:00 2001 From: JPeer264 Date: Sun, 25 Oct 2026 12:10:00 +0100 Subject: [PATCH 2/6] fixup! feat(server-utils): Add TanStack AI instrumentation (POC) --- .../gen-ai-libraries/package.json | 4 +- .../gen-ai-libraries/pnpm-workspace.yaml | 1 + .../gen-ai-libraries/src/libraries.ts | 51 +- .../gen-ai-libraries/tests/tools.test.ts | 70 ++- .../gen-ai-libraries/tests/utils.ts | 22 +- .../aws-serverless/bedrock/scenario.mjs | 13 +- .../suites/aws-serverless/bedrock/test.ts | 20 + .../tanstack-ai/scenario-interrupt.mjs | 28 + .../tracing/tanstack-ai/scenario-openai.mjs | 27 +- .../tanstack-ai/scenario-provider-error.mjs | 31 ++ .../tracing/tanstack-ai/scenario-retry.mjs | 19 + .../suites/tracing/tanstack-ai/scenario.mjs | 15 +- .../suites/tracing/tanstack-ai/test.ts | 173 +++++- packages/astro/src/index.server.ts | 2 + packages/aws-serverless/src/index.ts | 2 + packages/bun/src/index.ts | 2 + packages/cloudflare/src/index.ts | 2 +- packages/core/src/index.ts | 1 + packages/core/src/utils/ai/providerSkip.ts | 31 +- .../test/lib/utils/ai/providerSkip.test.ts | 35 ++ packages/deno/src/index.ts | 2 + packages/elysia/src/index.ts | 2 + packages/google-cloud-serverless/src/index.ts | 2 + packages/server-utils/src/ai/index.ts | 2 + .../server-utils/src/ai/tanstack-ai/index.ts | 260 +++++---- .../server-utils/src/ai/tanstack-ai/types.ts | 26 +- .../server-utils/src/ai/tanstack-ai/utils.ts | 153 +++++- .../src/integrations/aws-sdk/constants.ts | 8 + .../src/integrations/aws-sdk/index.ts | 10 +- .../src/integrations/tanstack-ai.ts | 16 +- .../test/ai/lib/tracing/tanstack-ai.test.ts | 514 ++++++++++++++++++ packages/vercel-edge/src/index.ts | 1 + 32 files changed, 1370 insertions(+), 175 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-interrupt.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-provider-error.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-retry.mjs create mode 100644 packages/server-utils/test/ai/lib/tracing/tanstack-ai.test.ts diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json index 2dbf1013a228..4e5f9fb6cf6b 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json @@ -1,6 +1,6 @@ { "name": "gen-ai-libraries", - "description": "Real gen_ai spans for every instrumented AI library (OpenAI, Anthropic, Mistral, Together, Vercel AI), each driven through OpenRouter with a chat query and a tool call, on Node and on Cloudflare", + "description": "Real gen_ai spans for every instrumented AI library (OpenAI, Anthropic, Mistral, Together, Vercel AI, TanStack AI), each driven through OpenRouter with a chat query and a tool call, on Node and on Cloudflare", "version": "1.0.0", "private": true, "type": "module", @@ -21,6 +21,8 @@ "@openrouter/ai-sdk-provider": "~3.0.0", "@sentry/cloudflare": "file:../../packed/sentry-cloudflare-packed.tgz", "@sentry/node": "file:../../packed/sentry-node-packed.tgz", + "@tanstack/ai": "^1.0.0", + "@tanstack/ai-openrouter": "^1.0.0", "ai": "~7.0.97", "express": "^4.21.2", "openai": "5.18.1", diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/pnpm-workspace.yaml b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/pnpm-workspace.yaml index fd29a82085e0..6f72f0ab7167 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/pnpm-workspace.yaml +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/pnpm-workspace.yaml @@ -1,4 +1,5 @@ allowBuilds: + '@openrouter/sdk': false esbuild: true workerd: true minimumReleaseAge: 0 diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts index 1b1490e7e9b4..abcc4ae7adf9 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts @@ -19,6 +19,8 @@ import Anthropic from '@anthropic-ai/sdk'; import { createOpenRouter } from '@openrouter/ai-sdk-provider'; import { Mistral } from '@mistralai/mistralai'; +import { chat, toolDefinition } from '@tanstack/ai'; +import { createOpenRouterText } from '@tanstack/ai-openrouter'; import OpenAI from 'openai'; import Together from 'together-ai'; import { generateText, tool } from 'ai'; @@ -54,9 +56,9 @@ export interface Library { id: string; /** The op of the model-call span; asserted by the tests. */ op: 'gen_ai.chat' | 'gen_ai.generate_content'; - /** `gen_ai.provider.name` for the direct-SDK libraries; unset for the Vercel AI SDK. */ + /** `gen_ai.provider.name` of the model-call span; unset for the Vercel AI SDK. */ provider?: string; - /** `sentry.origin` for the direct-SDK libraries; unset for the Vercel AI SDK. */ + /** `sentry.origin` of the model-call span; unset for the Vercel AI SDK. */ origin?: string; chat: (apiKey: string) => Promise; tools: (apiKey: string) => Promise; @@ -216,4 +218,49 @@ export const libraries: Library[] = [ return result.toolCalls ?? []; }, }, + + { + id: 'tanstack-ai', + // TanStack AI runs an agent loop like the Vercel AI SDK: an `invoke_agent` span with one `chat` span + // per model call, and an `execute_tool` span for each tool it runs. + op: 'gen_ai.chat', + provider: 'openrouter', + origin: 'auto.ai.tanstack_ai', + chat: async apiKey => { + const { text } = await chat({ + adapter: createOpenRouterText(MODEL, apiKey), + threadId: crypto.randomUUID(), + systemPrompts: [SYSTEM], + messages: [{ role: 'user', content: CHAT_PROMPT }], + modelOptions: { temperature: 0 }, + stream: false, + }); + return text; + }, + tools: async apiKey => { + const getWeather = toolDefinition({ + name: 'get_weather', + description: 'Get the current weather for a city.', + inputSchema: z.object({ city: z.string().describe('The city name') }), + }).server(async ({ city }) => `It is sunny in ${city}.`); + + const { text } = await chat({ + adapter: createOpenRouterText(MODEL, apiKey), + threadId: crypto.randomUUID(), + messages: [{ role: 'user', content: WEATHER_PROMPT }], + tools: [getWeather], + // A `toolChoice` option would apply to every model call of the loop, so the model could never + // answer. Forcing only the first call keeps the turn at one tool call and one answer. + middleware: [ + { + name: 'force-first-tool-call', + onConfig: ctx => + ctx.phase === 'beforeModel' && ctx.iteration === 0 ? { toolChoice: 'required' } : undefined, + }, + ], + stream: false, + }); + return text; + }, + }, ]; diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/tools.test.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/tools.test.ts index d2ff269ca0a7..3036fe40f456 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/tools.test.ts +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/tools.test.ts @@ -5,7 +5,7 @@ import { attr, describeTree, hasRecordedToolCalls, isExecuteToolSpan, isModelCal // The direct-SDK libraries record the model's tool call on the chat span's `gen_ai.response.tool_calls` // attribute, which only exists when the model actually returned tool calls. -const DIRECT_SDK_LIBRARIES = LIBRARIES.filter(library => library.provider); +const DIRECT_SDK_LIBRARIES = LIBRARIES.filter(library => !library.executesTools); for (const library of DIRECT_SDK_LIBRARIES) { test(`${library.id}: the model's tool call is recorded on the ${library.op} span`, async ({ baseURL }) => { @@ -28,20 +28,76 @@ for (const library of DIRECT_SDK_LIBRARIES) { }); } -// The Vercel AI SDK executes the tool and emits a dedicated `gen_ai.execute_tool` span instead. -test('vercel-ai: the tool call is captured as a gen_ai.execute_tool span', async ({ baseURL }) => { +// The libraries that run an agent loop execute the tool and emit a dedicated `gen_ai.execute_tool` span. +for (const library of LIBRARIES.filter(library => library.executesTools)) { + test(`${library.id}: the tool call is captured as a gen_ai.execute_tool span`, async ({ baseURL }) => { + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => spansOfTrace.some(isExecuteToolSpan) && spansOfTrace.some(span => isModelCallSpan(span, library)), + ); + + const response = await fetch(`${baseURL}/${library.id}/tools`); + expect(response.status).toBe(200); + + const spans = await spansPromise; + const toolSpan = spans.find(isExecuteToolSpan); + + expect(toolSpan, `expected a gen_ai.execute_tool span in:\n${describeTree(spans)}`).toBeDefined(); + expect(attr(toolSpan!, 'gen_ai.tool.name')).toBe('get_weather'); + }); +} + +// A real provider request goes over HTTP, which a fake adapter cannot show. TanStack AI wraps that fetch +// so the request nests under the `chat` span of its model call. +test('tanstack-ai: the agent loop nests its model calls, tool call and provider requests', async ({ baseURL }) => { + const library = LIBRARIES.find(library => library.id === 'tanstack-ai')!; const spansPromise = collectStreamedSpans( APP, spansOfTrace => - spansOfTrace.some(isExecuteToolSpan) && spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.generate_content'), + spansOfTrace.some(isExecuteToolSpan) && + spansOfTrace.filter(span => isModelCallSpan(span, library)).length >= 2 && + spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.invoke_agent'), ); - const response = await fetch(`${baseURL}/vercel-ai/tools`); + const response = await fetch(`${baseURL}/tanstack-ai/tools`); expect(response.status).toBe(200); const spans = await spansPromise; + const tree = describeTree(spans); + const agent = spans.find(span => getSpanOp(span) === 'gen_ai.invoke_agent'); + const chats = spans.filter(span => isModelCallSpan(span, library)); const toolSpan = spans.find(isExecuteToolSpan); - expect(toolSpan, `expected a gen_ai.execute_tool span in:\n${describeTree(spans)}`).toBeDefined(); - expect(attr(toolSpan!, 'gen_ai.tool.name')).toBe('get_weather'); + expect(agent, tree).toBeDefined(); + expect(attr(agent!, 'gen_ai.provider.name')).toBe('openrouter'); + expect(chats, tree).toHaveLength(2); + expect( + chats.every(span => span.parent_span_id === agent!.span_id), + tree, + ).toBe(true); + expect(toolSpan!.parent_span_id, tree).toBe(agent!.span_id); + + // The first call returns the tool call, the second answers with the tool result. + const [first, second] = [...chats].sort((a, b) => a.start_timestamp - b.start_timestamp); + expect(attr(first!, 'gen_ai.response.finish_reasons')).toBe('["tool_calls"]'); + expect(attr(first!, 'gen_ai.output.messages')).toContain('get_weather'); + expect(attr(second!, 'gen_ai.response.finish_reasons')).toBe('["stop"]'); + expect(typeof attr(second!, 'gen_ai.response.id')).toBe('string'); + + // The agent span sums the usage of both model calls. OpenRouter reports the cost itself, also for a BYOK + // key, where its own fee is 0. + expect(attr(agent!, 'gen_ai.usage.input_tokens')).toBe( + (attr(first!, 'gen_ai.usage.input_tokens') as number) + (attr(second!, 'gen_ai.usage.input_tokens') as number), + ); + expect(attr(agent!, 'gen_ai.cost.total_tokens')).toBeGreaterThan(0); + + const providerRequests = spans.filter( + span => + getSpanOp(span) === 'http.client' && String(attr(span, 'server.address') ?? span.name).includes('openrouter'), + ); + expect(providerRequests, tree).toHaveLength(2); + expect( + providerRequests.every(span => chats.some(chat => chat.span_id === span.parent_span_id)), + tree, + ).toBe(true); }); diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/utils.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/utils.ts index 4861a01c5334..51baab904857 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/utils.ts +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/utils.ts @@ -8,11 +8,14 @@ export interface LibraryUnderTest { /** The op of the model-call span: `gen_ai.chat` for the direct SDKs, `gen_ai.generate_content` for * the Vercel AI SDK. */ op: string; - /** `gen_ai.provider.name` for the direct-SDK libraries; unset for the Vercel AI SDK, whose provider - * name comes from the model id and is not asserted. */ + /** `gen_ai.provider.name` of the model-call span; unset for the Vercel AI SDK, whose provider name + * comes from the model id and is not asserted. */ provider?: string; - /** `sentry.origin` for the direct-SDK libraries; unset for the Vercel AI SDK. */ + /** `sentry.origin` of the model-call span; unset for the Vercel AI SDK. */ origin?: string; + /** The library runs the tool itself and records a `gen_ai.execute_tool` span for it, instead of only + * returning the model's tool call to the caller. */ + executesTools?: boolean; } export const LIBRARIES: LibraryUnderTest[] = [ @@ -20,24 +23,27 @@ export const LIBRARIES: LibraryUnderTest[] = [ { id: 'together', op: 'gen_ai.chat', provider: 'together_ai', origin: 'auto.ai.together_ai' }, { id: 'mistral', op: 'gen_ai.chat', provider: 'mistralai', origin: 'auto.ai.mistralai' }, { id: 'anthropic', op: 'gen_ai.chat', provider: 'anthropic', origin: 'auto.ai.anthropic' }, - { id: 'vercel-ai', op: 'gen_ai.generate_content' }, + { id: 'vercel-ai', op: 'gen_ai.generate_content', executesTools: true }, + { id: 'tanstack-ai', op: 'gen_ai.chat', provider: 'openrouter', origin: 'auto.ai.tanstack_ai', executesTools: true }, ]; export const attr = (span: SerializedStreamedSpan, key: string): unknown => span.attributes?.[key]?.value; /** - * The model-call span for a library. The four direct SDKs all report `gen_ai.chat`, so the provider - * name is what tells them apart — without it, a leftover span from another library's request could be + * The model-call span for a library. Most libraries report `gen_ai.chat`, so the provider name and origin + * are what tell them apart. Without them, a leftover span from another library's request could be * mistaken for this one's. */ export const isModelCallSpan = (span: SerializedStreamedSpan, library: LibraryUnderTest): boolean => - getSpanOp(span) === library.op && (!library.provider || attr(span, 'gen_ai.provider.name') === library.provider); + getSpanOp(span) === library.op && + (!library.provider || attr(span, 'gen_ai.provider.name') === library.provider) && + (!library.origin || attr(span, 'sentry.origin') === library.origin); /** A span that recorded the model returning tool calls on a chat-completions request (direct SDKs). */ export const hasRecordedToolCalls = (span: SerializedStreamedSpan): boolean => typeof attr(span, 'gen_ai.response.tool_calls') === 'string'; -/** The dedicated tool-execution span the Vercel AI SDK emits. */ +/** The dedicated tool-execution span the libraries that run tools emit. */ export const isExecuteToolSpan = (span: SerializedStreamedSpan): boolean => getSpanOp(span) === 'gen_ai.execute_tool'; /** Whether a trace shows any evidence of a tool call, used to tell a chat request from a tools one. */ diff --git a/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/scenario.mjs b/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/scenario.mjs index c1705ff0d403..a8238cdbdb5c 100644 --- a/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/scenario.mjs +++ b/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/scenario.mjs @@ -1,4 +1,5 @@ import * as Sentry from '@sentry/aws-serverless'; +import { _INTERNAL_withAiProviderWrappingSkipped } from '@sentry/core'; import { BedrockRuntimeClient, ConverseCommand, InvokeModelCommand } from '@aws-sdk/client-bedrock-runtime'; // Force the HTTP/1 handler so `nock` can intercept the request instead of hitting real AWS. import { NodeHttpHandler } from '@smithy/node-http-handler'; @@ -76,8 +77,16 @@ async function invokeModel() { async function run() { await Sentry.startSpan({ name: 'Test Transaction' }, async () => { - await converse(); - await invokeModel(); + if (process.env.SKIP_BEDROCK) { + // What an AI framework integration does around the Bedrock calls of its adapters, which it records itself. + await _INTERNAL_withAiProviderWrappingSkipped(['AwsBedrock'], async () => { + await converse(); + await invokeModel(); + }); + } else { + await converse(); + await invokeModel(); + } }); } diff --git a/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/test.ts b/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/test.ts index a7bc374499b4..0709b7f647b7 100644 --- a/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/test.ts +++ b/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/test.ts @@ -65,6 +65,26 @@ describe('awsIntegration - Bedrock', () => { test('auto-instruments Bedrock Converse and InvokeModel', { timeout: 90_000 }, async () => { await createTestRunner().ignore('event').expect({ span: assertBedrockSpans }).start().completed(); }); + + test( + 'records no Bedrock spans while an AI framework integration skips Bedrock', + { timeout: 90_000 }, + async () => { + await createTestRunner() + .withEnv({ SKIP_BEDROCK: '1' }) + .ignore('event') + .expect({ + span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'Test Transaction')).toBeDefined(); + expect( + container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.aws.aws_sdk'), + ).toEqual([]); + }, + }) + .start() + .completed(); + }, + ); }, { additionalDependencies: { '@aws-sdk/client-bedrock-runtime': '^3.1046.0' } }, ); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-interrupt.mjs b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-interrupt.mjs new file mode 100644 index 000000000000..a573bea99af7 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-interrupt.mjs @@ -0,0 +1,28 @@ +import * as Sentry from '@sentry/node'; +import { chat, defineInterrupt } from '@tanstack/ai'; +import { fakeText } from '@tanstack/ai/testing'; + +// A middleware pauses the run at the interrupt boundary that `INTERRUPT` names: `beforeModel` or `afterModel`. +const confirm = defineInterrupt({ id: 'confirm' }); +const pauseRun = { + name: 'confirm', + onInterruptBoundary: ctx => + ctx.phase === process.env.INTERRUPT + ? { interrupts: [confirm.interrupt({ key: 'confirm-1', reason: 'review', message: 'Please confirm.' })] } + : undefined, +}; + +const adapter = fakeText({ model: 'fake-model' }); +adapter.setResponses([{ text: 'It is 21 degrees in Vienna.' }]); + +await Sentry.startSpan({ name: 'main' }, async () => { + for await (const _chunk of chat({ + adapter, + threadId: 'thread-1', + messages: [{ role: 'user', content: 'What is the weather in Vienna?' }], + interrupts: [confirm], + middleware: [pauseRun], + })) { + // drain + } +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-openai.mjs b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-openai.mjs index d7d459dc034e..d9dc55fa9a59 100644 --- a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-openai.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-openai.mjs @@ -2,6 +2,7 @@ import express from 'express'; import * as Sentry from '@sentry/node'; import { chat } from '@tanstack/ai'; import { createOpenaiChatCompletions } from '@tanstack/ai-openai'; +import OpenAI from 'openai'; // Inlined OpenAI-compatible streaming mock: the runner copies only this file into its temp dir. function startMockProvider() { @@ -31,12 +32,30 @@ function startMockProvider() { } const server = await startMockProvider(); -const adapter = createOpenaiChatCompletions('gpt-4o', 'test-key', { - baseURL: `http://localhost:${server.address().port}/v1`, -}); +const baseURL = `http://localhost:${server.address().port}/v1`; +const adapter = createOpenaiChatCompletions('gpt-4o', 'test-key', { baseURL }); +const messages = [{ role: 'user', content: 'Hi' }]; await Sentry.startSpan({ name: 'main' }, async () => { - await chat({ adapter, threadId: 'thread-1', messages: [{ role: 'user', content: 'Hi' }], stream: false }); + if (process.env.STREAM) { + // The adapter calls the SDK while the app reads the stream, after `chat()` has returned. + for await (const _chunk of chat({ adapter, threadId: 'thread-1', messages, stream: true })) { + // drain + } + } else { + await chat({ adapter, threadId: 'thread-1', messages, stream: false }); + } + + await Sentry.startSpan({ name: 'direct' }, async () => { + const stream = await new OpenAI({ apiKey: 'test-key', baseURL }).chat.completions.create({ + model: 'gpt-4o', + messages, + stream: true, + }); + for await (const _chunk of stream) { + // drain + } + }); }); server.close(); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-provider-error.mjs b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-provider-error.mjs new file mode 100644 index 000000000000..9c691d6618e6 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-provider-error.mjs @@ -0,0 +1,31 @@ +import express from 'express'; +import * as Sentry from '@sentry/node'; +import { chat } from '@tanstack/ai'; +import { createOpenaiChatCompletions } from '@tanstack/ai-openai'; + +// Inlined OpenAI mock that rejects the key: the runner copies only this file into its temp dir. +function startMockProvider() { + const app = express(); + app.post('/v1/chat/completions', (_req, res) => { + res.status(401).json({ error: { message: 'Incorrect API key provided', type: 'invalid_request_error' } }); + }); + return new Promise(resolve => { + const server = app.listen(0, () => resolve(server)); + }); +} + +const server = await startMockProvider(); +const adapter = createOpenaiChatCompletions('gpt-4o', 'test-key', { + baseURL: `http://localhost:${server.address().port}/v1`, +}); + +await Sentry.startSpan({ name: 'main' }, async () => { + try { + // Without streaming, `chat()` rejects on the provider error and reads no further chunks. + await chat({ adapter, threadId: 'thread-1', messages: [{ role: 'user', content: 'Hi' }], stream: false }); + } catch { + // The 401 of the provider. + } +}); + +server.close(); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-retry.mjs b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-retry.mjs new file mode 100644 index 000000000000..927f56115b07 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-retry.mjs @@ -0,0 +1,19 @@ +import * as Sentry from '@sentry/node'; +import { chat } from '@tanstack/ai'; +import { fakeText } from '@tanstack/ai/testing'; + +// A middleware calls the model again after the first call fails. +const retry = { name: 'retry', onModelError: ctx => ctx.retry() }; + +const adapter = fakeText({ model: 'fake-model' }); +adapter.setResponses([{ error: 'Overloaded' }, { text: 'It is 21 degrees in Vienna.' }]); + +await Sentry.startSpan({ name: 'main' }, async () => { + await chat({ + adapter, + threadId: 'thread-1', + messages: [{ role: 'user', content: 'What is the weather in Vienna?' }], + middleware: [retry], + stream: false, + }); +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario.mjs index 3fc4afeb3fbf..a5d899bd9d21 100644 --- a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario.mjs @@ -2,9 +2,18 @@ import * as Sentry from '@sentry/node'; import { chat, toolDefinition } from '@tanstack/ai'; import { fakeText } from '@tanstack/ai/testing'; -// `MIDDLEWARE=manual` passes the middleware explicitly; otherwise the call carries none and relies -// on the integration injecting it. -const middleware = process.env.MIDDLEWARE === 'manual' ? [Sentry.createTanstackAiMiddleware()] : undefined; +// `MIDDLEWARE=manual` passes the middleware explicitly, and `MIDDLEWARE=transform-args` passes a middleware that +// changes the arguments of each tool call; otherwise the call carries none. Except with `manual`, it relies on the +// integration injecting the middleware. +const middleware = { + manual: [Sentry.createTanstackAiMiddleware()], + 'transform-args': [ + { + name: 'transform-args', + onBeforeToolCall: (_ctx, { args }) => ({ type: 'transformArgs', args: { ...args, unit: 'celsius' } }), + }, + ], +}[process.env.MIDDLEWARE]; const getWeather = toolDefinition({ name: 'get_weather', diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts index 94230c76587e..48d30d16f7aa 100644 --- a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts @@ -5,6 +5,7 @@ import { GEN_AI_PROVIDER_NAME, GEN_AI_REQUEST_MODEL, GEN_AI_RESPONSE_FINISH_REASONS, + GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_MODEL, GEN_AI_SYSTEM_INSTRUCTIONS, GEN_AI_TOOL_CALL_ARGUMENTS, @@ -20,12 +21,15 @@ import { } from '@sentry/conventions/attributes'; import type { SerializedStreamedSpan } from '@sentry/core'; import { afterAll, describe, expect } from 'vitest'; +import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; const TANSTACK_AI_DEPENDENCIES = { additionalDependencies: { '@tanstack/ai': '1.0.0', '@tanstack/ai-openai': '1.0.0', + // In the range of `@tanstack/ai-openai`, so that the scenarios and the adapter share one copy. + openai: '7.32.0', }, }; @@ -61,6 +65,7 @@ function expectAgentTree(spans: Spans): void { expect(chat.parent_span_id).toBe(agent.span_id); } expect(chats.map(chat => attr(chat, GEN_AI_RESPONSE_FINISH_REASONS))).toEqual(['["tool_calls"]', '["stop"]']); + expect(attr(agent, GEN_AI_RESPONSE_MODEL)).toBe('fake-model'); // The fake caches per thread: the first call writes the whole prompt, the second reads the // repeated prefix back. @@ -80,6 +85,9 @@ function expectAgentTree(spans: Spans): void { const tool = spans.find(span => span.name === 'execute_tool get_weather')!; expect(tool.status).toBe('ok'); + // Each model call ends before the tool that it asked for runs. + expect(chats[0]!.end_timestamp).toBeLessThanOrEqual(tool.start_timestamp); + expect(tool.end_timestamp).toBeLessThanOrEqual(chats[1]!.start_timestamp); expect(attr(tool, 'sentry.op')).toBe('gen_ai.execute_tool'); expect(attr(tool, GEN_AI_TOOL_NAME)).toBe('get_weather'); expect(tool.parent_span_id).toBe(agent.span_id); @@ -131,6 +139,7 @@ describe('TanStack AI integration', () => { finish_reason: 'tool_calls', }, ]); + const toolCallId = JSON.parse(String(attr(chats[0], GEN_AI_OUTPUT_MESSAGES)))[0].parts[1].id; const secondInput = JSON.parse(String(attr(chats[1], GEN_AI_INPUT_MESSAGES))); expect(secondInput.map((message: { role: string }) => message.role)).toEqual([ 'user', @@ -138,7 +147,20 @@ describe('TanStack AI integration', () => { 'tool', ]); + // The agent span has the input of the run and the answer of its last model call. + const agent = container.items.find(span => span.name === 'invoke_agent'); + expect(attr(agent, GEN_AI_INPUT_MESSAGES)).toBe(attr(chats[0], GEN_AI_INPUT_MESSAGES)); + expect(attr(agent, GEN_AI_SYSTEM_INSTRUCTIONS)).toBe('You are a helpful assistant.'); + expect(JSON.parse(String(attr(agent, GEN_AI_OUTPUT_MESSAGES)))).toEqual([ + { + role: 'assistant', + parts: [{ type: 'text', content: 'It is 21 degrees in Vienna.' }], + finish_reason: 'stop', + }, + ]); + const tool = container.items.find(span => span.name === 'execute_tool get_weather'); + expect(attr(tool, GEN_AI_TOOL_CALL_ID_ATTRIBUTE)).toBe(toolCallId); expect(attr(tool, GEN_AI_TOOL_CALL_ARGUMENTS)).toBe('{"city":"Vienna"}'); expect(attr(tool, GEN_AI_TOOL_CALL_RESULT)).toBe('{"city":"Vienna","temperature":21}'); }, @@ -162,6 +184,20 @@ describe('TanStack AI integration', () => { .start() .completed(); }); + + test('records the tool span when an earlier middleware changes the tool arguments', async () => { + await createRunner() + .withEnv({ MIDDLEWARE: 'transform-args' }) + .expect({ + span: container => { + expectAgentTree(container.items); + const tool = container.items.find(span => span.name === 'execute_tool get_weather'); + expect(attr(tool, GEN_AI_TOOL_CALL_ARGUMENTS)).toBe('{"city":"Vienna","unit":"celsius"}'); + }, + }) + .start() + .completed(); + }); }, TANSTACK_AI_DEPENDENCIES, ); @@ -212,23 +248,148 @@ describe('TanStack AI integration', () => { .expect({ span: container => { const spans = container.items; + const direct = spans.find(span => span.name === 'direct')!; const chats = spans.filter(span => attr(span, 'sentry.op') === 'gen_ai.chat'); - // Only ours: the OpenAI SDK integration is skipped for TanStack's model calls. - expect(chats).toHaveLength(1); - const chat = chats[0]!; - expect(attr(chat, 'sentry.origin')).toBe(ORIGIN); + // The OpenAI integration is skipped for TanStack's model call, but not for the direct call after it. + expect(chats.map(span => attr(span, 'sentry.origin')).sort()).toEqual(['auto.ai.openai', ORIGIN]); + const chat = chats.find(span => attr(span, 'sentry.origin') === ORIGIN)!; + const directChat = chats.find(span => attr(span, 'sentry.origin') === 'auto.ai.openai')!; + expect(directChat.parent_span_id).toBe(direct.span_id); // TanStack reports the adapter's name, which is per API, not the vendor. expect(attr(chat, GEN_AI_PROVIDER_NAME)).toBe('openai-chat'); expect(attr(chat, GEN_AI_REQUEST_MODEL)).toBe('gpt-4o'); expect(attr(chat, GEN_AI_RESPONSE_MODEL)).toBe('gpt-4o-2024-08-06'); + expect(attr(chat, GEN_AI_RESPONSE_ID)).toBe('chatcmpl-1'); + const agent = spans.find(span => attr(span, 'sentry.op') === 'gen_ai.invoke_agent'); + expect(attr(agent, GEN_AI_RESPONSE_ID)).toBe('chatcmpl-1'); expect(attr(chat, GEN_AI_USAGE_INPUT_TOKENS)).toBe(20); expect(attr(chat, GEN_AI_USAGE_OUTPUT_TOKENS)).toBe(5); expect(attr(chat, GEN_AI_USAGE_TOTAL_TOKENS)).toBe(25); expect(attr(chat, GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS)).toBe(8); expect(attr(chat, GEN_AI_USAGE_REASONING_OUTPUT_TOKENS)).toBe(2); - const http = spans.find(span => attr(span, 'sentry.op') === 'http.client'); - expect(http?.parent_span_id).toBe(chat.span_id); + const requests = spans.filter(span => attr(span, 'sentry.op') === 'http.client'); + expect(requests.map(span => span.parent_span_id).sort()).toEqual( + [chat.span_id, directChat.span_id].sort(), + ); + }, + }) + .start() + .completed(); + }); + + test('skips the OpenAI integration also while the app reads the stream', async () => { + await createRunner() + .withEnv({ STREAM: '1' }) + .expect({ + span: container => { + const spans = container.items; + const direct = spans.find(span => span.name === 'direct')!; + const chats = spans.filter(span => attr(span, 'sentry.op') === 'gen_ai.chat'); + expect(chats.map(span => attr(span, 'sentry.origin')).sort()).toEqual(['auto.ai.openai', ORIGIN]); + const directChat = chats.find(span => attr(span, 'sentry.origin') === 'auto.ai.openai')!; + expect(directChat.parent_span_id).toBe(direct.span_id); + }, + }) + .start() + .completed(); + }); + }, + TANSTACK_AI_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-interrupt.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + // TanStack calls no terminal hook for a run that pauses at an interrupt before or after a model call. + test('ends the spans of a run that pauses at an interrupt before the model call', async () => { + await createRunner() + .withEnv({ INTERRUPT: 'beforeModel' }) + .expect({ + span: container => { + const agent = container.items.find(span => span.name === 'invoke_agent'); + const chat = container.items.find(span => span.name === 'chat fake-model'); + expect(agent?.status).toBe('ok'); + expect(chat?.parent_span_id).toBe(agent?.span_id); + }, + }) + .start() + .completed(); + }); + + test('ends the spans of a run that pauses at an interrupt after the model call', async () => { + await createRunner() + .withEnv({ INTERRUPT: 'afterModel' }) + .expect({ + span: container => { + const agent = container.items.find(span => span.name === 'invoke_agent'); + const chat = container.items.find(span => span.name === 'chat fake-model'); + expect(agent?.status).toBe('ok'); + expect(chat?.parent_span_id).toBe(agent?.span_id); + expect(attr(chat, GEN_AI_RESPONSE_FINISH_REASONS)).toBe('["stop"]'); + }, + }) + .start() + .completed(); + }); + }, + TANSTACK_AI_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-retry.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + test('ends a model call that a middleware retried with an error status', async () => { + await createRunner() + .expect({ + span: container => { + const agent = container.items.find(span => span.name === 'invoke_agent'); + const chats = container.items + .filter(span => span.name === 'chat fake-model') + .sort((a, b) => a.start_timestamp - b.start_timestamp); + expect(chats.map(chat => chat.status)).toEqual(['error', 'ok']); + expect(chats.map(chat => chat.parent_span_id)).toEqual([agent?.span_id, agent?.span_id]); + expect(agent?.status).toBe('ok'); + }, + }) + .start() + .completed(); + }); + }, + TANSTACK_AI_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-provider-error.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + test('ends the spans with an error status when the provider call fails', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items; + const agent = spans.find(span => attr(span, 'sentry.op') === 'gen_ai.invoke_agent'); + const chat = spans.find(span => attr(span, 'sentry.op') === 'gen_ai.chat'); + expect(agent?.status).toBe('error'); + expect(chat?.status).toBe('error'); + expect(chat?.parent_span_id).toBe(agent?.span_id); }, }) .start() diff --git a/packages/astro/src/index.server.ts b/packages/astro/src/index.server.ts index 80158e311e49..6b458f67d643 100644 --- a/packages/astro/src/index.server.ts +++ b/packages/astro/src/index.server.ts @@ -99,6 +99,8 @@ export { groqIntegration, togetherAIIntegration, typesafeIntegration, + tanstackAiIntegration, + createTanstackAiMiddleware, langChainIntegration, langGraphIntegration, createFlueInstrumentation, diff --git a/packages/aws-serverless/src/index.ts b/packages/aws-serverless/src/index.ts index fd07e76a3fba..6cf94dfbf052 100644 --- a/packages/aws-serverless/src/index.ts +++ b/packages/aws-serverless/src/index.ts @@ -64,6 +64,8 @@ export { groqIntegration, togetherAIIntegration, typesafeIntegration, + tanstackAiIntegration, + createTanstackAiMiddleware, langChainIntegration, langGraphIntegration, mastraIntegration, diff --git a/packages/bun/src/index.ts b/packages/bun/src/index.ts index 4e91068d21f8..98d2d7aeebe4 100644 --- a/packages/bun/src/index.ts +++ b/packages/bun/src/index.ts @@ -86,6 +86,8 @@ export { groqIntegration, togetherAIIntegration, typesafeIntegration, + tanstackAiIntegration, + createTanstackAiMiddleware, langChainIntegration, langGraphIntegration, mastraIntegration, diff --git a/packages/cloudflare/src/index.ts b/packages/cloudflare/src/index.ts index 88ce8deaac01..2ddf149598b2 100644 --- a/packages/cloudflare/src/index.ts +++ b/packages/cloudflare/src/index.ts @@ -108,7 +108,7 @@ export { spanStreamingIntegration, } from '@sentry/core'; export { trpcMiddleware, wrapMcpServerWithSentry } from '@sentry/core/server'; -export { createFlueInstrumentation, instrumentPostgresJsSql } from '@sentry/server-utils'; +export { createFlueInstrumentation, createTanstackAiMiddleware, instrumentPostgresJsSql } from '@sentry/server-utils'; export type { FlueOptions } from '@sentry/server-utils'; export { withSentry } from './withSentry'; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 1eb1d0ff4d72..12e45cc0b4a7 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -77,6 +77,7 @@ export { } from './integration'; export { _INTERNAL_skipAiProviderWrapping, + _INTERNAL_withAiProviderWrappingSkipped, _INTERNAL_shouldSkipAiProviderWrapping, _INTERNAL_clearAiProviderSkips, } from './utils/ai/providerSkip'; diff --git a/packages/core/src/utils/ai/providerSkip.ts b/packages/core/src/utils/ai/providerSkip.ts index 0b7ca2a5c3bc..94529eb3a5c1 100644 --- a/packages/core/src/utils/ai/providerSkip.ts +++ b/packages/core/src/utils/ai/providerSkip.ts @@ -1,3 +1,4 @@ +import { getCurrentScope, withScope } from '../../currentScopes'; import { DEBUG_BUILD } from '../../debug-build'; import { debug } from '../debug-logger'; @@ -9,6 +10,9 @@ import { debug } from '../debug-logger'; */ const SKIPPED_AI_PROVIDERS = new Set(); +/** Scope metadata key of the modules that `_INTERNAL_withAiProviderWrappingSkipped` skips, as `{ [module]: true }`. */ +const SCOPED_SKIPPED_AI_PROVIDERS_KEY = '__SENTRY_SKIPPED_AI_PROVIDERS__'; + /** * Mark AI provider modules to skip instrumentation wrapping. * @@ -31,6 +35,25 @@ export function _INTERNAL_skipAiProviderWrapping(modules: string[]): void { }); } +/** + * Run `callback` with AI provider modules skipped, for the calls that it makes and the async work that + * it starts. A call to the same provider outside of `callback` is still instrumented. + * + * @internal + * @param modules - Array of npm module names to skip (e.g., '@anthropic-ai/sdk', 'openai') + */ +export function _INTERNAL_withAiProviderWrappingSkipped(modules: string[], callback: () => T): T { + return withScope(scope => { + const skipped: Record = {}; + for (const module of modules) { + skipped[module] = true; + } + // `setSDKProcessingMetadata` merges one level deep, so a nested call adds to the modules of the outer one. + scope.setSDKProcessingMetadata({ [SCOPED_SKIPPED_AI_PROVIDERS_KEY]: skipped }); + return callback(); + }); +} + /** * Check if an AI provider module should skip instrumentation wrapping. * @@ -47,7 +70,13 @@ export function _INTERNAL_skipAiProviderWrapping(modules: string[]): void { * ``` */ export function _INTERNAL_shouldSkipAiProviderWrapping(module: string): boolean { - return SKIPPED_AI_PROVIDERS.has(module); + if (SKIPPED_AI_PROVIDERS.has(module)) { + return true; + } + const scoped = getCurrentScope().getScopeData().sdkProcessingMetadata[SCOPED_SKIPPED_AI_PROVIDERS_KEY] as + | Record + | undefined; + return scoped?.[module] === true; } /** diff --git a/packages/core/test/lib/utils/ai/providerSkip.test.ts b/packages/core/test/lib/utils/ai/providerSkip.test.ts index de39778a2b5e..b072a73f7501 100644 --- a/packages/core/test/lib/utils/ai/providerSkip.test.ts +++ b/packages/core/test/lib/utils/ai/providerSkip.test.ts @@ -3,6 +3,7 @@ import { _INTERNAL_clearAiProviderSkips, _INTERNAL_shouldSkipAiProviderWrapping, _INTERNAL_skipAiProviderWrapping, + _INTERNAL_withAiProviderWrappingSkipped, } from '../../../../src/index'; const OPENAI_INTEGRATION_NAME = 'OpenAI'; @@ -36,6 +37,40 @@ describe('AI Provider Skip', () => { }); }); + describe('_INTERNAL_withAiProviderWrappingSkipped', () => { + it('skips the providers only inside the callback', () => { + const skippedInside = _INTERNAL_withAiProviderWrappingSkipped([OPENAI_INTEGRATION_NAME], () => [ + _INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME), + _INTERNAL_shouldSkipAiProviderWrapping(ANTHROPIC_AI_INTEGRATION_NAME), + ]); + + expect(skippedInside).toEqual([true, false]); + expect(_INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME)).toBe(false); + }); + + it('keeps the skip in the async work that the callback starts', async () => { + const skippedAfterAwait = await _INTERNAL_withAiProviderWrappingSkipped([OPENAI_INTEGRATION_NAME], async () => { + await Promise.resolve(); + return _INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME); + }); + + expect(skippedAfterAwait).toBe(true); + expect(_INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME)).toBe(false); + }); + + it('adds the providers of a nested call to the providers of the outer call', () => { + const skippedInside = _INTERNAL_withAiProviderWrappingSkipped([OPENAI_INTEGRATION_NAME], () => + _INTERNAL_withAiProviderWrappingSkipped([ANTHROPIC_AI_INTEGRATION_NAME], () => [ + _INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME), + _INTERNAL_shouldSkipAiProviderWrapping(ANTHROPIC_AI_INTEGRATION_NAME), + _INTERNAL_shouldSkipAiProviderWrapping(GOOGLE_GENAI_INTEGRATION_NAME), + ]), + ); + + expect(skippedInside).toEqual([true, true, false]); + }); + }); + describe('_INTERNAL_shouldSkipAiProviderWrapping', () => { it('returns false for unmarked providers', () => { expect(_INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME)).toBe(false); diff --git a/packages/deno/src/index.ts b/packages/deno/src/index.ts index f55fe14de14f..8033186eaa8c 100644 --- a/packages/deno/src/index.ts +++ b/packages/deno/src/index.ts @@ -180,6 +180,8 @@ export { groqIntegration, togetherAIIntegration, typesafeIntegration, + tanstackAiIntegration, + createTanstackAiMiddleware, postgresIntegration, postgresJsIntegration, neonIntegration, diff --git a/packages/elysia/src/index.ts b/packages/elysia/src/index.ts index 76c135d6f3f4..c557e3eecbd6 100644 --- a/packages/elysia/src/index.ts +++ b/packages/elysia/src/index.ts @@ -65,6 +65,8 @@ export { groqIntegration, togetherAIIntegration, typesafeIntegration, + tanstackAiIntegration, + createTanstackAiMiddleware, langChainIntegration, langGraphIntegration, createFlueInstrumentation, diff --git a/packages/google-cloud-serverless/src/index.ts b/packages/google-cloud-serverless/src/index.ts index b90e609bbfc1..ce5f6dee1995 100644 --- a/packages/google-cloud-serverless/src/index.ts +++ b/packages/google-cloud-serverless/src/index.ts @@ -64,6 +64,8 @@ export { groqIntegration, togetherAIIntegration, typesafeIntegration, + tanstackAiIntegration, + createTanstackAiMiddleware, langChainIntegration, langGraphIntegration, mastraIntegration, diff --git a/packages/server-utils/src/ai/index.ts b/packages/server-utils/src/ai/index.ts index 3fc0f4bd5fc9..b5dd51f0ef57 100644 --- a/packages/server-utils/src/ai/index.ts +++ b/packages/server-utils/src/ai/index.ts @@ -15,3 +15,5 @@ export { instrumentStateGraph, instrumentStateGraphCompile, instrumentCreateReac export { SentryMastraExporter } from './mastra'; export { createFlueInstrumentation } from './flue'; export type { FlueOptions } from './flue'; +export { createTanstackAiMiddleware } from './tanstack-ai'; +export type { TanstackAiOptions } from './tanstack-ai'; diff --git a/packages/server-utils/src/ai/tanstack-ai/index.ts b/packages/server-utils/src/ai/tanstack-ai/index.ts index 52cc1643ef18..95c2359790d1 100644 --- a/packages/server-utils/src/ai/tanstack-ai/index.ts +++ b/packages/server-utils/src/ai/tanstack-ai/index.ts @@ -1,13 +1,4 @@ -import type { Span } from '@sentry/core'; -import { - _INTERNAL_shouldSkipAiProviderWrapping, - _INTERNAL_skipAiProviderWrapping, - captureException, - SPAN_STATUS_ERROR, - startInactiveSpan, - stringify, - withActiveSpan, -} from '@sentry/core'; +import { captureException, SPAN_STATUS_ERROR, startInactiveSpan, stringify, withActiveSpan } from '@sentry/core'; import { GEN_AI_AGENT_NAME, GEN_AI_CONVERSATION_ID, @@ -15,6 +6,7 @@ import { GEN_AI_PROVIDER_NAME, GEN_AI_REQUEST_MODEL, GEN_AI_RESPONSE_FINISH_REASONS, + GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_MODEL, GEN_AI_RESPONSE_STREAMING, GEN_AI_TOOL_CALL_ARGUMENTS, @@ -23,22 +15,25 @@ import { GEN_AI_TOOL_NAME, SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; -import { ANTHROPIC_AI_INTEGRATION_NAME } from '../anthropic-ai/constants'; +import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../core/gen-ai-attributes'; import type { GenAiOptions } from '../core/utils'; import { getGenAiSpanOp, resolveAIRecordingOptions } from '../core/utils'; -import { GOOGLE_GENAI_INTEGRATION_NAME } from '../google-genai/constants'; -import { OPENAI_INTEGRATION_NAME } from '../openai/constants'; import { TANSTACK_AI_MIDDLEWARE_BRAND, TANSTACK_AI_ORIGIN } from './constants'; import type { TanstackAiChatMiddleware, TanstackAiFetch, TanstackAiMiddlewareContext, TanstackAiTool } from './types'; import type { RunState } from './utils'; -import { addUsage, endChatSpan, setOutputMessages, startChatSpan, usageAttributes } from './utils'; +import { + addUsage, + endChatSpan, + endSpan, + getProviderName, + inputAttributes, + setOutputMessages, + startChatSpan, + usageAttributes, +} from './utils'; export type TanstackAiOptions = GenAiOptions; -// TanStack's provider adapters call the vendor SDKs that Sentry instruments too, which would report -// each model call a second time beside our `chat` span. -const PROVIDER_INTEGRATIONS = [OPENAI_INTEGRATION_NAME, ANTHROPIC_AI_INTEGRATION_NAME, GOOGLE_GENAI_INTEGRATION_NAME]; - /** * Create a `@tanstack/ai` chat middleware that records `invoke_agent`, `chat` and `execute_tool` * spans for a `chat()` call. @@ -50,103 +45,74 @@ const PROVIDER_INTEGRATIONS = [OPENAI_INTEGRATION_NAME, ANTHROPIC_AI_INTEGRATION export function createTanstackAiMiddleware(options: TanstackAiOptions = {}): TanstackAiChatMiddleware { // Keyed by the middleware context, which TanStack creates once per `chat()` call. const runs = new WeakMap(); - // Tool call ids are unique per call, so one map serves every run this middleware sees. - const toolSpans = new Map(); - const wrappedTools = new WeakMap(); - - // TanStack runs `execute` outside any of the middleware hooks, so wrapping it is the only way to - // make the tool span active while the tool runs and have the tool's own spans nest under it. - const wrapTool = (tool: TanstackAiTool): TanstackAiTool => { - const known = wrappedTools.get(tool); - if (known) { - return known; - } - const { execute } = tool; - if (typeof execute !== 'function') { - wrappedTools.set(tool, tool); - return tool; - } - const wrapped: TanstackAiTool = { - ...tool, - execute: (args, context) => { - const span = context?.toolCallId ? toolSpans.get(context.toolCallId) : undefined; - return span ? withActiveSpan(span, () => execute.call(tool, args, context)) : execute.call(tool, args, context); - }, - }; - wrappedTools.set(tool, wrapped); - wrappedTools.set(wrapped, wrapped); - return wrapped; - }; - const endRun = (ctx: TanstackAiMiddlewareContext, error?: unknown): void => { + const endRun = (ctx: TanstackAiMiddlewareContext, errorStatus?: string): void => { const run = runs.get(ctx); if (!run) { return; } runs.delete(ctx); - for (const id of run.toolCallIds) { - toolSpans.get(id)?.end(); - toolSpans.delete(id); + for (const span of run.toolSpans.values()) { + endSpan(span, errorStatus); } - endChatSpan(run, error); - + endChatSpan(run, errorStatus); run.agentSpan.setAttributes(usageAttributes(run.usage)); - if (error !== undefined) { - run.agentSpan.setStatus({ code: SPAN_STATUS_ERROR, message: 'internal_error' }); - } - run.agentSpan.end(); + endSpan(run.agentSpan, errorStatus); }; return { name: 'sentry', [TANSTACK_AI_MIDDLEWARE_BRAND]: true, - onStart(ctx) { - if (ctx.activity !== 'chat') { - return; - } - if (!PROVIDER_INTEGRATIONS.every(name => _INTERNAL_shouldSkipAiProviderWrapping(name))) { - _INTERNAL_skipAiProviderWrapping(PROVIDER_INTEGRATIONS); - } - - const { recordInputs, recordOutputs } = resolveAIRecordingOptions(options); - const agentName = ctx.subagentName; - runs.set(ctx, { - agentSpan: startInactiveSpan({ - name: agentName ? `invoke_agent ${agentName}` : 'invoke_agent', - op: getGenAiSpanOp('invoke_agent'), - attributes: { - [SENTRY_ORIGIN]: TANSTACK_AI_ORIGIN, - [GEN_AI_OPERATION_NAME]: 'invoke_agent', - [GEN_AI_PROVIDER_NAME]: ctx.provider, - [GEN_AI_REQUEST_MODEL]: ctx.model, - [GEN_AI_CONVERSATION_ID]: ctx.threadId, - [GEN_AI_RESPONSE_STREAMING]: ctx.streaming, - ...(agentName ? { [GEN_AI_AGENT_NAME]: agentName } : {}), - }, - }), - recordInputs, - recordOutputs, - outputText: '', - outputToolCalls: [], - toolCallNames: new Map(), - usage: {}, - toolCallIds: new Set(), - }); - }, - onConfig(ctx, config) { - const tools = config.tools.some(tool => wrappedTools.get(tool) !== tool) ? config.tools.map(wrapTool) : undefined; + // TanStack runs the `init` phase once per `chat()` call, before `onStart` and before any tool can run, so the + // run starts here and its tools get the span of their call from the start. + if (ctx.phase === 'init' && ctx.activity === 'chat') { + const { recordInputs, recordOutputs } = resolveAIRecordingOptions(options); + const agentName = ctx.subagentName; + runs.set(ctx, { + agentSpan: startInactiveSpan({ + name: agentName ? `invoke_agent ${agentName}` : 'invoke_agent', + op: getGenAiSpanOp('invoke_agent'), + attributes: { + [SENTRY_ORIGIN]: TANSTACK_AI_ORIGIN, + [GEN_AI_OPERATION_NAME]: 'invoke_agent', + [GEN_AI_PROVIDER_NAME]: getProviderName(ctx), + [GEN_AI_REQUEST_MODEL]: ctx.model, + [GEN_AI_CONVERSATION_ID]: ctx.threadId, + [GEN_AI_RESPONSE_STREAMING]: ctx.streaming, + ...(agentName ? { [GEN_AI_AGENT_NAME]: agentName } : {}), + ...(recordInputs ? inputAttributes(config.messages, config.systemPrompts) : {}), + }, + }), + recordInputs, + recordOutputs, + outputText: '', + outputToolCalls: [], + toolCallNames: new Map(), + usage: {}, + toolSpans: new Map(), + wrappedTools: new WeakMap(), + }); + } const run = runs.get(ctx); + if (!run) { + return undefined; + } + const tools = config.tools.some(tool => run.wrappedTools.get(tool) !== tool) + ? config.tools.map(tool => wrapTool(run, tool)) + : undefined; // `beforeModel` opens each agent-loop model call; `structuredOutput` the separate final call - // made for `outputSchema`. `init` fires once before `onStart`, with no run yet. - if (!run || (ctx.phase !== 'beforeModel' && ctx.phase !== 'structuredOutput')) { + // made for `outputSchema`. + if (ctx.phase !== 'beforeModel' && ctx.phase !== 'structuredOutput') { return tools ? { tools } : undefined; } - endChatSpan(run); + // A model call that completes ends its span at the `afterModel` boundary. A span that is still open here + // belongs to a call that failed and that a middleware retried with `ctx.retry()`. + endChatSpan(run, 'internal_error'); const chatSpan = startChatSpan(ctx, config, run); run.chatSpan = chatSpan; @@ -163,8 +129,29 @@ export function createTanstackAiMiddleware(options: TanstackAiOptions = {}): Tan onChunk(ctx, chunk) { const run = runs.get(ctx); - const span = run?.chatSpan; - if (!run || !span) { + if (!run) { + return; + } + if (chunk.type === 'RUN_FINISHED' || chunk.type === 'RUN_ERROR') { + // TanStack sets the call metadata of the `structuredOutput` call only after `onConfig`. + const provider = getProviderName(ctx); + run.chatSpan?.setAttribute(GEN_AI_PROVIDER_NAME, provider); + run.agentSpan.setAttribute(GEN_AI_PROVIDER_NAME, provider); + } + // A provider error ends the run with this chunk. `chat({ stream: false })` throws on it and stops + // reading, so TanStack calls no terminal hook. + if (chunk.type === 'RUN_ERROR') { + endRun(ctx, ctx.signal?.aborted ? 'cancelled' : 'internal_error'); + return; + } + // A run that pauses at an interrupt ends with this chunk. For an interrupt at `beforeModel` or + // `afterModel`, TanStack calls no terminal hook. + if (chunk.type === 'RUN_FINISHED' && chunk.outcome?.type === 'interrupt') { + endRun(ctx); + return; + } + const span = run.chatSpan; + if (!span) { return; } @@ -180,13 +167,31 @@ export function createTanstackAiMiddleware(options: TanstackAiOptions = {}): Tan arguments: stringify(chunk.input ?? {}), }); } else if (chunk.type === 'RUN_FINISHED') { - if (chunk.model) { - span.setAttribute(GEN_AI_RESPONSE_MODEL, chunk.model); - } + // An adapter can report these only in the chunk's `metadata.tanstack`. TanStack copies them + // into `callMetadata` before middleware sees the chunk. + const responseModel = chunk.model ?? ctx.callMetadata?.model; + const responseId = chunk.responseId ?? ctx.callMetadata?.responseId; if (chunk.finishReason) { span.setAttribute(GEN_AI_RESPONSE_FINISH_REASONS, stringify([chunk.finishReason])); } - setOutputMessages(span, run, chunk.finishReason ?? undefined); + // The agent span keeps the response of the last model call, which is the answer of the run. + for (const target of [span, run.agentSpan]) { + if (responseModel) { + target.setAttribute(GEN_AI_RESPONSE_MODEL, responseModel); + } + if (responseId) { + target.setAttribute(GEN_AI_RESPONSE_ID, responseId); + } + setOutputMessages(target, run, chunk.finishReason ?? undefined); + } + } + }, + + // Runs after each model call, before the tools that it asked for, so the `chat` span does not cover them. + onInterruptBoundary(ctx) { + const run = runs.get(ctx); + if (run && ctx.phase === 'afterModel') { + endChatSpan(run); } }, @@ -199,15 +204,17 @@ export function createTanstackAiMiddleware(options: TanstackAiOptions = {}): Tan run.usage = addUsage(run.usage, usage); }, - onBeforeToolCall(ctx, { toolName, toolCallId, args, tool }) { + // Guards run after every `onBeforeToolCall` decision and after the input check, so `args` are the + // arguments the tool gets. A `transformArgs` decision of an earlier middleware ends the `onBeforeToolCall` + // chain, but not the guards. + onGuardToolCall(ctx, { toolName, toolCallId, args, tool }) { const run = runs.get(ctx); // A client tool runs in the browser, not here, so there is nothing to time. if (!run || (tool && typeof tool.execute !== 'function')) { return; } - run.toolCallIds.add(toolCallId); - toolSpans.set( + run.toolSpans.set( toolCallId, startInactiveSpan({ name: `execute_tool ${toolName}`, @@ -217,6 +224,7 @@ export function createTanstackAiMiddleware(options: TanstackAiOptions = {}): Tan [SENTRY_ORIGIN]: TANSTACK_AI_ORIGIN, [GEN_AI_OPERATION_NAME]: 'execute_tool', [GEN_AI_TOOL_NAME]: toolName, + [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: toolCallId, [GEN_AI_CONVERSATION_ID]: ctx.threadId, ...(tool?.description ? { [GEN_AI_TOOL_DESCRIPTION]: tool.description } : {}), ...(run.recordInputs ? { [GEN_AI_TOOL_CALL_ARGUMENTS]: stringify(args) } : {}), @@ -226,18 +234,20 @@ export function createTanstackAiMiddleware(options: TanstackAiOptions = {}): Tan }, onAfterToolCall(ctx, { toolCallId, ok, result, error }) { - const span = toolSpans.get(toolCallId); - if (!span) { + const run = runs.get(ctx); + const span = run?.toolSpans.get(toolCallId); + if (!run || !span) { return; } - toolSpans.delete(toolCallId); - const run = runs.get(ctx); - run?.toolCallIds.delete(toolCallId); + run.toolSpans.delete(toolCallId); - if (run?.recordOutputs && ok && result !== undefined) { + if (run.recordOutputs && ok && result !== undefined) { span.setAttribute(GEN_AI_TOOL_CALL_RESULT, typeof result === 'string' ? result : stringify(result)); } - if (!ok) { + if (!ok && ctx.signal?.aborted) { + // TanStack fails each tool that has not started yet with an `Operation aborted` error. + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'cancelled' }); + } else if (!ok) { span.setStatus({ code: SPAN_STATUS_ERROR, message: 'internal_error' }); // TanStack hands a tool failure back to the model as its result, so it never reaches the // app's error handlers. @@ -252,12 +262,38 @@ export function createTanstackAiMiddleware(options: TanstackAiOptions = {}): Tan endRun(ctx); }, + // As in the other AI integrations, an aborted call counts as failed. onAbort(ctx) { - endRun(ctx); + endRun(ctx, 'cancelled'); }, - onError(ctx, { error }) { - endRun(ctx, error); + onError(ctx) { + endRun(ctx, 'internal_error'); }, }; } + +// TanStack runs `execute` outside any of the middleware hooks, so wrapping it is the only way to make the +// tool span active while the tool runs and have the tool's own spans nest under it. +function wrapTool(run: RunState, tool: TanstackAiTool): TanstackAiTool { + const known = run.wrappedTools.get(tool); + if (known) { + return known; + } + const { execute } = tool; + const wrapped: TanstackAiTool = + typeof execute === 'function' + ? { + ...tool, + execute: (args, context) => { + const span = context?.toolCallId ? run.toolSpans.get(context.toolCallId) : undefined; + return span + ? withActiveSpan(span, () => execute.call(tool, args, context)) + : execute.call(tool, args, context); + }, + } + : tool; + run.wrappedTools.set(tool, wrapped); + run.wrappedTools.set(wrapped, wrapped); + return wrapped; +} diff --git a/packages/server-utils/src/ai/tanstack-ai/types.ts b/packages/server-utils/src/ai/tanstack-ai/types.ts index c50688c4660b..aae06d2e50d1 100644 --- a/packages/server-utils/src/ai/tanstack-ai/types.ts +++ b/packages/server-utils/src/ai/tanstack-ai/types.ts @@ -16,6 +16,8 @@ export interface TanstackAiUsage { completionTokens?: number; totalTokens?: number; cost?: number; + /** `upstreamCost` is the provider's charge that a gateway passes through, set by OpenRouter for BYOK keys. */ + costDetails?: { upstreamCost?: number }; promptTokensDetails?: { cachedTokens?: number; cacheWriteTokens?: number }; completionTokensDetails?: { reasoningTokens?: number }; } @@ -27,14 +29,29 @@ export interface TanstackAiTool { execute?: (args: unknown, context?: { toolCallId?: string }) => unknown; } +/** Mirrors the `TanStackMessageMetadata` fields of the current model call. */ +export interface TanstackAiCallMetadata { + model?: string; + responseId?: string; + /** `provider` is the adapter's `provider ?? name`, e.g. `openrouter` for the `openrouter-responses` adapter. */ + source?: { provider?: string; api?: string }; +} + export interface TanstackAiMiddlewareContext { activity: string; phase: string; + /** The adapter's `name`, e.g. `openai-chat`. */ provider: string; model: string; threadId: string; streaming: boolean; subagentName?: string; + callMetadata?: TanstackAiCallMetadata; + /** The signal of the `abortController` passed to `chat()`. */ + signal?: AbortSignal; + /** The conversation history of the run, as passed to `chat()`. */ + messages: ReadonlyArray; + systemPrompts: Array; } /** Mirrors `ModelMessage`. */ @@ -47,6 +64,8 @@ export interface TanstackAiModelMessage { export interface TanstackAiMiddlewareConfig { messages: TanstackAiModelMessage[]; + /** The messages sent to the provider, when a middleware sets them apart from `messages`. */ + providerMessages?: TanstackAiModelMessage[]; systemPrompts: Array; tools: TanstackAiTool[]; modelOptions?: Record; @@ -58,10 +77,13 @@ export interface TanstackAiChunk { type: string; delta?: string; model?: string; + responseId?: string; finishReason?: string | null; toolCallId?: string; toolCallName?: string; input?: unknown; + /** `{ type: 'interrupt' }` on the `RUN_FINISHED` chunk of a run that pauses at an interrupt. */ + outcome?: { type: string }; } export interface TanstackAiToolCallHookContext { @@ -85,10 +107,10 @@ export interface TanstackAiChatMiddleware { ctx: TanstackAiMiddlewareContext, config: TanstackAiMiddlewareConfig, ): Partial | undefined; - onStart(ctx: TanstackAiMiddlewareContext): void; onChunk(ctx: TanstackAiMiddlewareContext, chunk: TanstackAiChunk): void; + onInterruptBoundary(ctx: TanstackAiMiddlewareContext): void; onUsage(ctx: TanstackAiMiddlewareContext, usage: TanstackAiUsage): void; - onBeforeToolCall(ctx: TanstackAiMiddlewareContext, hookCtx: TanstackAiToolCallHookContext): void; + onGuardToolCall(ctx: TanstackAiMiddlewareContext, hookCtx: TanstackAiToolCallHookContext): void; onAfterToolCall(ctx: TanstackAiMiddlewareContext, info: TanstackAiAfterToolCallInfo): void; onFinish(ctx: TanstackAiMiddlewareContext): void; onAbort(ctx: TanstackAiMiddlewareContext): void; diff --git a/packages/server-utils/src/ai/tanstack-ai/utils.ts b/packages/server-utils/src/ai/tanstack-ai/utils.ts index f9438624cf57..27abd8ded38b 100644 --- a/packages/server-utils/src/ai/tanstack-ai/utils.ts +++ b/packages/server-utils/src/ai/tanstack-ai/utils.ts @@ -1,5 +1,11 @@ import type { Span } from '@sentry/core'; -import { SPAN_STATUS_ERROR, startInactiveSpan, stringify } from '@sentry/core'; +import { + _INTERNAL_withAiProviderWrappingSkipped, + isObjectLike, + SPAN_STATUS_ERROR, + startInactiveSpan, + stringify, +} from '@sentry/core'; import { GEN_AI_CONVERSATION_ID, GEN_AI_COST_TOTAL_TOKENS, @@ -17,12 +23,19 @@ import { GEN_AI_USAGE_REASONING_OUTPUT_TOKENS, SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; +import { AWS_BEDROCK_INTEGRATION_NAME } from '../../integrations/aws-sdk/constants'; +import { ANTHROPIC_AI_INTEGRATION_NAME } from '../anthropic-ai/constants'; import { getGenAiSpanOp, getTokenUsageAttributes } from '../core/utils'; +import { GOOGLE_GENAI_INTEGRATION_NAME } from '../google-genai/constants'; +import { MISTRAL_INTEGRATION_NAME } from '../mistral/constants'; +import { OPENAI_INTEGRATION_NAME } from '../openai/constants'; +import { WORKERS_AI_INTEGRATION_NAME } from '../workers-ai/constants'; import { TANSTACK_AI_ORIGIN } from './constants'; import type { TanstackAiMiddlewareConfig, TanstackAiMiddlewareContext, TanstackAiModelMessage, + TanstackAiTool, TanstackAiUsage, } from './types'; @@ -37,7 +50,93 @@ export interface RunState { // `TOOL_CALL_END` carries the parsed input but not the name, which only `TOOL_CALL_START` has. toolCallNames: Map; usage: TanstackAiUsage; - toolCallIds: Set; + /** The open `execute_tool` spans of the run, keyed by tool call id. */ + toolSpans: Map; + /** Each tool of the run mapped to the copy whose `execute` runs with the tool span active. */ + wrappedTools: WeakMap; +} + +// The Sentry instrumentations of the vendor SDKs and bindings that adapters call. Each one would report the +// model call of an adapter a second time beside our span. +const PROVIDER_INTEGRATIONS = [ + OPENAI_INTEGRATION_NAME, + ANTHROPIC_AI_INTEGRATION_NAME, + GOOGLE_GENAI_INTEGRATION_NAME, + MISTRAL_INTEGRATION_NAME, + // The Cloudflare adapter calls `env.AI.run()`, which `@sentry/cloudflare` instruments. + WORKERS_AI_INTEGRATION_NAME, + // The Bedrock adapters call `@aws-sdk/client-bedrock-runtime`, which `awsIntegration` instruments. + AWS_BEDROCK_INTEGRATION_NAME, +]; + +type AdapterMethod = (...args: unknown[]) => unknown; + +/** + * The adapter, with the Sentry integrations of the vendor SDKs skipped during its calls, as our spans record + * them. A direct call to the same SDK elsewhere in the app still gets its span. + */ +export function skipProviderIntegrations(adapter: T): T { + if (!isObjectLike(adapter)) { + return adapter; + } + const skipped = (callback: () => R): R => _INTERNAL_withAiProviderWrappingSkipped(PROVIDER_INTEGRATIONS, callback); + // So that each read of a method returns the same function. + const wrappedMethods = new WeakMap(); + + return new Proxy(adapter, { + get(target, property) { + const value: unknown = Reflect.get(target, property); + const descriptor = Reflect.getOwnPropertyDescriptor(target, property); + // A Proxy must return a read-only, non-configurable property (e.g. of a frozen adapter) as it is. + if ( + typeof value !== 'function' || + property === 'constructor' || + (descriptor && !descriptor.configurable && !descriptor.writable) + ) { + return value; + } + const known = wrappedMethods.get(value as AdapterMethod); + if (known) { + return known; + } + const wrapped: AdapterMethod = (...args) => { + const result: unknown = skipped(() => value.apply(target, args)); + // A text adapter returns an async generator, whose body runs on each `next()` in the async + // context of the caller, which for a streamed `chat()` is the app's code. + return isAsyncIterable(result) ? skipDuringIteration(result, skipped) : result; + }; + wrappedMethods.set(value as AdapterMethod, wrapped); + return wrapped; + }, + }); +} + +function isAsyncIterable(value: unknown): value is AsyncIterable { + return isObjectLike(value) && typeof value[Symbol.asyncIterator] === 'function'; +} + +function skipDuringIteration( + iterable: AsyncIterable, + skipped: (callback: () => R) => R, +): AsyncIterableIterator { + const iterator = iterable[Symbol.asyncIterator](); + return { + next: (...args) => skipped(() => iterator.next(...args)), + return: value => skipped(() => iterator.return?.(value) ?? Promise.resolve({ done: true, value })), + throw: error => skipped(() => iterator.throw?.(error) ?? Promise.reject(error)), + [Symbol.asyncIterator]() { + return this; + }, + }; +} + +/** + * `ctx.provider` is the adapter name, which for some adapters also names the wire API (e.g. + * `openrouter-responses`). The call metadata carries the adapter's provider identity, but only once the + * first model call has started. + */ +export function getProviderName(ctx: TanstackAiMiddlewareContext): string { + return ctx.callMetadata?.source?.provider ?? ctx.provider; } export function startChatSpan( @@ -48,25 +147,22 @@ export function startChatSpan( const attributes: Record = { [SENTRY_ORIGIN]: TANSTACK_AI_ORIGIN, [GEN_AI_OPERATION_NAME]: 'chat', - [GEN_AI_PROVIDER_NAME]: ctx.provider, + [GEN_AI_PROVIDER_NAME]: getProviderName(ctx), [GEN_AI_REQUEST_MODEL]: ctx.model, [GEN_AI_CONVERSATION_ID]: ctx.threadId, [GEN_AI_RESPONSE_STREAMING]: ctx.streaming, ...requestAttributes(config.modelOptions), }; - if (config.tools.length) { + // The `structuredOutput` call sends no tools, although its config lists them. + if (run.recordInputs && config.tools.length && ctx.phase !== 'structuredOutput') { attributes[GEN_AI_TOOL_DEFINITIONS] = stringify( config.tools.map(({ name, description, inputSchema }) => ({ name, description, inputSchema })), ); } if (run.recordInputs) { - attributes[GEN_AI_INPUT_MESSAGES] = stringify(config.messages.map(toGenAiMessage)); - if (config.systemPrompts.length) { - attributes[GEN_AI_SYSTEM_INSTRUCTIONS] = config.systemPrompts - .map(prompt => (typeof prompt === 'string' ? prompt : prompt.content)) - .join('\n'); - } + // A middleware can send the provider other messages than the history of the run. + Object.assign(attributes, inputAttributes(config.providerMessages ?? config.messages, config.systemPrompts)); } run.outputText = ''; @@ -80,14 +176,29 @@ export function startChatSpan( }); } -export function endChatSpan(run: RunState, error?: unknown): void { +export function inputAttributes( + messages: ReadonlyArray, + systemPrompts: TanstackAiMiddlewareConfig['systemPrompts'], +): Record { + return { + [GEN_AI_INPUT_MESSAGES]: stringify(messages.map(toGenAiMessage)), + [GEN_AI_SYSTEM_INSTRUCTIONS]: systemPrompts.length + ? systemPrompts.map(prompt => (typeof prompt === 'string' ? prompt : prompt.content)).join('\n') + : undefined, + }; +} + +export function endChatSpan(run: RunState, errorStatus?: string): void { const span = run.chatSpan; - if (!span) { - return; - } run.chatSpan = undefined; - if (error !== undefined) { - span.setStatus({ code: SPAN_STATUS_ERROR, message: 'internal_error' }); + if (span) { + endSpan(span, errorStatus); + } +} + +export function endSpan(span: Span, errorStatus?: string): void { + if (errorStatus) { + span.setStatus({ code: SPAN_STATUS_ERROR, message: errorStatus }); } span.end(); } @@ -155,9 +266,12 @@ export function usageAttributes(usage: TanstackAiUsage): Record attributes[GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] = usage.completionTokensDetails.reasoningTokens; } // The provider's own figure (e.g. OpenRouter) includes cache discounts and gateway markup that a - // price table cannot reproduce. - if (typeof usage.cost === 'number') { - attributes[GEN_AI_COST_TOTAL_TOKENS] = usage.cost; + // price table cannot reproduce. With a BYOK key, OpenRouter reports only its fee as `cost` and the + // provider's charge as `upstreamCost`. Relay keeps any cost it gets, so a 0 would replace the cost it + // computes from the token counts. + const cost = (usage.cost ?? 0) + (usage.costDetails?.upstreamCost ?? 0); + if (cost > 0) { + attributes[GEN_AI_COST_TOTAL_TOKENS] = cost; } return attributes; } @@ -170,6 +284,7 @@ export function addUsage(total: TanstackAiUsage, usage: TanstackAiUsage): Tansta completionTokens: add(total.completionTokens, usage.completionTokens), totalTokens: add(total.totalTokens, usage.totalTokens), cost: add(total.cost, usage.cost), + costDetails: { upstreamCost: add(total.costDetails?.upstreamCost, usage.costDetails?.upstreamCost) }, promptTokensDetails: { cachedTokens: add(total.promptTokensDetails?.cachedTokens, usage.promptTokensDetails?.cachedTokens), cacheWriteTokens: add(total.promptTokensDetails?.cacheWriteTokens, usage.promptTokensDetails?.cacheWriteTokens), diff --git a/packages/server-utils/src/integrations/aws-sdk/constants.ts b/packages/server-utils/src/integrations/aws-sdk/constants.ts index ae27a6e50315..27165dd153d3 100644 --- a/packages/server-utils/src/integrations/aws-sdk/constants.ts +++ b/packages/server-utils/src/integrations/aws-sdk/constants.ts @@ -17,3 +17,11 @@ export const MESSAGING_DESTINATION_KIND_VALUE_TOPIC = 'topic'; export const GEN_AI_OPERATION_NAME_VALUE_CHAT = 'chat'; export const GEN_AI_OPERATION_NAME_VALUE_GENERATE_CONTENT = 'generate_content'; export const GEN_AI_SYSTEM_VALUE_AWS_BEDROCK = 'aws.bedrock'; + +/** + * The key of Bedrock in the AI provider skip registry. An AI framework integration that records the Bedrock calls + * of its adapters itself skips it, so `awsIntegration` records no span for these calls. + * + * @see `_INTERNAL_withAiProviderWrappingSkipped` + */ +export const AWS_BEDROCK_INTEGRATION_NAME = 'AwsBedrock' as const; diff --git a/packages/server-utils/src/integrations/aws-sdk/index.ts b/packages/server-utils/src/integrations/aws-sdk/index.ts index 95b38b41eece..26f035726123 100644 --- a/packages/server-utils/src/integrations/aws-sdk/index.ts +++ b/packages/server-utils/src/integrations/aws-sdk/index.ts @@ -1,6 +1,6 @@ import * as diagnosticsChannel from '../../utils/diagnosticsChannel'; import type { IntegrationFn, Span } from '@sentry/core'; -import { defineIntegration, startInactiveSpan } from '@sentry/core'; +import { _INTERNAL_shouldSkipAiProviderWrapping, defineIntegration, startInactiveSpan } from '@sentry/core'; import { _AWS_REQUEST_ID as AWS_REQUEST_ID, AWS_REQUEST_EXTENDED_ID, @@ -16,7 +16,7 @@ import { awsSdkModuleNames } from '../../orchestrion/config/aws-sdk'; import { invokeOrchestrionInstrumentation } from '../../orchestrion/instrumentation'; import type { TracingChannelLifeCycleOptions } from '../../tracing-channel'; import { bindTracingChannelToSpan, safeChannelCallback } from '../../tracing-channel'; -import { AWS_SDK_ORIGIN } from './constants'; +import { AWS_BEDROCK_INTEGRATION_NAME, AWS_SDK_ORIGIN } from './constants'; import { ServicesExtensions } from './services'; import type { NormalizedRequest, NormalizedResponse, RequestMetadata } from './types'; import { extractAttributesFromNormalizedRequest, normalizeV3Request, removeSuffixFromStringIfExists } from './utils'; @@ -101,6 +101,12 @@ function instrumentAwsSdk(servicesExtensions: ServicesExtensions): void { command.input = {}; } const normalizedRequest = normalizeV3Request(serviceName, commandName, command.input, undefined); + if ( + normalizedRequest.serviceName === 'BedrockRuntime' && + _INTERNAL_shouldSkipAiProviderWrapping(AWS_BEDROCK_INTEGRATION_NAME) + ) { + return undefined; + } const requestMetadata = servicesExtensions.requestPreSpanHook(normalizedRequest); const span = startInactiveSpan({ diff --git a/packages/server-utils/src/integrations/tanstack-ai.ts b/packages/server-utils/src/integrations/tanstack-ai.ts index da38f1f50f74..d618d5e930b8 100644 --- a/packages/server-utils/src/integrations/tanstack-ai.ts +++ b/packages/server-utils/src/integrations/tanstack-ai.ts @@ -4,6 +4,7 @@ import { defineIntegration, isObjectLike } from '@sentry/core'; import type { TanstackAiOptions } from '../ai/tanstack-ai'; import { createTanstackAiMiddleware } from '../ai/tanstack-ai'; import { TANSTACK_AI_INTEGRATION_NAME, TANSTACK_AI_MIDDLEWARE_BRAND } from '../ai/tanstack-ai/constants'; +import { skipProviderIntegrations } from '../ai/tanstack-ai/utils'; import { CHANNELS } from '../orchestrion/channels'; import { tanstackAiModuleNames } from '../orchestrion/config/tanstack-ai'; import { invokeOrchestrionInstrumentation } from '../orchestrion/instrumentation'; @@ -22,7 +23,10 @@ const _tanstackAiIntegration = ((options: TanstackAiOptions = {}) => { }; }) satisfies IntegrationFn; -/** Add our middleware to every `chat()` call that does not already carry one. */ +/** + * Add our middleware to every `chat()` call that does not already carry one, and skip the integrations of + * the vendor SDKs during the calls of the adapter. + */ function injectMiddleware(options: TanstackAiOptions): void { const middleware = createTanstackAiMiddleware(options); @@ -38,11 +42,13 @@ function injectMiddleware(options: TanstackAiOptions): void { if (existing !== undefined && !Array.isArray(existing)) { return; } - if (existing?.some(entry => isObjectLike(entry) && entry[TANSTACK_AI_MIDDLEWARE_BRAND])) { - return; - } + const hasOurMiddleware = existing?.some(entry => isObjectLike(entry) && entry[TANSTACK_AI_MIDDLEWARE_BRAND]); // A copy, so options the app reuses across calls do not collect one middleware per call. - data.arguments[0] = { ...chatOptions, middleware: [...(existing ?? []), middleware] }; + data.arguments[0] = { + ...chatOptions, + adapter: skipProviderIntegrations(chatOptions.adapter), + ...(hasOurMiddleware ? {} : { middleware: [...(existing ?? []), middleware] }), + }; }); }); } diff --git a/packages/server-utils/test/ai/lib/tracing/tanstack-ai.test.ts b/packages/server-utils/test/ai/lib/tracing/tanstack-ai.test.ts new file mode 100644 index 000000000000..cb4502351626 --- /dev/null +++ b/packages/server-utils/test/ai/lib/tracing/tanstack-ai.test.ts @@ -0,0 +1,514 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Span } from '@sentry/core'; +import { + _INTERNAL_clearAiProviderSkips, + _INTERNAL_shouldSkipAiProviderWrapping, + captureMessage, + getActiveSpan, + getMainCarrier, + setCurrentClient, + spanToStaticSpanJSON, +} from '@sentry/core'; +import { ANTHROPIC_AI_INTEGRATION_NAME } from '../../../../src/ai/anthropic-ai/constants'; +import { OPENAI_INTEGRATION_NAME } from '../../../../src/ai/openai/constants'; +import { createTanstackAiMiddleware } from '../../../../src/ai/tanstack-ai'; +import type { TanstackAiMiddlewareConfig, TanstackAiMiddlewareContext } from '../../../../src/ai/tanstack-ai/types'; +import { skipProviderIntegrations } from '../../../../src/ai/tanstack-ai/utils'; +import { WORKERS_AI_INTEGRATION_NAME } from '../../../../src/ai/workers-ai/constants'; +import { AWS_BEDROCK_INTEGRATION_NAME } from '../../../../src/integrations/aws-sdk/constants'; +import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; + +// TanStack creates one context per `chat()` call and the middleware keys its state by it, so every test +// spreads its own copy. +const CTX: TanstackAiMiddlewareContext = { + activity: 'chat', + phase: 'init', + provider: 'openrouter-responses', + model: 'openai/gpt-4o-mini', + threadId: 'thread-1', + streaming: true, + callMetadata: {}, + messages: [], + systemPrompts: [], +}; + +const CONFIG: TanstackAiMiddlewareConfig = { + messages: [{ role: 'user', content: 'What is the weather in Vienna?' }], + systemPrompts: [], + tools: [], +}; + +describe('createTanstackAiMiddleware', () => { + let endedSpans: Span[]; + let client: TestClient; + + beforeEach(() => { + _INTERNAL_clearAiProviderSkips(); + getMainCarrier().__SENTRY__ = undefined; + client = new TestClient( + getDefaultTestClientOptions({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + tracesSampleRate: 1, + traceLifecycle: 'stream', + }), + ); + setCurrentClient(client); + client.init(); + + endedSpans = []; + client.on('spanEnd', span => endedSpans.push(span)); + }); + + afterEach(() => { + _INTERNAL_clearAiProviderSkips(); + getMainCarrier().__SENTRY__ = undefined; + }); + + it('reports the provider and response fields from the call metadata', () => { + const middleware = createTanstackAiMiddleware(); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, CONFIG); + // TanStack sets the call metadata when the model call starts, after `onStart`. + ctx.phase = 'beforeModel'; + ctx.callMetadata = { source: { provider: 'openrouter', api: 'responses' } }; + middleware.onConfig(ctx, CONFIG); + ctx.callMetadata = { ...ctx.callMetadata, model: 'openai/gpt-4o-mini-2024-07-18', responseId: 'gen-123' }; + middleware.onChunk(ctx, { type: 'RUN_FINISHED', finishReason: 'stop' }); + middleware.onFinish(ctx); + + const chat = endedSpans.map(span => spanToStaticSpanJSON(span)).find(span => span.op === 'gen_ai.chat'); + const agent = endedSpans.map(span => spanToStaticSpanJSON(span)).find(span => span.op === 'gen_ai.invoke_agent'); + expect(chat?.data).toMatchObject({ + 'gen_ai.provider.name': 'openrouter', + 'gen_ai.response.model': 'openai/gpt-4o-mini-2024-07-18', + 'gen_ai.response.id': 'gen-123', + 'gen_ai.response.finish_reasons': '["stop"]', + }); + expect(agent?.data['gen_ai.provider.name']).toBe('openrouter'); + }); + + it('records the input of the run and the answer of the last model call on the agent span', () => { + const middleware = createTanstackAiMiddleware({ recordInputs: true, recordOutputs: true }); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, { ...CONFIG, systemPrompts: ['Be brief.'] }); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + middleware.onChunk(ctx, { type: 'TOOL_CALL_START', toolCallId: 'call-1', toolCallName: 'get_weather' }); + middleware.onChunk(ctx, { type: 'TOOL_CALL_END', toolCallId: 'call-1', input: { city: 'Vienna' } }); + middleware.onChunk(ctx, { type: 'RUN_FINISHED', finishReason: 'tool_calls', responseId: 'gen-1' }); + ctx.phase = 'afterModel'; + middleware.onInterruptBoundary(ctx); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + middleware.onChunk(ctx, { type: 'TEXT_MESSAGE_CONTENT', delta: 'Sunny' }); + middleware.onChunk(ctx, { type: 'RUN_FINISHED', finishReason: 'stop', responseId: 'gen-2', model: 'gpt-4o-mini' }); + middleware.onFinish(ctx); + + const agent = endedSpans.map(span => spanToStaticSpanJSON(span)).find(span => span.op === 'gen_ai.invoke_agent'); + expect(agent?.data).toMatchObject({ + 'gen_ai.input.messages': '[{"role":"user","parts":[{"type":"text","content":"What is the weather in Vienna?"}]}]', + 'gen_ai.system_instructions': 'Be brief.', + 'gen_ai.response.id': 'gen-2', + 'gen_ai.response.model': 'gpt-4o-mini', + 'gen_ai.output.messages': + '[{"role":"assistant","parts":[{"type":"text","content":"Sunny"}],"finish_reason":"stop"}]', + }); + // The finish reason belongs to a single model call. + expect(agent?.data).not.toHaveProperty('gen_ai.response.finish_reasons'); + }); + + it('sums the usage and provider cost of every model call on the agent span', () => { + const middleware = createTanstackAiMiddleware(); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + middleware.onUsage(ctx, { promptTokens: 10, completionTokens: 2, totalTokens: 12, cost: 0.001 }); + ctx.phase = 'afterModel'; + middleware.onInterruptBoundary(ctx); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + // OpenRouter with a BYOK key: its fee is 0, and the provider's charge is the upstream cost. + middleware.onUsage(ctx, { + promptTokens: 20, + completionTokens: 5, + totalTokens: 25, + cost: 0, + costDetails: { upstreamCost: 0.002 }, + promptTokensDetails: { cachedTokens: 8 }, + }); + middleware.onFinish(ctx); + + const agent = endedSpans.map(span => spanToStaticSpanJSON(span)).find(span => span.op === 'gen_ai.invoke_agent'); + expect(agent?.data).toMatchObject({ + 'gen_ai.usage.input_tokens': 30, + 'gen_ai.usage.output_tokens': 7, + 'gen_ai.usage.total_tokens': 37, + 'gen_ai.usage.cache_read.input_tokens': 8, + 'gen_ai.cost.total_tokens': 0.003, + }); + expect(endedSpans.filter(span => spanToStaticSpanJSON(span).op === 'gen_ai.chat')).toHaveLength(2); + }); + + it('leaves out a cost of 0, so Relay computes the cost from the token counts', () => { + const middleware = createTanstackAiMiddleware(); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + middleware.onUsage(ctx, { promptTokens: 10, completionTokens: 2, totalTokens: 12, cost: 0 }); + middleware.onFinish(ctx); + + const spans = endedSpans.map(span => spanToStaticSpanJSON(span)); + expect(spans).toHaveLength(2); + expect(spans.every(span => span.data['gen_ai.usage.input_tokens'] === 10)).toBe(true); + expect(spans.some(span => 'gen_ai.cost.total_tokens' in span.data)).toBe(false); + }); + + it('marks a failed tool call and captures its error, which TanStack hands to the model', async () => { + const middleware = createTanstackAiMiddleware(); + const ctx = { ...CTX }; + const tool = { name: 'get_weather', execute: () => undefined }; + + middleware.onConfig(ctx, CONFIG); + middleware.onGuardToolCall(ctx, { toolName: 'get_weather', toolCallId: 'call-1', args: { city: 'Vienna' }, tool }); + middleware.onAfterToolCall(ctx, { toolCallId: 'call-1', ok: false, error: new TypeError('kaboom') }); + middleware.onFinish(ctx); + await client.flush(); + + const toolSpan = endedSpans.map(span => spanToStaticSpanJSON(span)).find(span => span.op === 'gen_ai.execute_tool'); + expect(toolSpan?.status).toBe('internal_error'); + expect(toolSpan?.data).toMatchObject({ + 'gen_ai.tool.call.id': 'call-1', + 'gen_ai.tool.call.arguments': '{"city":"Vienna"}', + }); + const exception = client.event?.exception?.values?.[0]; + expect(exception?.value).toBe('kaboom'); + expect(exception?.mechanism).toEqual({ type: 'auto.ai.tanstack_ai', handled: true }); + }); + + it('records no tool span for a client tool, which runs in the browser', () => { + const middleware = createTanstackAiMiddleware(); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, CONFIG); + middleware.onGuardToolCall(ctx, { + toolName: 'pick_color', + toolCallId: 'call-1', + args: {}, + tool: { name: 'pick_color' }, + }); + middleware.onFinish(ctx); + + expect(endedSpans.map(span => spanToStaticSpanJSON(span).op)).toEqual(['gen_ai.invoke_agent']); + }); + + it('ends every open span with an error status when the run fails', () => { + const middleware = createTanstackAiMiddleware(); + const ctx = { ...CTX }; + const tool = { name: 'get_weather', execute: () => undefined }; + + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + middleware.onGuardToolCall(ctx, { toolName: 'get_weather', toolCallId: 'call-1', args: {}, tool }); + middleware.onError(ctx, { error: new Error('provider down') }); + + expect(endedSpans.map(span => [spanToStaticSpanJSON(span).op, spanToStaticSpanJSON(span).status])).toEqual([ + ['gen_ai.execute_tool', 'internal_error'], + ['gen_ai.chat', 'internal_error'], + ['gen_ai.invoke_agent', 'internal_error'], + ]); + }); + + it('ends the run with an error status when the provider error arrives as a chunk', () => { + const middleware = createTanstackAiMiddleware(); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + // `chat({ stream: false })` throws on this chunk and stops reading, so no terminal hook follows. + middleware.onChunk(ctx, { type: 'RUN_ERROR' }); + + expect(endedSpans.map(span => [spanToStaticSpanJSON(span).op, spanToStaticSpanJSON(span).status])).toEqual([ + ['gen_ai.chat', 'internal_error'], + ['gen_ai.invoke_agent', 'internal_error'], + ]); + }); + + it('marks an aborted run as cancelled', () => { + const middleware = createTanstackAiMiddleware(); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + middleware.onAbort(ctx); + + expect(endedSpans.map(span => spanToStaticSpanJSON(span).status)).toEqual(['cancelled', 'cancelled']); + }); + + it('marks the provider error that ends an aborted run as cancelled', () => { + const middleware = createTanstackAiMiddleware(); + const abortController = new AbortController(); + const ctx = { ...CTX, signal: abortController.signal }; + + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + abortController.abort(); + // TanStack yields the held error of a rate-limited call when an abort ends the wait for the retry. + middleware.onChunk(ctx, { type: 'RUN_ERROR' }); + + expect(endedSpans.map(span => spanToStaticSpanJSON(span).status)).toEqual(['cancelled', 'cancelled']); + }); + + it('marks a tool call that fails because the run was aborted as cancelled, and captures no error', async () => { + const middleware = createTanstackAiMiddleware(); + const abortController = new AbortController(); + const ctx = { ...CTX, signal: abortController.signal }; + const tool = { name: 'get_weather', execute: () => undefined }; + const sendEvent = vi.spyOn(client, 'sendEvent'); + + middleware.onConfig(ctx, CONFIG); + middleware.onGuardToolCall(ctx, { toolName: 'get_weather', toolCallId: 'call-1', args: {}, tool }); + abortController.abort(); + middleware.onAfterToolCall(ctx, { toolCallId: 'call-1', ok: false, error: new Error('Operation aborted') }); + middleware.onAbort(ctx); + // Sentinel: once this event is sent, an event for the tool error would have been sent before it. + captureMessage('sentinel'); + await client.flush(); + + const toolSpan = endedSpans.map(span => spanToStaticSpanJSON(span)).find(span => span.op === 'gen_ai.execute_tool'); + expect(toolSpan?.status).toBe('cancelled'); + expect(sendEvent).toHaveBeenCalledTimes(1); + expect(sendEvent).toHaveBeenCalledWith(expect.objectContaining({ message: 'sentinel' }), expect.anything()); + }); + + it('ends a model call that a middleware retried with an error status', () => { + const middleware = createTanstackAiMiddleware(); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + // The `RUN_ERROR` of a call that `ctx.retry()` retries never reaches `onChunk`. TanStack runs `onConfig` again. + middleware.onConfig(ctx, CONFIG); + middleware.onChunk(ctx, { type: 'RUN_FINISHED', finishReason: 'stop' }); + ctx.phase = 'afterModel'; + middleware.onInterruptBoundary(ctx); + middleware.onFinish(ctx); + + expect(endedSpans.map(span => [spanToStaticSpanJSON(span).op, spanToStaticSpanJSON(span).status])).toEqual([ + ['gen_ai.chat', 'internal_error'], + ['gen_ai.chat', 'ok'], + ['gen_ai.invoke_agent', 'ok'], + ]); + }); + + it('ends the run when it pauses at an interrupt, for which TanStack calls no terminal hook', () => { + const middleware = createTanstackAiMiddleware(); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + middleware.onChunk(ctx, { type: 'RUN_FINISHED', finishReason: 'stop', outcome: { type: 'interrupt' } }); + + expect(endedSpans.map(span => [spanToStaticSpanJSON(span).op, spanToStaticSpanJSON(span).status])).toEqual([ + ['gen_ai.chat', 'ok'], + ['gen_ai.invoke_agent', 'ok'], + ]); + }); + + it('ends the chat span after the model call, before the tools run', () => { + const middleware = createTanstackAiMiddleware(); + const ctx = { ...CTX }; + const tool = { name: 'get_weather', execute: () => undefined }; + + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'afterModel'; + middleware.onInterruptBoundary(ctx); + middleware.onGuardToolCall(ctx, { toolName: 'get_weather', toolCallId: 'call-1', args: {}, tool }); + middleware.onAfterToolCall(ctx, { toolCallId: 'call-1', ok: true, result: 'sunny' }); + middleware.onFinish(ctx); + + expect(endedSpans.map(span => spanToStaticSpanJSON(span).op)).toEqual([ + 'gen_ai.chat', + 'gen_ai.execute_tool', + 'gen_ai.invoke_agent', + ]); + }); + + it('keeps the tool spans of two runs apart when their tool calls have the same id', () => { + const middleware = createTanstackAiMiddleware(); + const first = { ...CTX, threadId: 'thread-1' }; + const second = { ...CTX, threadId: 'thread-2' }; + const tool = { name: 'get_weather', execute: () => undefined }; + + middleware.onConfig(first, CONFIG); + middleware.onConfig(second, CONFIG); + middleware.onGuardToolCall(first, { toolName: 'get_weather', toolCallId: 'get_weather_1', args: {}, tool }); + middleware.onGuardToolCall(second, { toolName: 'get_weather', toolCallId: 'get_weather_1', args: {}, tool }); + middleware.onAfterToolCall(first, { toolCallId: 'get_weather_1', ok: true, result: 'sunny' }); + middleware.onAfterToolCall(second, { toolCallId: 'get_weather_1', ok: true, result: 'rainy' }); + + const tools = endedSpans.map(span => spanToStaticSpanJSON(span)).filter(span => span.op === 'gen_ai.execute_tool'); + expect(tools.map(span => [span.data['gen_ai.conversation.id'], span.data['gen_ai.tool.call.result']])).toEqual([ + ['thread-1', 'sunny'], + ['thread-2', 'rainy'], + ]); + }); + + it('records the messages that a middleware sends to the provider instead of the history', () => { + const middleware = createTanstackAiMiddleware({ recordInputs: true }); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, { ...CONFIG, providerMessages: [{ role: 'user', content: 'Weather in Vienna?' }] }); + middleware.onFinish(ctx); + + const chat = endedSpans.map(span => spanToStaticSpanJSON(span)).find(span => span.op === 'gen_ai.chat'); + expect(chat?.data['gen_ai.input.messages']).toBe( + '[{"role":"user","parts":[{"type":"text","content":"Weather in Vienna?"}]}]', + ); + }); + + it('records the provider of the structured output call, and no tools, which that call does not send', () => { + const middleware = createTanstackAiMiddleware({ recordInputs: true }); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, CONFIG); + // Without tools, the structured output call is the only model call, and TanStack sets its call metadata after + // `onConfig`. + ctx.phase = 'structuredOutput'; + middleware.onConfig(ctx, { ...CONFIG, tools: [{ name: 'get_weather', description: 'Get the weather.' }] }); + ctx.callMetadata = { source: { provider: 'openrouter', api: 'responses' } }; + middleware.onChunk(ctx, { type: 'RUN_FINISHED', finishReason: 'stop' }); + middleware.onFinish(ctx); + + const spans = endedSpans.map(span => spanToStaticSpanJSON(span)); + expect(spans.map(span => span.data['gen_ai.provider.name'])).toEqual(['openrouter', 'openrouter']); + expect(spans.find(span => span.op === 'gen_ai.chat')?.data).not.toHaveProperty('gen_ai.tool.definitions'); + }); + + it('leaves out the tool definitions when inputs are not recorded', () => { + const middleware = createTanstackAiMiddleware({ recordInputs: false }); + const ctx = { ...CTX }; + + middleware.onConfig(ctx, CONFIG); + ctx.phase = 'beforeModel'; + middleware.onConfig(ctx, { ...CONFIG, tools: [{ name: 'get_weather', description: 'Get the weather.' }] }); + middleware.onFinish(ctx); + + const chat = endedSpans.map(span => spanToStaticSpanJSON(span)).find(span => span.op === 'gen_ai.chat'); + expect(chat?.data).not.toHaveProperty('gen_ai.tool.definitions'); + }); + + it('runs each tool with the tool span of its own run active, also when two runs have the same tool call id', async () => { + const middleware = createTanstackAiMiddleware(); + const getWeather = { + name: 'get_weather', + execute: () => spanToStaticSpanJSON(getActiveSpan()!).data['gen_ai.conversation.id'], + }; + const first = { ...CTX, threadId: 'thread-1' }; + const second = { ...CTX, threadId: 'thread-2' }; + + const [firstTool] = middleware.onConfig(first, { ...CONFIG, tools: [getWeather] })!.tools!; + const [secondTool] = middleware.onConfig(second, { ...CONFIG, tools: [getWeather] })!.tools!; + middleware.onGuardToolCall(first, { + toolName: 'get_weather', + toolCallId: 'get_weather_1', + args: {}, + tool: firstTool, + }); + middleware.onGuardToolCall(second, { + toolName: 'get_weather', + toolCallId: 'get_weather_1', + args: {}, + tool: secondTool, + }); + + expect([ + await firstTool!.execute!({}, { toolCallId: 'get_weather_1' }), + await secondTool!.execute!({}, { toolCallId: 'get_weather_1' }), + ]).toEqual(['thread-1', 'thread-2']); + }); +}); + +describe('skipProviderIntegrations', () => { + it('skips every provider integration during a call, also after an await, but not after the call', async () => { + // `@tanstack/openai-base` lets an adapter have a name of its own. + const adapter = skipProviderIntegrations({ + name: 'deepseek', + async structuredOutput() { + await Promise.resolve(); + return [ + _INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME), + _INTERNAL_shouldSkipAiProviderWrapping(ANTHROPIC_AI_INTEGRATION_NAME), + _INTERNAL_shouldSkipAiProviderWrapping(WORKERS_AI_INTEGRATION_NAME), + _INTERNAL_shouldSkipAiProviderWrapping(AWS_BEDROCK_INTEGRATION_NAME), + ]; + }, + }); + + expect(await adapter.structuredOutput()).toEqual([true, true, true, true]); + expect(_INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME)).toBe(false); + }); + + it('skips the integrations in the body of a returned async generator, but not in the code that reads it', async () => { + const adapter = skipProviderIntegrations({ + name: 'openai-chat', + async *chatStream() { + yield _INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME); + await Promise.resolve(); + yield _INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME); + }, + }); + + const skipped: boolean[] = []; + for await (const skippedInAdapter of adapter.chatStream()) { + skipped.push(skippedInAdapter, _INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME)); + } + + expect(skipped).toEqual([true, false, true, false]); + }); + + it('calls the methods on the adapter itself, so that they can read its private fields', async () => { + class Adapter { + public readonly name = 'openai-chat'; + readonly #model = 'gpt-4o'; + + public async structuredOutput(): Promise { + return this.#model; + } + } + + const adapter = skipProviderIntegrations(new Adapter()); + + expect(await adapter.structuredOutput()).toBe('gpt-4o'); + }); + + it('returns the same function for each read of a method, and the constructor and frozen methods as they are', () => { + class Adapter { + public async structuredOutput(): Promise { + return 'ok'; + } + } + const frozen = Object.freeze({ chatStream: () => 'frozen' }); + + const adapter = skipProviderIntegrations(new Adapter()); + + expect(adapter.structuredOutput).toBe(adapter.structuredOutput); + expect(adapter.constructor).toBe(Adapter); + expect(skipProviderIntegrations(frozen).chatStream).toBe(frozen.chatStream); + }); +}); diff --git a/packages/vercel-edge/src/index.ts b/packages/vercel-edge/src/index.ts index d1f92c146338..91949ae846f8 100644 --- a/packages/vercel-edge/src/index.ts +++ b/packages/vercel-edge/src/index.ts @@ -115,6 +115,7 @@ export { instrumentAnthropicAiClient, instrumentGoogleGenAIClient, createLangChainCallbackHandler, + createTanstackAiMiddleware, instrumentLangChainEmbeddings, instrumentStateGraph, instrumentCreateReactAgent, From 028877e97a1555c19ba1d6c87b07bc540d8716fa Mon Sep 17 00:00:00 2001 From: JPeer264 Date: Sun, 25 Oct 2026 12:15:53 +0100 Subject: [PATCH 3/6] feat(server-utils): Instrument TanStack AI embed() calls Record a gen_ai.embeddings span for each @tanstack/ai embed() call, with the input, the response model and the usage. The span is active while the call runs, so the provider request nests under it. embed() takes observe-only middleware that gets neither the input nor the result and cannot make a span active, so the call itself is traced through an orchestrion channel, as for LangChain embeddings. As for chat(), the integrations of the vendor SDKs are skipped only during the calls of the adapter, and a call that the app aborts gets the cancelled status. Co-Authored-By: Claude Opus 5.5 --- .../tracing/tanstack-ai/scenario-embed.mjs | 39 ++++++ .../suites/tracing/tanstack-ai/test.ts | 50 +++++++ .../src/ai/tanstack-ai/generation.ts | 113 +++++++++++++++ .../server-utils/src/ai/tanstack-ai/utils.ts | 19 ++- .../src/integrations/tanstack-ai.ts | 41 +++++- .../src/orchestrion/config/tanstack-ai.ts | 8 ++ .../tracing/tanstack-ai-generation.test.ts | 131 ++++++++++++++++++ 7 files changed, 398 insertions(+), 3 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-embed.mjs create mode 100644 packages/server-utils/src/ai/tanstack-ai/generation.ts create mode 100644 packages/server-utils/test/ai/lib/tracing/tanstack-ai-generation.test.ts diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-embed.mjs b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-embed.mjs new file mode 100644 index 000000000000..ade74dbbf6f8 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-embed.mjs @@ -0,0 +1,39 @@ +import express from 'express'; +import * as Sentry from '@sentry/node'; +import { embed } from '@tanstack/ai'; +import { createOpenaiEmbedding } from '@tanstack/ai-openai'; +import OpenAI from 'openai'; + +// Inlined OpenAI-compatible mock: the runner copies only this file into its temp dir. +function startMockProvider() { + const app = express(); + app.use(express.json()); + app.post('/v1/embeddings', (req, res) => { + res.json({ + object: 'list', + data: req.body.input.map((_text, index) => ({ object: 'embedding', embedding: [0.1, 0.2, 0.3], index })), + model: 'text-embedding-3-small', + usage: { prompt_tokens: 8, total_tokens: 8 }, + }); + }); + return new Promise(resolve => { + const server = app.listen(0, () => resolve(server)); + }); +} + +const server = await startMockProvider(); +const baseURL = `http://localhost:${server.address().port}/v1`; +const adapter = createOpenaiEmbedding('text-embedding-3-small', 'test-key', { baseURL }); + +await Sentry.startSpan({ name: 'main' }, async () => { + await embed({ adapter, input: ['a red guitar', 'a blue drum kit'] }); + + await Sentry.startSpan({ name: 'direct' }, async () => { + await new OpenAI({ apiKey: 'test-key', baseURL }).embeddings.create({ + model: 'text-embedding-3-small', + input: ['a green piano'], + }); + }); +}); + +server.close(); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts index 48d30d16f7aa..8206fce769fa 100644 --- a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts @@ -1,6 +1,8 @@ import { GEN_AI_CONVERSATION_ID, + GEN_AI_EMBEDDINGS_INPUT, GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, GEN_AI_OUTPUT_MESSAGES, GEN_AI_PROVIDER_NAME, GEN_AI_REQUEST_MODEL, @@ -398,4 +400,52 @@ describe('TanStack AI integration', () => { }, TANSTACK_AI_DEPENDENCIES, ); + + createEsmAndCjsTests( + __dirname, + 'scenario-embed.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + test('records an embeddings span, with the provider request under it', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items; + const main = spans.find(span => span.name === 'main')!; + const direct = spans.find(span => span.name === 'direct')!; + const embeddings = spans.filter(span => attr(span, 'sentry.op') === 'gen_ai.embeddings'); + // The OpenAI integration is skipped for TanStack's call, but not for the direct call after it. + expect(embeddings.map(span => attr(span, 'sentry.origin')).sort()).toEqual(['auto.ai.openai', ORIGIN]); + const embedding = embeddings.find(span => attr(span, 'sentry.origin') === ORIGIN)!; + const directEmbedding = embeddings.find(span => attr(span, 'sentry.origin') === 'auto.ai.openai')!; + expect(directEmbedding.parent_span_id).toBe(direct.span_id); + expect(embedding.name).toBe('embeddings text-embedding-3-small'); + expect(embedding.parent_span_id).toBe(main.span_id); + expect(embedding.status).toBe('ok'); + expect(attr(embedding, 'sentry.origin')).toBe(ORIGIN); + expect(attr(embedding, GEN_AI_OPERATION_NAME)).toBe('embeddings'); + expect(attr(embedding, GEN_AI_PROVIDER_NAME)).toBe('openai'); + expect(attr(embedding, GEN_AI_REQUEST_MODEL)).toBe('text-embedding-3-small'); + expect(attr(embedding, GEN_AI_RESPONSE_MODEL)).toBe('text-embedding-3-small'); + expect(attr(embedding, GEN_AI_EMBEDDINGS_INPUT)).toBe('["a red guitar","a blue drum kit"]'); + expect(attr(embedding, GEN_AI_USAGE_INPUT_TOKENS)).toBe(8); + expect(attr(embedding, GEN_AI_USAGE_TOTAL_TOKENS)).toBe(8); + expect(attr(embedding, GEN_AI_USAGE_OUTPUT_TOKENS)).toBeUndefined(); + + const requests = spans.filter(span => attr(span, 'sentry.op') === 'http.client'); + expect(requests.map(span => span.parent_span_id).sort()).toEqual( + [embedding.span_id, directEmbedding.span_id].sort(), + ); + }, + }) + .start() + .completed(); + }); + }, + TANSTACK_AI_DEPENDENCIES, + ); }); diff --git a/packages/server-utils/src/ai/tanstack-ai/generation.ts b/packages/server-utils/src/ai/tanstack-ai/generation.ts new file mode 100644 index 000000000000..65456350458d --- /dev/null +++ b/packages/server-utils/src/ai/tanstack-ai/generation.ts @@ -0,0 +1,113 @@ +import type { Span, SpanAttributes } from '@sentry/core'; +import { isObjectLike, SPAN_STATUS_ERROR, startInactiveSpan, stringify } from '@sentry/core'; +import { + GEN_AI_EMBEDDINGS_INPUT, + GEN_AI_OPERATION_NAME, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_MODEL, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import type { OrchestrionChannelContext } from '../../orchestrion/types'; +import type { GenAiOptions } from '../core/utils'; +import { getGenAiSpanOp, resolveAIRecordingOptions } from '../core/utils'; +import { TANSTACK_AI_ORIGIN } from './constants'; +import { usageAttributes } from './utils'; + +/** + * A `@tanstack/ai` activity other than `chat()`, traced through its orchestrion channel. Its middleware + * gets neither the input of the call nor the result, and cannot make a span active during the provider + * request. + */ +export interface TanstackAiGenerationActivity { + /** The `gen_ai.operation.name` of the span. */ + operation: string; + /** The name of the adapter method that sends the provider request. */ + adapterMethod: string; + /** Attributes from the options of the call. */ + requestAttributes?: (options: Record) => SpanAttributes; + /** + * Attributes from the value that the adapter method resolves to. They are recorded also when TanStack AI throws + * after the provider answered, because the provider bills that call too. + */ + responseAttributes: (adapterResult: Record) => SpanAttributes; +} + +export const TANSTACK_AI_EMBED: TanstackAiGenerationActivity = { + operation: 'embeddings', + adapterMethod: 'createEmbeddings', + requestAttributes: ({ input }) => (input !== undefined ? { [GEN_AI_EMBEDDINGS_INPUT]: embeddingsInput(input) } : {}), + // The adapters make `id` themselves, so it is not the provider's response id. An embedding has no output + // tokens, but the OpenAI adapter reports 0. + responseAttributes: ({ model, usage }) => ({ + ...(typeof model === 'string' ? { [GEN_AI_RESPONSE_MODEL]: model } : {}), + ...(isObjectLike(usage) ? usageAttributes({ ...usage, completionTokens: undefined }) : {}), + }), +}; + +/** Start the span of an activity call, or return `undefined` for options that are not a call's options. */ +export function startGenerationSpan( + activity: TanstackAiGenerationActivity, + callOptions: unknown, + options: GenAiOptions, +): Span | undefined { + if (!isObjectLike(callOptions) || !isObjectLike(callOptions.adapter)) { + return undefined; + } + // As in `chat()`, `provider` is the vendor of an adapter whose `name` also names the wire API. + const { name, provider = name, model } = callOptions.adapter; + if (typeof provider !== 'string' || typeof model !== 'string') { + return undefined; + } + + const { recordInputs } = resolveAIRecordingOptions(options); + return startInactiveSpan({ + name: `${activity.operation} ${model}`, + op: getGenAiSpanOp(activity.operation), + attributes: { + [SENTRY_ORIGIN]: TANSTACK_AI_ORIGIN, + [GEN_AI_OPERATION_NAME]: activity.operation, + [GEN_AI_PROVIDER_NAME]: provider, + [GEN_AI_REQUEST_MODEL]: model, + ...(recordInputs ? activity.requestAttributes?.(callOptions) : {}), + }, + }); +} + +export function setGenerationResponseAttributes( + span: Span, + activity: TanstackAiGenerationActivity, + adapterResult: unknown, +): void { + if (isObjectLike(adapterResult)) { + span.setAttributes(activity.responseAttributes(adapterResult)); + } +} + +/** Record what the error that a call threw tells. */ +export function setGenerationEndAttributes(span: Span, call: OrchestrionChannelContext): void { + if ('result' in call) { + return; + } + // As for `chat()`, a call that the app aborted counts as cancelled. + const [callOptions] = call.arguments; + if (isObjectLike(callOptions) && (callOptions.abortSignal as AbortSignal | undefined)?.aborted) { + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'cancelled' }); + } +} + +/** An input item is a string, a content part, or an array of content parts. */ +function embeddingsInput(input: unknown): string | undefined { + if (typeof input === 'string') { + return input; + } + // Media parts carry inline bytes or provider handles; keep only their kind. + const withoutMedia = (part: unknown): unknown => + isObjectLike(part) && typeof part.type === 'string' + ? part.type === 'text' + ? part.content + : { type: part.type } + : part; + const items = Array.isArray(input) ? input : [input]; + return stringify(items.map(item => (Array.isArray(item) ? item.map(withoutMedia) : withoutMedia(item)))); +} diff --git a/packages/server-utils/src/ai/tanstack-ai/utils.ts b/packages/server-utils/src/ai/tanstack-ai/utils.ts index 27abd8ded38b..7e11b9381f66 100644 --- a/packages/server-utils/src/ai/tanstack-ai/utils.ts +++ b/packages/server-utils/src/ai/tanstack-ai/utils.ts @@ -2,6 +2,7 @@ import type { Span } from '@sentry/core'; import { _INTERNAL_withAiProviderWrappingSkipped, isObjectLike, + isThenable, SPAN_STATUS_ERROR, startInactiveSpan, stringify, @@ -74,8 +75,13 @@ type AdapterMethod = (...args: unknown[]) => unknown; /** * The adapter, with the Sentry integrations of the vendor SDKs skipped during its calls, as our spans record * them. A direct call to the same SDK elsewhere in the app still gets its span. + * + * `onResult.callback` gets the value that the adapter method named `onResult.method` resolves to. */ -export function skipProviderIntegrations(adapter: T): T { +export function skipProviderIntegrations( + adapter: T, + onResult?: { method: string; callback: (result: unknown) => void }, +): T { if (!isObjectLike(adapter)) { return adapter; } @@ -99,11 +105,20 @@ export function skipProviderIntegrations(adapter: T): T { if (known) { return known; } + const callback = property === onResult?.method ? onResult.callback : undefined; const wrapped: AdapterMethod = (...args) => { const result: unknown = skipped(() => value.apply(target, args)); // A text adapter returns an async generator, whose body runs on each `next()` in the async // context of the caller, which for a streamed `chat()` is the app's code. - return isAsyncIterable(result) ? skipDuringIteration(result, skipped) : result; + if (isAsyncIterable(result)) { + return skipDuringIteration(result, skipped); + } + return callback && isThenable(result) + ? result.then(resolved => { + callback(resolved); + return resolved; + }) + : result; }; wrappedMethods.set(value as AdapterMethod, wrapped); return wrapped; diff --git a/packages/server-utils/src/integrations/tanstack-ai.ts b/packages/server-utils/src/integrations/tanstack-ai.ts index d618d5e930b8..62efb8263964 100644 --- a/packages/server-utils/src/integrations/tanstack-ai.ts +++ b/packages/server-utils/src/integrations/tanstack-ai.ts @@ -4,12 +4,23 @@ import { defineIntegration, isObjectLike } from '@sentry/core'; import type { TanstackAiOptions } from '../ai/tanstack-ai'; import { createTanstackAiMiddleware } from '../ai/tanstack-ai'; import { TANSTACK_AI_INTEGRATION_NAME, TANSTACK_AI_MIDDLEWARE_BRAND } from '../ai/tanstack-ai/constants'; +import type { TanstackAiGenerationActivity } from '../ai/tanstack-ai/generation'; +import { + setGenerationEndAttributes, + setGenerationResponseAttributes, + startGenerationSpan, + TANSTACK_AI_EMBED, +} from '../ai/tanstack-ai/generation'; import { skipProviderIntegrations } from '../ai/tanstack-ai/utils'; import { CHANNELS } from '../orchestrion/channels'; import { tanstackAiModuleNames } from '../orchestrion/config/tanstack-ai'; import { invokeOrchestrionInstrumentation } from '../orchestrion/instrumentation'; import type { OrchestrionChannelContext } from '../orchestrion/types'; -import { safeChannelCallback } from '../tracing-channel'; +import { bindTracingChannelToSpan, safeChannelCallback } from '../tracing-channel'; + +const GENERATION_ACTIVITIES: Array<[channel: string, activity: TanstackAiGenerationActivity]> = [ + [CHANNELS.TANSTACK_AI_EMBED, TANSTACK_AI_EMBED], +]; const _tanstackAiIntegration = ((options: TanstackAiOptions = {}) => { return { @@ -19,6 +30,8 @@ const _tanstackAiIntegration = ((options: TanstackAiOptions = {}) => { invokeOrchestrionInstrumentation(client, tanstackAiModuleNames, injectMiddleware, [options], { requiresTracingChannelBinding: false, }); + // The other activities open their own spans via `bindTracingChannelToSpan`, which needs the binding. + invokeOrchestrionInstrumentation(client, tanstackAiModuleNames, instrumentGenerationActivities, [options]); }, }; }) satisfies IntegrationFn; @@ -53,4 +66,30 @@ function injectMiddleware(options: TanstackAiOptions): void { }); } +/** Record a span for each call of the activities other than `chat()`, active while the call runs. */ +function instrumentGenerationActivities(options: TanstackAiOptions): void { + for (const [channel, activity] of GENERATION_ACTIVITIES) { + bindTracingChannelToSpan( + diagnosticsChannel.tracingChannel(channel), + data => { + const callOptions = data.arguments[0]; + const span = startGenerationSpan(activity, callOptions, options); + // Only a call that our span records skips the integrations of the vendor SDKs. + if (span && isObjectLike(callOptions)) { + data.arguments[0] = { + ...callOptions, + adapter: skipProviderIntegrations(callOptions.adapter, { + method: activity.adapterMethod, + callback: adapterResult => + safeChannelCallback(() => setGenerationResponseAttributes(span, activity, adapterResult)), + }), + }; + } + return span; + }, + { beforeSpanEnd: (span, data) => setGenerationEndAttributes(span, data) }, + ); + } +} + export const tanstackAiIntegration = defineIntegration(_tanstackAiIntegration); diff --git a/packages/server-utils/src/orchestrion/config/tanstack-ai.ts b/packages/server-utils/src/orchestrion/config/tanstack-ai.ts index fda0f7eda09f..fbb4c0604dd3 100644 --- a/packages/server-utils/src/orchestrion/config/tanstack-ai.ts +++ b/packages/server-utils/src/orchestrion/config/tanstack-ai.ts @@ -11,10 +11,18 @@ export const tanstackAiConfig = [ module: { name: '@tanstack/ai', versionRange: '>=1.0.0 <2.0.0', filePath: 'dist/esm/activities/chat/index.js' }, functionQuery: { functionName: 'chat', kind: 'Sync' }, }, + // The other activities take observe-only middleware that never sees the input or the result, so we + // trace the call itself. + { + channelName: 'embed', + module: { name: '@tanstack/ai', versionRange: '>=1.0.0 <2.0.0', filePath: 'dist/esm/activities/embed/index.js' }, + functionQuery: { functionName: 'embed', kind: 'Async' }, + }, ] satisfies InstrumentationConfig[]; export const tanstackAiModuleNames = getModuleNames(tanstackAiConfig); export const tanstackAiChannels = { TANSTACK_AI_CHAT: 'orchestrion:@tanstack/ai:chat', + TANSTACK_AI_EMBED: 'orchestrion:@tanstack/ai:embed', } as const; diff --git a/packages/server-utils/test/ai/lib/tracing/tanstack-ai-generation.test.ts b/packages/server-utils/test/ai/lib/tracing/tanstack-ai-generation.test.ts new file mode 100644 index 000000000000..74deb1328b0c --- /dev/null +++ b/packages/server-utils/test/ai/lib/tracing/tanstack-ai-generation.test.ts @@ -0,0 +1,131 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; +import { + setGenerationEndAttributes, + setGenerationResponseAttributes, + startGenerationSpan, + TANSTACK_AI_EMBED, +} from '../../../../src/ai/tanstack-ai/generation'; +import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; + +describe('TanStack AI activities', () => { + beforeEach(() => { + getMainCarrier().__SENTRY__ = undefined; + const client = new TestClient( + getDefaultTestClientOptions({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + tracesSampleRate: 1, + traceLifecycle: 'stream', + }), + ); + setCurrentClient(client); + client.init(); + }); + + afterEach(() => { + getMainCarrier().__SENTRY__ = undefined; + }); + + it('opens no span for options without an adapter', () => { + expect(startGenerationSpan(TANSTACK_AI_EMBED, { input: 'a red guitar' }, {})).toBeUndefined(); + }); + + it('reports the provider of an adapter whose name also names the API', () => { + const span = startGenerationSpan( + TANSTACK_AI_EMBED, + { + adapter: { name: 'bedrock-embed', provider: 'amazon-bedrock', model: 'amazon.titan-embed-text-v2:0' }, + input: 'a', + }, + {}, + ); + + expect(spanToStaticSpanJSON(span!).data['gen_ai.provider.name']).toBe('amazon-bedrock'); + }); + + it('marks a call that the app aborted as cancelled', () => { + const abortController = new AbortController(); + abortController.abort(); + const callOptions = { + adapter: { name: 'openai', model: 'text-embedding-3-small' }, + input: 'a', + abortSignal: abortController.signal, + }; + const span = startGenerationSpan(TANSTACK_AI_EMBED, callOptions, {})!; + const error = Object.assign(new Error('Request was aborted.'), { name: 'APIUserAbortError' }); + setGenerationEndAttributes(span, { arguments: [callOptions], error }); + + expect(spanToStaticSpanJSON(span).status).toBe('cancelled'); + }); + + describe('embed', () => { + it('records the input without media', () => { + const span = startGenerationSpan( + TANSTACK_AI_EMBED, + { + adapter: { name: 'openai', model: 'text-embedding-3-small' }, + input: [ + 'a red guitar', + [ + { type: 'text', content: 'product photo' }, + { type: 'image', source: { type: 'data', value: 'iVBORw0KGgo', mimeType: 'image/png' } }, + ], + ], + }, + {}, + ); + + expect(spanToStaticSpanJSON(span!)).toMatchObject({ + description: 'embeddings text-embedding-3-small', + op: 'gen_ai.embeddings', + data: { + 'sentry.origin': 'auto.ai.tanstack_ai', + 'gen_ai.operation.name': 'embeddings', + 'gen_ai.provider.name': 'openai', + 'gen_ai.request.model': 'text-embedding-3-small', + 'gen_ai.embeddings.input': '["a red guitar",["product photo",{"type":"image"}]]', + }, + }); + }); + + it('keeps a single string input as it is', () => { + const span = startGenerationSpan( + TANSTACK_AI_EMBED, + { adapter: { name: 'ollama', model: 'nomic-embed-text' }, input: 'a red guitar' }, + {}, + ); + + expect(spanToStaticSpanJSON(span!).data['gen_ai.embeddings.input']).toBe('a red guitar'); + }); + + it('leaves out the input when inputs are not recorded', () => { + const span = startGenerationSpan( + TANSTACK_AI_EMBED, + { adapter: { name: 'openai', model: 'text-embedding-3-small' }, input: 'a red guitar' }, + { recordInputs: false }, + ); + + expect(spanToStaticSpanJSON(span!).data).not.toHaveProperty('gen_ai.embeddings.input'); + }); + + it('records the response model, usage and provider cost, but not the id and the output tokens that the adapter made', () => { + const span = startGenerationSpan(TANSTACK_AI_EMBED, { adapter: { name: 'openai', model: 'm' }, input: 'a' }, {})!; + setGenerationResponseAttributes(span, TANSTACK_AI_EMBED, { + id: 'openai-123', + model: 'text-embedding-3-small', + embeddings: [{ vector: [0.1], index: 0 }], + usage: { promptTokens: 8, completionTokens: 0, totalTokens: 8, cost: 0.0001 }, + }); + + const { data } = spanToStaticSpanJSON(span); + expect(data).toMatchObject({ + 'gen_ai.response.model': 'text-embedding-3-small', + 'gen_ai.usage.input_tokens': 8, + 'gen_ai.usage.total_tokens': 8, + 'gen_ai.cost.total_tokens': 0.0001, + }); + expect(data).not.toHaveProperty('gen_ai.response.id'); + expect(data).not.toHaveProperty('gen_ai.usage.output_tokens'); + }); + }); +}); From 3b6d6f080905c1aa8eabe29a7cf9a5c123d6cf46 Mon Sep 17 00:00:00 2001 From: JPeer264 Date: Sun, 25 Oct 2026 12:18:09 +0100 Subject: [PATCH 4/6] feat(server-utils): Instrument TanStack AI rerank() calls Record a gen_ai.rerank span for each @tanstack/ai rerank() call, with the total tokens and the provider cost. Like embed(), the call is traced through an orchestrion channel, so the provider request nests under the span. Rerank models bill search units. The OpenRouter adapter reports 0 input and 0 output tokens and makes the response id itself, so the span leaves these out. When rerank() throws because the provider returned an index out of range, the span still records the usage, because the provider bills the call. Co-Authored-By: Claude Opus 5.5 --- .../tracing/tanstack-ai/scenario-rerank.mjs | 48 +++++++++++++++++ .../suites/tracing/tanstack-ai/test.ts | 53 +++++++++++++++++++ .../src/ai/tanstack-ai/generation.ts | 32 +++++++++-- .../src/integrations/tanstack-ai.ts | 2 + .../src/orchestrion/config/tanstack-ai.ts | 6 +++ .../tracing/tanstack-ai-generation.test.ts | 32 +++++++++++ 6 files changed, 168 insertions(+), 5 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-rerank.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-rerank.mjs b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-rerank.mjs new file mode 100644 index 000000000000..5a4c2d9864e8 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-rerank.mjs @@ -0,0 +1,48 @@ +import express from 'express'; +import * as Sentry from '@sentry/node'; +import { rerank } from '@tanstack/ai'; +import { createOpenRouterRerank } from '@tanstack/ai-openrouter'; + +// Inlined OpenRouter rerank mock: the runner copies only this file into its temp dir. +function startMockProvider() { + const app = express(); + app.use(express.json()); + app.post('/rerank', (req, res) => { + const results = req.body.documents + .map((text, index) => ({ index, relevance_score: 1 - index / 10, document: { text } })) + .reverse(); + res.json({ + id: 'gen-rerank-1', + model: 'cohere/rerank-v3.5', + // An index that is out of range makes `rerank()` throw after the provider answered. + results: req.body.query.includes('out of range') + ? [...results, { index: results.length, relevance_score: 0, document: { text: '' } }] + : results, + usage: { total_tokens: 12, search_units: 1, cost: 0.002 }, + }); + }); + return new Promise(resolve => { + const server = app.listen(0, () => resolve(server)); + }); +} + +const server = await startMockProvider(); +const adapter = createOpenRouterRerank('cohere/rerank-v3.5', 'test-key', { + serverURL: `http://localhost:${server.address().port}`, +}); + +const documents = ['A heavy gaming desktop', 'A lightweight ultrabook with all-day battery']; + +await Sentry.startSpan({ name: 'main' }, async () => { + await rerank({ adapter, query: 'A laptop for travel', documents }); + + await Sentry.startSpan({ name: 'failed' }, async () => { + try { + await rerank({ adapter, query: 'Answer out of range', documents }); + } catch { + // `rerank()` throws on the index that is out of range. + } + }); +}); + +server.close(); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts index 8206fce769fa..e80f0c9e4103 100644 --- a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts @@ -1,5 +1,6 @@ import { GEN_AI_CONVERSATION_ID, + GEN_AI_COST_TOTAL_TOKENS, GEN_AI_EMBEDDINGS_INPUT, GEN_AI_INPUT_MESSAGES, GEN_AI_OPERATION_NAME, @@ -30,6 +31,7 @@ const TANSTACK_AI_DEPENDENCIES = { additionalDependencies: { '@tanstack/ai': '1.0.0', '@tanstack/ai-openai': '1.0.0', + '@tanstack/ai-openrouter': '1.0.0', // In the range of `@tanstack/ai-openai`, so that the scenarios and the adapter share one copy. openai: '7.32.0', }, @@ -448,4 +450,55 @@ describe('TanStack AI integration', () => { }, TANSTACK_AI_DEPENDENCIES, ); + + createEsmAndCjsTests( + __dirname, + 'scenario-rerank.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + test('records rerank spans with the usage, with the provider request under them', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items; + const main = spans.find(span => span.name === 'main')!; + const failed = spans.find(span => span.name === 'failed')!; + const reranks = spans.filter(span => attr(span, 'sentry.op') === 'gen_ai.rerank'); + expect(reranks).toHaveLength(2); + const rerank = reranks.find(span => span.parent_span_id === main.span_id)!; + expect(rerank.name).toBe('rerank cohere/rerank-v3.5'); + expect(rerank.status).toBe('ok'); + expect(attr(rerank, 'sentry.origin')).toBe(ORIGIN); + expect(attr(rerank, GEN_AI_OPERATION_NAME)).toBe('rerank'); + expect(attr(rerank, GEN_AI_PROVIDER_NAME)).toBe('openrouter'); + expect(attr(rerank, GEN_AI_REQUEST_MODEL)).toBe('cohere/rerank-v3.5'); + // The adapter reports no model, and makes the id itself when the response has none. + expect(attr(rerank, GEN_AI_RESPONSE_MODEL)).toBeUndefined(); + expect(attr(rerank, GEN_AI_RESPONSE_ID)).toBeUndefined(); + expect(attr(rerank, GEN_AI_USAGE_TOTAL_TOKENS)).toBe(12); + expect(attr(rerank, GEN_AI_USAGE_INPUT_TOKENS)).toBeUndefined(); + expect(attr(rerank, GEN_AI_COST_TOTAL_TOKENS)).toBe(0.002); + + // `rerank()` throws after the provider answered, and the provider bills the call. + const failedRerank = reranks.find(span => span.parent_span_id === failed.span_id)!; + expect(failedRerank.status).toBe('error'); + expect(attr(failedRerank, GEN_AI_USAGE_TOTAL_TOKENS)).toBe(12); + expect(attr(failedRerank, GEN_AI_COST_TOTAL_TOKENS)).toBe(0.002); + + const requests = spans.filter(span => attr(span, 'sentry.op') === 'http.client'); + expect(requests.map(span => span.parent_span_id).sort()).toEqual( + [rerank.span_id, failedRerank.span_id].sort(), + ); + }, + }) + .start() + .completed(); + }); + }, + TANSTACK_AI_DEPENDENCIES, + ); }); diff --git a/packages/server-utils/src/ai/tanstack-ai/generation.ts b/packages/server-utils/src/ai/tanstack-ai/generation.ts index 65456350458d..ab9edf38c8e2 100644 --- a/packages/server-utils/src/ai/tanstack-ai/generation.ts +++ b/packages/server-utils/src/ai/tanstack-ai/generation.ts @@ -5,6 +5,7 @@ import { GEN_AI_OPERATION_NAME, GEN_AI_PROVIDER_NAME, GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_MODEL, SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; @@ -28,7 +29,7 @@ export interface TanstackAiGenerationActivity { requestAttributes?: (options: Record) => SpanAttributes; /** * Attributes from the value that the adapter method resolves to. They are recorded also when TanStack AI throws - * after the provider answered, because the provider bills that call too. + * after the provider answered, e.g. for a rerank index out of range, because the provider bills that call too. */ responseAttributes: (adapterResult: Record) => SpanAttributes; } @@ -39,10 +40,23 @@ export const TANSTACK_AI_EMBED: TanstackAiGenerationActivity = { requestAttributes: ({ input }) => (input !== undefined ? { [GEN_AI_EMBEDDINGS_INPUT]: embeddingsInput(input) } : {}), // The adapters make `id` themselves, so it is not the provider's response id. An embedding has no output // tokens, but the OpenAI adapter reports 0. - responseAttributes: ({ model, usage }) => ({ - ...(typeof model === 'string' ? { [GEN_AI_RESPONSE_MODEL]: model } : {}), - ...(isObjectLike(usage) ? usageAttributes({ ...usage, completionTokens: undefined }) : {}), - }), + responseAttributes: ({ model, usage }) => + responseMetadataAttributes({ + model, + usage: isObjectLike(usage) ? { ...usage, completionTokens: undefined } : usage, + }), +}; + +export const TANSTACK_AI_RERANK: TanstackAiGenerationActivity = { + operation: 'rerank', + adapterMethod: 'rerank', + // Rerank models bill search units. The OpenRouter adapter reports 0 input and output tokens, so only the + // total and the cost are real, and it makes `id` itself when the response has none. + responseAttributes: ({ model, usage }) => + responseMetadataAttributes({ + model, + usage: isObjectLike(usage) ? { ...usage, promptTokens: undefined, completionTokens: undefined } : usage, + }), }; /** Start the span of an activity call, or return `undefined` for options that are not a call's options. */ @@ -96,6 +110,14 @@ export function setGenerationEndAttributes(span: Span, call: OrchestrionChannelC } } +function responseMetadataAttributes({ id, model, usage }: Record): SpanAttributes { + return { + ...(typeof id === 'string' ? { [GEN_AI_RESPONSE_ID]: id } : {}), + ...(typeof model === 'string' ? { [GEN_AI_RESPONSE_MODEL]: model } : {}), + ...(isObjectLike(usage) ? usageAttributes(usage) : {}), + }; +} + /** An input item is a string, a content part, or an array of content parts. */ function embeddingsInput(input: unknown): string | undefined { if (typeof input === 'string') { diff --git a/packages/server-utils/src/integrations/tanstack-ai.ts b/packages/server-utils/src/integrations/tanstack-ai.ts index 62efb8263964..bc2a5c34e6ce 100644 --- a/packages/server-utils/src/integrations/tanstack-ai.ts +++ b/packages/server-utils/src/integrations/tanstack-ai.ts @@ -10,6 +10,7 @@ import { setGenerationResponseAttributes, startGenerationSpan, TANSTACK_AI_EMBED, + TANSTACK_AI_RERANK, } from '../ai/tanstack-ai/generation'; import { skipProviderIntegrations } from '../ai/tanstack-ai/utils'; import { CHANNELS } from '../orchestrion/channels'; @@ -20,6 +21,7 @@ import { bindTracingChannelToSpan, safeChannelCallback } from '../tracing-channe const GENERATION_ACTIVITIES: Array<[channel: string, activity: TanstackAiGenerationActivity]> = [ [CHANNELS.TANSTACK_AI_EMBED, TANSTACK_AI_EMBED], + [CHANNELS.TANSTACK_AI_RERANK, TANSTACK_AI_RERANK], ]; const _tanstackAiIntegration = ((options: TanstackAiOptions = {}) => { diff --git a/packages/server-utils/src/orchestrion/config/tanstack-ai.ts b/packages/server-utils/src/orchestrion/config/tanstack-ai.ts index fbb4c0604dd3..c9d2b6533c2d 100644 --- a/packages/server-utils/src/orchestrion/config/tanstack-ai.ts +++ b/packages/server-utils/src/orchestrion/config/tanstack-ai.ts @@ -18,6 +18,11 @@ export const tanstackAiConfig = [ module: { name: '@tanstack/ai', versionRange: '>=1.0.0 <2.0.0', filePath: 'dist/esm/activities/embed/index.js' }, functionQuery: { functionName: 'embed', kind: 'Async' }, }, + { + channelName: 'rerank', + module: { name: '@tanstack/ai', versionRange: '>=1.0.0 <2.0.0', filePath: 'dist/esm/activities/rerank/index.js' }, + functionQuery: { functionName: 'rerank', kind: 'Async' }, + }, ] satisfies InstrumentationConfig[]; export const tanstackAiModuleNames = getModuleNames(tanstackAiConfig); @@ -25,4 +30,5 @@ export const tanstackAiModuleNames = getModuleNames(tanstackAiConfig); export const tanstackAiChannels = { TANSTACK_AI_CHAT: 'orchestrion:@tanstack/ai:chat', TANSTACK_AI_EMBED: 'orchestrion:@tanstack/ai:embed', + TANSTACK_AI_RERANK: 'orchestrion:@tanstack/ai:rerank', } as const; diff --git a/packages/server-utils/test/ai/lib/tracing/tanstack-ai-generation.test.ts b/packages/server-utils/test/ai/lib/tracing/tanstack-ai-generation.test.ts index 74deb1328b0c..41b686f7164a 100644 --- a/packages/server-utils/test/ai/lib/tracing/tanstack-ai-generation.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/tanstack-ai-generation.test.ts @@ -5,6 +5,7 @@ import { setGenerationResponseAttributes, startGenerationSpan, TANSTACK_AI_EMBED, + TANSTACK_AI_RERANK, } from '../../../../src/ai/tanstack-ai/generation'; import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; @@ -128,4 +129,35 @@ describe('TanStack AI activities', () => { expect(data).not.toHaveProperty('gen_ai.usage.output_tokens'); }); }); + + describe('rerank', () => { + it('records the total tokens and provider cost, but not the id and the input and output tokens that the adapter made', () => { + const span = startGenerationSpan( + TANSTACK_AI_RERANK, + { adapter: { name: 'openrouter', model: 'cohere/rerank-v3.5' }, query: 'travel laptop', documents: ['a', 'b'] }, + {}, + )!; + setGenerationResponseAttributes(span, TANSTACK_AI_RERANK, { + id: 'rerank-1792850464143-k3j9x2a', + ranking: [{ index: 1, score: 0.9 }], + usage: { promptTokens: 0, completionTokens: 0, totalTokens: 12, cost: 0.002 }, + }); + + expect(spanToStaticSpanJSON(span)).toMatchObject({ + description: 'rerank cohere/rerank-v3.5', + op: 'gen_ai.rerank', + data: { + 'gen_ai.operation.name': 'rerank', + 'gen_ai.provider.name': 'openrouter', + 'gen_ai.request.model': 'cohere/rerank-v3.5', + 'gen_ai.usage.total_tokens': 12, + 'gen_ai.cost.total_tokens': 0.002, + }, + }); + const { data } = spanToStaticSpanJSON(span); + expect(data).not.toHaveProperty('gen_ai.response.id'); + expect(data).not.toHaveProperty('gen_ai.usage.input_tokens'); + expect(data).not.toHaveProperty('gen_ai.usage.output_tokens'); + }); + }); }); From d827a51c68852ab8239e5b01361f79bf66eb6fa6 Mon Sep 17 00:00:00 2001 From: JPeer264 Date: Sun, 25 Oct 2026 12:22:11 +0100 Subject: [PATCH 5/6] feat(server-utils): Instrument TanStack AI decide() calls Record a gen_ai.decide span for each @tanstack/ai decide() call, with the state and questions as input messages, the answers as output messages, the response model and the usage, in the same form as for Vercel AI's experimental_decide(). Like embed(), the call is traced through an orchestrion channel, and the integrations of the vendor SDKs are skipped during the calls of the decider. A refused decide() call is still billed. TanStack AI puts the usage and the model on the EvaluateError that it throws, so the span records them with the error status. Co-Authored-By: Claude Opus 5.5 --- .../tracing/tanstack-ai/scenario-decide.mjs | 66 ++++++++++++++ .../suites/tracing/tanstack-ai/test.ts | 88 +++++++++++++++++++ .../src/ai/tanstack-ai/generation.ts | 40 ++++++++- .../src/integrations/tanstack-ai.ts | 4 +- .../src/orchestrion/config/tanstack-ai.ts | 6 ++ .../tracing/tanstack-ai-generation.test.ts | 71 ++++++++++++++- 6 files changed, 270 insertions(+), 5 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-decide.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-decide.mjs b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-decide.mjs new file mode 100644 index 000000000000..58ca3db3930e --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/scenario-decide.mjs @@ -0,0 +1,66 @@ +import express from 'express'; +import * as Sentry from '@sentry/node'; +import { boolean, choice, decide } from '@tanstack/ai'; +import { createOpenaiDecider } from '@tanstack/ai-openai'; +import OpenAI from 'openai'; + +// Inlined OpenAI decisions mock: the runner copies only this file into its temp dir. It refuses the question +// about a refund when the input asks it to. +function startMockProvider() { + const app = express(); + app.use(express.json()); + app.post('/v1/decisions', (req, res) => { + res.json({ + model: 'gpt-6-luna-2026-09-01', + answers: [ + { + type: 'choice', + name: 'queue', + choice: 'billing', + probabilities: [ + { value: 'billing', probability: 0.9 }, + { value: 'tech', probability: 0.1 }, + ], + confidence: 0.85, + }, + req.body.input.includes('refuse') + ? { type: 'refusal', name: 'refund', refusal: 'I cannot answer this.' } + : { type: 'predicate', name: 'refund', probability: 0.8 }, + ], + usage: { input_tokens: 30, output_tokens: 4, total_tokens: 34 }, + }); + }); + return new Promise(resolve => { + const server = app.listen(0, () => resolve(server)); + }); +} + +const server = await startMockProvider(); +const baseURL = `http://localhost:${server.address().port}/v1`; +const adapter = createOpenaiDecider('gpt-6-luna', 'test-key', { baseURL }); +const questions = { + queue: choice({ instructions: 'Which team handles this ticket?', options: { billing: null, tech: null } }), + refund: boolean({ instructions: 'Does the customer ask for a refund?' }), +}; + +await Sentry.startSpan({ name: 'main' }, async () => { + await decide({ adapter, state: 'I was charged twice for my subscription.', questions }); + + await Sentry.startSpan({ name: 'refused' }, async () => { + try { + await decide({ adapter, state: 'Please refuse to answer.', questions }); + } catch { + // The model refused the question about a refund. + } + }); + + await Sentry.startSpan({ name: 'direct' }, () => + new OpenAI({ apiKey: 'test-key', baseURL }).decisions.create({ + model: 'gpt-6-luna', + input: 'I was charged twice for my subscription.', + questions: [{ type: 'predicate', name: 'refund', instructions: 'Does the customer ask for a refund?' }], + }), + ); +}); + +server.close(); diff --git a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts index e80f0c9e4103..0f0d6bb0163e 100644 --- a/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/tanstack-ai/test.ts @@ -501,4 +501,92 @@ describe('TanStack AI integration', () => { }, TANSTACK_AI_DEPENDENCIES, ); + + createEsmAndCjsTests( + __dirname, + 'scenario-decide.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + test('records decide spans with the questions, the answers and the usage, with the provider request under them', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items; + const main = spans.find(span => span.name === 'main')!; + const refused = spans.find(span => span.name === 'refused')!; + const direct = spans.find(span => span.name === 'direct')!; + const decisions = spans.filter(span => attr(span, 'sentry.op') === 'gen_ai.decide'); + // The OpenAI integration is skipped for TanStack's calls, but not for the direct call after them. + expect(decisions.map(span => attr(span, 'sentry.origin')).sort()).toEqual([ + 'auto.ai.openai', + ORIGIN, + ORIGIN, + ]); + const decision = decisions.find(span => span.parent_span_id === main.span_id)!; + const refusal = decisions.find(span => span.parent_span_id === refused.span_id)!; + const directDecision = decisions.find(span => span.parent_span_id === direct.span_id)!; + expect(attr(directDecision, 'sentry.origin')).toBe('auto.ai.openai'); + expect(decision.name).toBe('decide gpt-6-luna'); + expect(decision.status).toBe('ok'); + expect(attr(decision, 'sentry.origin')).toBe(ORIGIN); + expect(attr(decision, GEN_AI_OPERATION_NAME)).toBe('decide'); + expect(attr(decision, GEN_AI_PROVIDER_NAME)).toBe('openai'); + expect(attr(decision, GEN_AI_REQUEST_MODEL)).toBe('gpt-6-luna'); + expect(attr(decision, GEN_AI_RESPONSE_MODEL)).toBe('gpt-6-luna-2026-09-01'); + expect(attr(decision, GEN_AI_USAGE_INPUT_TOKENS)).toBe(30); + expect(attr(decision, GEN_AI_USAGE_OUTPUT_TOKENS)).toBe(4); + expect(attr(decision, GEN_AI_USAGE_TOTAL_TOKENS)).toBe(34); + expect(JSON.parse(String(attr(decision, GEN_AI_INPUT_MESSAGES)))).toEqual([ + { + type: 'evaluation', + state: 'I was charged twice for my subscription.', + questions: { + queue: { + type: 'choice', + instructions: 'Which team handles this ticket?', + criteria: { billing: null, tech: null }, + }, + refund: { type: 'noul', instructions: 'Does the customer ask for a refund?' }, + }, + }, + ]); + expect(JSON.parse(String(attr(decision, GEN_AI_OUTPUT_MESSAGES)))).toEqual([ + { + type: 'evaluation', + answers: { + queue: { + type: 'choice', + value: 'billing', + probability: 0.9, + confidence: 0.85, + probabilities: { billing: 0.9, tech: 0.1 }, + }, + refund: { type: 'boolean', value: true, probability: 0.8 }, + }, + }, + ]); + + // A refused call fails, but is still billed. + expect(refusal.status).toBe('error'); + expect(attr(refusal, 'sentry.origin')).toBe(ORIGIN); + expect(attr(refusal, GEN_AI_RESPONSE_MODEL)).toBe('gpt-6-luna-2026-09-01'); + expect(attr(refusal, GEN_AI_USAGE_INPUT_TOKENS)).toBe(30); + expect(attr(refusal, GEN_AI_USAGE_OUTPUT_TOKENS)).toBe(4); + + const requests = spans.filter(span => attr(span, 'sentry.op') === 'http.client'); + expect(requests.map(span => span.parent_span_id).sort()).toEqual( + [decision.span_id, refusal.span_id, directDecision.span_id].sort(), + ); + }, + }) + .start() + .completed(); + }); + }, + TANSTACK_AI_DEPENDENCIES, + ); }); diff --git a/packages/server-utils/src/ai/tanstack-ai/generation.ts b/packages/server-utils/src/ai/tanstack-ai/generation.ts index ab9edf38c8e2..7319447abf2e 100644 --- a/packages/server-utils/src/ai/tanstack-ai/generation.ts +++ b/packages/server-utils/src/ai/tanstack-ai/generation.ts @@ -2,7 +2,9 @@ import type { Span, SpanAttributes } from '@sentry/core'; import { isObjectLike, SPAN_STATUS_ERROR, startInactiveSpan, stringify } from '@sentry/core'; import { GEN_AI_EMBEDDINGS_INPUT, + GEN_AI_INPUT_MESSAGES, GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, GEN_AI_PROVIDER_NAME, GEN_AI_REQUEST_MODEL, GEN_AI_RESPONSE_ID, @@ -11,7 +13,12 @@ import { } from '@sentry/conventions/attributes'; import type { OrchestrionChannelContext } from '../../orchestrion/types'; import type { GenAiOptions } from '../core/utils'; -import { getGenAiSpanOp, resolveAIRecordingOptions } from '../core/utils'; +import { + getEvaluationInputMessages, + getEvaluationOutputMessages, + getGenAiSpanOp, + resolveAIRecordingOptions, +} from '../core/utils'; import { TANSTACK_AI_ORIGIN } from './constants'; import { usageAttributes } from './utils'; @@ -32,6 +39,8 @@ export interface TanstackAiGenerationActivity { * after the provider answered, e.g. for a rerank index out of range, because the provider bills that call too. */ responseAttributes: (adapterResult: Record) => SpanAttributes; + /** Attributes from the value that the call resolves to, recorded only when outputs are recorded. */ + outputAttributes?: (result: Record) => SpanAttributes; } export const TANSTACK_AI_EMBED: TanstackAiGenerationActivity = { @@ -59,6 +68,19 @@ export const TANSTACK_AI_RERANK: TanstackAiGenerationActivity = { }), }; +export const TANSTACK_AI_DECIDE: TanstackAiGenerationActivity = { + operation: 'decide', + adapterMethod: 'evaluate', + requestAttributes: ({ state, questions }) => ({ + [GEN_AI_INPUT_MESSAGES]: getEvaluationInputMessages({ state, questions }), + }), + responseAttributes: responseMetadataAttributes, + // `decide()` resolves to the answers keyed by question name, plus `meta` with the model, the usage and the id. + outputAttributes: ({ meta: _meta, ...answers }) => ({ + [GEN_AI_OUTPUT_MESSAGES]: getEvaluationOutputMessages(answers), + }), +}; + /** Start the span of an activity call, or return `undefined` for options that are not a call's options. */ export function startGenerationSpan( activity: TanstackAiGenerationActivity, @@ -98,11 +120,23 @@ export function setGenerationResponseAttributes( } } -/** Record what the error that a call threw tells. */ -export function setGenerationEndAttributes(span: Span, call: OrchestrionChannelContext): void { +/** Record the value that a call resolved to, or what the error that it threw tells. */ +export function setGenerationEndAttributes( + span: Span, + activity: TanstackAiGenerationActivity, + call: OrchestrionChannelContext, + options: GenAiOptions, +): void { if ('result' in call) { + if (activity.outputAttributes && isObjectLike(call.result) && resolveAIRecordingOptions(options).recordOutputs) { + span.setAttributes(activity.outputAttributes(call.result)); + } return; } + if (isObjectLike(call.error) && call.error.name === 'EvaluateError') { + // A refused `decide()` is still billed. TanStack AI puts the usage and the model on the error. + setGenerationResponseAttributes(span, activity, call.error); + } // As for `chat()`, a call that the app aborted counts as cancelled. const [callOptions] = call.arguments; if (isObjectLike(callOptions) && (callOptions.abortSignal as AbortSignal | undefined)?.aborted) { diff --git a/packages/server-utils/src/integrations/tanstack-ai.ts b/packages/server-utils/src/integrations/tanstack-ai.ts index bc2a5c34e6ce..8686095d5274 100644 --- a/packages/server-utils/src/integrations/tanstack-ai.ts +++ b/packages/server-utils/src/integrations/tanstack-ai.ts @@ -9,6 +9,7 @@ import { setGenerationEndAttributes, setGenerationResponseAttributes, startGenerationSpan, + TANSTACK_AI_DECIDE, TANSTACK_AI_EMBED, TANSTACK_AI_RERANK, } from '../ai/tanstack-ai/generation'; @@ -22,6 +23,7 @@ import { bindTracingChannelToSpan, safeChannelCallback } from '../tracing-channe const GENERATION_ACTIVITIES: Array<[channel: string, activity: TanstackAiGenerationActivity]> = [ [CHANNELS.TANSTACK_AI_EMBED, TANSTACK_AI_EMBED], [CHANNELS.TANSTACK_AI_RERANK, TANSTACK_AI_RERANK], + [CHANNELS.TANSTACK_AI_DECIDE, TANSTACK_AI_DECIDE], ]; const _tanstackAiIntegration = ((options: TanstackAiOptions = {}) => { @@ -89,7 +91,7 @@ function instrumentGenerationActivities(options: TanstackAiOptions): void { } return span; }, - { beforeSpanEnd: (span, data) => setGenerationEndAttributes(span, data) }, + { beforeSpanEnd: (span, data) => setGenerationEndAttributes(span, activity, data, options) }, ); } } diff --git a/packages/server-utils/src/orchestrion/config/tanstack-ai.ts b/packages/server-utils/src/orchestrion/config/tanstack-ai.ts index c9d2b6533c2d..482055e46f0b 100644 --- a/packages/server-utils/src/orchestrion/config/tanstack-ai.ts +++ b/packages/server-utils/src/orchestrion/config/tanstack-ai.ts @@ -23,6 +23,11 @@ export const tanstackAiConfig = [ module: { name: '@tanstack/ai', versionRange: '>=1.0.0 <2.0.0', filePath: 'dist/esm/activities/rerank/index.js' }, functionQuery: { functionName: 'rerank', kind: 'Async' }, }, + { + channelName: 'decide', + module: { name: '@tanstack/ai', versionRange: '>=1.0.0 <2.0.0', filePath: 'dist/esm/activities/evaluate/index.js' }, + functionQuery: { functionName: 'decide', kind: 'Async' }, + }, ] satisfies InstrumentationConfig[]; export const tanstackAiModuleNames = getModuleNames(tanstackAiConfig); @@ -31,4 +36,5 @@ export const tanstackAiChannels = { TANSTACK_AI_CHAT: 'orchestrion:@tanstack/ai:chat', TANSTACK_AI_EMBED: 'orchestrion:@tanstack/ai:embed', TANSTACK_AI_RERANK: 'orchestrion:@tanstack/ai:rerank', + TANSTACK_AI_DECIDE: 'orchestrion:@tanstack/ai:decide', } as const; diff --git a/packages/server-utils/test/ai/lib/tracing/tanstack-ai-generation.test.ts b/packages/server-utils/test/ai/lib/tracing/tanstack-ai-generation.test.ts index 41b686f7164a..3ce4944b3ca2 100644 --- a/packages/server-utils/test/ai/lib/tracing/tanstack-ai-generation.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/tanstack-ai-generation.test.ts @@ -4,6 +4,7 @@ import { setGenerationEndAttributes, setGenerationResponseAttributes, startGenerationSpan, + TANSTACK_AI_DECIDE, TANSTACK_AI_EMBED, TANSTACK_AI_RERANK, } from '../../../../src/ai/tanstack-ai/generation'; @@ -54,7 +55,7 @@ describe('TanStack AI activities', () => { }; const span = startGenerationSpan(TANSTACK_AI_EMBED, callOptions, {})!; const error = Object.assign(new Error('Request was aborted.'), { name: 'APIUserAbortError' }); - setGenerationEndAttributes(span, { arguments: [callOptions], error }); + setGenerationEndAttributes(span, TANSTACK_AI_EMBED, { arguments: [callOptions], error }, {}); expect(spanToStaticSpanJSON(span).status).toBe('cancelled'); }); @@ -160,4 +161,72 @@ describe('TanStack AI activities', () => { expect(data).not.toHaveProperty('gen_ai.usage.output_tokens'); }); }); + + describe('decide', () => { + const callOptions = { + adapter: { name: 'openai', model: 'gpt-6-luna' }, + state: 'I was charged twice.', + questions: { refund: { type: 'noul', instructions: 'Does the customer ask for a refund?' } }, + }; + const adapterResult = { + id: 'decision-1', + model: 'gpt-6-luna-2026-09-01', + answers: { refund: { type: 'noul', noul: 0.8 } }, + usage: { promptTokens: 30, completionTokens: 4, totalTokens: 34 }, + }; + const result = { + refund: { type: 'boolean', value: true, probability: 0.8 }, + meta: { id: adapterResult.id, model: adapterResult.model, usage: adapterResult.usage }, + }; + + it('records the questions and answers', () => { + const span = startGenerationSpan(TANSTACK_AI_DECIDE, callOptions, {})!; + setGenerationResponseAttributes(span, TANSTACK_AI_DECIDE, adapterResult); + setGenerationEndAttributes(span, TANSTACK_AI_DECIDE, { arguments: [callOptions], result }, {}); + + expect(spanToStaticSpanJSON(span)).toMatchObject({ + description: 'decide gpt-6-luna', + op: 'gen_ai.decide', + data: { + 'gen_ai.operation.name': 'decide', + 'gen_ai.input.messages': + '[{"type":"evaluation","state":"I was charged twice.","questions":{"refund":{"type":"noul","instructions":"Does the customer ask for a refund?"}}}]', + 'gen_ai.output.messages': + '[{"type":"evaluation","answers":{"refund":{"type":"boolean","value":true,"probability":0.8}}}]', + 'gen_ai.response.id': 'decision-1', + 'gen_ai.response.model': 'gpt-6-luna-2026-09-01', + 'gen_ai.usage.input_tokens': 30, + 'gen_ai.usage.output_tokens': 4, + }, + }); + }); + + it('leaves out the questions and answers when inputs and outputs are not recorded', () => { + const options = { recordInputs: false, recordOutputs: false }; + const span = startGenerationSpan(TANSTACK_AI_DECIDE, callOptions, options)!; + setGenerationResponseAttributes(span, TANSTACK_AI_DECIDE, adapterResult); + setGenerationEndAttributes(span, TANSTACK_AI_DECIDE, { arguments: [callOptions], result }, options); + + const { data } = spanToStaticSpanJSON(span); + expect(data).not.toHaveProperty('gen_ai.input.messages'); + expect(data).not.toHaveProperty('gen_ai.output.messages'); + expect(data['gen_ai.response.model']).toBe('gpt-6-luna-2026-09-01'); + }); + + it('records the model and usage of a refused call, which the provider bills', () => { + const span = startGenerationSpan(TANSTACK_AI_DECIDE, callOptions, {})!; + const error = Object.assign(new Error('The model refused to answer "refund".'), { + name: 'EvaluateError', + model: 'gpt-6-luna-2026-09-01', + usage: { promptTokens: 30, completionTokens: 4, totalTokens: 34 }, + }); + setGenerationEndAttributes(span, TANSTACK_AI_DECIDE, { arguments: [callOptions], error }, {}); + + expect(spanToStaticSpanJSON(span).data).toMatchObject({ + 'gen_ai.response.model': 'gpt-6-luna-2026-09-01', + 'gen_ai.usage.input_tokens': 30, + 'gen_ai.usage.output_tokens': 4, + }); + }); + }); }); From cd29ea761b708bb23c5ce41c4cc0ffdfa586e99b Mon Sep 17 00:00:00 2001 From: JPeer264 Date: Mon, 26 Oct 2026 12:10:18 -0600 Subject: [PATCH 6/6] test(e2e): Add TanStack AI embed(), rerank() and decide() tests The gen-ai-libraries app now calls embed(), rerank() and decide(), so CI checks these activities on Node (runtime injection) and on Cloudflare (build-time injection). embed() goes through the OpenAI adapter, and the openai tests that run after it check that the OpenAI integration is skipped only during the TanStack AI call. Co-Authored-By: Claude Opus 5.5 --- .../gen-ai-libraries/package.json | 1 + .../gen-ai-libraries/src/entry.cloudflare.ts | 12 +- .../gen-ai-libraries/src/entry.node.ts | 12 ++ .../src/tanstack-ai-activities.ts | 52 ++++++++ .../tests/tanstack-ai-activities.test.ts | 114 ++++++++++++++++++ 5 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/tanstack-ai-activities.ts create mode 100644 dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/tanstack-ai-activities.test.ts diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json index 4e5f9fb6cf6b..bdf2c25ca0a7 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json @@ -22,6 +22,7 @@ "@sentry/cloudflare": "file:../../packed/sentry-cloudflare-packed.tgz", "@sentry/node": "file:../../packed/sentry-node-packed.tgz", "@tanstack/ai": "^1.0.0", + "@tanstack/ai-openai": "^1.0.0", "@tanstack/ai-openrouter": "^1.0.0", "ai": "~7.0.97", "express": "^4.21.2", diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts index 265b8122f0df..e7cca78ad3fb 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts @@ -2,8 +2,10 @@ // `@sentry/cloudflare/vite` bundler plugin (build-time channel injection) and run on workerd. import * as Sentry from '@sentry/cloudflare'; import { libraries } from './libraries'; +import { tanstackAiActivities } from './tanstack-ai-activities'; const byId = new Map(libraries.map(library => [library.id, library])); +const tanstackAiActivityById = new Map(tanstackAiActivities.map(activity => [activity.id, activity])); export default Sentry.withSentry( (env: Env) => ({ @@ -17,8 +19,14 @@ export default Sentry.withSentry( const url = new URL(request.url); const [, id, action] = url.pathname.split('/'); const library = id ? byId.get(id) : undefined; + const run = + library && (action === 'chat' || action === 'tools') + ? library[action] + : id === 'tanstack-ai' && action + ? tanstackAiActivityById.get(action)?.run + : undefined; - if (!library || (action !== 'chat' && action !== 'tools')) { + if (!run) { return new Response('Not found', { status: 404 }); } @@ -29,7 +37,7 @@ export default Sentry.withSentry( try { const spanName = action === 'tools' ? 'ai-tool-workflow' : 'ai-workflow'; - const result = await Sentry.startSpan({ name: spanName, op: 'function' }, () => library[action](apiKey)); + const result = await Sentry.startSpan({ name: spanName, op: 'function' }, () => run(apiKey)); return Response.json({ result }); } catch (error) { return Response.json({ message: (error as Error).message }, { status: 500 }); diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.node.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.node.ts index 726924dbd69d..17a3ebbdae3a 100644 --- a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.node.ts +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.node.ts @@ -2,6 +2,7 @@ import * as Sentry from '@sentry/node'; import express from 'express'; import { libraries } from './libraries'; +import { tanstackAiActivities } from './tanstack-ai-activities'; const apiKey = process.env.E2E_OPENROUTER_API_KEY; if (!apiKey) { @@ -35,6 +36,17 @@ for (const library of libraries) { }); } +for (const activity of tanstackAiActivities) { + app.get(`/tanstack-ai/${activity.id}`, async (_req, res, next) => { + try { + const result = await Sentry.startSpan({ name: 'ai-workflow', op: 'function' }, () => activity.run(apiKey)); + res.send({ result }); + } catch (error) { + next(error); + } + }); +} + Sentry.setupExpressErrorHandler(app); app.use((error: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => { diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/tanstack-ai-activities.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/tanstack-ai-activities.ts new file mode 100644 index 000000000000..f9c02b796705 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/tanstack-ai-activities.ts @@ -0,0 +1,52 @@ +// The TanStack AI activities other than `chat()`, run against OpenRouter like the libraries in +// `libraries.ts`. The embedding adapter calls the OpenAI SDK, whose Sentry integration TanStack AI skips +// only during its own call, so the `openai` tests that run later still get their spans. +import { boolean, choice, decide, embed, rerank } from '@tanstack/ai'; +import { createOpenaiEmbedding } from '@tanstack/ai-openai'; +import { createOpenRouterDecider, createOpenRouterRerank } from '@tanstack/ai-openrouter'; + +export interface TanstackAiActivity { + id: string; + run: (apiKey: string) => Promise; +} + +export const tanstackAiActivities: TanstackAiActivity[] = [ + { + id: 'embed', + run: async apiKey => { + const { embeddings } = await embed({ + // OpenRouter names the model with its vendor, which the model type of the adapter does not allow. + adapter: createOpenaiEmbedding('openai/text-embedding-3-small' as 'text-embedding-3-small', apiKey, { + baseURL: 'https://openrouter.ai/api/v1', + }), + input: ['a red guitar', 'a blue drum kit'], + }); + return embeddings.length; + }, + }, + { + id: 'decide', + run: async apiKey => { + const result = await decide({ + adapter: createOpenRouterDecider('~typesafe/jev-latest', apiKey), + state: 'I was charged twice for my subscription.', + questions: { + queue: choice({ instructions: 'Which team handles this ticket?', options: { billing: null, tech: null } }), + refund: boolean({ instructions: 'Does the customer ask for a refund?' }), + }, + }); + return { queue: result.queue.value, refund: result.refund.value }; + }, + }, + { + id: 'rerank', + run: async apiKey => { + const { ranking } = await rerank({ + adapter: createOpenRouterRerank('cohere/rerank-v3.5', apiKey), + query: 'A laptop for travel', + documents: ['A heavy gaming desktop', 'A lightweight ultrabook with all-day battery'], + }); + return ranking.map(entry => entry.index); + }, + }, +]; diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/tanstack-ai-activities.test.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/tanstack-ai-activities.test.ts new file mode 100644 index 000000000000..fe7594c69352 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/tanstack-ai-activities.test.ts @@ -0,0 +1,114 @@ +import { expect, test } from '@playwright/test'; +import { collectStreamedSpans, getSpanOp } from '@sentry-internal/test-utils'; +import { APP } from './constants'; +import { attr, describeTree } from './utils'; + +// The span of an activity is active while the activity runs, so the provider request nests under it. +// Each test waits for the segment span, which ends last, so that all spans of the trace are there. + +test('tanstack-ai: an embed() call emits a gen_ai.embeddings span, and no span of the OpenAI integration', async ({ + baseURL, +}) => { + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => + spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.embeddings') && spansOfTrace.some(span => span.is_segment), + ); + + const response = await fetch(`${baseURL}/tanstack-ai/embed`); + expect(response.status).toBe(200); + + const spans = await spansPromise; + const tree = describeTree(spans); + const workflow = spans.find(span => span.name === 'ai-workflow'); + const embeddings = spans.filter(span => getSpanOp(span) === 'gen_ai.embeddings'); + const providerRequests = spans.filter( + span => getSpanOp(span) === 'http.client' && span.parent_span_id === embeddings[0]?.span_id, + ); + + expect(embeddings, tree).toHaveLength(1); + expect(embeddings[0]!.parent_span_id, tree).toBe(workflow?.span_id); + expect(embeddings[0]!.name).toBe('embeddings openai/text-embedding-3-small'); + expect(embeddings[0]!.status).toBe('ok'); + expect(attr(embeddings[0]!, 'sentry.origin')).toBe('auto.ai.tanstack_ai'); + expect(attr(embeddings[0]!, 'gen_ai.operation.name')).toBe('embeddings'); + expect(attr(embeddings[0]!, 'gen_ai.provider.name')).toBe('openai'); + expect(attr(embeddings[0]!, 'gen_ai.request.model')).toBe('openai/text-embedding-3-small'); + expect(typeof attr(embeddings[0]!, 'gen_ai.response.model')).toBe('string'); + expect(attr(embeddings[0]!, 'gen_ai.usage.input_tokens')).toBeGreaterThan(0); + expect(attr(embeddings[0]!, 'gen_ai.embeddings.input')).toBe('["a red guitar","a blue drum kit"]'); + + expect(providerRequests, tree).toHaveLength(1); + expect(String(attr(providerRequests[0]!, 'server.address') ?? providerRequests[0]!.name)).toContain('openrouter'); +}); + +test('tanstack-ai: a decide() call emits a gen_ai.decide span', async ({ baseURL }) => { + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => + spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.decide') && spansOfTrace.some(span => span.is_segment), + ); + + const response = await fetch(`${baseURL}/tanstack-ai/decide`); + expect(response.status).toBe(200); + + const spans = await spansPromise; + const tree = describeTree(spans); + const workflow = spans.find(span => span.name === 'ai-workflow'); + const decideSpan = spans.find(span => getSpanOp(span) === 'gen_ai.decide'); + const providerRequests = spans.filter( + span => getSpanOp(span) === 'http.client' && span.parent_span_id === decideSpan?.span_id, + ); + + expect(decideSpan, tree).toBeDefined(); + expect(decideSpan!.parent_span_id, tree).toBe(workflow?.span_id); + expect(decideSpan!.name).toBe('decide ~typesafe/jev-latest'); + expect(decideSpan!.status).toBe('ok'); + expect(attr(decideSpan!, 'sentry.origin')).toBe('auto.ai.tanstack_ai'); + expect(attr(decideSpan!, 'gen_ai.operation.name')).toBe('decide'); + expect(attr(decideSpan!, 'gen_ai.provider.name')).toBe('openrouter'); + expect(attr(decideSpan!, 'gen_ai.request.model')).toBe('~typesafe/jev-latest'); + expect(typeof attr(decideSpan!, 'gen_ai.response.model')).toBe('string'); + expect(typeof attr(decideSpan!, 'gen_ai.response.id')).toBe('string'); + expect(attr(decideSpan!, 'gen_ai.usage.input_tokens')).toBeGreaterThan(0); + expect(attr(decideSpan!, 'gen_ai.usage.output_tokens')).toBeGreaterThan(0); + expect(attr(decideSpan!, 'gen_ai.cost.total_tokens')).toBeGreaterThan(0); + expect(attr(decideSpan!, 'gen_ai.input.messages')).toContain('I was charged twice for my subscription.'); + expect(attr(decideSpan!, 'gen_ai.output.messages')).toContain('"queue":{"type":"choice"'); + expect(attr(decideSpan!, 'gen_ai.output.messages')).toContain('"refund":{"type":"boolean"'); + + expect(providerRequests, tree).toHaveLength(1); + expect(String(attr(providerRequests[0]!, 'server.address') ?? providerRequests[0]!.name)).toContain('openrouter'); +}); + +// The rerank models of OpenRouter come from Cohere, and the zero data retention guardrail of the account +// of the test key blocks them, so OpenRouter refuses the request. Thus the test checks only what the span +// records when OpenRouter answers and when it refuses. +test('tanstack-ai: a rerank() call emits a gen_ai.rerank span', async ({ baseURL }) => { + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => + spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.rerank') && spansOfTrace.some(span => span.is_segment), + ); + + await fetch(`${baseURL}/tanstack-ai/rerank`); + + const spans = await spansPromise; + const tree = describeTree(spans); + const workflow = spans.find(span => span.name === 'ai-workflow'); + const rerankSpan = spans.find(span => getSpanOp(span) === 'gen_ai.rerank'); + const providerRequests = spans.filter( + span => getSpanOp(span) === 'http.client' && span.parent_span_id === rerankSpan?.span_id, + ); + + expect(rerankSpan, tree).toBeDefined(); + expect(rerankSpan!.parent_span_id, tree).toBe(workflow?.span_id); + expect(rerankSpan!.name).toBe('rerank cohere/rerank-v3.5'); + expect(attr(rerankSpan!, 'sentry.origin')).toBe('auto.ai.tanstack_ai'); + expect(attr(rerankSpan!, 'gen_ai.operation.name')).toBe('rerank'); + expect(attr(rerankSpan!, 'gen_ai.provider.name')).toBe('openrouter'); + expect(attr(rerankSpan!, 'gen_ai.request.model')).toBe('cohere/rerank-v3.5'); + + expect(providerRequests, tree).toHaveLength(1); + expect(String(attr(providerRequests[0]!, 'server.address') ?? providerRequests[0]!.name)).toContain('openrouter'); +});