diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/.gitignore b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/.gitignore new file mode 100644 index 000000000000..fdc614d4c5e0 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/.gitignore @@ -0,0 +1,6 @@ +node_modules +pnpm-lock.yaml +dist +.wrangler +test-results +playwright-report diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json new file mode 100644 index 000000000000..89364cbbe423 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json @@ -0,0 +1,56 @@ +{ + "name": "gen-ai-libraries", + "description": "Real gen_ai spans for every instrumented AI library (OpenAI, Anthropic, Mistral, Together, Vercel AI), each driven through OpenRouter with a chat query and a tool call, on Node and on Cloudflare", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "dev:node": "node --import tsx/esm --import ./src/instrument.node.ts src/entry.node.ts", + "dev:cloudflare": "wrangler dev --config ./dist/gen_ai_libraries/wrangler.json --var \"E2E_TEST_DSN:$E2E_TEST_DSN\" --var \"E2E_OPENROUTER_API_KEY:$E2E_OPENROUTER_API_KEY\" --port 38787", + "preview": "vite preview --port 38787", + "test": "playwright test", + "clean": "npx rimraf node_modules dist pnpm-lock.yaml", + "test:build": "pnpm install", + "test:build:cloudflare": "pnpm install && vite build", + "test:assert": "pnpm test", + "test:assert:cloudflare": "RUNTIME=cloudflare pnpm test" + }, + "dependencies": { + "@anthropic-ai/sdk": "0.63.0", + "@mistralai/mistralai": "^2.6.4", + "@openrouter/ai-sdk-provider": "~3.0.0", + "@sentry/cloudflare": "file:../../packed/sentry-cloudflare-packed.tgz", + "@sentry/node": "file:../../packed/sentry-node-packed.tgz", + "ai": "~7.0.97", + "express": "^4.21.2", + "openai": "5.18.1", + "together-ai": "0.54.0", + "zod": "4.5.4" + }, + "devDependencies": { + "@cloudflare/vite-plugin": "1.52.0", + "@cloudflare/workers-types": "^4.20260426.0", + "@playwright/test": "~1.63.0", + "@sentry-internal/test-utils": "link:../../../test-utils", + "@types/express": "^4.17.21", + "@types/node": "^18.19.1", + "tsx": "4.21.0", + "typescript": "^5.5.2", + "vite": "8.3.0", + "wrangler": "^4.86.0" + }, + "sentryTest": { + "optional": true, + "optionalVariants": [ + { + "build-command": "pnpm test:build:cloudflare", + "assert-command": "pnpm test:assert:cloudflare", + "label": "gen-ai-libraries (cloudflare)" + } + ] + }, + "volta": { + "node": "24.15.0", + "extends": "../../package.json" + } +} diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/playwright.config.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/playwright.config.ts new file mode 100644 index 000000000000..e3119c8f0ce2 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/playwright.config.ts @@ -0,0 +1,21 @@ +import { getPlaywrightConfig } from '@sentry-internal/test-utils'; +import { RUNTIME } from './tests/constants'; + +// The same suite runs against both runtimes, selected by the `RUNTIME` env var (see the `sentryTest` +// variants in package.json): the Node entry (runtime channel injection) or the Cloudflare entry (Vite +// build + `@sentry/cloudflare/vite` plugin at build time, the prebuilt bundle served by `wrangler dev`). +const CF_PORT = 38787; +const NODE_PORT = 3030; + +const config = getPlaywrightConfig( + { + startCommand: RUNTIME === 'cloudflare' ? 'pnpm dev:cloudflare' : 'pnpm dev:node', + port: RUNTIME === 'cloudflare' ? CF_PORT : NODE_PORT, + }, + // Every test drives a real OpenRouter model call (a tool-calling turn does two) and then waits for + // the gen_ai spans to flush, which does not fit the default 30s test timeout when the provider is + // slow. + { timeout: 90_000, retries: 0 }, +); + +export default config; diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts new file mode 100644 index 000000000000..265b8122f0df --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts @@ -0,0 +1,39 @@ +// The Cloudflare variant: the same libraries and routes as the Node entry, but instrumented by the +// `@sentry/cloudflare/vite` bundler plugin (build-time channel injection) and run on workerd. +import * as Sentry from '@sentry/cloudflare'; +import { libraries } from './libraries'; + +const byId = new Map(libraries.map(library => [library.id, library])); + +export default Sentry.withSentry( + (env: Env) => ({ + dsn: env.E2E_TEST_DSN, + environment: 'qa', + tunnel: 'http://localhost:3031/', + tracesSampleRate: 1.0, + }), + { + async fetch(request, env, _ctx) { + const url = new URL(request.url); + const [, id, action] = url.pathname.split('/'); + const library = id ? byId.get(id) : undefined; + + if (!library || (action !== 'chat' && action !== 'tools')) { + return new Response('Not found', { status: 404 }); + } + + const apiKey = env.E2E_OPENROUTER_API_KEY; + if (!apiKey) { + return new Response('E2E_OPENROUTER_API_KEY is not set', { status: 500 }); + } + + try { + const spanName = action === 'tools' ? 'ai-tool-workflow' : 'ai-workflow'; + const result = await Sentry.startSpan({ name: spanName, op: 'function' }, () => library[action](apiKey)); + return Response.json({ result }); + } catch (error) { + return Response.json({ message: (error as Error).message }, { status: 500 }); + } + }, + } satisfies ExportedHandler, +); diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.node.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.node.ts new file mode 100644 index 000000000000..726924dbd69d --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.node.ts @@ -0,0 +1,48 @@ +// `instrument.node.ts` is preloaded via `node --import`, so Sentry is already initialised here. +import * as Sentry from '@sentry/node'; +import express from 'express'; +import { libraries } from './libraries'; + +const apiKey = process.env.E2E_OPENROUTER_API_KEY; +if (!apiKey) { + throw new Error('E2E_OPENROUTER_API_KEY is not set'); +} + +const app = express(); + +// One `/:lib/chat` and `/:lib/tools` per instrumented library. Each SDK call is wrapped in a manual +// `ai-workflow` span, so the gen_ai span nests inside it, and it inside the auto-instrumented request +// span. +for (const library of libraries) { + app.get(`/${library.id}/chat`, async (_req, res, next) => { + try { + const answer = await Sentry.startSpan({ name: 'ai-workflow', op: 'function' }, () => library.chat(apiKey)); + res.send({ answer }); + } catch (error) { + next(error); + } + }); + + app.get(`/${library.id}/tools`, async (_req, res, next) => { + try { + const toolCalls = await Sentry.startSpan({ name: 'ai-tool-workflow', op: 'function' }, () => + library.tools(apiKey), + ); + res.send({ toolCalls }); + } catch (error) { + next(error); + } + }); +} + +Sentry.setupExpressErrorHandler(app); + +app.use((error: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => { + res.status(500).send({ message: error.message }); +}); + +const port = Number(process.env.PORT ?? 3030); +app.listen(port, () => { + // eslint-disable-next-line no-console + console.log(`gen-ai-libraries (Node) listening on port ${port}`); +}); diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/env.d.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/env.d.ts new file mode 100644 index 000000000000..b7f1170dd7e5 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/env.d.ts @@ -0,0 +1,4 @@ +interface Env { + E2E_TEST_DSN: ''; + E2E_OPENROUTER_API_KEY: ''; +} diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/instrument.node.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/instrument.node.ts new file mode 100644 index 000000000000..4d8f7f625877 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/instrument.node.ts @@ -0,0 +1,12 @@ +import * as Sentry from '@sentry/node'; + +// Loaded through `node --import`, so the runtime channel-injection hook transforms the AI SDKs and +// express as they load. (The Cloudflare variant covers the build-time bundler-plugin injection path.) +Sentry.init({ + environment: 'qa', + dsn: process.env.E2E_TEST_DSN, + debug: !!process.env.DEBUG, + tunnel: 'http://localhost:3031/', + tracesSampleRate: 1, + enableRuntimeChannelInjection: true, +}); diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts new file mode 100644 index 000000000000..1b1490e7e9b4 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/libraries.ts @@ -0,0 +1,219 @@ +// Each gen-AI *library* (not framework) we instrument, exercised against a real model through +// OpenRouter — the single `E2E_OPENROUTER_API_KEY` the other AI e2e apps already use. Every entry runs +// a plain chat query and a forced tool call, so the same two assertions apply to all of them. These +// handlers are framework- and runtime-agnostic: `entry.node.ts` (express) and `entry.cloudflare.ts` +// (workerd) both call them, passing the key from their respective environments. +// +// OpenRouter serves an OpenAI-compatible `/api/v1/chat/completions` and an Anthropic-compatible +// `/api/v1/messages` ("Anthropic skin"), which is why the OpenAI, Together, Mistral, Anthropic and +// Vercel AI SDKs can all point at it. Every request uses the same `openai/gpt-4o-mini` model — the +// model is incidental; what is under test is each SDK's own request/response code path, the thing +// Sentry instruments. +// +// Two libraries we instrument are intentionally absent because they cannot reach OpenRouter: +// - Google GenAI (`@google/genai`) speaks the native Gemini `generateContent` format, which +// OpenRouter does not serve. +// - Groq (`groq-sdk`) hardcodes a `/openai/v1/...` request path that OpenRouter (served under +// `/api/v1`) does not expose. Its instrumentation is the shared OpenAI-compatible code path that +// Together exercises here, and it is covered by the node-integration-tests. +import Anthropic from '@anthropic-ai/sdk'; +import { createOpenRouter } from '@openrouter/ai-sdk-provider'; +import { Mistral } from '@mistralai/mistralai'; +import OpenAI from 'openai'; +import Together from 'together-ai'; +import { generateText, tool } from 'ai'; +import { z } from 'zod'; + +const OPENROUTER_V1 = 'https://openrouter.ai/api/v1'; +const OPENROUTER_BASE = 'https://openrouter.ai/api'; +const MODEL = 'openai/gpt-4o-mini'; + +const SHORT_ANSWER = 'Answer in at most five words.'; +const CHAT_PROMPT = `What is the capital of France? ${SHORT_ANSWER}`; +// Deliberately does not name the tool: `tool_choice: 'required'` forces the call, and keeping +// "get_weather" out of the prompt means the string only appears in an actual recorded tool call, not +// in `gen_ai.input.messages`. +const WEATHER_PROMPT = `What is the weather in Paris? ${SHORT_ANSWER}`; +const SYSTEM = 'You are a helpful assistant used by an automated test.'; + +// OpenAI-style function tool, shared by the OpenAI-compatible SDKs. +const OPENAI_TOOL = { + type: 'function' as const, + function: { + name: 'get_weather', + description: 'Get the current weather for a city.', + parameters: { + type: 'object', + properties: { city: { type: 'string', description: 'The city name' } }, + required: ['city'], + }, + }, +}; + +export interface Library { + id: string; + /** The op of the model-call span; asserted by the tests. */ + op: 'gen_ai.chat' | 'gen_ai.generate_content'; + /** `gen_ai.provider.name` for the direct-SDK libraries; unset for the Vercel AI SDK. */ + provider?: string; + /** `sentry.origin` for the direct-SDK libraries; unset for the Vercel AI SDK. */ + origin?: string; + chat: (apiKey: string) => Promise; + tools: (apiKey: string) => Promise; +} + +/** Chat + forced tool call for an OpenAI-compatible chat-completions client (OpenAI, Together). */ +function openAiCompatible( + id: string, + provider: string, + origin: string, + makeClient: (apiKey: string) => { chat: { completions: { create: (body: unknown) => Promise } } }, +): Library { + return { + id, + op: 'gen_ai.chat', + provider, + origin, + chat: async apiKey => { + const completion = await makeClient(apiKey).chat.completions.create({ + model: MODEL, + messages: [ + { role: 'system', content: SYSTEM }, + { role: 'user', content: CHAT_PROMPT }, + ], + temperature: 0, + max_tokens: 32, + }); + return completion.choices?.[0]?.message?.content ?? ''; + }, + tools: async apiKey => { + const completion = await makeClient(apiKey).chat.completions.create({ + model: MODEL, + messages: [{ role: 'user', content: WEATHER_PROMPT }], + tools: [OPENAI_TOOL], + tool_choice: 'required', + max_tokens: 64, + }); + return completion.choices?.[0]?.message?.tool_calls ?? []; + }, + }; +} + +export const libraries: Library[] = [ + openAiCompatible('openai', 'openai', 'auto.ai.openai', apiKey => new OpenAI({ apiKey, baseURL: OPENROUTER_V1 })), + openAiCompatible( + 'together', + 'together_ai', + 'auto.ai.together_ai', + apiKey => new Together({ apiKey, baseURL: OPENROUTER_V1 }) as any, + ), + + { + id: 'mistral', + op: 'gen_ai.chat', + provider: 'mistralai', + origin: 'auto.ai.mistralai', + chat: async apiKey => { + const client = new Mistral({ apiKey, serverURL: OPENROUTER_BASE }); + const completion = await client.chat.complete({ + model: MODEL, + messages: [ + { role: 'system', content: SYSTEM }, + { role: 'user', content: CHAT_PROMPT }, + ], + temperature: 0, + maxTokens: 32, + }); + return completion.choices?.[0]?.message?.content ?? ''; + }, + tools: async apiKey => { + const client = new Mistral({ apiKey, serverURL: OPENROUTER_BASE }); + const completion = await client.chat.complete({ + model: MODEL, + messages: [{ role: 'user', content: WEATHER_PROMPT }], + tools: [OPENAI_TOOL] as any, + // OpenRouter's OpenAI-compatible endpoint only accepts none/auto/required, not Mistral's `any`. + toolChoice: 'required', + maxTokens: 64, + }); + return completion.choices?.[0]?.message?.toolCalls ?? []; + }, + }, + + { + id: 'anthropic', + op: 'gen_ai.chat', + provider: 'anthropic', + origin: 'auto.ai.anthropic', + chat: async apiKey => { + // OpenRouter's Anthropic skin authenticates with a bearer token, so the key goes in `authToken` + // (Authorization: Bearer) rather than `apiKey` (x-api-key). + const client = new Anthropic({ authToken: apiKey, baseURL: OPENROUTER_BASE }); + const message = await client.messages.create({ + model: MODEL, + max_tokens: 32, + temperature: 0, + system: SYSTEM, + messages: [{ role: 'user', content: CHAT_PROMPT }], + }); + const first = message.content?.[0]; + return first && first.type === 'text' ? first.text : ''; + }, + tools: async apiKey => { + const client = new Anthropic({ authToken: apiKey, baseURL: OPENROUTER_BASE }); + const message = await client.messages.create({ + model: MODEL, + max_tokens: 64, + messages: [{ role: 'user', content: WEATHER_PROMPT }], + tools: [ + { + name: 'get_weather', + description: 'Get the current weather for a city.', + input_schema: { + type: 'object', + properties: { city: { type: 'string', description: 'The city name' } }, + required: ['city'], + }, + }, + ], + tool_choice: { type: 'tool', name: 'get_weather' }, + }); + return (message.content ?? []).filter(block => block.type === 'tool_use'); + }, + }, + + { + id: 'vercel-ai', + // The Vercel AI SDK emits `gen_ai.generate_content` (nested in a `gen_ai.invoke_agent` span), + // reports the provider from the model id, and uses its own span origin. + op: 'gen_ai.generate_content', + chat: async apiKey => { + const openrouter = createOpenRouter({ apiKey }); + const { text } = await generateText({ + model: openrouter(MODEL), + system: SYSTEM, + prompt: CHAT_PROMPT, + temperature: 0, + experimental_telemetry: { isEnabled: true }, + }); + return text; + }, + tools: async apiKey => { + const openrouter = createOpenRouter({ apiKey }); + const result = await generateText({ + model: openrouter(MODEL), + prompt: WEATHER_PROMPT, + toolChoice: 'required', + experimental_telemetry: { isEnabled: true }, + tools: { + get_weather: tool({ + description: 'Get the current weather for a city.', + inputSchema: z.object({ city: z.string().describe('The city name') }), + execute: async ({ city }) => `It is sunny in ${city}.`, + }), + }, + }); + return result.toolCalls ?? []; + }, + }, +]; diff --git a/dev-packages/e2e-tests/test-applications/node-mistral/start-event-proxy.mjs b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/start-event-proxy.mjs similarity index 74% rename from dev-packages/e2e-tests/test-applications/node-mistral/start-event-proxy.mjs rename to dev-packages/e2e-tests/test-applications/gen-ai-libraries/start-event-proxy.mjs index 2c8fdc947553..ad68d9340fce 100644 --- a/dev-packages/e2e-tests/test-applications/node-mistral/start-event-proxy.mjs +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/start-event-proxy.mjs @@ -2,5 +2,5 @@ import { startEventProxyServer } from '@sentry-internal/test-utils'; startEventProxyServer({ port: 3031, - proxyServerName: 'node-mistral', + proxyServerName: 'gen-ai-libraries', }); diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/chat.test.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/chat.test.ts new file mode 100644 index 000000000000..59d406841821 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/chat.test.ts @@ -0,0 +1,24 @@ +import { expect, test } from '@playwright/test'; +import { collectStreamedSpans } from '@sentry-internal/test-utils'; +import { APP } from './constants'; +import { describeTree, expectCommonChatAttributes, isModelCallSpan, LIBRARIES, traceHasToolEvidence } from './utils'; + +for (const library of LIBRARIES) { + test(`${library.id}: a chat query emits a ${library.op} span`, async ({ baseURL }) => { + // Scope to this chat request's own trace: it carries this library's model-call span and, unlike the + // tools request, no tool-call evidence — so a leftover trace from another request cannot satisfy it. + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => spansOfTrace.some(span => isModelCallSpan(span, library)) && !traceHasToolEvidence(spansOfTrace), + ); + + const response = await fetch(`${baseURL}/${library.id}/chat`); + expect(response.status).toBe(200); + + const spans = await spansPromise; + const modelSpan = spans.find(span => isModelCallSpan(span, library)); + + expect(modelSpan, `expected a ${library.op} span in:\n${describeTree(spans)}`).toBeDefined(); + expectCommonChatAttributes(modelSpan!, library); + }); +} diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/constants.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/constants.ts new file mode 100644 index 000000000000..fb2a2d8999ec --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/constants.ts @@ -0,0 +1,5 @@ +export type Runtime = 'node' | 'cloudflare'; + +export const RUNTIME = (process.env.RUNTIME || 'node') as Runtime; + +export const APP = 'gen-ai-libraries'; diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/tools.test.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/tools.test.ts new file mode 100644 index 000000000000..d2ff269ca0a7 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/tools.test.ts @@ -0,0 +1,47 @@ +import { expect, test } from '@playwright/test'; +import { collectStreamedSpans, getSpanOp } from '@sentry-internal/test-utils'; +import { APP } from './constants'; +import { attr, describeTree, hasRecordedToolCalls, isExecuteToolSpan, isModelCallSpan, LIBRARIES } from './utils'; + +// The direct-SDK libraries record the model's tool call on the chat span's `gen_ai.response.tool_calls` +// attribute, which only exists when the model actually returned tool calls. +const DIRECT_SDK_LIBRARIES = LIBRARIES.filter(library => library.provider); + +for (const library of DIRECT_SDK_LIBRARIES) { + test(`${library.id}: the model's tool call is recorded on the ${library.op} span`, async ({ baseURL }) => { + // Scope to this tools request's trace: this library's model-call span, carrying recorded tool calls. + const spansPromise = collectStreamedSpans(APP, spansOfTrace => + spansOfTrace.some(span => isModelCallSpan(span, library) && hasRecordedToolCalls(span)), + ); + + const response = await fetch(`${baseURL}/${library.id}/tools`); + expect(response.status).toBe(200); + + const spans = await spansPromise; + const modelSpan = spans.find(span => isModelCallSpan(span, library) && hasRecordedToolCalls(span)); + + expect( + modelSpan, + `expected a ${library.op} span with recorded tool calls in:\n${describeTree(spans)}`, + ).toBeDefined(); + expect(attr(modelSpan!, 'gen_ai.response.tool_calls')).toContain('get_weather'); + }); +} + +// The Vercel AI SDK executes the tool and emits a dedicated `gen_ai.execute_tool` span instead. +test('vercel-ai: the tool call is captured as a gen_ai.execute_tool span', async ({ baseURL }) => { + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => + spansOfTrace.some(isExecuteToolSpan) && spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.generate_content'), + ); + + const response = await fetch(`${baseURL}/vercel-ai/tools`); + expect(response.status).toBe(200); + + const spans = await spansPromise; + const toolSpan = spans.find(isExecuteToolSpan); + + expect(toolSpan, `expected a gen_ai.execute_tool span in:\n${describeTree(spans)}`).toBeDefined(); + expect(attr(toolSpan!, 'gen_ai.tool.name')).toBe('get_weather'); +}); diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/utils.ts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/utils.ts new file mode 100644 index 000000000000..4861a01c5334 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tests/utils.ts @@ -0,0 +1,80 @@ +import { expect } from '@playwright/test'; +import type { SerializedStreamedSpan } from '@sentry-internal/test-utils'; +import { getSpanOp } from '@sentry-internal/test-utils'; + +/** Mirrors `src/libraries.ts`, and records how strictly each library's spans can be asserted. */ +export interface LibraryUnderTest { + id: string; + /** The op of the model-call span: `gen_ai.chat` for the direct SDKs, `gen_ai.generate_content` for + * the Vercel AI SDK. */ + op: string; + /** `gen_ai.provider.name` for the direct-SDK libraries; unset for the Vercel AI SDK, whose provider + * name comes from the model id and is not asserted. */ + provider?: string; + /** `sentry.origin` for the direct-SDK libraries; unset for the Vercel AI SDK. */ + origin?: string; +} + +export const LIBRARIES: LibraryUnderTest[] = [ + { id: 'openai', op: 'gen_ai.chat', provider: 'openai', origin: 'auto.ai.openai' }, + { id: 'together', op: 'gen_ai.chat', provider: 'together_ai', origin: 'auto.ai.together_ai' }, + { id: 'mistral', op: 'gen_ai.chat', provider: 'mistralai', origin: 'auto.ai.mistralai' }, + { id: 'anthropic', op: 'gen_ai.chat', provider: 'anthropic', origin: 'auto.ai.anthropic' }, + { id: 'vercel-ai', op: 'gen_ai.generate_content' }, +]; + +export const attr = (span: SerializedStreamedSpan, key: string): unknown => span.attributes?.[key]?.value; + +/** + * The model-call span for a library. The four direct SDKs all report `gen_ai.chat`, so the provider + * name is what tells them apart — without it, a leftover span from another library's request could be + * mistaken for this one's. + */ +export const isModelCallSpan = (span: SerializedStreamedSpan, library: LibraryUnderTest): boolean => + getSpanOp(span) === library.op && (!library.provider || attr(span, 'gen_ai.provider.name') === library.provider); + +/** A span that recorded the model returning tool calls on a chat-completions request (direct SDKs). */ +export const hasRecordedToolCalls = (span: SerializedStreamedSpan): boolean => + typeof attr(span, 'gen_ai.response.tool_calls') === 'string'; + +/** The dedicated tool-execution span the Vercel AI SDK emits. */ +export const isExecuteToolSpan = (span: SerializedStreamedSpan): boolean => getSpanOp(span) === 'gen_ai.execute_tool'; + +/** Whether a trace shows any evidence of a tool call, used to tell a chat request from a tools one. */ +export const traceHasToolEvidence = (spansOfTrace: SerializedStreamedSpan[]): boolean => + spansOfTrace.some(span => hasRecordedToolCalls(span) || isExecuteToolSpan(span)); + +/** A readable span tree, used as a failure message so a broken assertion is diagnosable. */ +export function describeTree(spans: SerializedStreamedSpan[]): string { + return spans + .map(span => `${span.name} [${getSpanOp(span) ?? '-'}] id=${span.span_id} parent=${span.parent_span_id ?? '-'}`) + .join('\n'); +} + +/** + * The attributes every successful gen_ai model-call span carries, whatever the model happens to + * answer. Model-dependent values (token counts, response text) are checked for shape, not content. + */ +export function expectCommonChatAttributes(span: SerializedStreamedSpan, library: LibraryUnderTest): void { + const operationName = library.op.replace('gen_ai.', ''); + + expect(getSpanOp(span), describeTree([span])).toBe(library.op); + expect(attr(span, 'gen_ai.operation.name')).toBe(operationName); + expect(span.status).toBe('ok'); + + expect(typeof attr(span, 'gen_ai.provider.name')).toBe('string'); + expect(typeof attr(span, 'gen_ai.request.model')).toBe('string'); + expect(typeof attr(span, 'gen_ai.response.model')).toBe('string'); + expect(attr(span, 'gen_ai.usage.input_tokens')).toBeGreaterThan(0); + expect(attr(span, 'gen_ai.usage.output_tokens')).toBeGreaterThan(0); + expect(attr(span, 'gen_ai.usage.total_tokens')).toBeGreaterThan(0); + + // The direct-SDK libraries carry a stable provider name and origin; the Vercel AI SDK does not. + if (library.provider) { + expect(attr(span, 'gen_ai.provider.name')).toBe(library.provider); + expect(span.name).toBe(`${operationName} ${attr(span, 'gen_ai.request.model')}`); + } + if (library.origin) { + expect(attr(span, 'sentry.origin')).toBe(library.origin); + } +} diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tsconfig.json b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tsconfig.json new file mode 100644 index 000000000000..91fe6c743269 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "es2021", + "lib": ["es2021"], + "module": "es2022", + "moduleResolution": "Bundler", + "resolveJsonModule": true, + "allowJs": true, + "checkJs": false, + "noEmit": true, + "isolatedModules": true, + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipLibCheck": true, + "types": ["node", "@cloudflare/workers-types/experimental"] + }, + "exclude": ["tests"], + "include": ["src/**/*.ts"] +} diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/vite.config.mts b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/vite.config.mts new file mode 100644 index 000000000000..1c4c4863046d --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/vite.config.mts @@ -0,0 +1,9 @@ +import { cloudflare } from '@cloudflare/vite-plugin'; +import { sentryCloudflareVitePlugin } from '@sentry/cloudflare/vite'; +import { defineConfig } from 'vite'; + +// Builds the Cloudflare variant (`src/entry.cloudflare.ts`, per `wrangler.toml`). The Node variant runs +// straight from source via tsx and does not use this config. +export default defineConfig({ + plugins: [cloudflare(), sentryCloudflareVitePlugin()], +}); diff --git a/dev-packages/e2e-tests/test-applications/gen-ai-libraries/wrangler.jsonc b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/wrangler.jsonc new file mode 100644 index 000000000000..07b7bc9ee832 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/gen-ai-libraries/wrangler.jsonc @@ -0,0 +1,7 @@ +{ + "$schema": "node_modules/wrangler/config-schema.json", + "name": "gen-ai-libraries", + "main": "src/entry.cloudflare.ts", + "compatibility_date": "2026-04-20", + "compatibility_flags": ["nodejs_compat"], +} diff --git a/dev-packages/e2e-tests/test-applications/node-mistral/.gitignore b/dev-packages/e2e-tests/test-applications/node-mistral/.gitignore deleted file mode 100644 index 1521c8b7652b..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-mistral/.gitignore +++ /dev/null @@ -1 +0,0 @@ -dist diff --git a/dev-packages/e2e-tests/test-applications/node-mistral/build.mjs b/dev-packages/e2e-tests/test-applications/node-mistral/build.mjs deleted file mode 100644 index acb282f9c706..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-mistral/build.mjs +++ /dev/null @@ -1,40 +0,0 @@ -// Produces the prod-mode artifact: a single bundle whose `@mistralai/mistralai`, `dataloader` and -// `express` copies were transformed at build time by `sentryEsbuildPlugin`. Nothing is left for a -// runtime hook to do, which is what `enableRuntimeChannelInjection: false` in `instrument.mjs` -// asserts. -// -// `@sentry/node` stays external: the SDK is the subscriber, not a transform target, and inlining it -// would force its CommonJS `require('node:async_hooks')` through esbuild's ESM interop for no gain. -// CJS output for the same reason the `node-esbuild` app uses it. Left unminified so the injected -// snippet keeps its identifiers. -import { rmSync } from 'node:fs'; -import { dirname, join } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { sentryEsbuildPlugin } from '@sentry/node/esbuild'; -import { build } from 'esbuild'; - -const __dirname = dirname(fileURLToPath(import.meta.url)); - -rmSync(join(__dirname, 'dist'), { recursive: true, force: true }); - -await build({ - entryPoints: [join(__dirname, 'src', 'app.mjs')], - outfile: join(__dirname, 'dist', 'app.cjs'), - bundle: true, - platform: 'node', - format: 'cjs', - target: 'node18', - external: ['@sentry/node'], - minify: false, - logLevel: 'info', - plugins: [ - sentryEsbuildPlugin({ - telemetry: false, - sourcemaps: { disable: true }, - release: { create: false, finalize: false, inject: false }, - }), - ], -}); - -// eslint-disable-next-line no-console -console.log('built dist/app.cjs with sentryEsbuildPlugin'); diff --git a/dev-packages/e2e-tests/test-applications/node-mistral/package.json b/dev-packages/e2e-tests/test-applications/node-mistral/package.json deleted file mode 100644 index 16b53b297d36..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-mistral/package.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "name": "node-mistral", - "description": "Mistral AI gen_ai spans, errors, span nesting and co-instrumented dataloader spans, exercised through both the runtime loader (dev) and a bundler-instrumented build (prod)", - "version": "1.0.0", - "private": true, - "type": "module", - "scripts": { - "start": "node --import ./src/instrument.mjs src/app.mjs", - "start:bundled": "node dist/app.cjs", - "build": "node build.mjs", - "clean": "npx rimraf node_modules dist pnpm-lock.yaml", - "test:build": "pnpm install && pnpm build", - "test:assert": "pnpm test:prod && pnpm test:dev", - "test:prod": "TEST_ENV=production playwright test", - "test:dev": "TEST_ENV=development playwright test" - }, - "dependencies": { - "@mistralai/mistralai": "^2.6.4", - "@sentry/node": "file:../../packed/sentry-node-packed.tgz", - "dataloader": "^2.2.2", - "express": "^4.21.2" - }, - "devDependencies": { - "@playwright/test": "~1.56.0", - "@sentry-internal/test-utils": "link:../../../test-utils", - "@sentry/bundler-plugins": "file:../../packed/sentry-bundler-plugins-packed.tgz", - "@sentry/core": "file:../../packed/sentry-core-packed.tgz", - "esbuild": "0.28.2" - }, - "sentryTest": { - "optional": true - }, - "volta": { - "extends": "../../package.json" - } -} diff --git a/dev-packages/e2e-tests/test-applications/node-mistral/playwright.config.mjs b/dev-packages/e2e-tests/test-applications/node-mistral/playwright.config.mjs deleted file mode 100644 index 39daff08107f..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-mistral/playwright.config.mjs +++ /dev/null @@ -1,15 +0,0 @@ -import { getPlaywrightConfig } from '@sentry-internal/test-utils'; - -// The suite runs twice, once per instrumentation path, the way `node-mastra` splits dev and prod: -// -// production - `dist/app.cjs`, whose Mistral, dataloader and express copies were transformed at -// build time by `sentryEsbuildPlugin`. `instrument.mjs` turns runtime injection off -// there, so the bundler plugin is the only thing that can have instrumented them. -// development - unbundled ESM behind the runtime `--import` hook. -const isDev = process.env.TEST_ENV === 'development'; - -const config = getPlaywrightConfig({ - startCommand: isDev ? 'pnpm start' : 'pnpm start:bundled', -}); - -export default config; diff --git a/dev-packages/e2e-tests/test-applications/node-mistral/src/app.mjs b/dev-packages/e2e-tests/test-applications/node-mistral/src/app.mjs deleted file mode 100644 index f9e408f057b4..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-mistral/src/app.mjs +++ /dev/null @@ -1,180 +0,0 @@ -// `instrument.mjs` is imported for its side effect in the prod bundle; in dev `--import` has already -// run it, and a second import is a no-op because ES modules are evaluated once. -import './instrument.mjs'; - -import { Mistral } from '@mistralai/mistralai'; -import * as Sentry from '@sentry/node'; -import DataLoader from 'dataloader'; -import express from 'express'; - -const apiKey = process.env.E2E_OPENROUTER_API_KEY; -if (!apiKey) { - throw new Error('E2E_OPENROUTER_API_KEY is not set'); -} - -// The Mistral SDK talks to OpenRouter rather than api.mistral.ai, so the suite needs only the one -// OpenRouter key the other AI e2e apps already use. OpenRouter serves an OpenAI-compatible -// `/v1/chat/completions`, which is the endpoint `chat.complete` and `chat.stream` post to, and the -// SDK's response schemas are lenient enough to accept it (`usage` has a `catchall`, `finish_reason` -// is an open enum). What is under test is the SDK's own code path, which is what Sentry instruments. -const client = new Mistral({ apiKey, serverURL: 'https://openrouter.ai/api' }); - -// Same model the eve and mastra apps drive through this key. The model is incidental here; the -// Mistral SDK request/response path is the thing being instrumented. -const MODEL = 'openai/gpt-4o-mini'; - -// Kept short so a live model stays cheap and quick, and so streamed responses still arrive in more -// than one chunk. -const SHORT_ANSWER = 'Answer in at most five words.'; - -const userLoader = new DataLoader(async keys => keys.map(key => ({ id: key, name: `user-${key}` }))); - -async function main() { - const port = Number(process.env.PORT ?? 3030); - const app = express(); - - app.get('/chat', async (req, res) => { - // A manual span wrapping the SDK call: the gen_ai span has to nest inside this one, and this one - // has to nest inside the auto-instrumented request span. - const answer = await Sentry.startSpan({ name: 'ai-workflow', op: 'function' }, async () => { - const completion = await client.chat.complete({ - model: MODEL, - messages: [ - { role: 'system', content: 'You are a helpful assistant used by an automated test.' }, - { role: 'user', content: `What is the capital of France? ${SHORT_ANSWER}` }, - ], - temperature: 0, - maxTokens: 32, - }); - - // A manual sibling of the gen_ai span, so the assertions can tell "child of the manual span" - // apart from "child of whatever ran last". - return Sentry.startSpan( - { name: 'post-process', op: 'function' }, - () => completion.choices?.[0]?.message?.content ?? '', - ); - }); - - res.send({ answer }); - }); - - app.get('/chat-stream', async (req, res) => { - const chunks = []; - - await Sentry.startSpan({ name: 'ai-stream-workflow', op: 'function' }, async () => { - const stream = await client.chat.stream({ - model: MODEL, - messages: [{ role: 'user', content: `Name three colours. ${SHORT_ANSWER}` }], - temperature: 0, - maxTokens: 32, - }); - - for await (const event of stream) { - const content = event.data?.choices?.[0]?.delta?.content; - if (typeof content === 'string') { - chunks.push(content); - } - } - }); - - res.send({ answer: chunks.join('') }); - }); - - // `tee()` acquires its reader through internal slots rather than the public `getReader`, so it is - // the drain path most likely to escape instrumentation. Both branches are drained so the response - // only comes back once the stream is finished. - app.get('/chat-stream-tee', async (req, res) => { - const branches = await Sentry.startSpan({ name: 'ai-tee-workflow', op: 'function' }, async () => { - const stream = await client.chat.stream({ - model: MODEL, - messages: [{ role: 'user', content: `Name three colours. ${SHORT_ANSWER}` }], - temperature: 0, - maxTokens: 32, - }); - - const [left, right] = stream.tee(); - - const drain = async branch => { - const parts = []; - for await (const event of branch) { - const content = event.data?.choices?.[0]?.delta?.content; - if (typeof content === 'string') { - parts.push(content); - } - } - return parts.join(''); - }; - - return Promise.all([drain(left), drain(right)]); - }); - - res.send({ left: branches[0], right: branches[1] }); - }); - - // Relays the stream through a transform, the shape an edge handler would use to forward tokens. - app.get('/chat-stream-pipe', async (req, res) => { - const answer = await Sentry.startSpan({ name: 'ai-pipe-workflow', op: 'function' }, async () => { - const stream = await client.chat.stream({ - model: MODEL, - messages: [{ role: 'user', content: `Name three colours. ${SHORT_ANSWER}` }], - temperature: 0, - maxTokens: 32, - }); - - const relayed = stream.pipeThrough( - new TransformStream({ - transform(event, controller) { - controller.enqueue(event.data?.choices?.[0]?.delta?.content ?? ''); - }, - }), - ); - - const parts = []; - for await (const part of relayed) { - parts.push(part); - } - return parts.join(''); - }); - - res.send({ answer }); - }); - - // A model id the upstream will reject, so the failure is a real API error rather than a simulated - // one. The caller-supplied id makes each request identifiable in the spans it produces. - app.get('/chat-error', async (req, res, next) => { - const model = `no-such-model/${req.query.id ?? 'default'}`; - - try { - await client.chat.complete({ model, messages: [{ role: 'user', content: 'This will fail' }] }); - res.send({ ok: true }); - } catch (error) { - // Rethrown through the express error handler so the SDK captures it the way a real app would. - next(new Error(`Mistral call failed for ${model}: ${error.message}`)); - } - }); - - // A dataloader (orchestrion-instrumented, like Mistral) and a Mistral call in one request, so the - // assertions can prove both sets of spans land in the same trace. - app.get('/dataloader-and-chat', async (req, res) => { - const user = await userLoader.load(`${req.query.id ?? '1'}`); - - const completion = await client.chat.complete({ - model: MODEL, - messages: [{ role: 'user', content: `Say hello to ${user.name}. ${SHORT_ANSWER}` }], - temperature: 0, - maxTokens: 32, - }); - - res.send({ user, answer: completion.choices?.[0]?.message?.content ?? '' }); - }); - - Sentry.setupExpressErrorHandler(app); - - app.use((error, req, res, _next) => { - res.status(500).send({ message: error.message }); - }); - - app.listen(port); -} - -void main(); diff --git a/dev-packages/e2e-tests/test-applications/node-mistral/src/instrument.mjs b/dev-packages/e2e-tests/test-applications/node-mistral/src/instrument.mjs deleted file mode 100644 index 30abaa918aa1..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-mistral/src/instrument.mjs +++ /dev/null @@ -1,24 +0,0 @@ -// Shared Sentry bootstrap for both modes. -// -// dev - loaded through `node --import`, so the runtime channel-injection hook transforms -// `@mistralai/mistralai`, `dataloader` and `express` as they load. -// prod - bundled into `dist/app.cjs` by `build.mjs`, where `sentryEsbuildPlugin` applies the same -// transforms at build time. Runtime injection is switched off there so the bundler plugin is -// the only possible injector and a passing prod test really proves the build-time path. -import * as Sentry from '@sentry/node'; - -// `production` is the bundled build, where `sentryEsbuildPlugin` already injected the channels. -const isDev = process.env.TEST_ENV === 'development'; - -Sentry.init({ - environment: 'qa', - dsn: process.env.E2E_TEST_DSN, - debug: !!process.env.DEBUG, - tunnel: 'http://localhost:3031/', - tracesSampleRate: 1, - traceLifecycle: 'stream', - enableRuntimeChannelInjection: isDev, - integrations: [Sentry.spanStreamingIntegration()], -}); - -Sentry.setTag('e2e.mode', isDev ? 'development' : 'production'); diff --git a/dev-packages/e2e-tests/test-applications/node-mistral/tests/ai-spans.test.ts b/dev-packages/e2e-tests/test-applications/node-mistral/tests/ai-spans.test.ts deleted file mode 100644 index aabe2af66f5c..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-mistral/tests/ai-spans.test.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { collectStreamedSpansUntilSegment } from '@sentry-internal/test-utils'; -import { APP, attr, expectCommonChatAttributes, isChatSpan } from './utils'; - -test('emits a gen_ai.chat span for a non-streaming call', async ({ baseURL, request }) => { - const spansPromise = collectStreamedSpansUntilSegment(APP, 'GET /chat'); - - const response = await request.get(`${baseURL}/chat`); - expect(response.status()).toBe(200); - expect((await response.json()).answer).toBeTruthy(); - - const spans = await spansPromise; - const chatSpan = spans.find(isChatSpan); - - expect(chatSpan).toBeDefined(); - expectCommonChatAttributes(chatSpan!); - expect(attr(chatSpan!, 'gen_ai.request.stream')).toBe(false); - expect(attr(chatSpan!, 'gen_ai.request.temperature')).toBe(0); - expect(attr(chatSpan!, 'gen_ai.request.max_tokens')).toBe(32); -}); - -test('emits a gen_ai.chat span for a streaming call', async ({ baseURL, request }) => { - const spansPromise = collectStreamedSpansUntilSegment(APP, 'GET /chat-stream'); - - const response = await request.get(`${baseURL}/chat-stream`); - expect(response.status()).toBe(200); - expect((await response.json()).answer).toBeTruthy(); - - const spans = await spansPromise; - const streamSpan = spans.find(isChatSpan); - - expect(streamSpan).toBeDefined(); - expectCommonChatAttributes(streamSpan!); - // Set from the called method: v2's `stream` request field is optional and the app never passes it. - expect(attr(streamSpan!, 'gen_ai.request.stream')).toBe(true); - expect(attr(streamSpan!, 'gen_ai.response.streaming')).toBe(true); -}); - -test('records inputs and outputs in the shape the gen_ai conventions specify', async ({ baseURL, request }) => { - const spansPromise = collectStreamedSpansUntilSegment(APP, 'GET /chat'); - - await request.get(`${baseURL}/chat`); - - const spans = await spansPromise; - const chatSpan = spans.find(isChatSpan)!; - - // The system message is split out from the rest of the prompt. - expect(attr(chatSpan, 'gen_ai.system_instructions')).toContain('automated test'); - expect(attr(chatSpan, 'gen_ai.input.messages')).toContain('capital of France'); - - // A stringified array of messages, not one concatenated string. - const responseText = JSON.parse(attr(chatSpan, 'gen_ai.response.text') as string); - expect(Array.isArray(responseText)).toBe(true); - expect(responseText).toHaveLength(1); - expect(typeof responseText[0]).toBe('string'); - - const outputMessages = JSON.parse(attr(chatSpan, 'gen_ai.output.messages') as string); - expect(outputMessages).toEqual([ - { - role: 'assistant', - parts: [{ type: 'text', content: expect.any(String) }], - finish_reason: expect.any(String), - }, - ]); -}); diff --git a/dev-packages/e2e-tests/test-applications/node-mistral/tests/co-instrumentation.test.ts b/dev-packages/e2e-tests/test-applications/node-mistral/tests/co-instrumentation.test.ts deleted file mode 100644 index 9888e154f7cf..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-mistral/tests/co-instrumentation.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { collectStreamedSpansUntilSegment, getSpanOp } from '@sentry-internal/test-utils'; -import { APP, attr, isChatSpan } from './utils'; - -// Mistral and dataloader are both instrumented through orchestrion, so one request that touches -// both proves the Mistral channels coexist with the rest of the injected set rather than displacing -// them. dataloader is also CommonJS where Mistral is ESM-only, so this covers both module formats -// going through the same transform in one process. -test('emits dataloader spans alongside gen_ai spans in one trace', async ({ baseURL, request }) => { - const spansPromise = collectStreamedSpansUntilSegment(APP, 'GET /dataloader-and-chat'); - - const response = await request.get(`${baseURL}/dataloader-and-chat?id=7`); - expect(response.status()).toBe(200); - expect((await response.json()).user).toEqual({ id: '7', name: 'user-7' }); - - const spans = await spansPromise; - const segment = spans.find(span => span.is_segment && span.name === 'GET /dataloader-and-chat')!; - - const chatSpan = spans.find(isChatSpan); - const dataloaderSpans = spans.filter(span => attr(span, 'sentry.origin') === 'auto.db.dataloader'); - - expect(chatSpan).toBeDefined(); - expect(dataloaderSpans.length).toBeGreaterThan(0); - - // `load` is recorded as a cache read. - expect(dataloaderSpans.some(span => getSpanOp(span) === 'cache.get')).toBe(true); - - // Both instrumentations contribute to the same trace, under the same request. - expect(chatSpan!.trace_id).toBe(segment.trace_id); - for (const span of dataloaderSpans) { - expect(span.trace_id).toBe(segment.trace_id); - } -}); diff --git a/dev-packages/e2e-tests/test-applications/node-mistral/tests/drain-paths.test.ts b/dev-packages/e2e-tests/test-applications/node-mistral/tests/drain-paths.test.ts deleted file mode 100644 index 40563d376a4d..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-mistral/tests/drain-paths.test.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { collectStreamedSpansUntilSegment } from '@sentry-internal/test-utils'; -import { APP, attr, byName, expectCommonChatAttributes, isChatSpan } from './utils'; - -// `tee`, `pipeTo` and `pipeThrough` take their reader from internal slots rather than the public -// `getReader`, so they bypass a stream instrumented only through `getReader` and the async iterator. -// These cover the two an app is realistically built on: teeing to relay and persist at once, and -// piping through a transform to forward tokens to a client. - -test('records a gen_ai span for a teed stream, once', async ({ baseURL, request }) => { - const spansPromise = collectStreamedSpansUntilSegment(APP, 'GET /chat-stream-tee'); - - const response = await request.get(`${baseURL}/chat-stream-tee`); - expect(response.status()).toBe(200); - - // Both branches receive the same stream. - const { left, right } = await response.json(); - expect(left).toBeTruthy(); - expect(left).toBe(right); - - const spans = await spansPromise; - const chatSpans = spans.filter(isChatSpan); - - // One span, not one per tee branch. - expect(chatSpans).toHaveLength(1); - const chatSpan = chatSpans[0]!; - - expectCommonChatAttributes(chatSpan); - expect(attr(chatSpan, 'gen_ai.request.stream')).toBe(true); - expect(attr(chatSpan, 'gen_ai.response.streaming')).toBe(true); - - // Nesting still holds on this drain path. - const segment = spans.find(span => span.is_segment && span.name === 'GET /chat-stream-tee')!; - const workflow = byName(spans, 'ai-tee-workflow'); - expect(chatSpan.parent_span_id).toBe(workflow.span_id); - expect(chatSpan.trace_id).toBe(segment.trace_id); -}); - -test('records a gen_ai span for a stream relayed through a transform', async ({ baseURL, request }) => { - const spansPromise = collectStreamedSpansUntilSegment(APP, 'GET /chat-stream-pipe'); - - const response = await request.get(`${baseURL}/chat-stream-pipe`); - expect(response.status()).toBe(200); - expect((await response.json()).answer).toBeTruthy(); - - const spans = await spansPromise; - const chatSpan = spans.find(isChatSpan); - - expect(chatSpan).toBeDefined(); - expectCommonChatAttributes(chatSpan!); - expect(attr(chatSpan!, 'gen_ai.response.streaming')).toBe(true); - - const segment = spans.find(span => span.is_segment && span.name === 'GET /chat-stream-pipe')!; - const workflow = byName(spans, 'ai-pipe-workflow'); - expect(chatSpan!.parent_span_id).toBe(workflow.span_id); - expect(chatSpan!.trace_id).toBe(segment.trace_id); -}); diff --git a/dev-packages/e2e-tests/test-applications/node-mistral/tests/errors.test.ts b/dev-packages/e2e-tests/test-applications/node-mistral/tests/errors.test.ts deleted file mode 100644 index c2dc4067d9d5..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-mistral/tests/errors.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { collectStreamedSpans, waitForError } from '@sentry-internal/test-utils'; -import { APP, attr, isChatSpan } from './utils'; - -test('captures an error thrown by a failed Mistral call', async ({ baseURL, request }) => { - const model = 'no-such-model/capture'; - const errorPromise = waitForError( - APP, - event => !event.type && !!event.exception?.values?.[0]?.value?.includes(model), - ); - - const response = await request.get(`${baseURL}/chat-error?id=capture`); - expect(response.status()).toBe(500); - - const errorEvent = await errorPromise; - - expect(errorEvent.exception?.values?.[0]?.value).toContain('Mistral call failed'); - expect(errorEvent.transaction).toBe('GET /chat-error'); - expect(errorEvent.contexts?.trace?.trace_id).toMatch(/[a-f0-9]{32}/); -}); - -test('marks the gen_ai span errored and ties it to the captured error', async ({ baseURL, request }) => { - const id = 'linked'; - const model = `no-such-model/${id}`; - - const errorPromise = waitForError( - APP, - event => !event.type && !!event.exception?.values?.[0]?.value?.includes(model), - ); - // Every request to this route produces an equivalent-looking trace, so the predicate names the - // per-request model rather than the route. - const spansPromise = collectStreamedSpans( - APP, - spansOfTrace => - spansOfTrace.some(span => span.is_segment && span.name === 'GET /chat-error') && - spansOfTrace.some(span => attr(span, 'gen_ai.request.model') === model), - ); - - await request.get(`${baseURL}/chat-error?id=${id}`); - - const [errorEvent, spans] = await Promise.all([errorPromise, spansPromise]); - const chatSpan = spans.find(isChatSpan)!; - - expect(chatSpan).toBeDefined(); - expect(chatSpan.status).not.toBe('ok'); - // No response was produced, so nothing should have been recorded from one. - expect(attr(chatSpan, 'gen_ai.response.text')).toBeUndefined(); - expect(attr(chatSpan, 'gen_ai.output.messages')).toBeUndefined(); - - expect(chatSpan.trace_id).toBe(errorEvent.contexts?.trace?.trace_id); -}); diff --git a/dev-packages/e2e-tests/test-applications/node-mistral/tests/instrumentation-path.test.ts b/dev-packages/e2e-tests/test-applications/node-mistral/tests/instrumentation-path.test.ts deleted file mode 100644 index 1c53b9d332ae..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-mistral/tests/instrumentation-path.test.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { readFileSync } from 'node:fs'; -import { expect, test } from '@playwright/test'; - -// Guards the premise the prod run rests on. `Sentry.init` registers the runtime injection hook -// unless `enableRuntimeChannelInjection` is false, which `instrument.mjs` sets outside dev. With -// that off and no `--import` on the bundled start command, the bundler plugin is the only thing -// that can have injected these channels, so finding them in the built file is what makes a passing -// production run mean build-time instrumentation rather than a silent fallback. -test('the bundle carries build-time injected channels', () => { - test.skip(process.env.TEST_ENV === 'development', 'the dev run is instrumented by the runtime hook'); - - const bundle = readFileSync('dist/app.cjs', 'utf8'); - - expect(bundle).toContain('orchestrion:@mistralai/mistralai:chat'); - expect(bundle).toContain('orchestrion:@mistralai/mistralai:chat-stream'); - expect(bundle).toContain('orchestrion:dataloader:load'); -}); diff --git a/dev-packages/e2e-tests/test-applications/node-mistral/tests/nesting.test.ts b/dev-packages/e2e-tests/test-applications/node-mistral/tests/nesting.test.ts deleted file mode 100644 index 8cb72ffa1728..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-mistral/tests/nesting.test.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { collectStreamedSpansUntilSegment } from '@sentry-internal/test-utils'; -import { ancestorIds, APP, byName, describeTree, isChatSpan } from './utils'; - -test('nests the manual span under the request span and the gen_ai span under the manual span', async ({ - baseURL, - request, -}) => { - const spansPromise = collectStreamedSpansUntilSegment(APP, 'GET /chat'); - - await request.get(`${baseURL}/chat`); - - const spans = await spansPromise; - const tree = describeTree(spans); - - const segment = spans.find(span => span.is_segment && span.name === 'GET /chat')!; - const workflow = byName(spans, 'ai-workflow'); - const postProcess = byName(spans, 'post-process'); - const chatSpan = spans.find(isChatSpan)!; - - // Manual span inside the generated request span. Express contributes its own middleware and - // request-handler spans in between, so this is an ancestry check, not a direct-parent one. - expect(ancestorIds(spans, workflow), `ai-workflow is not under the request span:\n${tree}`).toContain( - segment.span_id, - ); - - // Generated span directly inside the manual one: nothing should slip between them. - expect(chatSpan.parent_span_id, `gen_ai span is not a child of ai-workflow:\n${tree}`).toBe(workflow.span_id); - - // A second manual span, sibling of the gen_ai span rather than its child. - expect(postProcess.parent_span_id, `post-process is not a child of ai-workflow:\n${tree}`).toBe(workflow.span_id); - - for (const span of [workflow, postProcess, chatSpan]) { - expect(span.trace_id).toBe(segment.trace_id); - } -}); - -test('nests the streaming gen_ai span under its manual parent', async ({ baseURL, request }) => { - const spansPromise = collectStreamedSpansUntilSegment(APP, 'GET /chat-stream'); - - await request.get(`${baseURL}/chat-stream`); - - const spans = await spansPromise; - const tree = describeTree(spans); - - const segment = spans.find(span => span.is_segment && span.name === 'GET /chat-stream')!; - const workflow = byName(spans, 'ai-stream-workflow'); - const streamSpan = spans.find(isChatSpan)!; - - expect(ancestorIds(spans, workflow), `ai-stream-workflow is not under the request span:\n${tree}`).toContain( - segment.span_id, - ); - - // The stream is drained inside the manual span, so the gen_ai span has to close under it rather - // than escaping to the request root. - expect(streamSpan.parent_span_id, `streamed gen_ai span is not a child of ai-stream-workflow:\n${tree}`).toBe( - workflow.span_id, - ); - expect(streamSpan.trace_id).toBe(segment.trace_id); -}); diff --git a/dev-packages/e2e-tests/test-applications/node-mistral/tests/utils.ts b/dev-packages/e2e-tests/test-applications/node-mistral/tests/utils.ts deleted file mode 100644 index 5172a74beb54..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-mistral/tests/utils.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { expect } from '@playwright/test'; -import type { SerializedStreamedSpan } from '@sentry-internal/test-utils'; -import { getSpanOp } from '@sentry-internal/test-utils'; - -export const APP = 'node-mistral'; - -export const attr = (span: SerializedStreamedSpan, key: string): unknown => span.attributes?.[key]?.value; - -export const isChatSpan = (span: SerializedStreamedSpan): boolean => getSpanOp(span) === 'gen_ai.chat'; - -/** A readable span tree, used as a failure message so a broken assertion is diagnosable. */ -export function describeTree(spans: SerializedStreamedSpan[]): string { - return spans - .map(span => `${span.name} [${getSpanOp(span) ?? '-'}] id=${span.span_id} parent=${span.parent_span_id ?? '-'}`) - .join('\n'); -} - -/** Walk to the trace root, so assertions can allow auto-instrumented spans in between. */ -export function ancestorIds(spans: SerializedStreamedSpan[], span: SerializedStreamedSpan): string[] { - const byId = new Map(spans.map(candidate => [candidate.span_id, candidate])); - const ids: string[] = []; - - let current: SerializedStreamedSpan | undefined = span; - while (current?.parent_span_id) { - ids.push(current.parent_span_id); - current = byId.get(current.parent_span_id); - } - - return ids; -} - -export function byName(spans: SerializedStreamedSpan[], name: string): SerializedStreamedSpan { - const span = spans.find(candidate => candidate.name === name); - expect(span, `expected a span named "${name}" in:\n${describeTree(spans)}`).toBeDefined(); - return span!; -} - -/** - * Attributes every successful gen_ai span carries, whatever the model happens to answer. Values that - * depend on the model (token counts, response text) are checked for shape and not for content. - */ -export function expectCommonChatAttributes(span: SerializedStreamedSpan): void { - expect(attr(span, 'sentry.origin')).toBe('auto.ai.mistralai'); - expect(attr(span, 'gen_ai.provider.name')).toBe('mistralai'); - expect(attr(span, 'gen_ai.operation.name')).toBe('chat'); - expect(span.name).toBe(`chat ${attr(span, 'gen_ai.request.model')}`); - expect(span.status).toBe('ok'); - - expect(typeof attr(span, 'gen_ai.response.model')).toBe('string'); - expect(attr(span, 'gen_ai.usage.input_tokens')).toBeGreaterThan(0); - expect(attr(span, 'gen_ai.usage.output_tokens')).toBeGreaterThan(0); - expect(attr(span, 'gen_ai.usage.total_tokens')).toBeGreaterThan(0); -}