diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 53c2cf9ab6b4..2229874d327f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,7 +28,8 @@ concurrency: cancel-in-progress: true env: - HEAD_COMMIT: ${{ github.event.inputs.commit || github.sha }} + # Keep the SDK source fixed while benchmark commits change test-file boundaries. + HEAD_COMMIT: ${{ github.event.inputs.commit || github.event.pull_request.head.sha || github.sha }} CACHED_DEPENDENCY_PATHS: | ${{ github.workspace }}/node_modules @@ -575,7 +576,7 @@ jobs: run: pnpm rebuild @sentry/node-cpu-profiler @sentry/node-native-stacktrace - name: Run affected tests - run: pnpm run test:pr:node --base=${{ github.event.pull_request.base.sha }} + run: pnpm run test:ci:node if: github.event_name == 'pull_request' env: NODE_VERSION: ${{ matrix.node }} @@ -856,6 +857,16 @@ jobs: working-directory: dev-packages/node-integration-tests # Tedious runs in the dedicated SQL Server job because its database container requires x86. run: pnpm run test --shard=${{ matrix.shard }}/${{ matrix.shards }} --exclude '**/tracing/tedious/**' + env: + INTEGRATION_TIMINGS: ${{ runner.temp }}/integration-timings.json + - name: Upload integration timings + if: always() + uses: actions/upload-artifact@v7 + with: + name: timings-node-${{ matrix.node }}-${{ matrix.shard }} + path: ${{ runner.temp }}/integration-timings.json + if-no-files-found: error + retention-days: 7 job_sql_server_integration_tests: name: SQL Server Integration Tests (Node 24) @@ -919,6 +930,16 @@ jobs: - name: Run integration tests working-directory: dev-packages/cloudflare-integration-tests run: pnpm run test --shard=${{ matrix.shard }}/${{ matrix.shards }} + env: + INTEGRATION_TIMINGS: ${{ runner.temp }}/integration-timings.json + - name: Upload integration timings + if: always() + uses: actions/upload-artifact@v7 + with: + name: timings-cloudflare-${{ matrix.shard }} + path: ${{ runner.temp }}/integration-timings.json + if-no-files-found: error + retention-days: 7 # Shows breakage from new wrangler and workerd releases early. It is not in `job_required_jobs_passed`, so it cannot # block a PR, and like the optional E2E tests it does not run on release branches. @@ -954,7 +975,7 @@ jobs: - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) uses: actions/checkout@v7 with: - ref: ${{ env.HEAD_COMMIT }} + ref: ${{ github.sha }} - name: Set up Node uses: actions/setup-node@v7 with: @@ -999,6 +1020,16 @@ jobs: - name: Run integration tests working-directory: dev-packages/bun-integration-tests run: pnpm run test --shard=${{ matrix.shard }}/4 + env: + INTEGRATION_TIMINGS: ${{ runner.temp }}/integration-timings.json + - name: Upload integration timings + if: always() + uses: actions/upload-artifact@v7 + with: + name: timings-bun-${{ matrix.bun }}-${{ matrix.shard }} + path: ${{ runner.temp }}/integration-timings.json + if-no-files-found: error + retention-days: 7 job_deno_integration_tests: name: Deno Integration Tests (${{ matrix.shard }}/4) @@ -1039,6 +1070,16 @@ jobs: working-directory: dev-packages/deno-integration-tests # Tedious runs in the dedicated SQL Server job because its database container requires x86. run: pnpm run test:node-suites --shard=${{ matrix.shard }}/4 --exclude '**/tracing/tedious/**' + env: + INTEGRATION_TIMINGS: ${{ runner.temp }}/integration-timings.json + - name: Upload integration timings + if: always() + uses: actions/upload-artifact@v7 + with: + name: timings-deno-${{ matrix.shard }} + path: ${{ runner.temp }}/integration-timings.json + if-no-files-found: error + retention-days: 7 # GitHub Actions allows at most 256 jobs in one matrix, so the E2E tests are split into two jobs that share `env` # and `steps`. diff --git a/dev-packages/bun-integration-tests/node-suites/excludes.ts b/dev-packages/bun-integration-tests/node-suites/excludes.ts index ff7e526ad3db..03fd699753d9 100644 --- a/dev-packages/bun-integration-tests/node-suites/excludes.ts +++ b/dev-packages/bun-integration-tests/node-suites/excludes.ts @@ -96,7 +96,7 @@ export const NO_AUTO_INSTRUMENTATION = [ 'suites/tracing/langchain/**', 'suites/tracing/langgraph/**', 'suites/tracing/lru-memoizer/test.ts', - 'suites/tracing/mastra/test.ts', + 'suites/tracing/mastra/**', 'suites/tracing/mcp-handler-exact-once/test.ts', 'suites/tracing/mcp-server/**', 'suites/tracing/mistral-v2/test.ts', @@ -116,7 +116,7 @@ export const NO_AUTO_INSTRUMENTATION = [ 'suites/tracing/mysql2-tracing-channel/test.ts', 'suites/tracing/mysql2/test.ts', 'suites/tracing/neon/test.ts', - 'suites/tracing/openai/test.ts', + 'suites/tracing/openai/v5/**', 'suites/tracing/openai/v6/test.ts', 'suites/tracing/openai/v7/test.ts', 'suites/tracing/orchestrion-lazy-registration/test.ts', @@ -235,15 +235,16 @@ const BUN_BUILD_UNBUNDLEABLE_DEPENDENCY = ['suites/tracing/mistral/test.ts']; // `@sentry/bun` the request body comes from `bunHttpServerIntegration`. const BUN_BUILD_NOT_TRIAGED = [ 'suites/express/sentry-trace/test.ts', - 'suites/express/tracing/test.ts', + 'suites/express/tracing/routing/test.ts', + 'suites/express/tracing/request-data/test.ts', 'suites/express/with-http/**', 'suites/hono-sdk/test.ts', 'suites/pino/test.ts', 'suites/tracing/google-genai-v2/test.ts', 'suites/tracing/google-genai/test.ts', - 'suites/tracing/langchain/v1/test.ts', + 'suites/tracing/langchain/v1/**', 'suites/tracing/langgraph/v1/test.ts', - 'suites/tracing/mastra/test.ts', + 'suites/tracing/mastra/**', 'suites/tracing/mcp-handler-exact-once/test.ts', 'suites/tracing/mcp-server-streamed/test.ts', 'suites/tracing/mongodb-v4/test.ts', @@ -260,7 +261,7 @@ const BUN_BUILD_NOT_TRIAGED = [ 'suites/tracing/mysql/test.ts', 'suites/tracing/prisma-orm-v8/test.ts', 'suites/tracing/together-ai/test.ts', - 'suites/tracing/vercelai/v6_v7/test.ts', + 'suites/tracing/vercelai/v6_v7/**', ]; export const BUN_BUILD_EXCLUDE = [ diff --git a/dev-packages/bun-integration-tests/vite.config.mts b/dev-packages/bun-integration-tests/vite.config.mts index f339cd917a02..c20276d5f8a8 100644 --- a/dev-packages/bun-integration-tests/vite.config.mts +++ b/dev-packages/bun-integration-tests/vite.config.mts @@ -41,7 +41,11 @@ export default defineConfig({ reporters: process.env.DEBUG ? ['default', { summary: false }] : process.env.GITHUB_ACTIONS - ? ['dot', 'github-actions'] + ? [ + 'dot', + 'github-actions', + ...(process.env.INTEGRATION_TIMINGS ? ['../node-integration-tests/benchmark-reporter.ts'] : []), + ] : ['verbose'], projects: [ { diff --git a/dev-packages/cloudflare-integration-tests/vite.config.mts b/dev-packages/cloudflare-integration-tests/vite.config.mts index 600cd6a118ee..98a7ffe75da2 100644 --- a/dev-packages/cloudflare-integration-tests/vite.config.mts +++ b/dev-packages/cloudflare-integration-tests/vite.config.mts @@ -33,7 +33,11 @@ export default defineConfig({ reporters: process.env.DEBUG ? ['default', { summary: false }] : process.env.GITHUB_ACTIONS - ? ['dot', 'github-actions'] + ? [ + 'dot', + 'github-actions', + ...(process.env.INTEGRATION_TIMINGS ? ['../node-integration-tests/benchmark-reporter.ts'] : []), + ] : ['verbose'], }, }); diff --git a/dev-packages/deno-integration-tests/node-suites/excludes.ts b/dev-packages/deno-integration-tests/node-suites/excludes.ts index e6d5906bc2c8..c7e7d700f3a9 100644 --- a/dev-packages/deno-integration-tests/node-suites/excludes.ts +++ b/dev-packages/deno-integration-tests/node-suites/excludes.ts @@ -47,7 +47,7 @@ const PRISMA_ESM_INTEROP = ['suites/tracing/prisma-orm-v5/test.ts', 'suites/trac // which any module load hook breaks on Deno 2.8.3 (the CI version); it passes on Deno 2.9.0. const REQUIRE_OF_ESM_ONLY_DEPENDENCY = [ 'suites/tracing/graphql-tracing-channel/**', - 'suites/tracing/mastra/test.ts', + 'suites/tracing/mastra/**', 'suites/tracing/prisma-orm-v8/test.ts', ]; @@ -72,12 +72,12 @@ const NOT_TRIAGED = [ 'suites/tracing/mongodb/test.ts', 'suites/tracing/mongoose-v5/test.ts', 'suites/tracing/mysql/test.ts', - 'suites/tracing/openai/test.ts', + 'suites/tracing/openai/v5/**', 'suites/tracing/openai/v7/test.ts', 'suites/tracing/orchestrion-lazy-registration/test.ts', 'suites/tracing/prisma-orm-v7/test.ts', 'suites/tracing/together-ai/test.ts', - 'suites/tracing/vercelai/v6_v7/test.ts', + 'suites/tracing/vercelai/v6_v7/**', ]; // Passes on Deno when run alone, but failed in about 1 of 3 full runs of this package. diff --git a/dev-packages/deno-integration-tests/vite.config.mts b/dev-packages/deno-integration-tests/vite.config.mts index 6dccfdc74a5e..c8a0115bed62 100644 --- a/dev-packages/deno-integration-tests/vite.config.mts +++ b/dev-packages/deno-integration-tests/vite.config.mts @@ -32,7 +32,11 @@ export default defineConfig({ reporters: process.env.DEBUG ? ['default', { summary: false }] : process.env.GITHUB_ACTIONS - ? ['dot', 'github-actions'] + ? [ + 'dot', + 'github-actions', + ...(process.env.INTEGRATION_TIMINGS ? ['../node-integration-tests/benchmark-reporter.ts'] : []), + ] : ['verbose'], }, }); diff --git a/dev-packages/node-integration-tests/benchmark-reporter.ts b/dev-packages/node-integration-tests/benchmark-reporter.ts new file mode 100644 index 000000000000..c64c2e28a9a7 --- /dev/null +++ b/dev-packages/node-integration-tests/benchmark-reporter.ts @@ -0,0 +1,32 @@ +import { mkdirSync, writeFileSync } from 'node:fs'; +import { dirname } from 'node:path'; +import type { Reporter, TestModule } from 'vitest/node'; + +export default class BenchmarkReporter implements Reporter { + onTestRunEnd(modules: ReadonlyArray, errors: ReadonlyArray, reason: string): void { + const path = process.env.INTEGRATION_TIMINGS; + if (!path) return; + mkdirSync(dirname(path), { recursive: true }); + writeFileSync( + path, + JSON.stringify( + { + source: process.env.HEAD_COMMIT, + node: process.version, + reason, + errors: errors.length, + files: modules.map(module => ({ + project: module.project.name, + file: module.relativeModuleId, + durationMs: module.diagnostic().duration, + collectMs: module.diagnostic().collectDuration, + state: module.state(), + tests: [...module.children.allTests()].map(test => ({ name: test.fullName, state: test.result().state })), + })), + }, + null, + 2, + ), + ); + } +} diff --git a/dev-packages/node-integration-tests/suites/express/tracing/request-data/test.ts b/dev-packages/node-integration-tests/suites/express/tracing/request-data/test.ts new file mode 100644 index 000000000000..192796f1bae7 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/express/tracing/request-data/test.ts @@ -0,0 +1,4 @@ +import { describe } from 'vitest'; +import { registerExpressRequestDataTests } from '../suite'; + +describe('express tracing', registerExpressRequestDataTests); diff --git a/dev-packages/node-integration-tests/suites/express/tracing/routing/test.ts b/dev-packages/node-integration-tests/suites/express/tracing/routing/test.ts new file mode 100644 index 000000000000..56b1d9bc82ee --- /dev/null +++ b/dev-packages/node-integration-tests/suites/express/tracing/routing/test.ts @@ -0,0 +1,4 @@ +import { describe } from 'vitest'; +import { registerExpressRoutingTests } from '../suite'; + +describe('express tracing', registerExpressRoutingTests); diff --git a/dev-packages/node-integration-tests/suites/express/tracing/test.ts b/dev-packages/node-integration-tests/suites/express/tracing/suite.ts similarity index 98% rename from dev-packages/node-integration-tests/suites/express/tracing/test.ts rename to dev-packages/node-integration-tests/suites/express/tracing/suite.ts index b5637d14ab1c..eaa355456942 100644 --- a/dev-packages/node-integration-tests/suites/express/tracing/test.ts +++ b/dev-packages/node-integration-tests/suites/express/tracing/suite.ts @@ -10,11 +10,10 @@ function findExpressSpan(container: SerializedStreamedSpanContainer, type: strin return container.items.find(item => item.attributes['express.type']?.value === type); } -describe('express tracing', () => { +export function registerExpressRoutingTests(): void { afterAll(() => { cleanupChildProcesses(); }); - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { test('should create and send segment spans for Express routes and spans for middlewares.', async () => { const runner = createRunner() @@ -63,7 +62,6 @@ describe('express tracing', () => { runner.makeRequest('get', '/test/express'); await runner.completed(); }); - test('names router and request handler spans after their route', async () => { const runner = createRunner() .expect({ @@ -90,7 +88,6 @@ describe('express tracing', () => { await runner.completed(); }); - test('should set a correct segment name for routes specified in RegEx', async () => { const runner = createRunner() .expect({ @@ -113,7 +110,6 @@ describe('express tracing', () => { runner.makeRequest('get', '/test/regex'); await runner.completed(); }); - test('nests a sub-router route handler span under the router span', async () => { const runner = createRunner() .expect({ @@ -141,7 +137,6 @@ describe('express tracing', () => { runner.makeRequest('get', '/test/router/user/42'); await runner.completed(); }); - test('keeps the parameter in a route mounted under a parameterized sub-router path', async () => { const runner = createRunner() .expect({ @@ -157,7 +152,6 @@ describe('express tracing', () => { runner.makeRequest('get', '/test/version/v1/user'); await runner.completed(); }); - test('handles root page correctly', async () => { const runner = createRunner() .expect({ @@ -182,9 +176,6 @@ describe('express tracing', () => { runner.makeRequest('get', '/'); await runner.completed(); }); - - // With span streaming, child spans are sent as they end, before the response status code is - // known, so `ignoreStatusCodes` has no effect and these routes are captured like any other. test.each([ { status_code: 401, url: '/401', status_message: 'unauthenticated', name: 'GET /401', source: 'route' }, { status_code: 402, url: '/402', status_message: 'invalid_argument', name: 'GET /402', source: 'route' }, @@ -231,7 +222,6 @@ describe('express tracing', () => { await runner.completed(); }, ); - test.each([['array1'], ['array5']])( 'should set a correct segment name for routes consisting of arrays of routes for %p', async (segment: string) => { @@ -257,7 +247,6 @@ describe('express tracing', () => { await runner.completed(); }, ); - test.each([ ['arr/545'], ['arr/required'], @@ -289,7 +278,14 @@ describe('express tracing', () => { await runner.makeRequest('get', `/test/${segment}`); await runner.completed(); }); + }); +} +export function registerExpressRequestDataTests(): void { + afterAll(() => { + cleanupChildProcesses(); + }); + createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { describe('request data', () => { test('correctly captures JSON request data', async () => { const runner = createRunner() @@ -427,4 +423,4 @@ describe('express tracing', () => { }); }); }); -}); +} diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/decisions/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/decisions/test.ts new file mode 100644 index 000000000000..6ee25e97c042 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/decisions/test.ts @@ -0,0 +1,4 @@ +import { describe } from 'vitest'; +import { registerLangChainDecisionTests } from '../suite'; + +describe('LangChain integration (v1)', registerLangChainDecisionTests); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/suite.ts similarity index 99% rename from dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts rename to dev-packages/node-integration-tests/suites/tracing/langchain/v1/suite.ts index 6abf7a4dcc84..ecd00667d384 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/suite.ts @@ -1,4 +1,4 @@ -import { afterAll, describe, expect } from 'vitest'; +import { afterAll, expect } from 'vitest'; import { GEN_AI_AGENT_NAME, GEN_AI_CONVERSATION_ID, @@ -25,7 +25,7 @@ import { GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE } from '../../../../../../package import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; import { createEsmTests } from '../../../../utils/runner/createEsmAndCjsTests'; -describe('LangChain integration (v1)', () => { +export function registerLangChainTracingTests(): void { afterAll(() => { cleanupChildProcesses(); }); @@ -299,6 +299,108 @@ describe('LangChain integration (v1)', () => { }, ); + createEsmTests( + __dirname, + 'scenario-openai-before-langchain.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('demonstrates timing issue with duplicate spans', async () => { + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = (await spansPromise).filter( + span => + span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain' || + span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic', + ); + expect(spans).toHaveLength(2); + const anthropicSpan = spans.find(span => span.attributes[SENTRY_ORIGIN].value === 'auto.ai.anthropic'); + expect(anthropicSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + + const langchainSpan = spans.find(span => span.attributes[SENTRY_ORIGIN].value === 'auto.ai.langchain'); + expect(langchainSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + }); + }, + { + additionalDependencies: { + langchain: '^1.0.0', + '@langchain/core': '^1.0.0', + '@langchain/anthropic': '^1.0.0', + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-init-chat-model.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('creates langchain spans using initChatModel with OpenAI', async () => { + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = (await spansPromise).filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain', + ); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual(['chat error-model', 'chat gpt-3.5-turbo', 'chat gpt-4o']); + + const gpt4oSpan = spans.find(span => span.name === 'chat gpt-4o'); + expect(gpt4oSpan!.status).toBe('ok'); + expect(gpt4oSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(gpt4oSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(gpt4oSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(gpt4oSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); + expect(gpt4oSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gpt-4o'); + expect(gpt4oSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); + expect(gpt4oSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); + expect(gpt4oSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(8); + expect(gpt4oSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); + expect(gpt4oSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); + expect(gpt4oSpan!.attributes[GEN_AI_RESPONSE_ID]).toBeDefined(); + expect(gpt4oSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('gpt-4o'); + expect(gpt4oSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE].value).toBe('stop'); + + const gpt35Span = spans.find(span => span.name === 'chat gpt-3.5-turbo'); + expect(gpt35Span!.status).toBe('ok'); + expect(gpt35Span!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(gpt35Span!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(gpt35Span!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); + expect(gpt35Span!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gpt-3.5-turbo'); + expect(gpt35Span!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.5); + expect(gpt35Span!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(8); + expect(gpt35Span!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); + expect(gpt35Span!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); + expect(gpt35Span!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('gpt-3.5-turbo'); + expect(gpt35Span!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE].value).toBe('stop'); + + const errorSpan = spans.find(span => span.name === 'chat error-model'); + expect(errorSpan!.status).toBe('error'); + expect(errorSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(errorSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(errorSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); + expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); + }); + }, + { + additionalDependencies: { + langchain: '^1.0.0', + '@langchain/core': '^1.0.0', + '@langchain/openai': '^1.0.0', + }, + }, + ); +} + +export function registerLangChainDecisionTests(): void { + afterAll(() => { + cleanupChildProcesses(); + }); + createEsmAndCjsTests( __dirname, 'scenario-typesafe-classifier.mjs', @@ -439,100 +541,4 @@ describe('LangChain integration (v1)', () => { }, }, ); - - createEsmTests( - __dirname, - 'scenario-openai-before-langchain.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('demonstrates timing issue with duplicate spans', async () => { - const runner = createRunner().ignore('event'); - const spansPromise = runner.collectStreamedSpansUntilSegment('main'); - - await runner.start().completed(); - - const spans = (await spansPromise).filter( - span => - span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain' || - span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic', - ); - expect(spans).toHaveLength(2); - const anthropicSpan = spans.find(span => span.attributes[SENTRY_ORIGIN].value === 'auto.ai.anthropic'); - expect(anthropicSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); - - const langchainSpan = spans.find(span => span.attributes[SENTRY_ORIGIN].value === 'auto.ai.langchain'); - expect(langchainSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); - }); - }, - { - additionalDependencies: { - langchain: '^1.0.0', - '@langchain/core': '^1.0.0', - '@langchain/anthropic': '^1.0.0', - }, - }, - ); - - createEsmAndCjsTests( - __dirname, - 'scenario-init-chat-model.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('creates langchain spans using initChatModel with OpenAI', async () => { - const runner = createRunner().ignore('event'); - const spansPromise = runner.collectStreamedSpansUntilSegment('main'); - - await runner.start().completed(); - - const spans = (await spansPromise).filter( - span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain', - ); - expect(spans).toHaveLength(3); - expect(spans.map(span => span.name).sort()).toEqual(['chat error-model', 'chat gpt-3.5-turbo', 'chat gpt-4o']); - - const gpt4oSpan = spans.find(span => span.name === 'chat gpt-4o'); - expect(gpt4oSpan!.status).toBe('ok'); - expect(gpt4oSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); - expect(gpt4oSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); - expect(gpt4oSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(gpt4oSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); - expect(gpt4oSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gpt-4o'); - expect(gpt4oSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); - expect(gpt4oSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); - expect(gpt4oSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(8); - expect(gpt4oSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); - expect(gpt4oSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); - expect(gpt4oSpan!.attributes[GEN_AI_RESPONSE_ID]).toBeDefined(); - expect(gpt4oSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('gpt-4o'); - expect(gpt4oSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE].value).toBe('stop'); - - const gpt35Span = spans.find(span => span.name === 'chat gpt-3.5-turbo'); - expect(gpt35Span!.status).toBe('ok'); - expect(gpt35Span!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); - expect(gpt35Span!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); - expect(gpt35Span!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); - expect(gpt35Span!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gpt-3.5-turbo'); - expect(gpt35Span!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.5); - expect(gpt35Span!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(8); - expect(gpt35Span!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); - expect(gpt35Span!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); - expect(gpt35Span!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('gpt-3.5-turbo'); - expect(gpt35Span!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE].value).toBe('stop'); - - const errorSpan = spans.find(span => span.name === 'chat error-model'); - expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); - expect(errorSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); - expect(errorSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); - expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); - }); - }, - { - additionalDependencies: { - langchain: '^1.0.0', - '@langchain/core': '^1.0.0', - '@langchain/openai': '^1.0.0', - }, - }, - ); -}); +} diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/tracing/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/tracing/test.ts new file mode 100644 index 000000000000..2af80b510634 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/tracing/test.ts @@ -0,0 +1,4 @@ +import { describe } from 'vitest'; +import { registerLangChainTracingTests } from '../suite'; + +describe('LangChain integration (v1)', registerLangChainTracingTests); diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/classifier/test.ts b/dev-packages/node-integration-tests/suites/tracing/mastra/classifier/test.ts new file mode 100644 index 000000000000..8a6006d19a1d --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/classifier/test.ts @@ -0,0 +1,5 @@ +import { describe } from 'vitest'; +import { supports } from '../../../../utils'; +import { registerMastraClassifierTests } from '../suite'; + +describe.runIf(supports({ min: 22 }))('Mastra integration', registerMastraClassifierTests); diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/core/agents/test.ts b/dev-packages/node-integration-tests/suites/tracing/mastra/core/agents/test.ts new file mode 100644 index 000000000000..a070e6786a20 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/core/agents/test.ts @@ -0,0 +1,5 @@ +import { describe } from 'vitest'; +import { supports } from '../../../../../utils'; +import { registerMastraAgentTests } from '../../suite'; + +describe.runIf(supports({ min: 22 }))('Mastra integration', registerMastraAgentTests); diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/core/observability/test.ts b/dev-packages/node-integration-tests/suites/tracing/mastra/core/observability/test.ts new file mode 100644 index 000000000000..3a2ec7a21f2e --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/core/observability/test.ts @@ -0,0 +1,5 @@ +import { describe } from 'vitest'; +import { supports } from '../../../../../utils'; +import { registerMastraObservabilityTests } from '../../suite'; + +describe.runIf(supports({ min: 22 }))('Mastra integration', registerMastraObservabilityTests); diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/nesting/test.ts b/dev-packages/node-integration-tests/suites/tracing/mastra/nesting/test.ts new file mode 100644 index 000000000000..7707be7525d6 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/nesting/test.ts @@ -0,0 +1,5 @@ +import { describe } from 'vitest'; +import { supports } from '../../../../utils'; +import { registerMastraNestingTests } from '../suite'; + +describe.runIf(supports({ min: 22 }))('Mastra integration', registerMastraNestingTests); diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts b/dev-packages/node-integration-tests/suites/tracing/mastra/suite.ts similarity index 98% rename from dev-packages/node-integration-tests/suites/tracing/mastra/test.ts rename to dev-packages/node-integration-tests/suites/tracing/mastra/suite.ts index 0c3778daa368..4d2951fdc46a 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/suite.ts @@ -21,8 +21,7 @@ import { URL_FULL, } from '@sentry/conventions/attributes'; import { GEN_AI_CHAT, GEN_AI_DECIDE, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op'; -import { afterAll, describe, expect } from 'vitest'; -import { supports } from '../../../utils'; +import { afterAll, expect } from 'vitest'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; // `@mastra/core` declares `engines.node >= 22.13`, so it can't live in the package's root @@ -52,11 +51,10 @@ const MASTRA_CLASSIFIER_DEPENDENCIES = { }, }; -describe.runIf(supports({ min: 22 }))('Mastra integration', () => { +export function registerMastraAgentTests(): void { afterAll(() => { cleanupChildProcesses(); }); - createEsmAndCjsTests( __dirname, 'scenario.mjs', @@ -118,7 +116,6 @@ describe.runIf(supports({ min: 22 }))('Mastra integration', () => { }, MASTRA_DEPENDENCIES, ); - createEsmAndCjsTests( __dirname, 'scenario.mjs', @@ -146,7 +143,6 @@ describe.runIf(supports({ min: 22 }))('Mastra integration', () => { }, MASTRA_DEPENDENCIES, ); - createEsmAndCjsTests( __dirname, 'scenario-tools.mjs', @@ -189,7 +185,6 @@ describe.runIf(supports({ min: 22 }))('Mastra integration', () => { }, MASTRA_DEPENDENCIES, ); - createEsmAndCjsTests( __dirname, 'scenario-workflow.mjs', @@ -215,6 +210,83 @@ describe.runIf(supports({ min: 22 }))('Mastra integration', () => { }, MASTRA_DEPENDENCIES, ); +} + +export function registerMastraObservabilityTests(): void { + afterAll(() => { + cleanupChildProcesses(); + }); + createEsmAndCjsTests( + __dirname, + 'scenario-auto.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('auto-instruments a Mastra app that configured no observability at all', async () => { + await createRunner() + .expect({ + span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra'); + expect(spans.map(span => span.name).sort()).toEqual(['chat gpt-4o-mini', 'invoke_agent weather_agent']); + for (const span of spans) { + expect(span.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra'); + } + }, + }) + .start() + .completed(); + }); + }, + MASTRA_DEPENDENCIES, + ); + createEsmAndCjsTests( + __dirname, + 'scenario-community-exporter.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('still exports its own spans when the community @mastra/sentry exporter is present', async () => { + await createRunner() + .expect({ + span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra'); + expect(spans.map(span => span.name).sort()).toEqual(['chat gpt-4o-mini', 'invoke_agent weather_agent']); + }, + }) + .start() + .completed(); + }); + }, + MASTRA_DEPENDENCIES, + ); + createEsmAndCjsTests( + __dirname, + 'scenario-auto.mjs', + 'instrument-no-bootstrap.mjs', + (createRunner, test) => { + test('leaves an unconfigured app alone when bootstrapObservability is false', async () => { + await createRunner() + .expect({ + span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); + const genAiSpans = container.items.filter(span => + String(span.attributes[SENTRY_OP]?.value ?? '').startsWith('gen_ai.'), + ); + expect(genAiSpans).toEqual([]); + }, + }) + .start() + .completed(); + }); + }, + MASTRA_DEPENDENCIES, + ); +} + +export function registerMastraClassifierTests(): void { + afterAll(() => { + cleanupChildProcesses(); + }); createEsmAndCjsTests( __dirname, @@ -307,73 +379,12 @@ describe.runIf(supports({ min: 22 }))('Mastra integration', () => { }, MASTRA_CLASSIFIER_DEPENDENCIES, ); +} - createEsmAndCjsTests( - __dirname, - 'scenario-auto.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('auto-instruments a Mastra app that configured no observability at all', async () => { - await createRunner() - .expect({ - span: container => { - expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra'); - expect(spans.map(span => span.name).sort()).toEqual(['chat gpt-4o-mini', 'invoke_agent weather_agent']); - for (const span of spans) { - expect(span.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra'); - } - }, - }) - .start() - .completed(); - }); - }, - MASTRA_DEPENDENCIES, - ); - - createEsmAndCjsTests( - __dirname, - 'scenario-community-exporter.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('still exports its own spans when the community @mastra/sentry exporter is present', async () => { - await createRunner() - .expect({ - span: container => { - expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra'); - expect(spans.map(span => span.name).sort()).toEqual(['chat gpt-4o-mini', 'invoke_agent weather_agent']); - }, - }) - .start() - .completed(); - }); - }, - MASTRA_DEPENDENCIES, - ); - createEsmAndCjsTests( - __dirname, - 'scenario-auto.mjs', - 'instrument-no-bootstrap.mjs', - (createRunner, test) => { - test('leaves an unconfigured app alone when bootstrapObservability is false', async () => { - await createRunner() - .expect({ - span: container => { - expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); - const genAiSpans = container.items.filter(span => - String(span.attributes[SENTRY_OP]?.value ?? '').startsWith('gen_ai.'), - ); - expect(genAiSpans).toEqual([]); - }, - }) - .start() - .completed(); - }); - }, - MASTRA_DEPENDENCIES, - ); +export function registerMastraNestingTests(): void { + afterAll(() => { + cleanupChildProcesses(); + }); createEsmAndCjsTests( __dirname, @@ -550,4 +561,4 @@ describe.runIf(supports({ min: 22 }))('Mastra integration', () => { }, MASTRA_NESTING_DEPENDENCIES, ); -}); +} diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/suite.ts similarity index 99% rename from dev-packages/node-integration-tests/suites/tracing/openai/test.ts rename to dev-packages/node-integration-tests/suites/tracing/openai/suite.ts index 98b9f8755573..246c4f2a005e 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/suite.ts @@ -1,5 +1,5 @@ import type { SerializedStreamedSpanContainer } from '@sentry/core'; -import { afterAll, describe, expect } from 'vitest'; +import { afterAll, expect } from 'vitest'; import { GEN_AI_CONVERSATION_ID, GEN_AI_EMBEDDINGS_INPUT, @@ -28,11 +28,10 @@ import { } from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; -describe('OpenAI integration', () => { +export function registerOpenAiChatTests(): void { afterAll(() => { cleanupChildProcesses(); }); - createEsmAndCjsTests(__dirname, 'scenario-chat.mjs', 'instrument.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording disabled', async () => { const runner = createRunner(); @@ -441,7 +440,6 @@ describe('OpenAI integration', () => { } }); }); - createEsmAndCjsTests(__dirname, 'scenario-chat.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording enabled', async () => { const runner = createRunner(); @@ -911,7 +909,6 @@ describe('OpenAI integration', () => { }); }); }); - createEsmAndCjsTests(__dirname, 'scenario-chat.mjs', 'instrument-with-options.mjs', (createRunner, test) => { test('creates openai related spans with custom options', async () => { const runner = createRunner(); @@ -996,7 +993,136 @@ describe('OpenAI integration', () => { }); }); }); + createEsmAndCjsTests(__dirname, 'scenario-azure-openai.mjs', 'instrument.mjs', (createRunner, test) => { + test('it works with Azure OpenAI', async () => { + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; + + await createRunner() + .unordered() + .expect({ + span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo')).toBeDefined(); + receivedSpans = container.items; + }, + }) + .start() + .completed(); + + const segment = receivedSpans.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); + expect(segment?.span_id).toEqual(expect.any(String)); + expect(segment?.trace_id).toEqual(expect.any(String)); + expect(segment?.attributes[GEN_AI_OPERATION_NAME]).toEqual({ value: 'chat', type: 'string' }); + expect(segment?.attributes[SENTRY_OP]).toEqual({ value: 'gen_ai.chat', type: 'string' }); + expect(segment?.attributes[SENTRY_ORIGIN]).toEqual({ value: 'auto.ai.openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ value: 'openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ value: 0.7, type: 'double' }); + expect(segment?.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_ID]).toEqual({ value: 'chatcmpl-mock123', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ value: '["stop"]', type: 'string' }); + expect(segment?.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ value: 10, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ value: 15, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ value: 25, type: 'integer' }); + expect(segment?.status).toEqual('ok'); + }); + }); + createEsmAndCjsTests(__dirname, 'scenario-with-response.mjs', 'instrument.mjs', (createRunner, test) => { + test('preserves .withResponse() method and works correctly', async () => { + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; + + await createRunner() + .unordered() + .expect({ + span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; + }, + }) + .start() + .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(2); + + for (const span of spans) { + expect(span!.name).toBe('chat gpt-4'); + expect(span!.status).toBe('ok'); + expect(span!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'chat' }); + expect(span!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ type: 'string', value: 'gpt-4' }); + } + + const parsedSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID] !== undefined); + expect(parsedSpans).toHaveLength(1); + expect(parsedSpans[0]!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-withresponse', + }); + const rawSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID] === undefined); + expect(rawSpans).toHaveLength(1); + expect(rawSpans[0]!.attributes[GEN_AI_RESPONSE_TEXT]).toBeUndefined(); + expect(rawSpans[0]!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toBeUndefined(); + }); + }); + createEsmAndCjsTests( + __dirname, + 'scenario-stream-midway-error.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('marks the span as errored when the stream fails midway, but not when the caller breaks early', async () => { + await createRunner() + .unordered() + .expect({ + span: container => { + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(2); + + const midwaySpan = spans.find(span => span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'midway-error'); + expect(midwaySpan!.status).toBe('error'); + expect(midwaySpan!.attributes[GEN_AI_RESPONSE_STREAMING]?.value).toBe(true); + expect(midwaySpan!.attributes[GEN_AI_RESPONSE_ID]?.value).toBe('chatcmpl-midway'); + expect(midwaySpan!.attributes[GEN_AI_RESPONSE_TEXT]?.value).toBe('This stream will be cut off'); + + const earlyBreakSpan = spans.find(span => span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'early-break'); + expect(earlyBreakSpan!.status).toBe('ok'); + expect(earlyBreakSpan!.attributes[GEN_AI_RESPONSE_TEXT]?.value).toBe('This stream will '); + }, + }) + .start() + .completed(); + }); + }, + ); + createEsmAndCjsTests(__dirname, 'scenario-abort.mjs', 'instrument.mjs', (createRunner, test) => { + test('ends a call the caller aborts as cancelled, but an SDK timeout as an error', async () => { + await createRunner() + .unordered() + .expect({ + span: container => { + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(4); + const statusByModel = Object.fromEntries( + spans.map(span => [span.attributes[GEN_AI_REQUEST_MODEL]?.value, span.status]), + ); + // Span streaming sends `cancelled` as `ok`. + expect(statusByModel).toEqual({ + 'abort-before-stream': 'ok', + 'abort-non-streaming': 'ok', + 'timeout-before-stream': 'error', + 'timeout-non-streaming': 'error', + }); + }, + }) + .start() + .completed(); + }); + }); +} + +export function registerOpenAiOtherApiTests(): void { + afterAll(() => { + cleanupChildProcesses(); + }); createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording disabled', async () => { let receivedSpans: SerializedStreamedSpanContainer['items'] = []; @@ -1160,7 +1286,6 @@ describe('OpenAI integration', () => { expect(rawEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toBeUndefined(); }); }); - createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording enabled', async () => { let receivedSpans: SerializedStreamedSpanContainer['items'] = []; @@ -1333,7 +1458,6 @@ describe('OpenAI integration', () => { }); }); }); - createEsmAndCjsTests(__dirname, 'scenario-root-span.mjs', 'instrument.mjs', (createRunner, test) => { test('it works without a wrapping span', async () => { let receivedSpans: SerializedStreamedSpanContainer['items'] = []; @@ -1367,42 +1491,6 @@ describe('OpenAI integration', () => { expect(segment?.status).toEqual('ok'); }); }); - - createEsmAndCjsTests(__dirname, 'scenario-azure-openai.mjs', 'instrument.mjs', (createRunner, test) => { - test('it works with Azure OpenAI', async () => { - let receivedSpans: SerializedStreamedSpanContainer['items'] = []; - - await createRunner() - .unordered() - .expect({ - span: container => { - expect(container.items.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo')).toBeDefined(); - receivedSpans = container.items; - }, - }) - .start() - .completed(); - - const segment = receivedSpans.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); - expect(segment?.span_id).toEqual(expect.any(String)); - expect(segment?.trace_id).toEqual(expect.any(String)); - expect(segment?.attributes[GEN_AI_OPERATION_NAME]).toEqual({ value: 'chat', type: 'string' }); - expect(segment?.attributes[SENTRY_OP]).toEqual({ value: 'gen_ai.chat', type: 'string' }); - expect(segment?.attributes[SENTRY_ORIGIN]).toEqual({ value: 'auto.ai.openai', type: 'string' }); - expect(segment?.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ value: 'openai', type: 'string' }); - expect(segment?.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); - expect(segment?.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ value: 0.7, type: 'double' }); - expect(segment?.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); - expect(segment?.attributes[GEN_AI_RESPONSE_ID]).toEqual({ value: 'chatcmpl-mock123', type: 'string' }); - expect(segment?.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ value: '["stop"]', type: 'string' }); - expect(segment?.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ value: 10, type: 'integer' }); - expect(segment?.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ value: 15, type: 'integer' }); - expect(segment?.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ value: 25, type: 'integer' }); - expect(segment?.status).toEqual('ok'); - }); - }); - - // Conversation id: read from the Conversations API, not from `previous_response_id` chaining createEsmAndCjsTests(__dirname, 'scenario-conversation.mjs', 'instrument.mjs', (createRunner, test) => { test('captures the conversation ID from the Conversations API and ignores previous_response_id', async () => { let receivedSpans: SerializedStreamedSpanContainer['items'] = []; @@ -1514,8 +1602,6 @@ describe('OpenAI integration', () => { expect(rawConversationSpan!.attributes[GEN_AI_CONVERSATION_ID]).toBeUndefined(); }); }); - - // Test for manual conversation ID setting using setConversationId() createEsmAndCjsTests(__dirname, 'scenario-manual-conversation-id.mjs', 'instrument.mjs', (createRunner, test) => { test('attaches manual conversation ID set via setConversationId() to all chat spans', async () => { let receivedSpans: SerializedStreamedSpanContainer['items'] = []; @@ -1551,7 +1637,6 @@ describe('OpenAI integration', () => { } }); }); - createEsmAndCjsTests(__dirname, 'scenario-separate-scope-1.mjs', 'instrument.mjs', (createRunner, test) => { test('isolates conversation IDs across separate scopes - conversation 1', async () => { let receivedSpans: SerializedStreamedSpanContainer['items'] = []; @@ -1586,7 +1671,6 @@ describe('OpenAI integration', () => { } }); }); - createEsmAndCjsTests(__dirname, 'scenario-separate-scope-2.mjs', 'instrument.mjs', (createRunner, test) => { test('isolates conversation IDs across separate scopes - conversation 2', async () => { let receivedSpans: SerializedStreamedSpanContainer['items'] = []; @@ -1621,7 +1705,6 @@ describe('OpenAI integration', () => { } }); }); - createEsmAndCjsTests( __dirname, 'scenario-system-instructions.mjs', @@ -1653,98 +1736,4 @@ describe('OpenAI integration', () => { }); }, ); - - createEsmAndCjsTests(__dirname, 'scenario-with-response.mjs', 'instrument.mjs', (createRunner, test) => { - test('preserves .withResponse() method and works correctly', async () => { - let receivedSpans: SerializedStreamedSpanContainer['items'] = []; - - await createRunner() - .unordered() - .expect({ - span: container => { - expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); - receivedSpans = container.items; - }, - }) - .start() - .completed(); - - const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); - expect(spans).toHaveLength(2); - - for (const span of spans) { - expect(span!.name).toBe('chat gpt-4'); - expect(span!.status).toBe('ok'); - expect(span!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'chat' }); - expect(span!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ type: 'string', value: 'gpt-4' }); - } - - const parsedSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID] !== undefined); - expect(parsedSpans).toHaveLength(1); - expect(parsedSpans[0]!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-withresponse', - }); - - const rawSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID] === undefined); - expect(rawSpans).toHaveLength(1); - expect(rawSpans[0]!.attributes[GEN_AI_RESPONSE_TEXT]).toBeUndefined(); - expect(rawSpans[0]!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toBeUndefined(); - }); - }); - - createEsmAndCjsTests( - __dirname, - 'scenario-stream-midway-error.mjs', - 'instrument-with-pii.mjs', - (createRunner, test) => { - test('marks the span as errored when the stream fails midway, but not when the caller breaks early', async () => { - await createRunner() - .unordered() - .expect({ - span: container => { - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); - expect(spans).toHaveLength(2); - - const midwaySpan = spans.find(span => span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'midway-error'); - expect(midwaySpan!.status).toBe('error'); - expect(midwaySpan!.attributes[GEN_AI_RESPONSE_STREAMING]?.value).toBe(true); - expect(midwaySpan!.attributes[GEN_AI_RESPONSE_ID]?.value).toBe('chatcmpl-midway'); - expect(midwaySpan!.attributes[GEN_AI_RESPONSE_TEXT]?.value).toBe('This stream will be cut off'); - - const earlyBreakSpan = spans.find(span => span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'early-break'); - expect(earlyBreakSpan!.status).toBe('ok'); - expect(earlyBreakSpan!.attributes[GEN_AI_RESPONSE_TEXT]?.value).toBe('This stream will '); - }, - }) - .start() - .completed(); - }); - }, - ); - - createEsmAndCjsTests(__dirname, 'scenario-abort.mjs', 'instrument.mjs', (createRunner, test) => { - test('ends a call the caller aborts as cancelled, but an SDK timeout as an error', async () => { - await createRunner() - .unordered() - .expect({ - span: container => { - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); - expect(spans).toHaveLength(4); - const statusByModel = Object.fromEntries( - spans.map(span => [span.attributes[GEN_AI_REQUEST_MODEL]?.value, span.status]), - ); - // Span streaming sends `cancelled` as `ok`. - expect(statusByModel).toEqual({ - 'abort-before-stream': 'ok', - 'abort-non-streaming': 'ok', - 'timeout-before-stream': 'error', - 'timeout-non-streaming': 'error', - }); - }, - }) - .start() - .completed(); - }); - }); -}); +} diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v5/chat/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v5/chat/test.ts new file mode 100644 index 000000000000..bbf127f875ef --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v5/chat/test.ts @@ -0,0 +1,4 @@ +import { describe } from 'vitest'; +import { registerOpenAiChatTests } from '../../suite'; + +describe('OpenAI integration', registerOpenAiChatTests); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v5/other-apis/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v5/other-apis/test.ts new file mode 100644 index 000000000000..2b8c2a143c76 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v5/other-apis/test.ts @@ -0,0 +1,4 @@ +import { describe } from 'vitest'; +import { registerOpenAiOtherApiTests } from '../../suite'; + +describe('OpenAI integration', registerOpenAiOtherApiTests); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/suite.ts similarity index 99% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/suite.ts index d4774f9d34d4..8731eb25e9b5 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/suite.ts @@ -1,6 +1,6 @@ import type { SerializedStreamedSpanContainer } from '@sentry/core'; import type { Event } from '@sentry/node'; -import { afterAll, describe, expect } from 'vitest'; +import { afterAll, expect } from 'vitest'; import { GEN_AI_CONVERSATION_ID, GEN_AI_EMBEDDINGS_INPUT, @@ -35,20 +35,11 @@ import { import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests, createEsmTests } from '../../../../utils/runner'; -const matrix = [ - ['6', '^6.0.0'], - ['7', '^7.0.0'], -] as const; - -describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVersion) => { +export function registerVercelAiGenerationTests(version: '6' | '7', vercelAiVersion: string): void { afterAll(() => { cleanupChildProcesses(); }); - const expectedOrigin = 'auto.ai.vercelai'; - - // We only run this in ESM and CJS to verify full support - // Other suites we only run in ESM to simplify the test setup createEsmAndCjsTests( __dirname, 'scenario.mjs', @@ -301,7 +292,6 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, }, ); - createEsmTests( __dirname, 'scenario.mjs', @@ -517,7 +507,6 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, }, ); - createEsmTests( __dirname, 'scenario-error-in-tool.mjs', @@ -603,7 +592,6 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, }, ); - createEsmTests( __dirname, 'scenario.mjs', @@ -643,7 +631,6 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, }, ); - createEsmTests( __dirname, 'scenario-tool-loop-agent.mjs', @@ -699,7 +686,6 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, }, ); - createEsmTests( __dirname, 'scenario-concurrent.mjs', @@ -745,16 +731,12 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, }, ); - createEsmTests( __dirname, - 'scenario-concurrent-stream.mjs', + 'scenario-rejected-model.mjs', 'instrument.mjs', (createRunner, test) => { - // A single model instance shared by two concurrent `streamText` calls carries only one - // captured-parent slot, so both model calls must still land under their own `invoke_agent` — not - // collapse onto whichever operation resolved the shared model last. - test('parents concurrent streamText calls that share one model instance correctly', async () => { + test('finishes spans with an error status when the operation rejects', async () => { await createRunner() .expect({ span: container => { @@ -763,25 +745,15 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe const spans = container.items.filter( span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.vercelai', ); - const invokeAgents = spans.filter(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent'); - const generateContents = spans.filter( - span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', - ); - - // Two concurrent operations -> two invoke_agent + two generate_content spans. - expect(invokeAgents).toHaveLength(2); - expect(generateContents).toHaveLength(2); - - const agentSpanIds = new Set(invokeAgents.map(span => span.span_id)); + // The model throws, so the operation rejects. The spans must still be *finished* (and + // therefore present in the transaction) with an error status — not left open. + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; + expect(invokeAgent.status).toBe('error'); - // Each model call lands under an invoke_agent span... - for (const span of generateContents) { - expect(agentSpanIds.has(span.parent_span_id!)).toBe(true); - } - // ...a distinct one each (no cross-attribution despite the shared model instance)... - expect(new Set(generateContents.map(span => span.parent_span_id)).size).toBe(2); - // ...and both operations sit under the same `main` parent. - expect(new Set(invokeAgents.map(span => span.parent_span_id)).size).toBe(1); + const generateContent = spans.find( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', + )!; + expect(generateContent.status).toBe('error'); }, }) .start() @@ -794,13 +766,12 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, }, ); - createEsmTests( __dirname, - 'scenario-stream-text.mjs', + 'scenario-provider-metadata.mjs', 'instrument.mjs', (createRunner, test) => { - test('creates streamText spans with the model call parented to invoke_agent', async () => { + test('derives provider-metadata token breakdown and system instructions', async () => { await createRunner() .expect({ span: container => { @@ -809,34 +780,28 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe const spans = container.items.filter( span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.vercelai', ); - const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; - expect(invokeAgent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText'); - const generateContent = spans.find( span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', )!; - expect(generateContent.parent_span_id).toBe(invokeAgent.span_id); - expect(generateContent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText.doStream'); - // The stream's final usage/finish/output arrive only as the stream drains, after the - // channel already resolved the model call. Tapping the stream recovers them onto the - // model-call span on every path (v7 channel, v6 OTel, v6 orchestrion). - expect(generateContent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); - expect(generateContent.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20); - expect(generateContent.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(30); - expect(generateContent.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value).toBe('["stop"]'); - expect(generateContent.attributes[GEN_AI_OUTPUT_MESSAGES]?.value).toBe( - '[{"role":"assistant","parts":[{"type":"text","content":"Stream response!"}],"finish_reason":"stop"}]', - ); + // Cache/reasoning token breakdown is derived from the model's `providerMetadata` by the + // channel subscriber, which v6 reaches through the orchestrion adapter, so the shape is the + // same on both versions. + expect(generateContent.attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]?.value).toBe(5); + expect(generateContent.attributes[GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]?.value).toBe(7); + // The per-response `responseId` is not a conversation id and must not be recorded as one. + expect(generateContent.attributes[GEN_AI_CONVERSATION_ID]).toBeUndefined(); - // The summed usage and output also land on the parent invoke_agent span, whose own - // channel result is otherwise undefined for a stream. - expect(invokeAgent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); - expect(invokeAgent.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20); - expect(invokeAgent.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(30); - expect(invokeAgent.attributes[GEN_AI_OUTPUT_MESSAGES]?.value).toBe( - '[{"role":"assistant","parts":[{"type":"text","content":"Stream response!"}],"finish_reason":"stop"}]', - ); + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; + expect(invokeAgent).toBeDefined(); + + // The system prompt is supplied via the v7-only `instructions` option. Only the channel + // instrumentation surfaces it (as `gen_ai.system_instructions`); v6 has no such option. + if (version === '7') { + const expected = '[{"type":"text","content":"You are a helpful assistant."}]'; + expect(invokeAgent.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]?.value).toBe(expected); + expect(generateContent.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]?.value).toBe(expected); + } }, }) .start() @@ -849,45 +814,28 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, }, ); - createEsmTests( __dirname, - 'scenario-stream-object.mjs', + 'scenario-system-message.mjs', 'instrument.mjs', (createRunner, test) => { - // v7's native `ai:telemetry` channel does not publish `streamObject`, same as `generateObject`. - test.skipIf(version === '7')( - 'creates streamObject spans with the model call parented to invoke_agent', - async () => { - await createRunner() - .expect({ - span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = container.items.filter( - span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.vercelai', - ); - const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_INVOKE_AGENT)!; - expect(invokeAgent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamObject'); - expect(invokeAgent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); - expect(invokeAgent.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(25); - expect(invokeAgent.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(40); + test('records system instructions from system message objects', async () => { + await createRunner() + .expect({ + span: container => { + const systemInstructions = container.items + .filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_INVOKE_AGENT) + .map(span => span.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]?.value); - const generateContent = spans.find( - span => span.attributes[SENTRY_OP]?.value === GEN_AI_GENERATE_CONTENT, - )!; - expect(generateContent.parent_span_id).toBe(invokeAgent.span_id); - expect(generateContent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamObject.doStream'); - expect(generateContent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); - expect(generateContent.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(25); - expect(generateContent.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(40); - }, - }) - .unordered() - .start() - .completed(); - }, - ); + expect(systemInstructions).toEqual([ + '[{"type":"text","content":"Be brief."}]', + '[{"type":"text","content":"Be brief."},{"type":"text","content":"Be kind."}]', + ]); + }, + }) + .start() + .completed(); + }); }, { additionalDependencies: { @@ -895,58 +843,51 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, }, ); - createEsmTests( __dirname, - 'scenario-stream-tools.mjs', + 'scenario-openai-conversation.mjs', 'instrument.mjs', (createRunner, test) => { - test('captures usage, tool calls and output across a multi-step streamText', async () => { + test('derives gen_ai.conversation.id from the OpenAI `conversation` provider option', async () => { await createRunner() .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = container.items.filter( - span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.vercelai', + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const genAiSpans = container.items.filter(s => + String(s.attributes[SENTRY_OP]?.value ?? '').startsWith('gen_ai.'), ); - const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; - expect(invokeAgent.status).toBe('ok'); - expect(invokeAgent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText'); - // Usage is summed across the two streamed model calls (10+15, 20+25, 30+40). - expect(invokeAgent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(25); - expect(invokeAgent.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(45); - expect(invokeAgent.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(70); - - const generateContents = spans.filter( - span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', + const conversationIdOf = (span: (typeof genAiSpans)[number]) => + span.attributes[GEN_AI_CONVERSATION_ID]?.value; + const invokeAgentSpans = genAiSpans.filter(s => s.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent'); + expect(invokeAgentSpans).toHaveLength(4); + const [firstTurn, secondTurn, chainedTurn, apiTurn] = invokeAgentSpans.sort( + (a, b) => a.start_timestamp - b.start_timestamp, ); - expect(generateContents).toHaveLength(2); - generateContents.forEach(span => expect(span.parent_span_id).toBe(invokeAgent.span_id)); - - // The step that streamed a tool call: tool-call output part + tool-calls finish reason. - const toolStep = generateContents.find( - span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["tool-calls"]', - )!; - expect(toolStep.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); - expect(toolStep.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20); - const toolStepOutput = toolStep.attributes[GEN_AI_OUTPUT_MESSAGES]?.value as string; - expect(toolStepOutput).toContain('"type":"tool_call"'); - expect(toolStepOutput).toContain('getWeather'); - // The step that streamed the final answer text. - const textStep = generateContents.find( - span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["stop"]', - )!; - expect(textStep.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); - expect(textStep.attributes[GEN_AI_OUTPUT_MESSAGES]?.value).toContain('Sunny, 72°F.'); + // `providerOptions.openai.conversation` is the Conversations API id: the same on every turn. + expect(conversationIdOf(firstTurn!)).toBe('conv_abc123'); + // The Azure Responses API uses the `azure` key for the same option. + expect(conversationIdOf(secondTurn!)).toBe('conv_azure'); + // `previousResponseId` names a response rather than a thread, and the response's own + // `responseId` is recorded as `gen_ai.response.id` only. + expect(conversationIdOf(chainedTurn!)).toBeUndefined(); + // `Sentry.setConversationId()` beats the provider option. + expect(conversationIdOf(apiTurn!)).toBe('conv-from-api'); - // A tool span is emitted for the streamed tool call. Its parent and recorded input/output - // vary by path during stream consumption (tool i/o is covered by the non-stream scenario - // and the v7 path), so here we just assert the span exists with the right name/status. - const executeTool = spans.find(span => span.name === 'execute_tool getWeather')!; - expect(executeTool.status).toBe('ok'); - expect(executeTool.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); + // Model-call and tool spans carry their operation's id, even though their start events + // do not carry `providerOptions`. + const modelCallSpans = genAiSpans.filter( + s => s.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', + ); + expect(modelCallSpans.map(conversationIdOf).sort()).toEqual([ + 'conv-from-api', + 'conv_abc123', + 'conv_azure', + undefined, + ]); + const toolSpan = genAiSpans.find(s => s.attributes[SENTRY_OP]?.value === 'gen_ai.execute_tool')!; + expect(toolSpan).toBeDefined(); + expect(conversationIdOf(toolSpan)).toBe('conv_azure'); }, }) .start() @@ -959,13 +900,12 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, }, ); - createEsmTests( __dirname, - 'scenario-stream-structured-output.mjs', + 'scenario-cache-tokens.mjs', 'instrument.mjs', (createRunner, test) => { - test('captures streamed structured output (streamText with experimental_output)', async () => { + test('reads cache token counts from the SDK usage object', async () => { await createRunner() .expect({ span: container => { @@ -974,23 +914,16 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe const spans = container.items.filter( span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.vercelai', ); - const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; - expect(invokeAgent.status).toBe('ok'); - expect(invokeAgent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText'); - expect(invokeAgent.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(30); - const generateContent = spans.find( span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', )!; - expect(generateContent.parent_span_id).toBe(invokeAgent.span_id); - expect(generateContent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(12); - expect(generateContent.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(18); - expect(generateContent.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value).toBe('["stop"]'); - // The streamed JSON object is accumulated from the text deltas and captured as the - // model's output text (embedded as an escaped JSON string in the output message). - const output = generateContent.attributes[GEN_AI_OUTPUT_MESSAGES]?.value as string; - expect(output).toContain('San Francisco'); - expect(output).toContain('sunny'); + expect(generateContent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(120); + expect(generateContent.attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]?.value).toBe(80); + expect(generateContent.attributes[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]?.value).toBe(20); + + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; + expect(invokeAgent.attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]?.value).toBe(80); + expect(invokeAgent.attributes[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]?.value).toBe(20); }, }) .start() @@ -1003,29 +936,49 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, }, ); - createEsmTests( __dirname, - 'scenario-stream-object-invalid.mjs', - 'instrument.mjs', + 'scenario-embeddings.mjs', + 'instrument-with-pii.mjs', (createRunner, test) => { - // v7's native `ai:telemetry` channel does not publish `streamObject`, same as `generateObject`. - test.skipIf(version === '7')('keeps streamObject usage when the object fails schema validation', async () => { + // `ai` v7 only routes `embed` through its telemetry tracing channel — `embedMany` is dispatched + // via the callback-only path and never published — so the channel-based integration (v7 and v6 + // orchestrion) cannot see it there. On v6 both the OTel processor and the orchestrion channels + // instrument `embedMany`, so its span is expected only on v6. + const embedManyInstrumented = version === '6'; + + test('creates embeddings spans for embed and embedMany', async () => { await createRunner() .expect({ span: container => { + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); const spans = container.items.filter( span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.vercelai', ); - const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_INVOKE_AGENT)!; - expect(invokeAgent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamObject'); - expect(invokeAgent.status).toBe('error'); - expect(invokeAgent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); - expect(invokeAgent.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(25); - expect(invokeAgent.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(40); + + const embedSpan = spans.find( + span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', + )!; + expect(embedSpan.name).toBe('embeddings mock-model-id'); + expect(embedSpan.status).toBe('ok'); + expect(embedSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.embeddings'); + expect(embedSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('mock-model-id'); + expect(embedSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); + + const embedManySpan = spans.find( + span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === '["First input","Second input"]', + ); + if (embedManyInstrumented) { + expect(embedManySpan!.name).toBe('embeddings mock-model-id'); + expect(embedManySpan!.status).toBe('ok'); + expect(embedManySpan!.attributes[SENTRY_OP]?.value).toBe('gen_ai.embeddings'); + expect(embedManySpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(20); + } else { + expect(embedManySpan).toBeUndefined(); + } }, }) - .unordered() .start() .completed(); }); @@ -1036,13 +989,16 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, }, ); - createEsmTests( __dirname, - 'scenario-rejected-model.mjs', - 'instrument.mjs', + 'scenario-generate-object.mjs', + 'instrument-with-pii.mjs', (createRunner, test) => { - test('finishes spans with an error status when the operation rejects', async () => { + // ai v7's native `ai:telemetry` channel does not publish a top-level `generateObject` operation + // (like `embedMany`, it's dispatched via a path the channel never sees), so the channel-based + // integration can't surface it on v7. v4/v5/v6 use orchestrion, which injects the `generateObject` + // channel directly — v6 is exercised here, v4 in the base suite. + test.skipIf(version === '7')('creates spans for generateObject', async () => { await createRunner() .expect({ span: container => { @@ -1051,15 +1007,24 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe const spans = container.items.filter( span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.vercelai', ); - // The model throws, so the operation rejects. The spans must still be *finished* (and - // therefore present in the transaction) with an error status — not left open. - const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; - expect(invokeAgent.status).toBe('error'); - const generateContent = spans.find( - span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', - )!; - expect(generateContent.status).toBe('error'); + const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); + expect(invokeAgentSpan!.status).toBe('ok'); + expect(invokeAgentSpan!.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes['vercel.ai.operationId']?.value).toBe('ai.generateObject'); + expect(invokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('mock-model-id'); + expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); + expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(25); + expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(40); + + const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); + expect(generateContentSpan!.status).toBe('ok'); + expect(generateContentSpan!.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes['vercel.ai.operationId']?.value).toBe( + 'ai.generateObject.doGenerate', + ); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('mock-model-id'); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); }, }) .start() @@ -1072,13 +1037,21 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, }, ); +} +export function registerVercelAiStreamingTests(version: '6' | '7', vercelAiVersion: string): void { + afterAll(() => { + cleanupChildProcesses(); + }); createEsmTests( __dirname, - 'scenario-provider-metadata.mjs', + 'scenario-concurrent-stream.mjs', 'instrument.mjs', (createRunner, test) => { - test('derives provider-metadata token breakdown and system instructions', async () => { + // A single model instance shared by two concurrent `streamText` calls carries only one + // captured-parent slot, so both model calls must still land under their own `invoke_agent` — not + // collapse onto whichever operation resolved the shared model last. + test('parents concurrent streamText calls that share one model instance correctly', async () => { await createRunner() .expect({ span: container => { @@ -1087,28 +1060,25 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe const spans = container.items.filter( span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.vercelai', ); - const generateContent = spans.find( + const invokeAgents = spans.filter(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent'); + const generateContents = spans.filter( span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', - )!; + ); - // Cache/reasoning token breakdown is derived from the model's `providerMetadata` by the - // channel subscriber, which v6 reaches through the orchestrion adapter, so the shape is the - // same on both versions. - expect(generateContent.attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]?.value).toBe(5); - expect(generateContent.attributes[GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]?.value).toBe(7); - // The per-response `responseId` is not a conversation id and must not be recorded as one. - expect(generateContent.attributes[GEN_AI_CONVERSATION_ID]).toBeUndefined(); + // Two concurrent operations -> two invoke_agent + two generate_content spans. + expect(invokeAgents).toHaveLength(2); + expect(generateContents).toHaveLength(2); - const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; - expect(invokeAgent).toBeDefined(); + const agentSpanIds = new Set(invokeAgents.map(span => span.span_id)); - // The system prompt is supplied via the v7-only `instructions` option. Only the channel - // instrumentation surfaces it (as `gen_ai.system_instructions`); v6 has no such option. - if (version === '7') { - const expected = '[{"type":"text","content":"You are a helpful assistant."}]'; - expect(invokeAgent.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]?.value).toBe(expected); - expect(generateContent.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]?.value).toBe(expected); + // Each model call lands under an invoke_agent span... + for (const span of generateContents) { + expect(agentSpanIds.has(span.parent_span_id!)).toBe(true); } + // ...a distinct one each (no cross-attribution despite the shared model instance)... + expect(new Set(generateContents.map(span => span.parent_span_id)).size).toBe(2); + // ...and both operations sit under the same `main` parent. + expect(new Set(invokeAgents.map(span => span.parent_span_id)).size).toBe(1); }, }) .start() @@ -1121,24 +1091,48 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, }, ); - createEsmTests( __dirname, - 'scenario-system-message.mjs', + 'scenario-stream-text.mjs', 'instrument.mjs', (createRunner, test) => { - test('records system instructions from system message objects', async () => { + test('creates streamText spans with the model call parented to invoke_agent', async () => { await createRunner() .expect({ span: container => { - const systemInstructions = container.items - .filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_INVOKE_AGENT) - .map(span => span.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]?.value); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.vercelai', + ); + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; + expect(invokeAgent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText'); - expect(systemInstructions).toEqual([ - '[{"type":"text","content":"Be brief."}]', - '[{"type":"text","content":"Be brief."},{"type":"text","content":"Be kind."}]', - ]); + const generateContent = spans.find( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', + )!; + expect(generateContent.parent_span_id).toBe(invokeAgent.span_id); + expect(generateContent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText.doStream'); + + // The stream's final usage/finish/output arrive only as the stream drains, after the + // channel already resolved the model call. Tapping the stream recovers them onto the + // model-call span on every path (v7 channel, v6 OTel, v6 orchestrion). + expect(generateContent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); + expect(generateContent.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20); + expect(generateContent.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(30); + expect(generateContent.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value).toBe('["stop"]'); + expect(generateContent.attributes[GEN_AI_OUTPUT_MESSAGES]?.value).toBe( + '[{"role":"assistant","parts":[{"type":"text","content":"Stream response!"}],"finish_reason":"stop"}]', + ); + + // The summed usage and output also land on the parent invoke_agent span, whose own + // channel result is otherwise undefined for a stream. + expect(invokeAgent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); + expect(invokeAgent.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20); + expect(invokeAgent.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(30); + expect(invokeAgent.attributes[GEN_AI_OUTPUT_MESSAGES]?.value).toBe( + '[{"role":"assistant","parts":[{"type":"text","content":"Stream response!"}],"finish_reason":"stop"}]', + ); }, }) .start() @@ -1151,57 +1145,44 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, }, ); - createEsmTests( __dirname, - 'scenario-openai-conversation.mjs', + 'scenario-stream-object.mjs', 'instrument.mjs', (createRunner, test) => { - test('derives gen_ai.conversation.id from the OpenAI `conversation` provider option', async () => { - await createRunner() - .expect({ - span: container => { - expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); - const genAiSpans = container.items.filter(s => - String(s.attributes[SENTRY_OP]?.value ?? '').startsWith('gen_ai.'), - ); - const conversationIdOf = (span: (typeof genAiSpans)[number]) => - span.attributes[GEN_AI_CONVERSATION_ID]?.value; - const invokeAgentSpans = genAiSpans.filter(s => s.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent'); - expect(invokeAgentSpans).toHaveLength(4); - const [firstTurn, secondTurn, chainedTurn, apiTurn] = invokeAgentSpans.sort( - (a, b) => a.start_timestamp - b.start_timestamp, - ); - - // `providerOptions.openai.conversation` is the Conversations API id: the same on every turn. - expect(conversationIdOf(firstTurn!)).toBe('conv_abc123'); - // The Azure Responses API uses the `azure` key for the same option. - expect(conversationIdOf(secondTurn!)).toBe('conv_azure'); - // `previousResponseId` names a response rather than a thread, and the response's own - // `responseId` is recorded as `gen_ai.response.id` only. - expect(conversationIdOf(chainedTurn!)).toBeUndefined(); - // `Sentry.setConversationId()` beats the provider option. - expect(conversationIdOf(apiTurn!)).toBe('conv-from-api'); + // v7's native `ai:telemetry` channel does not publish `streamObject`, same as `generateObject`. + test.skipIf(version === '7')( + 'creates streamObject spans with the model call parented to invoke_agent', + async () => { + await createRunner() + .expect({ + span: container => { + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.vercelai', + ); + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_INVOKE_AGENT)!; + expect(invokeAgent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamObject'); + expect(invokeAgent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); + expect(invokeAgent.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(25); + expect(invokeAgent.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(40); - // Model-call and tool spans carry their operation's id, even though their start events - // do not carry `providerOptions`. - const modelCallSpans = genAiSpans.filter( - s => s.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', - ); - expect(modelCallSpans.map(conversationIdOf).sort()).toEqual([ - 'conv-from-api', - 'conv_abc123', - 'conv_azure', - undefined, - ]); - const toolSpan = genAiSpans.find(s => s.attributes[SENTRY_OP]?.value === 'gen_ai.execute_tool')!; - expect(toolSpan).toBeDefined(); - expect(conversationIdOf(toolSpan)).toBe('conv_azure'); - }, - }) - .start() - .completed(); - }); + const generateContent = spans.find( + span => span.attributes[SENTRY_OP]?.value === GEN_AI_GENERATE_CONTENT, + )!; + expect(generateContent.parent_span_id).toBe(invokeAgent.span_id); + expect(generateContent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamObject.doStream'); + expect(generateContent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); + expect(generateContent.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(25); + expect(generateContent.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(40); + }, + }) + .unordered() + .start() + .completed(); + }, + ); }, { additionalDependencies: { @@ -1209,13 +1190,12 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, }, ); - createEsmTests( __dirname, - 'scenario-cache-tokens.mjs', + 'scenario-stream-tools.mjs', 'instrument.mjs', (createRunner, test) => { - test('reads cache token counts from the SDK usage object', async () => { + test('captures usage, tool calls and output across a multi-step streamText', async () => { await createRunner() .expect({ span: container => { @@ -1224,16 +1204,43 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe const spans = container.items.filter( span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.vercelai', ); - const generateContent = spans.find( + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; + expect(invokeAgent.status).toBe('ok'); + expect(invokeAgent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText'); + // Usage is summed across the two streamed model calls (10+15, 20+25, 30+40). + expect(invokeAgent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(25); + expect(invokeAgent.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(45); + expect(invokeAgent.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(70); + + const generateContents = spans.filter( span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', + ); + expect(generateContents).toHaveLength(2); + generateContents.forEach(span => expect(span.parent_span_id).toBe(invokeAgent.span_id)); + + // The step that streamed a tool call: tool-call output part + tool-calls finish reason. + const toolStep = generateContents.find( + span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["tool-calls"]', )!; - expect(generateContent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(120); - expect(generateContent.attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]?.value).toBe(80); - expect(generateContent.attributes[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]?.value).toBe(20); + expect(toolStep.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); + expect(toolStep.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20); + const toolStepOutput = toolStep.attributes[GEN_AI_OUTPUT_MESSAGES]?.value as string; + expect(toolStepOutput).toContain('"type":"tool_call"'); + expect(toolStepOutput).toContain('getWeather'); - const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; - expect(invokeAgent.attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]?.value).toBe(80); - expect(invokeAgent.attributes[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]?.value).toBe(20); + // The step that streamed the final answer text. + const textStep = generateContents.find( + span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["stop"]', + )!; + expect(textStep.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); + expect(textStep.attributes[GEN_AI_OUTPUT_MESSAGES]?.value).toContain('Sunny, 72°F.'); + + // A tool span is emitted for the streamed tool call. Its parent and recorded input/output + // vary by path during stream consumption (tool i/o is covered by the non-stream scenario + // and the v7 path), so here we just assert the span exists with the right name/status. + const executeTool = spans.find(span => span.name === 'execute_tool getWeather')!; + expect(executeTool.status).toBe('ok'); + expect(executeTool.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); }, }) .start() @@ -1246,19 +1253,12 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, }, ); - createEsmTests( __dirname, - 'scenario-embeddings.mjs', - 'instrument-with-pii.mjs', + 'scenario-stream-structured-output.mjs', + 'instrument.mjs', (createRunner, test) => { - // `ai` v7 only routes `embed` through its telemetry tracing channel — `embedMany` is dispatched - // via the callback-only path and never published — so the channel-based integration (v7 and v6 - // orchestrion) cannot see it there. On v6 both the OTel processor and the orchestrion channels - // instrument `embedMany`, so its span is expected only on v6. - const embedManyInstrumented = version === '6'; - - test('creates embeddings spans for embed and embedMany', async () => { + test('captures streamed structured output (streamText with experimental_output)', async () => { await createRunner() .expect({ span: container => { @@ -1267,27 +1267,23 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe const spans = container.items.filter( span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.vercelai', ); + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; + expect(invokeAgent.status).toBe('ok'); + expect(invokeAgent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText'); + expect(invokeAgent.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(30); - const embedSpan = spans.find( - span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', + const generateContent = spans.find( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', )!; - expect(embedSpan.name).toBe('embeddings mock-model-id'); - expect(embedSpan.status).toBe('ok'); - expect(embedSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.embeddings'); - expect(embedSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('mock-model-id'); - expect(embedSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); - - const embedManySpan = spans.find( - span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === '["First input","Second input"]', - ); - if (embedManyInstrumented) { - expect(embedManySpan!.name).toBe('embeddings mock-model-id'); - expect(embedManySpan!.status).toBe('ok'); - expect(embedManySpan!.attributes[SENTRY_OP]?.value).toBe('gen_ai.embeddings'); - expect(embedManySpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(20); - } else { - expect(embedManySpan).toBeUndefined(); - } + expect(generateContent.parent_span_id).toBe(invokeAgent.span_id); + expect(generateContent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(12); + expect(generateContent.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(18); + expect(generateContent.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value).toBe('["stop"]'); + // The streamed JSON object is accumulated from the text deltas and captured as the + // model's output text (embedded as an escaped JSON string in the output message). + const output = generateContent.attributes[GEN_AI_OUTPUT_MESSAGES]?.value as string; + expect(output).toContain('San Francisco'); + expect(output).toContain('sunny'); }, }) .start() @@ -1300,45 +1296,28 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, }, ); - createEsmTests( __dirname, - 'scenario-generate-object.mjs', - 'instrument-with-pii.mjs', + 'scenario-stream-object-invalid.mjs', + 'instrument.mjs', (createRunner, test) => { - // ai v7's native `ai:telemetry` channel does not publish a top-level `generateObject` operation - // (like `embedMany`, it's dispatched via a path the channel never sees), so the channel-based - // integration can't surface it on v7. v4/v5/v6 use orchestrion, which injects the `generateObject` - // channel directly — v6 is exercised here, v4 in the base suite. - test.skipIf(version === '7')('creates spans for generateObject', async () => { + // v7's native `ai:telemetry` channel does not publish `streamObject`, same as `generateObject`. + test.skipIf(version === '7')('keeps streamObject usage when the object fails schema validation', async () => { await createRunner() .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); const spans = container.items.filter( span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.vercelai', ); - - const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan!.status).toBe('ok'); - expect(invokeAgentSpan!.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); - expect(invokeAgentSpan!.attributes['vercel.ai.operationId']?.value).toBe('ai.generateObject'); - expect(invokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('mock-model-id'); - expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); - expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(25); - expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(40); - - const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); - expect(generateContentSpan!.attributes['vercel.ai.operationId']?.value).toBe( - 'ai.generateObject.doGenerate', - ); - expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('mock-model-id'); - expect(generateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_INVOKE_AGENT)!; + expect(invokeAgent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamObject'); + expect(invokeAgent.status).toBe('error'); + expect(invokeAgent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); + expect(invokeAgent.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(25); + expect(invokeAgent.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(40); }, }) + .unordered() .start() .completed(); }); @@ -1349,7 +1328,6 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, }, ); - createEsmTests( __dirname, 'scenario-aborted-stream-text.mjs', @@ -1388,9 +1366,9 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, }, ); -}); +} -describe('Vercel AI integration experimental_decide', () => { +export function registerDecideTests(): void { afterAll(() => { cleanupChildProcesses(); }); @@ -1493,9 +1471,9 @@ describe('Vercel AI integration experimental_decide', () => { }, }, ); -}); +} -describe('Vercel AI telemetry metadata (v6)', () => { +export function registerMetadataTests(): void { afterAll(() => { cleanupChildProcesses(); }); @@ -1542,9 +1520,9 @@ describe('Vercel AI telemetry metadata (v6)', () => { }, { additionalDependencies: { ai: '^6.0.0' } }, ); -}); +} -describe('Vercel AI runtime and tool context (v7)', () => { +export function registerRuntimeTests(): void { afterAll(() => { cleanupChildProcesses(); }); @@ -1601,9 +1579,9 @@ describe('Vercel AI runtime and tool context (v7)', () => { }, { additionalDependencies: { ai: '^7.0.154' } }, ); -}); +} -describe('Vercel AI calls with recording off (v7)', () => { +export function registerRecordingOffTests(): void { afterAll(() => { cleanupChildProcesses(); }); @@ -1768,4 +1746,4 @@ describe('Vercel AI calls with recording off (v7)', () => { }, { additionalDependencies: { ai: '^7.0.163' } }, ); -}); +} diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/v6/generation/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/v6/generation/test.ts new file mode 100644 index 000000000000..d17670e049a8 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/v6/generation/test.ts @@ -0,0 +1,8 @@ +import { describe } from 'vitest'; +import { registerVercelAiGenerationTests, registerMetadataTests } from '../../suite'; + +describe('Vercel AI integration (version 6)', () => { + registerVercelAiGenerationTests('6', '^6.0.0'); +}); + +describe('Vercel AI telemetry metadata (v6)', registerMetadataTests); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/v6/streaming/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/v6/streaming/test.ts new file mode 100644 index 000000000000..daeee727e4db --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/v6/streaming/test.ts @@ -0,0 +1,6 @@ +import { describe } from 'vitest'; +import { registerVercelAiStreamingTests } from '../../suite'; + +describe('Vercel AI integration (version 6)', () => { + registerVercelAiStreamingTests('6', '^6.0.0'); +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/v7/generation/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/v7/generation/test.ts new file mode 100644 index 000000000000..0b6f1253535f --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/v7/generation/test.ts @@ -0,0 +1,17 @@ +import { describe } from 'vitest'; +import { + registerVercelAiGenerationTests, + registerDecideTests, + registerRuntimeTests, + registerRecordingOffTests, +} from '../../suite'; + +describe('Vercel AI integration (version 7)', () => { + registerVercelAiGenerationTests('7', '^7.0.0'); +}); + +describe('Vercel AI integration experimental_decide', registerDecideTests); + +describe('Vercel AI runtime and tool context (v7)', registerRuntimeTests); + +describe('Vercel AI calls with recording off (v7)', registerRecordingOffTests); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/v7/streaming/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/v7/streaming/test.ts new file mode 100644 index 000000000000..2294d7c323c2 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/v7/streaming/test.ts @@ -0,0 +1,6 @@ +import { describe } from 'vitest'; +import { registerVercelAiStreamingTests } from '../../suite'; + +describe('Vercel AI integration (version 7)', () => { + registerVercelAiStreamingTests('7', '^7.0.0'); +}); diff --git a/dev-packages/node-integration-tests/vite.config.ts b/dev-packages/node-integration-tests/vite.config.ts index 5a8b87af8dce..fbc580836f1a 100644 --- a/dev-packages/node-integration-tests/vite.config.ts +++ b/dev-packages/node-integration-tests/vite.config.ts @@ -25,7 +25,7 @@ export default defineConfig({ reporters: process.env.DEBUG ? [['default', { summary: false }]] : process.env.GITHUB_ACTIONS - ? ['dot', 'github-actions'] + ? ['dot', 'github-actions', ...(process.env.INTEGRATION_TIMINGS ? ['./benchmark-reporter.ts'] : [])] : ['verbose'], }, });