From a2184276dfccbabf0235c6955c42959cffd35ab5 Mon Sep 17 00:00:00 2001 From: Lukas Stracke Date: Wed, 7 Oct 2026 12:00:55 +0200 Subject: [PATCH 01/47] ref(eslint-plugin-sdk): Improve `no-unsafe-random-apis` rule (#25123) makes the `no-unsafe-random-apis` rule a bit more robust. We now catch: - `crypto.randomUUID!()` and TS casts (`as`, `satisfies`,` `) - Optional calls: `performance?.now()`, `crypto?.randomUUID?.()` - Calls through a global object: `globalThis.performance.now()`, `GLOBAL_OBJ.crypto.randomUUID()`, `WINDOW.performance?.now()` --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .../src/rules/no-unsafe-random-apis.js | 160 ++++++++---------- .../lib/rules/no-unsafe-random-apis.test.ts | 105 +++++++++++- 2 files changed, 179 insertions(+), 86 deletions(-) diff --git a/packages/eslint-plugin-sdk/src/rules/no-unsafe-random-apis.js b/packages/eslint-plugin-sdk/src/rules/no-unsafe-random-apis.js index e90f24f2003d..420338d86804 100644 --- a/packages/eslint-plugin-sdk/src/rules/no-unsafe-random-apis.js +++ b/packages/eslint-plugin-sdk/src/rules/no-unsafe-random-apis.js @@ -37,6 +37,76 @@ const UNSAFE_MEMBER_CALLS = [ }, ]; +// `@sentry/core` exports the wrapper as `_INTERNAL_withRandomSafeContext` to other packages +const WRAPPER_NAMES = ['withRandomSafeContext', '_INTERNAL_withRandomSafeContext']; + +const TRANSPARENT_WRAPPERS = [ + 'ChainExpression', + 'ParenthesizedExpression', + 'TSNonNullExpression', + 'TSAsExpression', + 'TSSatisfiesExpression', + 'TSTypeAssertion', +]; + +function unwrap(node) { + let current = node; + while (current && TRANSPARENT_WRAPPERS.includes(current.type)) { + current = current.expression; + } + return current; +} + +function getPropertyName(memberExpression) { + const property = memberExpression.property; + if (!memberExpression.computed && property.type === 'Identifier') { + return property.name; + } + if (memberExpression.computed && property.type === 'Literal') { + return property.value; + } + return null; +} + +/** + * Returns the name of the global an expression refers to, e.g. `performance` for + * `performance`, `globalThis.performance` or `GLOBAL_OBJ.performance`. + */ +function getObjectName(node) { + const object = unwrap(node); + if (object?.type === 'Identifier') { + return object.name; + } + if (object?.type === 'MemberExpression') { + return getPropertyName(object); + } + return null; +} + +function getUnsafeApi(callee) { + const member = unwrap(callee); + if (member?.type !== 'MemberExpression') { + return undefined; + } + const objectName = getObjectName(member.object); + const propertyName = getPropertyName(member); + return UNSAFE_MEMBER_CALLS.find(api => api.object === objectName && api.property === propertyName); +} + +function isInsideWithRandomSafeContext(node) { + let current = node.parent; + while (current) { + if (current.type === 'CallExpression') { + const callee = unwrap(current.callee); + if (callee.type === 'Identifier' && WRAPPER_NAMES.includes(callee.name)) { + return true; + } + } + current = current.parent; + } + return false; +} + module.exports = { meta: { type: 'problem', @@ -64,103 +134,23 @@ module.exports = { }, }, create: function (context) { - /** - * Check if a node is inside a withRandomSafeContext call - */ - function isInsidewithRandomSafeContext(node) { - let current = node.parent; - - while (current) { - // Check if we're inside a callback passed to withRandomSafeContext - if ( - current.type === 'CallExpression' && - current.callee.type === 'Identifier' && - current.callee.name === 'withRandomSafeContext' - ) { - return true; - } - - // Also check for arrow functions or regular functions passed to withRandomSafeContext - if ( - (current.type === 'ArrowFunctionExpression' || current.type === 'FunctionExpression') && - current.parent?.type === 'CallExpression' && - current.parent.callee.type === 'Identifier' && - current.parent.callee.name === 'withRandomSafeContext' - ) { - return true; - } - - current = current.parent; - } - - return false; - } - - /** - * Check if a node is inside the safeRandomGeneratorRunner.ts file (the definition file) - */ - function isInSafeRandomGeneratorRunner(_node) { - const filename = context.getFilename(); - return filename.includes('safeRandomGeneratorRunner'); - } - return { CallExpression(node) { - // Skip if we're in the safeRandomGeneratorRunner.ts file itself - if (isInSafeRandomGeneratorRunner(node)) { - return; - } - - // Check for member expression calls like Date.now(), Math.random(), etc. - if (node.callee.type === 'MemberExpression') { - const callee = node.callee; - - // Get the object name (e.g., 'Date', 'Math', 'performance', 'crypto') - let objectName = null; - if (callee.object.type === 'Identifier') { - objectName = callee.object.name; - } - - // Get the property name (e.g., 'now', 'random', 'randomUUID') - let propertyName = null; - if (callee.property.type === 'Identifier') { - propertyName = callee.property.name; - } else if (callee.computed && callee.property.type === 'Literal') { - propertyName = callee.property.value; - } - - if (!objectName || !propertyName) { - return; - } - - // Check if this is one of the unsafe APIs - const unsafeApi = UNSAFE_MEMBER_CALLS.find(api => api.object === objectName && api.property === propertyName); - - if (unsafeApi && !isInsidewithRandomSafeContext(node)) { - context.report({ - node, - messageId: unsafeApi.messageId, - }); - } + const api = getUnsafeApi(node.callee); + if (api && !isInsideWithRandomSafeContext(node)) { + context.report({ node, messageId: api.messageId }); } }, // Flag the `new Date()` constructor with no arguments, which reads the ambient clock. // `new Date()` is safe because it does not read the current time. NewExpression(node) { - if (isInSafeRandomGeneratorRunner(node)) { - return; - } - if ( node.callee.type === 'Identifier' && node.callee.name === 'Date' && node.arguments.length === 0 && - !isInsidewithRandomSafeContext(node) + !isInsideWithRandomSafeContext(node) ) { - context.report({ - node, - messageId: 'unsafeDateConstructor', - }); + context.report({ node, messageId: 'unsafeDateConstructor' }); } }, }; diff --git a/packages/eslint-plugin-sdk/test/lib/rules/no-unsafe-random-apis.test.ts b/packages/eslint-plugin-sdk/test/lib/rules/no-unsafe-random-apis.test.ts index 533c048f979c..b24f2a96ac5b 100644 --- a/packages/eslint-plugin-sdk/test/lib/rules/no-unsafe-random-apis.test.ts +++ b/packages/eslint-plugin-sdk/test/lib/rules/no-unsafe-random-apis.test.ts @@ -1,5 +1,5 @@ import { RuleTester } from 'eslint'; -import { describe, test } from 'vitest'; +import { describe, expect, test } from 'vitest'; // @ts-expect-error untyped module import rule from '../../../src/rules/no-unsafe-random-apis'; @@ -67,6 +67,15 @@ describe('no-unsafe-random-apis', () => { { code: 'withRandomSafeContext(() => new Date())', }, + { + code: '_INTERNAL_withRandomSafeContext(() => performance.now())', + }, + { + code: 'withRandomSafeContext(() => globalThis.performance.now())', + }, + { + code: 'withRandomSafeContext(() => performance?.now())', + }, ], invalid: [ // Direct Date.now() calls @@ -178,7 +187,101 @@ describe('no-unsafe-random-apis', () => { }, ], }, + // Optional chaining + { + code: 'const perf = performance?.now()', + errors: [{ messageId: 'unsafePerformanceNow' }], + }, + { + code: 'const uuid = crypto?.randomUUID?.()', + errors: [{ messageId: 'unsafeCryptoRandomUUID' }], + }, + // Access through a global object + { + code: 'const perf = globalThis.performance.now()', + errors: [{ messageId: 'unsafePerformanceNow' }], + }, + { + code: 'const uuid = GLOBAL_OBJ.crypto.randomUUID()', + errors: [{ messageId: 'unsafeCryptoRandomUUID' }], + }, + { + code: 'const perf = WINDOW.performance?.now()', + errors: [{ messageId: 'unsafePerformanceNow' }], + }, + { + code: 'const random = Math["random"]()', + errors: [{ messageId: 'unsafeMathRandom' }], + }, ], }); }); + + // RuleTester can't parse TypeScript syntax: the repo has no working TypeScript parser for ESLint + // (`@typescript-eslint/parser@5` can't run against `typescript@7`), so we drive the visitors with + // the AST shape that oxlint produces. + describe('TypeScript syntax', () => { + function withParents(node: any, parent: any = null): any { + node.parent = parent; + for (const [key, value] of Object.entries(node)) { + if (key === 'parent') continue; + for (const child of Array.isArray(value) ? value : [value]) { + if (child && typeof child.type === 'string') withParents(child, node); + } + } + return node; + } + + function run(root: any): string[] { + const messageIds: string[] = []; + const visitor = rule.create({ + report: ({ messageId }: { messageId: string }) => messageIds.push(messageId), + }); + (function walk(node: any) { + visitor[node.type]?.(node); + for (const [key, value] of Object.entries(node)) { + if (key === 'parent') continue; + for (const child of Array.isArray(value) ? value : [value]) { + if (child && typeof child.type === 'string') walk(child); + } + } + })(withParents(root)); + return messageIds; + } + + const id = (name: string) => ({ type: 'Identifier', name }); + const member = (object: unknown, property: string) => ({ + type: 'MemberExpression', + object, + property: id(property), + computed: false, + optional: false, + }); + const call = (callee: unknown) => ({ type: 'CallExpression', callee, arguments: [], optional: false }); + const nonNull = (expression: unknown) => ({ type: 'TSNonNullExpression', expression }); + const as = (expression: unknown) => ({ type: 'TSAsExpression', expression, typeAnnotation: null }); + const statement = (expression: unknown) => ({ + type: 'Program', + body: [{ type: 'ExpressionStatement', expression }], + }); + + test('reports `crypto.randomUUID!()`', () => { + expect(run(statement(call(nonNull(member(id('crypto'), 'randomUUID')))))).toEqual(['unsafeCryptoRandomUUID']); + }); + + test('reports `(globalThis as any).performance.now()`', () => { + expect(run(statement(call(member(member(as(id('globalThis')), 'performance'), 'now'))))).toEqual([ + 'unsafePerformanceNow', + ]); + }); + + test('allows `withRandomSafeContext(() => crypto.randomUUID!())`', () => { + const callback = { + type: 'ArrowFunctionExpression', + params: [], + body: call(nonNull(member(id('crypto'), 'randomUUID'))), + }; + expect(run(statement({ ...call(id('withRandomSafeContext')), arguments: [callback] }))).toEqual([]); + }); + }); }); From 5c7550191fcaa14ee5d682bd89565ba7ede5c059 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Wed, 7 Oct 2026 14:05:20 +0200 Subject: [PATCH 02/47] test(node): Add streamed span collection helpers (#25131) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add Node runner equivalents of Cloudflare’s `collectStreamedSpans` and `collectStreamedSpansUntilSegment` so tests can collect across envelopes. Migrate AWS, feature-flag, GrowthBook, and sampler fallback tests using the segment helper. Part of #24141 --------- Co-authored-by: GPT-6 --- dev-packages/node-integration-tests/README.md | 29 ++++ .../aws-serverless/aws-integration/test.ts | 18 +- .../traces-sampler-throws/test.ts | 18 +- .../featureFlagsIntegration/onSpan/test.ts | 46 +++-- .../featureFlags/growthbook/onSpan/test.ts | 48 +++--- .../utils/runner/createRunner.ts | 159 ++++++++++++++++-- 6 files changed, 232 insertions(+), 86 deletions(-) diff --git a/dev-packages/node-integration-tests/README.md b/dev-packages/node-integration-tests/README.md index e8e9aedf5620..ac35f0ea54c0 100644 --- a/dev-packages/node-integration-tests/README.md +++ b/dev-packages/node-integration-tests/README.md @@ -34,6 +34,35 @@ Nock interceptors are internally used to capture envelope requests by `getEnvelo be used inside the test cases to intercept requests but should be removed before the test ends, as not to cause flakiness. +### Collecting streamed spans + +Register collectors before starting the scenario. `collectStreamedSpansUntilSegment` collects spans across envelopes +until it sees a segment matching a name or predicate, then returns the collected spans from that trace: + +```ts +const runner = createRunner(__dirname, 'scenario.ts'); +const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + +await runner.start().completed(); + +const spans = await spansPromise; +expect(spans.filter(span => !span.is_segment)).toHaveLength(2); +``` + +A segment does not guarantee that all its children have arrived. If a scenario sends children later, use +`collectStreamedSpans` with a predicate describing which spans must be present: + +```ts +const spansPromise = runner.collectStreamedSpans( + spans => spans.some(span => span.is_segment && span.name === 'main') && spans.length >= 3, +); +``` + +The predicate is evaluated separately for each trace. Returning `false` keeps waiting; throwing fails the runner +immediately, including in unordered mode. Keep assertions after collection so incorrect attributes produce assertion +failures. A predicate that never matches still fails on scenario exit or timeout. Neither helper waits for spans +arriving after its condition is satisfied. + ## Other Runtimes `dev-packages/bun-integration-tests` and `dev-packages/deno-integration-tests` run every suite of this package on Bun diff --git a/dev-packages/node-integration-tests/suites/aws-serverless/aws-integration/test.ts b/dev-packages/node-integration-tests/suites/aws-serverless/aws-integration/test.ts index e12bf1ecaff2..951a0e437e38 100644 --- a/dev-packages/node-integration-tests/suites/aws-serverless/aws-integration/test.ts +++ b/dev-packages/node-integration-tests/suites/aws-serverless/aws-integration/test.ts @@ -258,18 +258,12 @@ describe('awsIntegration', () => { 'instrument.mjs', (createTestRunner, test) => { test('auto-instruments aws-sdk service operations', { timeout: 90_000 }, async () => { - const spans: SerializedStreamedSpanContainer['items'] = []; - await createTestRunner() - .ignore('event') - .unordered() - .expect({ - span: container => { - spans.push(...container.items); - assertAwsServiceSpans(spans); - }, - }) - .start() - .completed(); + const runner = createTestRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('Test Transaction'); + + await runner.start().completed(); + + assertAwsServiceSpans(await spansPromise); }); }, { additionalDependencies }, diff --git a/dev-packages/node-integration-tests/suites/client-reports/drop-reasons/traces-sampler-throws/test.ts b/dev-packages/node-integration-tests/suites/client-reports/drop-reasons/traces-sampler-throws/test.ts index 965985ea78d0..07d5b7a82de2 100644 --- a/dev-packages/node-integration-tests/suites/client-reports/drop-reasons/traces-sampler-throws/test.ts +++ b/dev-packages/node-integration-tests/suites/client-reports/drop-reasons/traces-sampler-throws/test.ts @@ -24,14 +24,12 @@ test('records a client report and no error event when tracesSampler throws', asy }); test('sends the span when tracesSampler throws but tracesSampleRate is 1', async () => { - await createRunner(__dirname, 'scenario-fallback.ts') - .unordered() - .expect({ - span: container => { - const segment = container.items.find(span => span.is_segment); - expect(segment?.name).toBe('sampled via tracesSampleRate fallback'); - }, - }) - .start() - .completed(); + const runner = createRunner(__dirname, 'scenario-fallback.ts'); + const spansPromise = runner.collectStreamedSpansUntilSegment(); + + await runner.start().completed(); + + const spans = await spansPromise; + const segment = spans.find(span => span.is_segment); + expect(segment?.name).toBe('sampled via tracesSampleRate fallback'); }); diff --git a/dev-packages/node-integration-tests/suites/featureFlags/featureFlagsIntegration/onSpan/test.ts b/dev-packages/node-integration-tests/suites/featureFlags/featureFlagsIntegration/onSpan/test.ts index b62c330ce81e..e2caf9ed4fc1 100644 --- a/dev-packages/node-integration-tests/suites/featureFlags/featureFlagsIntegration/onSpan/test.ts +++ b/dev-packages/node-integration-tests/suites/featureFlags/featureFlagsIntegration/onSpan/test.ts @@ -1,4 +1,4 @@ -import { _INTERNAL_MAX_FLAGS_PER_SPAN as MAX_FLAGS_PER_SPAN, type SerializedStreamedSpanContainer } from '@sentry/core'; +import { _INTERNAL_MAX_FLAGS_PER_SPAN as MAX_FLAGS_PER_SPAN } from '@sentry/core'; import { afterAll, expect, test } from 'vitest'; import { cleanupChildProcesses, createRunner } from '../../../../utils/runner'; @@ -12,29 +12,25 @@ test('Flags captured on span attributes with max limit', async () => { for (let i = 1; i <= MAX_FLAGS_PER_SPAN; i++) { expectedFlags[`flag.evaluation.feat${i}`] = { type: 'boolean', value: i === 3 }; } - const spans: SerializedStreamedSpanContainer['items'] = []; - await createRunner(__dirname, 'scenario.ts') - .unordered() - .expect({ - span: container => { - spans.push(...container.items); - const children = spans.filter(span => !span.is_segment); - expect(children).toHaveLength(2); - expect(children).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - name: 'test-span', - attributes: expect.objectContaining({}), - }), - expect.objectContaining({ - name: 'test-nested-span', - attributes: expect.objectContaining(expectedFlags), - }), - ]), - ); - }, - }) - .start() - .completed(); + const runner = createRunner(__dirname, 'scenario.ts'); + const spansPromise = runner.collectStreamedSpansUntilSegment('test-root-span'); + + await runner.start().completed(); + + const spans = await spansPromise; + const children = spans.filter(span => !span.is_segment); + expect(children).toHaveLength(2); + expect(children).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: 'test-span', + attributes: expect.objectContaining({}), + }), + expect.objectContaining({ + name: 'test-nested-span', + attributes: expect.objectContaining(expectedFlags), + }), + ]), + ); }); diff --git a/dev-packages/node-integration-tests/suites/featureFlags/growthbook/onSpan/test.ts b/dev-packages/node-integration-tests/suites/featureFlags/growthbook/onSpan/test.ts index 8f5d4fdbdf9b..75a920581eec 100644 --- a/dev-packages/node-integration-tests/suites/featureFlags/growthbook/onSpan/test.ts +++ b/dev-packages/node-integration-tests/suites/featureFlags/growthbook/onSpan/test.ts @@ -6,29 +6,27 @@ afterAll(() => { }); test('GrowthBook flags are added to active span attributes on span end', async () => { - await createRunner(__dirname, 'scenario.ts') - .unordered() - .expect({ - span: container => { - expect(container.items).toHaveLength(1); - const span = container.items.find(item => item.is_segment); - expect(span).toMatchObject({ - name: 'test-span', - status: 'ok', - span_id: expect.stringMatching(/[a-f\d]{16}/), - trace_id: expect.stringMatching(/[a-f\d]{32}/), - attributes: { - 'flag.evaluation.feat1': { type: 'boolean', value: true }, - 'flag.evaluation.feat2': { type: 'boolean', value: false }, - 'flag.evaluation.bool-feat': { type: 'boolean', value: true }, - 'sentry.op': { type: 'string', value: 'function' }, - 'sentry.origin': { type: 'string', value: 'manual' }, - }, - }); - // string-feat should NOT be here since it's not boolean - expect(span?.attributes['flag.evaluation.string-feat']).toBeUndefined(); - }, - }) - .start() - .completed(); + const runner = createRunner(__dirname, 'scenario.ts'); + const spansPromise = runner.collectStreamedSpansUntilSegment(); + + await runner.start().completed(); + + const spans = await spansPromise; + expect(spans).toHaveLength(1); + const span = spans.find(item => item.is_segment); + expect(span).toMatchObject({ + name: 'test-span', + status: 'ok', + span_id: expect.stringMatching(/[a-f\d]{16}/), + trace_id: expect.stringMatching(/[a-f\d]{32}/), + attributes: { + 'flag.evaluation.feat1': { type: 'boolean', value: true }, + 'flag.evaluation.feat2': { type: 'boolean', value: false }, + 'flag.evaluation.bool-feat': { type: 'boolean', value: true }, + 'sentry.op': { type: 'string', value: 'function' }, + 'sentry.origin': { type: 'string', value: 'manual' }, + }, + }); + // string-feat should NOT be here since it's not boolean + expect(span?.attributes['flag.evaluation.string-feat']).toBeUndefined(); }); diff --git a/dev-packages/node-integration-tests/utils/runner/createRunner.ts b/dev-packages/node-integration-tests/utils/runner/createRunner.ts index ed94c1e1bd5a..de845c3063ac 100644 --- a/dev-packages/node-integration-tests/utils/runner/createRunner.ts +++ b/dev-packages/node-integration-tests/utils/runner/createRunner.ts @@ -8,6 +8,7 @@ import type { SerializedLogContainer, SerializedMetricContainer, SerializedSession, + SerializedStreamedSpan, SerializedStreamedSpanContainer, SessionAggregates, TransactionEvent, @@ -19,7 +20,7 @@ import { existsSync } from 'fs'; import { tmpdir } from 'os'; import { join } from 'path'; import { inspect } from 'util'; -import { onTestFailed } from 'vitest'; +import { onTestFailed, onTestFinished } from 'vitest'; import type { DeepPartial } from './../assertions'; import { assertEnvelopeHeader, @@ -37,6 +38,15 @@ import { type VoidFunction = () => void; +type SegmentMatcher = string | ((span: SerializedStreamedSpan) => boolean); + +type SpanCollector = { + done: boolean; + spanCount: number; + add(spans: SerializedStreamedSpan[]): void; + reject(error: Error): void; +}; + type ExpectedEvent = Partial | ((event: Event) => void); type ExpectedTransaction = DeepPartial | ((event: TransactionEvent) => void); type ExpectedSession = Partial | ((event: SerializedSession) => void); @@ -158,6 +168,10 @@ export function createRunner(...paths: string[]) { } const expectedEnvelopes: Expected[] = []; + const spanCollectors: SpanCollector[] = []; + let started = false; + let cleanupError: Error | undefined; + let stopRunner: ((error: Error) => void) | undefined; let expectedEnvelopeHeaders: ExpectedEnvelopeHeader[] | undefined = undefined; const flags: string[] = []; // By default, we ignore session & sessions @@ -190,14 +204,72 @@ export function createRunner(...paths: string[]) { CLEANUP_STEPS.add(step); } + function cleanup(): void { + cleanupError ??= new Error('Test runner cleaned up before collection completed.'); + stopRunner?.(cleanupError); + for (const collector of spanCollectors) { + if (!collector.done) collector.reject(cleanupError); + } + for (const step of runnerCleanupSteps) { + step(); + CLEANUP_STEPS.delete(step); + } + runnerCleanupSteps.clear(); + } + + function collectStreamedSpans( + isDone: (spansOfTrace: SerializedStreamedSpan[]) => boolean, + ): Promise { + if (started || cleanupError) { + throw new Error('Register span collectors before calling start() or cleanup().'); + } + + const promise = new Promise((resolve, reject) => { + const spansByTrace = new Map(); + const collector: SpanCollector = { + done: false, + spanCount: 0, + reject(error) { + spansByTrace.clear(); + reject(error); + }, + add(spans) { + collector.spanCount += spans.length; + for (const span of spans) { + const traceSpans = spansByTrace.get(span.trace_id); + if (traceSpans) { + traceSpans.push(span); + } else { + spansByTrace.set(span.trace_id, [span]); + } + } + for (const traceSpans of spansByTrace.values()) { + if (isDone(traceSpans)) { + collector.done = true; + resolve(traceSpans); + spansByTrace.clear(); + return; + } + } + }, + }; + spanCollectors.push(collector); + }); + // completed() can reject before the caller reaches await spansPromise. + promise.catch(() => {}); + return promise; + } + return { /** Run (and de-register) only the cleanup steps registered by this runner. */ - cleanup: function (): void { - for (const step of runnerCleanupSteps) { - step(); - CLEANUP_STEPS.delete(step); - } - runnerCleanupSteps.clear(); + cleanup, + /** Register before start(). Collect across envelopes until one trace satisfies the predicate. */ + collectStreamedSpans, + /** Collect until a matching segment arrives; children arriving later are not included. */ + collectStreamedSpansUntilSegment(segment: SegmentMatcher = () => true): Promise { + const matchesSegment = + typeof segment === 'string' ? (span: SerializedStreamedSpan) => span.name === segment : segment; + return collectStreamedSpans(spans => spans.some(span => span.is_segment && matchesSegment(span))); }, expect: function (expected: Expected) { if (ensureNoErrorOutput) { @@ -274,10 +346,25 @@ export function createRunner(...paths: string[]) { return this; }, start: function (): StartResult { + if (started || cleanupError) { + throw new Error('A test runner can only be started once, before cleanup().'); + } + if ( + spanCollectors.length > 0 && + (ensureNoErrorOutput || expectedEnvelopeHeaders || expectedEnvelopes.some(expected => 'span' in expected)) + ) { + const error = new Error( + 'Span collectors cannot be combined with span/header expectations or ensureNoErrorOutput().', + ); + for (const collector of spanCollectors) collector.reject(error); + throw error; + } + started = true; let isComplete = false; let completeError: Error | undefined; const expectedEnvelopeCount = Math.max(expectedEnvelopes.length, (expectedEnvelopeHeaders || []).length); + const hasExpectations = expectedEnvelopeCount > 0 || spanCollectors.length > 0; let envelopeCount = 0; let scenarioServerPort: number | undefined; @@ -301,6 +388,10 @@ export function createRunner(...paths: string[]) { dumpCapturedLogs(); } }); + if (spanCollectors.length > 0) { + onTestFinished(cleanup); + } + stopRunner = complete; function complete(error?: Error): void { if (isComplete) { @@ -309,6 +400,11 @@ export function createRunner(...paths: string[]) { isComplete = true; completeError = error || undefined; + if (error) { + for (const collector of spanCollectors) { + if (!collector.done) collector.reject(error); + } + } stopChild(); completedDeferred.resolve(); } @@ -350,6 +446,7 @@ export function createRunner(...paths: string[]) { `pid=${child?.pid ?? 'none'}`, hasExited ? `exited (${exitStatus})` : 'running', `envelopes=${envelopeCount}/${expectedEnvelopeCount}`, + ...(spanCollectors.length > 0 ? [collectorProgress()] : []), `ms since spawn=${spawnedAt ? now - spawnedAt : 'not spawned'}`, `ms since last output=${lastOutputAt ? now - lastOutputAt : 'no output'}`, ].join(', '); @@ -369,20 +466,48 @@ export function createRunner(...paths: string[]) { console.log('--- End of captured child process output ---\n'); } + function collectorProgress(): string { + const pending = spanCollectors.filter(collector => !collector.done); + return `pending span collectors=${pending.length}, received spans=[${pending.map(c => c.spanCount).join(', ')}]`; + } + + function checkComplete(): void { + if ( + hasExpectations && + envelopeCount === expectedEnvelopeCount && + spanCollectors.every(collector => collector.done) + ) { + complete(); + } + } + /** Called after each expect callback to check if we're complete */ function expectCallbackCalled(): void { envelopeCount++; - if (envelopeCount === expectedEnvelopeCount) { - complete(); - } + checkComplete(); } function newEnvelope(envelope: Envelope): void { if (process.env.DEBUG) log('newEnvelope', inspect(envelope, false, null, true)); for (const item of envelope[1]) { + if (isComplete) return; const envelopeItemType = item[0].type; + if (envelopeItemType === 'span' && spanCollectors.length > 0) { + try { + const container = item[1] as SerializedStreamedSpanContainer; + for (const collector of spanCollectors) { + if (!collector.done) collector.add(container.items); + } + } catch (error) { + complete(error instanceof Error ? error : new Error(String(error))); + return; + } + checkComplete(); + continue; + } + if (ignored.has(envelopeItemType)) { continue; } @@ -425,6 +550,7 @@ export function createRunner(...paths: string[]) { }); if (matchIndex < 0) { + if (spanCollectors.length > 0) continue; return; } @@ -436,6 +562,7 @@ export function createRunner(...paths: string[]) { // Catch any error or failed assertions and pass them to done to end the test quickly try { if (!expected) { + if (spanCollectors.length > 0) continue; return; } @@ -466,6 +593,10 @@ export function createRunner(...paths: string[]) { serverStartup .then(([mockServerPort, mockServerClose]) => { + if (isComplete) { + mockServerClose?.(); + return; + } if (mockServerClose) { registerCleanupStep(() => { mockServerClose(); @@ -497,8 +628,7 @@ export function createRunner(...paths: string[]) { // `should-exit`, `ensureNoErrorOutput`) verify the child exits // naturally and auto-flush would delay that with retrying HTTP // requests to the fake DSN. - const wantsAutoFlush = - !ensureNoErrorOutput && (expectedEnvelopes.length > 0 || (expectedEnvelopeHeaders?.length ?? 0) > 0); + const wantsAutoFlush = !ensureNoErrorOutput && hasExpectations; const runtime = getRuntime(); const childFlags = wantsAutoFlush ? [...buildAutoFlushFlags(flags, testPath, runtime), ...flags] : flags; const entryPath = buildScenario(runtime, testPath); @@ -550,12 +680,12 @@ export function createRunner(...paths: string[]) { // child output is dumped by `completed()`. In the success path `complete()` has already // run (so `isComplete` short-circuits this), server-style tests are killed by `complete()` // first, and tests that expect no envelopes drive completion some other way. - if (!isComplete && expectedEnvelopeCount > 0) { + if (!isComplete && hasExpectations) { const how = signal ? `signal ${signal}` : `code ${code ?? 'unknown'}`; complete( new Error( `Scenario exited (${how}) after ${envelopeCount}/${expectedEnvelopeCount} expected ` + - 'envelope(s), before the test completed.', + `envelope(s), before the test completed.${spanCollectors.length > 0 ? ` ${collectorProgress()}` : ''}`, ), ); } @@ -616,6 +746,7 @@ export function createRunner(...paths: string[]) { try { await waitForEvent(completedDeferred.promise, 120_000, 'Timed out waiting for test to complete'); } catch (e) { + complete(e instanceof Error ? e : new Error(String(e))); // On timeout, dump the captured child output (same info `DEBUG=1` would have streamed live) // so CI failures are diagnosable without re-running locally with DEBUG enabled. dumpCapturedLogs(); From ce18d900565657d702f14dca76cea944da7fe7a4 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Wed, 7 Oct 2026 14:41:57 +0200 Subject: [PATCH 03/47] test(node): Port OpenAI integration tests to span streaming (#25081) Port the OpenAI v5/v6/v7, Azure and tool-call integration tests to the default span-streaming lifecycle. I know this looks huge but it really isn't. This just uses the new `collectStreamedSpansUntilSegment` helper and that changes indentation that's why the diff is so large mostly. Part of #24138 --------- Co-authored-by: GPT-6 --- .../tracing/openai/instrument-root-span.mjs | 11 - .../openai/instrument-span-streaming.mjs | 11 - .../openai/instrument-with-options.mjs | 7 - .../tracing/openai/instrument-with-pii.mjs | 7 - .../suites/tracing/openai/instrument.mjs | 7 - .../openai-tool-calls/instrument-with-pii.mjs | 1 - .../openai/openai-tool-calls/instrument.mjs | 1 - .../tracing/openai/openai-tool-calls/test.ts | 36 +- .../suites/tracing/openai/test.ts | 3177 ++++++++--------- .../openai/v6/instrument-root-span.mjs | 11 - .../openai/v6/instrument-with-options.mjs | 7 - .../tracing/openai/v6/instrument-with-pii.mjs | 7 - .../suites/tracing/openai/v6/instrument.mjs | 7 - .../suites/tracing/openai/v6/test.ts | 1931 +++++----- .../openai/v7/instrument-span-streaming.mjs | 11 - .../suites/tracing/openai/v7/instrument.mjs | 7 - .../suites/tracing/openai/v7/test.ts | 224 +- 17 files changed, 2564 insertions(+), 2899 deletions(-) delete mode 100644 dev-packages/node-integration-tests/suites/tracing/openai/instrument-root-span.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/openai/instrument-span-streaming.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-root-span.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument-span-streaming.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/instrument-root-span.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/instrument-root-span.mjs deleted file mode 100644 index 42052d281304..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/openai/instrument-root-span.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - traceLifecycle: 'static', - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: false, outputs: false } }, - transport: loggingTransport, -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/instrument-span-streaming.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/instrument-span-streaming.mjs deleted file mode 100644 index 657bed0a3a8c..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/openai/instrument-span-streaming.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: true, outputs: true } }, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-options.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-options.mjs index 0d6548af1291..913758cd6078 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-options.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-options.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, @@ -14,10 +13,4 @@ Sentry.init({ recordOutputs: true, }), ], - beforeSendTransaction: event => { - if (event.transaction.includes('/openai/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-pii.mjs index 2b78a3ea1987..1c507ba84b9c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/instrument-with-pii.mjs @@ -2,16 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: true, outputs: true } }, transport: loggingTransport, - beforeSendTransaction: event => { - if (event.transaction.includes('/openai/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/instrument.mjs index 2097d76a4eff..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/instrument.mjs @@ -2,16 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, - beforeSendTransaction: event => { - if (event.transaction.includes('/openai/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/instrument-with-pii.mjs index bdf24d6bbd0e..1c507ba84b9c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/instrument-with-pii.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/instrument.mjs index 42052d281304..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/instrument.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts index 5dfd13c21f7f..59473fcea8fe 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts @@ -79,14 +79,15 @@ describe('OpenAI Tool Calls integration', () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { test('creates openai tool calls related spans with genAI recording disabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(4); - const chatToolsSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(4); + const chatToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-tools-123', ); - expect(chatToolsSpan).toBeDefined(); expect(chatToolsSpan!.name).toBe('chat gpt-4'); expect(chatToolsSpan!.status).toBe('ok'); expect(chatToolsSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -133,10 +134,9 @@ describe('OpenAI Tool Calls integration', () => { value: 40, }); - const streamingChatToolsSpan = container.items.find( + const streamingChatToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-tools-123', ); - expect(streamingChatToolsSpan).toBeDefined(); expect(streamingChatToolsSpan!.name).toBe('chat gpt-4'); expect(streamingChatToolsSpan!.status).toBe('ok'); expect(streamingChatToolsSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -193,10 +193,9 @@ describe('OpenAI Tool Calls integration', () => { value: 40, }); - const responsesToolsSpan = container.items.find( + const responsesToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_tools_789', ); - expect(responsesToolsSpan).toBeDefined(); expect(responsesToolsSpan!.name).toBe('chat gpt-4'); expect(responsesToolsSpan!.status).toBe('ok'); expect(responsesToolsSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -246,10 +245,9 @@ describe('OpenAI Tool Calls integration', () => { value: 20, }); - const streamingResponsesToolsSpan = container.items.find( + const streamingResponsesToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_tools_789', ); - expect(streamingResponsesToolsSpan).toBeDefined(); expect(streamingResponsesToolsSpan!.name).toBe('chat gpt-4'); expect(streamingResponsesToolsSpan!.status).toBe('ok'); expect(streamingResponsesToolsSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -315,14 +313,15 @@ describe('OpenAI Tool Calls integration', () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates openai tool calls related spans with genAI recording enabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(4); - const chatToolsSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(4); + const chatToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-tools-123', ); - expect(chatToolsSpan).toBeDefined(); expect(chatToolsSpan!.name).toBe('chat gpt-4'); expect(chatToolsSpan!.status).toBe('ok'); expect(chatToolsSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -383,10 +382,9 @@ describe('OpenAI Tool Calls integration', () => { value: 40, }); - const streamingChatToolsSpan = container.items.find( + const streamingChatToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-tools-123', ); - expect(streamingChatToolsSpan).toBeDefined(); expect(streamingChatToolsSpan!.name).toBe('chat gpt-4'); expect(streamingChatToolsSpan!.status).toBe('ok'); expect(streamingChatToolsSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -454,10 +452,9 @@ describe('OpenAI Tool Calls integration', () => { value: 40, }); - const responsesToolsSpan = container.items.find( + const responsesToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_tools_789', ); - expect(responsesToolsSpan).toBeDefined(); expect(responsesToolsSpan!.name).toBe('chat gpt-4'); expect(responsesToolsSpan!.status).toBe('ok'); expect(responsesToolsSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ @@ -518,10 +515,9 @@ describe('OpenAI Tool Calls integration', () => { value: 20, }); - const streamingResponsesToolsSpan = container.items.find( + const streamingResponsesToolsSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_tools_789', ); - expect(streamingResponsesToolsSpan).toBeDefined(); expect(streamingResponsesToolsSpan!.name).toBe('chat gpt-4'); expect(streamingResponsesToolsSpan!.status).toBe('ok'); expect(streamingResponsesToolsSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts index 2da282ebabb4..754328af131a 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/test.ts @@ -34,1634 +34,1587 @@ describe('OpenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-chat.mjs', 'instrument.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording disabled', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(8); - const chatCompletionSpan = container.items.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', - ); - expect(chatCompletionSpan).toBeDefined(); - expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(chatCompletionSpan!.status).toBe('ok'); - expect(chatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ - type: 'double', - value: 0.7, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-mock123', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["stop"]', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 15, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 25, - }); - - const responsesSpan = container.items.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', - ); - expect(responsesSpan).toBeDefined(); - expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(responsesSpan!.status).toBe('ok'); - expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(responsesSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(responsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(responsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); - expect(responsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'resp_mock456', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["completed"]', - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 5, - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 8, - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 13, - }); - - const nonStreamingErrorSpan = container.items.find( - span => - span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, - ); - expect(nonStreamingErrorSpan).toBeDefined(); - expect(nonStreamingErrorSpan!.name).toBe('chat error-model'); - expect(nonStreamingErrorSpan!.status).toBe('error'); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(nonStreamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(nonStreamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); - - const streamingChatCompletionSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', - ); - expect(streamingChatCompletionSpan).toBeDefined(); - expect(streamingChatCompletionSpan!.name).toBe('chat gpt-4'); - expect(streamingChatCompletionSpan!.status).toBe('ok'); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ - type: 'double', - value: 0.8, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-stream-123', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["stop"]', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 12, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 18, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 30, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ - type: 'boolean', - value: true, - }); - - const streamingResponsesSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', - ); - expect(streamingResponsesSpan).toBeDefined(); - expect(streamingResponsesSpan!.name).toBe('chat gpt-4'); - expect(streamingResponsesSpan!.status).toBe('ok'); - expect(streamingResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingResponsesSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'resp_stream_456', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["in_progress","completed"]', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 6, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 16, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ - type: 'boolean', - value: true, - }); - - const streamingErrorSpan = container.items.find( - span => - span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, - ); - expect(streamingErrorSpan).toBeDefined(); - expect(streamingErrorSpan!.name).toBe('chat error-model'); - expect(streamingErrorSpan!.status).toBe('error'); - expect(streamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); - expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - - const parsedChatCompletionSpan = container.items.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', - ); - expect(parsedChatCompletionSpan).toBeDefined(); - expect(parsedChatCompletionSpan!.name).toBe('chat gpt-4o'); - expect(parsedChatCompletionSpan!.status).toBe('ok'); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(parsedChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(parsedChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4o', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4o', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-mock123', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["stop"]', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 15, - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 25, - }); - - const parsedResponsesSpan = container.items.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', - ); - expect(parsedResponsesSpan).toBeDefined(); - expect(parsedResponsesSpan!.name).toBe('chat gpt-4o'); - expect(parsedResponsesSpan!.status).toBe('ok'); - expect(parsedResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(parsedResponsesSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(parsedResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); - expect(parsedResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4o', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4o', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'resp_mock456', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["completed"]', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 5, - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 8, - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 13, - }); - - // GenAI inputs and outputs must never be recorded when `dataCollection.genAI` disables them. - // Asserting over every span in the envelope catches attributes leaking onto - // spans which are not individually inspected above. - for (const span of container.items) { - expect(span.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); - expect(span.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toBeUndefined(); - expect(span.attributes[GEN_AI_RESPONSE_TEXT]).toBeUndefined(); - expect(span.attributes[GEN_AI_EMBEDDINGS_INPUT]).toBeUndefined(); - } - }, - }) - .start() - .completed(); + const runner = createRunner(); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const allSpans = await spansPromise; + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(8); + const chatCompletionSpan = spans.find( + span => + span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', + ); + expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(chatCompletionSpan!.status).toBe('ok'); + expect(chatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ + type: 'double', + value: 0.7, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-mock123', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + + const responsesSpan = spans.find( + span => + span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', + ); + expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(responsesSpan!.status).toBe('ok'); + expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(responsesSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(responsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(responsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(responsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'resp_mock456', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["completed"]', + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 5, + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 8, + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 13, + }); + + const nonStreamingErrorSpan = spans.find( + span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, + ); + expect(nonStreamingErrorSpan!.status).toBe('error'); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(nonStreamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(nonStreamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); + + const streamingChatCompletionSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', + ); + expect(streamingChatCompletionSpan!.name).toBe('chat gpt-4'); + expect(streamingChatCompletionSpan!.status).toBe('ok'); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ + type: 'double', + value: 0.8, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-stream-123', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 12, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 18, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ + type: 'boolean', + value: true, + }); + + const streamingResponsesSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', + ); + expect(streamingResponsesSpan!.name).toBe('chat gpt-4'); + expect(streamingResponsesSpan!.status).toBe('ok'); + expect(streamingResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingResponsesSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'resp_stream_456', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["in_progress","completed"]', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 6, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 16, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ + type: 'boolean', + value: true, + }); + + const streamingErrorSpan = spans.find( + span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, + ); + expect(streamingErrorSpan!.status).toBe('error'); + expect(streamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + + const parsedChatCompletionSpan = spans.find( + span => + span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', + ); + expect(parsedChatCompletionSpan!.name).toBe('chat gpt-4o'); + expect(parsedChatCompletionSpan!.status).toBe('ok'); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(parsedChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(parsedChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4o', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4o', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-mock123', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + + const parsedResponsesSpan = spans.find( + span => + span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', + ); + expect(parsedResponsesSpan!.name).toBe('chat gpt-4o'); + expect(parsedResponsesSpan!.status).toBe('ok'); + expect(parsedResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(parsedResponsesSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(parsedResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(parsedResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4o', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4o', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'resp_mock456', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["completed"]', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 5, + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 8, + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 13, + }); + + // GenAI inputs and outputs must never be recorded when `dataCollection.genAI` disables them. + // Asserting over every GenAI span catches attributes leaking onto + // spans which are not individually inspected above. + for (const span of spans) { + expect(span.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); + expect(span.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toBeUndefined(); + expect(span.attributes[GEN_AI_RESPONSE_TEXT]).toBeUndefined(); + expect(span.attributes[GEN_AI_EMBEDDINGS_INPUT]).toBeUndefined(); + } }); }); createEsmAndCjsTests(__dirname, 'scenario-chat.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording enabled', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(8); - const chatCompletionSpan = container.items.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', - ); - expect(chatCompletionSpan).toBeDefined(); - expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(chatCompletionSpan!.status).toBe('ok'); - expect(chatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ - type: 'double', - value: 0.7, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: '[{"role":"user","content":"What is the capital of France?"}]', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toEqual({ - type: 'string', - value: JSON.stringify([{ type: 'text', content: 'You are a helpful assistant.' }]), - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-mock123', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["stop"]', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ - type: 'string', - value: '["Hello from OpenAI mock!"]', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 15, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 25, - }); - - const responsesSpan = container.items.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', - ); - expect(responsesSpan).toBeDefined(); - expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(responsesSpan!.status).toBe('ok'); - expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(responsesSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(responsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(responsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); - expect(responsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(responsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: 'Translate this to French: Hello', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ - type: 'string', - value: 'Response to: Translate this to French: Hello', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["completed"]', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'resp_mock456', - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 5, - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 8, - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 13, - }); - - const nonStreamingErrorSpan = container.items.find( - span => - span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, - ); - expect(nonStreamingErrorSpan).toBeDefined(); - expect(nonStreamingErrorSpan!.name).toBe('chat error-model'); - expect(nonStreamingErrorSpan!.status).toBe('error'); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(nonStreamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(nonStreamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: '[{"role":"user","content":"This will fail"}]', - }); - - const streamingChatCompletionSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', - ); - expect(streamingChatCompletionSpan).toBeDefined(); - expect(streamingChatCompletionSpan!.name).toBe('chat gpt-4'); - expect(streamingChatCompletionSpan!.status).toBe('ok'); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ - type: 'double', - value: 0.8, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: '[{"role":"user","content":"Tell me about streaming"}]', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toEqual({ - type: 'string', - value: JSON.stringify([{ type: 'text', content: 'You are a helpful assistant.' }]), - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ - type: 'string', - value: 'Hello from OpenAI streaming!', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["stop"]', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-stream-123', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 12, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 18, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 30, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ - type: 'boolean', - value: true, - }); - - const streamingResponsesSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', - ); - expect(streamingResponsesSpan).toBeDefined(); - expect(streamingResponsesSpan!.name).toBe('chat gpt-4'); - expect(streamingResponsesSpan!.status).toBe('ok'); - expect(streamingResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingResponsesSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: 'Test streaming responses API', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ - type: 'string', - value: 'Streaming response to: Test streaming responses APITest streaming responses API', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["in_progress","completed"]', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'resp_stream_456', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 6, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 16, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ - type: 'boolean', - value: true, - }); - - const streamingErrorSpan = container.items.find( - span => - span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, - ); - expect(streamingErrorSpan).toBeDefined(); - expect(streamingErrorSpan!.name).toBe('chat error-model'); - expect(streamingErrorSpan!.status).toBe('error'); - expect(streamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); - expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingErrorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: '[{"role":"user","content":"This will fail"}]', - }); - expect(streamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - - const parsedChatCompletionSpan = container.items.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', - ); - expect(parsedChatCompletionSpan).toBeDefined(); - expect(parsedChatCompletionSpan!.name).toBe('chat gpt-4o'); - expect(parsedChatCompletionSpan!.status).toBe('ok'); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(parsedChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(parsedChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4o', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: '[{"role":"user","content":"What is the capital of France?"}]', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toBeUndefined(); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4o', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-mock123', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["stop"]', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ - type: 'string', - value: '["Hello from OpenAI mock!"]', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 15, - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 25, - }); - - const parsedResponsesSpan = container.items.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', - ); - expect(parsedResponsesSpan).toBeDefined(); - expect(parsedResponsesSpan!.name).toBe('chat gpt-4o'); - expect(parsedResponsesSpan!.status).toBe('ok'); - expect(parsedResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(parsedResponsesSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(parsedResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); - expect(parsedResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4o', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: 'What is the capital of France?', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ - type: 'string', - value: 'Response to: What is the capital of France?', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["completed"]', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4o', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'resp_mock456', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 5, - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 8, - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 13, - }); - }, - }) - .start() - .completed(); + const runner = createRunner(); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const allSpans = await spansPromise; + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(8); + const chatCompletionSpan = spans.find( + span => + span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', + ); + expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(chatCompletionSpan!.status).toBe('ok'); + expect(chatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ + type: 'double', + value: 0.7, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"What is the capital of France?"}]', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toEqual({ + type: 'string', + value: JSON.stringify([{ type: 'text', content: 'You are a helpful assistant.' }]), + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-mock123', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ + type: 'string', + value: '["Hello from OpenAI mock!"]', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + + const responsesSpan = spans.find( + span => + span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', + ); + expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(responsesSpan!.status).toBe('ok'); + expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(responsesSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(responsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(responsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(responsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(responsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: 'Translate this to French: Hello', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ + type: 'string', + value: 'Response to: Translate this to French: Hello', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["completed"]', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'resp_mock456', + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 5, + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 8, + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 13, + }); + + const nonStreamingErrorSpan = spans.find( + span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, + ); + expect(nonStreamingErrorSpan!.status).toBe('error'); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(nonStreamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(nonStreamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"This will fail"}]', + }); + + const streamingChatCompletionSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', + ); + expect(streamingChatCompletionSpan!.name).toBe('chat gpt-4'); + expect(streamingChatCompletionSpan!.status).toBe('ok'); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ + type: 'double', + value: 0.8, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"Tell me about streaming"}]', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toEqual({ + type: 'string', + value: JSON.stringify([{ type: 'text', content: 'You are a helpful assistant.' }]), + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ + type: 'string', + value: 'Hello from OpenAI streaming!', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-stream-123', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 12, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 18, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ + type: 'boolean', + value: true, + }); + + const streamingResponsesSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', + ); + expect(streamingResponsesSpan!.name).toBe('chat gpt-4'); + expect(streamingResponsesSpan!.status).toBe('ok'); + expect(streamingResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingResponsesSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: 'Test streaming responses API', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ + type: 'string', + value: 'Streaming response to: Test streaming responses APITest streaming responses API', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["in_progress","completed"]', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'resp_stream_456', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 6, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 16, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ + type: 'boolean', + value: true, + }); + + const streamingErrorSpan = spans.find( + span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, + ); + expect(streamingErrorSpan!.status).toBe('error'); + expect(streamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingErrorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"This will fail"}]', + }); + expect(streamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + + const parsedChatCompletionSpan = spans.find( + span => + span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', + ); + expect(parsedChatCompletionSpan!.name).toBe('chat gpt-4o'); + expect(parsedChatCompletionSpan!.status).toBe('ok'); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(parsedChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(parsedChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4o', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"What is the capital of France?"}]', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toBeUndefined(); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4o', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-mock123', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ + type: 'string', + value: '["Hello from OpenAI mock!"]', + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + + const parsedResponsesSpan = spans.find( + span => + span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', + ); + expect(parsedResponsesSpan!.name).toBe('chat gpt-4o'); + expect(parsedResponsesSpan!.status).toBe('ok'); + expect(parsedResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(parsedResponsesSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(parsedResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(parsedResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4o', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: 'What is the capital of France?', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ + type: 'string', + value: 'Response to: What is the capital of France?', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["completed"]', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4o', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'resp_mock456', + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 5, + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 8, + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 13, + }); }); }); createEsmAndCjsTests(__dirname, 'scenario-chat.mjs', 'instrument-with-options.mjs', (createRunner, test) => { test('creates openai related spans with custom options', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(8); - const chatCompletionSpan = container.items.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', - ); - expect(chatCompletionSpan).toBeDefined(); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); - expect(chatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - - const streamingChatCompletionSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', - ); - expect(streamingChatCompletionSpan).toBeDefined(); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - - const parsedChatCompletionSpan = container.items.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', - ); - expect(parsedChatCompletionSpan).toBeDefined(); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - - const parsedResponsesSpan = container.items.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', - ); - expect(parsedResponsesSpan).toBeDefined(); - expect(parsedResponsesSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); - expect(parsedResponsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - }, - }) - .start() - .completed(); + const runner = createRunner(); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const allSpans = await spansPromise; + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(8); + const chatCompletionSpan = spans.find( + span => + span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', + ); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(chatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + + const streamingChatCompletionSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', + ); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + + const parsedChatCompletionSpan = spans.find( + span => + span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', + ); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + expect(parsedChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + + const parsedResponsesSpan = spans.find( + span => + span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', + ); + expect(parsedResponsesSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(parsedResponsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + expect(parsedResponsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); }); }); createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording disabled', async () => { + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() - .expect({ - transaction: { - transaction: 'main', - }, - }) + .unordered() .expect({ span: container => { - expect(container.items).toHaveLength(4); - const singleEmbeddingSpan = container.items.find( - span => - span.name === 'embeddings text-embedding-3-small' && - span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] !== undefined, - ); - expect(singleEmbeddingSpan).toBeDefined(); - expect(singleEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); - expect(singleEmbeddingSpan!.status).toBe('ok'); - expect(singleEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(singleEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(singleEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]).toEqual({ - type: 'string', - value: 'float', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]).toEqual({ - type: 'integer', - value: 1536, - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - - const errorEmbeddingSpan = container.items.find(span => span.name === 'embeddings error-model'); - expect(errorEmbeddingSpan).toBeDefined(); - expect(errorEmbeddingSpan!.name).toBe('embeddings error-model'); - expect(errorEmbeddingSpan!.status).toBe('error'); - expect(errorEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(errorEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(errorEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(errorEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(errorEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); - - const multiEmbeddingSpan = container.items.find( - span => - span.name === 'embeddings text-embedding-3-small' && - span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] === undefined, - ); - expect(multiEmbeddingSpan).toBeDefined(); - expect(multiEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); - expect(multiEmbeddingSpan!.status).toBe('ok'); - expect(multiEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(multiEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(multiEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - - // GenAI inputs and outputs must never be recorded when `dataCollection.genAI` disables them. - // Asserting over every span in the envelope catches attributes leaking onto - // spans which are not individually inspected above. - for (const span of container.items) { - expect(span.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); - expect(span.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toBeUndefined(); - expect(span.attributes[GEN_AI_RESPONSE_TEXT]).toBeUndefined(); - expect(span.attributes[GEN_AI_EMBEDDINGS_INPUT]).toBeUndefined(); - } - - const rawEmbeddingSpan = container.items.find(span => span.name === 'embeddings text-embedding-3-large'); - expect(rawEmbeddingSpan).toBeDefined(); - expect(rawEmbeddingSpan!.name).toBe('embeddings text-embedding-3-large'); - expect(rawEmbeddingSpan!.status).toBe('ok'); - expect(rawEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(rawEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(rawEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(rawEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(rawEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-large', - }); - expect(rawEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined(); - expect(rawEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toBeUndefined(); - expect(rawEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toBeUndefined(); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(4); + const singleEmbeddingSpan = spans.find( + span => + span.name === 'embeddings text-embedding-3-small' && + span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] !== undefined, + ); + expect(singleEmbeddingSpan!.status).toBe('ok'); + expect(singleEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(singleEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(singleEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'float', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1536, + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + + const errorEmbeddingSpan = spans.find(span => span.name === 'embeddings error-model'); + expect(errorEmbeddingSpan!.status).toBe('error'); + expect(errorEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(errorEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(errorEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(errorEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(errorEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); + + const multiEmbeddingSpan = spans.find( + span => + span.name === 'embeddings text-embedding-3-small' && + span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] === undefined, + ); + expect(multiEmbeddingSpan!.status).toBe('ok'); + expect(multiEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(multiEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(multiEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + + // GenAI inputs and outputs must never be recorded when `dataCollection.genAI` disables them. + // Asserting over every span in the envelope catches attributes leaking onto + // spans which are not individually inspected above. + for (const span of spans) { + expect(span.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); + expect(span.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toBeUndefined(); + expect(span.attributes[GEN_AI_RESPONSE_TEXT]).toBeUndefined(); + expect(span.attributes[GEN_AI_EMBEDDINGS_INPUT]).toBeUndefined(); + } + + const rawEmbeddingSpan = spans.find(span => span.name === 'embeddings text-embedding-3-large'); + expect(rawEmbeddingSpan!.status).toBe('ok'); + expect(rawEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(rawEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(rawEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(rawEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(rawEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-large', + }); + expect(rawEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined(); + expect(rawEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toBeUndefined(); + expect(rawEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toBeUndefined(); }); }); createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording enabled', async () => { + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() - .expect({ - transaction: { - transaction: 'main', - }, - }) + .unordered() .expect({ span: container => { - expect(container.items).toHaveLength(4); - const singleEmbeddingSpan = container.items.find( - span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', - ); - expect(singleEmbeddingSpan).toBeDefined(); - expect(singleEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); - expect(singleEmbeddingSpan!.status).toBe('ok'); - expect(singleEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(singleEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(singleEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]).toEqual({ - type: 'string', - value: 'float', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]).toEqual({ - type: 'integer', - value: 1536, - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ - type: 'string', - value: 'Embedding test!', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - - const errorEmbeddingSpan = container.items.find( - span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Error embedding test!', - ); - expect(errorEmbeddingSpan).toBeDefined(); - expect(errorEmbeddingSpan!.name).toBe('embeddings error-model'); - expect(errorEmbeddingSpan!.status).toBe('error'); - expect(errorEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(errorEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(errorEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(errorEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(errorEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); - expect(errorEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ - type: 'string', - value: 'Error embedding test!', - }); - - const multiEmbeddingSpan = container.items.find( - span => - span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === - '["First input text","Second input text","Third input text"]', - ); - expect(multiEmbeddingSpan).toBeDefined(); - expect(multiEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); - expect(multiEmbeddingSpan!.status).toBe('ok'); - expect(multiEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(multiEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(multiEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ - type: 'string', - value: '["First input text","Second input text","Third input text"]', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - - const rawEmbeddingSpan = container.items.find(span => span.name === 'embeddings text-embedding-3-large'); - expect(rawEmbeddingSpan).toBeDefined(); - expect(rawEmbeddingSpan!.name).toBe('embeddings text-embedding-3-large'); - expect(rawEmbeddingSpan!.status).toBe('ok'); - expect(rawEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(rawEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(rawEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(rawEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(rawEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-large', - }); - expect(rawEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined(); - expect(rawEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toBeUndefined(); - expect(rawEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toBeUndefined(); - expect(rawEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ - type: 'string', - value: 'Raw embedding test!', - }); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(4); + const singleEmbeddingSpan = spans.find( + span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', + ); + expect(singleEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); + expect(singleEmbeddingSpan!.status).toBe('ok'); + expect(singleEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(singleEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(singleEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'float', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1536, + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ + type: 'string', + value: 'Embedding test!', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + + const errorEmbeddingSpan = spans.find( + span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Error embedding test!', + ); + expect(errorEmbeddingSpan!.name).toBe('embeddings error-model'); + expect(errorEmbeddingSpan!.status).toBe('error'); + expect(errorEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(errorEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(errorEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(errorEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(errorEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(errorEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ + type: 'string', + value: 'Error embedding test!', + }); + + const multiEmbeddingSpan = spans.find( + span => + span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === + '["First input text","Second input text","Third input text"]', + ); + expect(multiEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); + expect(multiEmbeddingSpan!.status).toBe('ok'); + expect(multiEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(multiEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(multiEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ + type: 'string', + value: '["First input text","Second input text","Third input text"]', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + + const rawEmbeddingSpan = spans.find(span => span.name === 'embeddings text-embedding-3-large'); + expect(rawEmbeddingSpan!.status).toBe('ok'); + expect(rawEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(rawEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(rawEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(rawEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(rawEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-large', + }); + expect(rawEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined(); + expect(rawEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toBeUndefined(); + expect(rawEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toBeUndefined(); + expect(rawEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ + type: 'string', + value: 'Raw embedding test!', + }); }); }); - createEsmAndCjsTests(__dirname, 'scenario-root-span.mjs', 'instrument-root-span.mjs', (createRunner, test) => { + createEsmAndCjsTests(__dirname, 'scenario-root-span.mjs', 'instrument.mjs', (createRunner, test) => { test('it works without a wrapping span', async () => { + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() - // First the span that our mock express server is emitting, unrelated to this test - .expect({ - transaction: { - transaction: 'POST /openai/chat/completions', - }, - }) + .unordered() .expect({ - transaction: { - transaction: 'chat gpt-3.5-turbo', - contexts: { - trace: { - span_id: expect.any(String), - trace_id: expect.any(String), - data: { - [GEN_AI_OPERATION_NAME]: 'chat', - [SENTRY_OP]: 'gen_ai.chat', - [SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_PROVIDER_NAME]: 'openai', - [GEN_AI_REQUEST_MODEL]: 'gpt-3.5-turbo', - [GEN_AI_REQUEST_TEMPERATURE]: 0.7, - [GEN_AI_RESPONSE_MODEL]: 'gpt-3.5-turbo', - [GEN_AI_RESPONSE_ID]: 'chatcmpl-mock123', - [GEN_AI_RESPONSE_FINISH_REASONS]: '["stop"]', - [GEN_AI_USAGE_INPUT_TOKENS]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS]: 25, - }, - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }, - }, + span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const segment = receivedSpans.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); + expect(segment?.span_id).toEqual(expect.any(String)); + expect(segment?.trace_id).toEqual(expect.any(String)); + expect(segment?.attributes[GEN_AI_OPERATION_NAME]).toEqual({ value: 'chat', type: 'string' }); + expect(segment?.attributes[SENTRY_OP]).toEqual({ value: 'gen_ai.chat', type: 'string' }); + expect(segment?.attributes[SENTRY_ORIGIN]).toEqual({ value: 'auto.ai.openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ value: 'openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ value: 0.7, type: 'double' }); + expect(segment?.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_ID]).toEqual({ value: 'chatcmpl-mock123', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ value: '["stop"]', type: 'string' }); + expect(segment?.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ value: 10, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ value: 15, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ value: 25, type: 'integer' }); + expect(segment?.status).toEqual('ok'); }); }); createEsmAndCjsTests(__dirname, 'scenario-azure-openai.mjs', 'instrument.mjs', (createRunner, test) => { test('it works with Azure OpenAI', async () => { + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() - // First the span that our mock express server is emitting, unrelated to this test - .expect({ - transaction: { - transaction: 'POST /azureopenai/deployments/:model/chat/completions', - }, - }) + .unordered() .expect({ - transaction: { - transaction: 'chat gpt-3.5-turbo', - contexts: { - trace: { - span_id: expect.any(String), - trace_id: expect.any(String), - data: { - [GEN_AI_OPERATION_NAME]: 'chat', - [SENTRY_OP]: 'gen_ai.chat', - [SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_PROVIDER_NAME]: 'openai', - [GEN_AI_REQUEST_MODEL]: 'gpt-3.5-turbo', - [GEN_AI_REQUEST_TEMPERATURE]: 0.7, - [GEN_AI_RESPONSE_MODEL]: 'gpt-3.5-turbo', - [GEN_AI_RESPONSE_ID]: 'chatcmpl-mock123', - [GEN_AI_RESPONSE_FINISH_REASONS]: '["stop"]', - [GEN_AI_USAGE_INPUT_TOKENS]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS]: 25, - }, - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }, - }, + span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const segment = receivedSpans.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); + expect(segment?.span_id).toEqual(expect.any(String)); + expect(segment?.trace_id).toEqual(expect.any(String)); + expect(segment?.attributes[GEN_AI_OPERATION_NAME]).toEqual({ value: 'chat', type: 'string' }); + expect(segment?.attributes[SENTRY_OP]).toEqual({ value: 'gen_ai.chat', type: 'string' }); + expect(segment?.attributes[SENTRY_ORIGIN]).toEqual({ value: 'auto.ai.openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ value: 'openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ value: 0.7, type: 'double' }); + expect(segment?.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_ID]).toEqual({ value: 'chatcmpl-mock123', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ value: '["stop"]', type: 'string' }); + expect(segment?.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ value: 10, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ value: 15, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ value: 25, type: 'integer' }); + expect(segment?.status).toEqual('ok'); }); }); // Test for conversation ID support (Conversations API and previous_response_id) createEsmAndCjsTests(__dirname, 'scenario-conversation.mjs', 'instrument.mjs', (createRunner, test) => { test('captures conversation ID from Conversations API and previous_response_id', async () => { + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() - .expect({ - transaction: { - transaction: 'conversation-test', - }, - }) + .unordered() .expect({ span: container => { - expect(container.items).toHaveLength(5); - const conversationCreateSpan = container.items.find(span => span.name === 'chat unknown'); - expect(conversationCreateSpan).toBeDefined(); - expect(conversationCreateSpan!.name).toBe('chat unknown'); - expect(conversationCreateSpan!.status).toBe('ok'); - expect(conversationCreateSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(conversationCreateSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(conversationCreateSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(conversationCreateSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(conversationCreateSpan!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ - type: 'string', - value: 'conv_689667905b048191b4740501625afd940c7533ace33a2dab', - }); - - const conversationResponseSpan = container.items.find( - span => - span.attributes[GEN_AI_CONVERSATION_ID]?.value === - 'conv_689667905b048191b4740501625afd940c7533ace33a2dab' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4', - ); - expect(conversationResponseSpan).toBeDefined(); - expect(conversationResponseSpan!.status).toBe('ok'); - expect(conversationResponseSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(conversationResponseSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(conversationResponseSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(conversationResponseSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(conversationResponseSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(conversationResponseSpan!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ - type: 'string', - value: 'conv_689667905b048191b4740501625afd940c7533ace33a2dab', - }); - - const unlinkedResponseSpan = container.items.find( - span => - span.attributes[SENTRY_OP]?.value === 'gen_ai.chat' && - span.attributes[GEN_AI_CONVERSATION_ID] === undefined, - ); - expect(unlinkedResponseSpan).toBeDefined(); - expect(unlinkedResponseSpan!.status).toBe('ok'); - expect(unlinkedResponseSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(unlinkedResponseSpan!.attributes[GEN_AI_CONVERSATION_ID]).toBeUndefined(); - - const previousResponseSpan = container.items.find( - span => span.attributes[GEN_AI_CONVERSATION_ID]?.value === 'resp_mock_conv_123', - ); - expect(previousResponseSpan).toBeDefined(); - expect(previousResponseSpan!.status).toBe('ok'); - expect(previousResponseSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(previousResponseSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(previousResponseSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(previousResponseSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(previousResponseSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(previousResponseSpan!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ - type: 'string', - value: 'resp_mock_conv_123', - }); - - const rawConversationSpan = container.items.find( - span => span.name === 'chat unknown' && span.attributes[GEN_AI_CONVERSATION_ID] === undefined, - ); - expect(rawConversationSpan).toBeDefined(); - expect(rawConversationSpan!.name).toBe('chat unknown'); - expect(rawConversationSpan!.status).toBe('ok'); - expect(rawConversationSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(rawConversationSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(rawConversationSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(rawConversationSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(rawConversationSpan!.attributes[GEN_AI_CONVERSATION_ID]).toBeUndefined(); + expect(container.items.find(span => span.is_segment && span.name === 'conversation-test')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(5); + const conversationCreateSpan = spans.find( + span => span.name === 'chat' && span.attributes[GEN_AI_RESPONSE_ID] !== undefined, + ); + expect(conversationCreateSpan!.status).toBe('ok'); + expect(conversationCreateSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(conversationCreateSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(conversationCreateSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(conversationCreateSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(conversationCreateSpan!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ + type: 'string', + value: 'conv_689667905b048191b4740501625afd940c7533ace33a2dab', + }); + + const conversationResponseSpan = spans.find( + span => + span.attributes[GEN_AI_CONVERSATION_ID]?.value === 'conv_689667905b048191b4740501625afd940c7533ace33a2dab' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4', + ); + expect(conversationResponseSpan!.status).toBe('ok'); + expect(conversationResponseSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(conversationResponseSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(conversationResponseSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(conversationResponseSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(conversationResponseSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(conversationResponseSpan!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ + type: 'string', + value: 'conv_689667905b048191b4740501625afd940c7533ace33a2dab', + }); + + const unlinkedResponseSpan = spans.find( + span => + span.attributes[SENTRY_OP]?.value === 'gen_ai.chat' && + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4' && + span.attributes[GEN_AI_CONVERSATION_ID] === undefined, + ); + expect(unlinkedResponseSpan!.status).toBe('ok'); + expect(unlinkedResponseSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(unlinkedResponseSpan!.attributes[GEN_AI_CONVERSATION_ID]).toBeUndefined(); + + const previousResponseSpan = spans.find( + span => span.attributes[GEN_AI_CONVERSATION_ID]?.value === 'resp_mock_conv_123', + ); + expect(previousResponseSpan!.status).toBe('ok'); + expect(previousResponseSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(previousResponseSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(previousResponseSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(previousResponseSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(previousResponseSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(previousResponseSpan!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ + type: 'string', + value: 'resp_mock_conv_123', + }); + + const rawConversationSpan = spans.find( + span => span.name === 'chat' && span.attributes[GEN_AI_CONVERSATION_ID] === undefined, + ); + expect(rawConversationSpan!.status).toBe('ok'); + expect(rawConversationSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(rawConversationSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(rawConversationSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(rawConversationSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(rawConversationSpan!.attributes[GEN_AI_CONVERSATION_ID]).toBeUndefined(); }); }); // Test for manual conversation ID setting using setConversationId() createEsmAndCjsTests(__dirname, 'scenario-manual-conversation-id.mjs', 'instrument.mjs', (createRunner, test) => { test('attaches manual conversation ID set via setConversationId() to all chat spans', async () => { + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() - .expect({ - transaction: { - transaction: 'chat-with-manual-conversation-id', - }, - }) + .unordered() .expect({ span: container => { - expect(container.items).toHaveLength(3); - - // All three chat completion spans should have the same manually-set conversation ID - for (const span of container.items) { - expect(span!.name).toBe('chat gpt-4'); - expect(span!.status).toBe('ok'); - expect(span!.attributes['gen_ai.conversation.id']).toEqual({ - type: 'string', - value: 'user_chat_session_abc123', - }); - expect(span!.attributes['gen_ai.provider.name']).toEqual({ type: 'string', value: 'openai' }); - expect(span!.attributes['gen_ai.request.model']).toEqual({ type: 'string', value: 'gpt-4' }); - expect(span!.attributes['gen_ai.operation.name']).toEqual({ type: 'string', value: 'chat' }); - expect(span!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' }); - } + expect( + container.items.find(span => span.is_segment && span.name === 'chat-with-manual-conversation-id'), + ).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(3); + + // All three chat completion spans should have the same manually-set conversation ID + for (const span of spans) { + expect(span!.name).toBe('chat gpt-4'); + expect(span!.status).toBe('ok'); + expect(span!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ + type: 'string', + value: 'user_chat_session_abc123', + }); + expect(span!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(span!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(span!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'chat' }); + expect(span!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); + } }); }); createEsmAndCjsTests(__dirname, 'scenario-separate-scope-1.mjs', 'instrument.mjs', (createRunner, test) => { test('isolates conversation IDs across separate scopes - conversation 1', async () => { + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() - .expect({ - transaction: { - transaction: 'GET /chat/conversation-1', - }, - }) + .unordered() .expect({ span: container => { - expect(container.items).toHaveLength(2); - - // Both chat completion spans should have the expected conversation ID - for (const span of container.items) { - expect(span!.name).toBe('chat gpt-4'); - expect(span!.status).toBe('ok'); - expect(span!.attributes['gen_ai.conversation.id']).toEqual({ - type: 'string', - value: 'conv_user1_session_abc', - }); - expect(span!.attributes['gen_ai.provider.name']).toEqual({ type: 'string', value: 'openai' }); - expect(span!.attributes['gen_ai.request.model']).toEqual({ type: 'string', value: 'gpt-4' }); - expect(span!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' }); - } + expect( + container.items.find(span => span.is_segment && span.name === 'GET /chat/conversation-1'), + ).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(2); + + // Both chat completion spans should have the expected conversation ID + for (const span of spans) { + expect(span!.name).toBe('chat gpt-4'); + expect(span!.status).toBe('ok'); + expect(span!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ + type: 'string', + value: 'conv_user1_session_abc', + }); + expect(span!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(span!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(span!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); + } }); }); createEsmAndCjsTests(__dirname, 'scenario-separate-scope-2.mjs', 'instrument.mjs', (createRunner, test) => { test('isolates conversation IDs across separate scopes - conversation 2', async () => { + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() - .expect({ - transaction: { - transaction: 'GET /chat/conversation-2', - }, - }) + .unordered() .expect({ span: container => { - expect(container.items).toHaveLength(2); - - // Both chat completion spans should have the expected conversation ID - for (const span of container.items) { - expect(span!.name).toBe('chat gpt-4'); - expect(span!.status).toBe('ok'); - expect(span!.attributes['gen_ai.conversation.id']).toEqual({ - type: 'string', - value: 'conv_user2_session_xyz', - }); - expect(span!.attributes['gen_ai.provider.name']).toEqual({ type: 'string', value: 'openai' }); - expect(span!.attributes['gen_ai.request.model']).toEqual({ type: 'string', value: 'gpt-4' }); - expect(span!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' }); - } + expect( + container.items.find(span => span.is_segment && span.name === 'GET /chat/conversation-2'), + ).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(2); + + // Both chat completion spans should have the expected conversation ID + for (const span of spans) { + expect(span!.name).toBe('chat gpt-4'); + expect(span!.status).toBe('ok'); + expect(span!.attributes[GEN_AI_CONVERSATION_ID]).toEqual({ + type: 'string', + value: 'conv_user2_session_xyz', + }); + expect(span!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(span!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ type: 'string', value: 'gpt-4' }); + expect(span!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); + } }); }); @@ -1671,166 +1624,68 @@ describe('OpenAI integration', () => { 'instrument-with-pii.mjs', (createRunner, test) => { test('extracts system instructions from messages', async () => { + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() - .expect({ - transaction: { - transaction: 'main', - }, - }) + .unordered() .expect({ span: container => { - expect(container.items).toHaveLength(1); - const [firstSpan] = container.items; - - // [0] chat completion with system instructions extracted from messages - expect(firstSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toEqual({ - type: 'string', - value: JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]), - }); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(1); + const [firstSpan] = spans; + + // [0] chat completion with system instructions extracted from messages + expect(firstSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toEqual({ + type: 'string', + value: JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]), + }); }); }, ); createEsmAndCjsTests(__dirname, 'scenario-with-response.mjs', 'instrument.mjs', (createRunner, test) => { test('preserves .withResponse() method and works correctly', async () => { - await createRunner() - .expect({ - transaction: { - transaction: 'main', - }, - }) - .expect({ - span: container => { - expect(container.items).toHaveLength(2); - - for (const span of container.items) { - expect(span!.name).toBe('chat gpt-4'); - expect(span!.status).toBe('ok'); - expect(span!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'chat' }); - expect(span!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ type: 'string', value: 'gpt-4' }); - } - - const parsedSpans = container.items.filter(span => span.attributes[GEN_AI_RESPONSE_ID] !== undefined); - expect(parsedSpans).toHaveLength(1); - expect(parsedSpans[0]!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-withresponse', - }); - - const rawSpans = container.items.filter(span => span.attributes[GEN_AI_RESPONSE_ID] === undefined); - expect(rawSpans).toHaveLength(1); - expect(rawSpans[0]!.attributes[GEN_AI_RESPONSE_TEXT]).toBeUndefined(); - expect(rawSpans[0]!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toBeUndefined(); - }, - }) - .start() - .completed(); - }); - }); - - createEsmAndCjsTests(__dirname, 'scenario-chat.mjs', 'instrument-span-streaming.mjs', (createRunner, test) => { - test('creates openai related spans with span streaming enabled', async () => { - const spans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .expect({ span: container => { - spans.push(...container.items); - const chatCompletionSpan = spans.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-3.5-turbo', - ); - expect(chatCompletionSpan).toBeDefined(); - expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(chatCompletionSpan!.status).toBe('ok'); - expect(chatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'chat' }); - expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: '[{"role":"user","content":"What is the capital of France?"}]', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); - - const parsedChatCompletionSpan = spans.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', - ); - expect(parsedChatCompletionSpan).toBeDefined(); - expect(parsedChatCompletionSpan!.name).toBe('chat gpt-4o'); - expect(parsedChatCompletionSpan!.status).toBe('ok'); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(parsedChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(parsedChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4o', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: '[{"role":"user","content":"What is the capital of France?"}]', - }); - expect(parsedChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - - const parsedResponsesSpan = spans.find( - span => - span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456' && - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'gpt-4o', - ); - expect(parsedResponsesSpan).toBeDefined(); - expect(parsedResponsesSpan!.name).toBe('chat gpt-4o'); - expect(parsedResponsesSpan!.status).toBe('ok'); - expect(parsedResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'chat' }); - expect(parsedResponsesSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(parsedResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4o', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: 'What is the capital of France?', - }); - expect(parsedResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(2); + + for (const span of spans) { + expect(span!.name).toBe('chat gpt-4'); + expect(span!.status).toBe('ok'); + expect(span!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'chat' }); + expect(span!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ type: 'string', value: 'gpt-4' }); + } + + const parsedSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID] !== undefined); + expect(parsedSpans).toHaveLength(1); + expect(parsedSpans[0]!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-withresponse', + }); + + const rawSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID] === undefined); + expect(rawSpans).toHaveLength(1); + expect(rawSpans[0]!.attributes[GEN_AI_RESPONSE_TEXT]).toBeUndefined(); + expect(rawSpans[0]!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toBeUndefined(); }); }); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-root-span.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-root-span.mjs deleted file mode 100644 index 42052d281304..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-root-span.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - traceLifecycle: 'static', - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: false, outputs: false } }, - transport: loggingTransport, -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-with-options.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-with-options.mjs index 0169719e5d59..8bdeb0ef483f 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-with-options.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-with-options.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, @@ -15,10 +14,4 @@ Sentry.init({ }), ], debug: true, - beforeSendTransaction: event => { - if (event.transaction.includes('/openai/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-with-pii.mjs index 2b78a3ea1987..1c507ba84b9c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument-with-pii.mjs @@ -2,16 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: true, outputs: true } }, transport: loggingTransport, - beforeSendTransaction: event => { - if (event.transaction.includes('/openai/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument.mjs index 2097d76a4eff..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v6/instrument.mjs @@ -2,16 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, - beforeSendTransaction: event => { - if (event.transaction.includes('/openai/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts index 49d5cb48bd9d..c93586010b03 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v6/test.ts @@ -1,3 +1,4 @@ +import type { SerializedStreamedSpanContainer } from '@sentry/core'; import { afterAll, describe, expect } from 'vitest'; import { GEN_AI_EMBEDDINGS_INPUT, @@ -36,300 +37,287 @@ describe('OpenAI integration (V6)', () => { 'instrument.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording disabled (v6)', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(6); - const chatCompletionSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', - ); - expect(chatCompletionSpan).toBeDefined(); - expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(chatCompletionSpan!.status).toBe('ok'); - expect(chatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ - type: 'double', - value: 0.7, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-mock123', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["stop"]', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 15, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 25, - }); + const runner = createRunner(); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); - const responsesSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456', - ); - expect(responsesSpan).toBeDefined(); - expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(responsesSpan!.status).toBe('ok'); - expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(responsesSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(responsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(responsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); - expect(responsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'resp_mock456', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["completed"]', - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 5, - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 8, - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 13, - }); + await runner.start().completed(); - const nonStreamingErrorSpan = container.items.find( - span => - span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, - ); - expect(nonStreamingErrorSpan).toBeDefined(); - expect(nonStreamingErrorSpan!.name).toBe('chat error-model'); - expect(nonStreamingErrorSpan!.status).toBe('error'); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(nonStreamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(nonStreamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); + const allSpans = await spansPromise; + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(6); + const chatCompletionSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', + ); + expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(chatCompletionSpan!.status).toBe('ok'); + expect(chatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ + type: 'double', + value: 0.7, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-mock123', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); - const streamingChatCompletionSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', - ); - expect(streamingChatCompletionSpan).toBeDefined(); - expect(streamingChatCompletionSpan!.name).toBe('chat gpt-4'); - expect(streamingChatCompletionSpan!.status).toBe('ok'); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ - type: 'double', - value: 0.8, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-stream-123', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["stop"]', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 12, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 18, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 30, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ - type: 'boolean', - value: true, - }); + const responsesSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456'); + expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(responsesSpan!.status).toBe('ok'); + expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(responsesSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(responsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(responsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(responsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'resp_mock456', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["completed"]', + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 5, + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 8, + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 13, + }); - const streamingResponsesSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', - ); - expect(streamingResponsesSpan).toBeDefined(); - expect(streamingResponsesSpan!.name).toBe('chat gpt-4'); - expect(streamingResponsesSpan!.status).toBe('ok'); - expect(streamingResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingResponsesSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'resp_stream_456', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["in_progress","completed"]', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 6, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 16, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ - type: 'boolean', - value: true, - }); + const nonStreamingErrorSpan = spans.find( + span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, + ); + expect(nonStreamingErrorSpan!.status).toBe('error'); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(nonStreamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(nonStreamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); - const streamingErrorSpan = container.items.find( - span => - span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, - ); - expect(streamingErrorSpan).toBeDefined(); - expect(streamingErrorSpan!.name).toBe('chat error-model'); - expect(streamingErrorSpan!.status).toBe('error'); - expect(streamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); - expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - }, - }) - .start() - .completed(); + const streamingChatCompletionSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', + ); + expect(streamingChatCompletionSpan!.name).toBe('chat gpt-4'); + expect(streamingChatCompletionSpan!.status).toBe('ok'); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ + type: 'double', + value: 0.8, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-stream-123', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 12, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 18, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ + type: 'boolean', + value: true, + }); + + const streamingResponsesSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', + ); + expect(streamingResponsesSpan!.name).toBe('chat gpt-4'); + expect(streamingResponsesSpan!.status).toBe('ok'); + expect(streamingResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingResponsesSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'resp_stream_456', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["in_progress","completed"]', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 6, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 16, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ + type: 'boolean', + value: true, + }); + + const streamingErrorSpan = spans.find( + span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, + ); + expect(streamingErrorSpan!.status).toBe('error'); + expect(streamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); }); }, { @@ -345,348 +333,335 @@ describe('OpenAI integration (V6)', () => { 'instrument-with-pii.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording enabled (v6)', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(6); - const chatCompletionSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', - ); - expect(chatCompletionSpan).toBeDefined(); - expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(chatCompletionSpan!.status).toBe('ok'); - expect(chatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ - type: 'double', - value: 0.7, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: '[{"role":"user","content":"What is the capital of France?"}]', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toEqual({ - type: 'string', - value: '[{"type":"text","content":"You are a helpful assistant."}]', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-mock123', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["stop"]', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ - type: 'string', - value: '["Hello from OpenAI mock!"]', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 15, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 25, - }); + const runner = createRunner(); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); - const responsesSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456', - ); - expect(responsesSpan).toBeDefined(); - expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(responsesSpan!.status).toBe('ok'); - expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(responsesSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(responsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(responsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); - expect(responsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(responsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: 'Translate this to French: Hello', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ - type: 'string', - value: 'Response to: Translate this to French: Hello', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["completed"]', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'resp_mock456', - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 5, - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 8, - }); - expect(responsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 13, - }); + await runner.start().completed(); - const nonStreamingErrorSpan = container.items.find( - span => - span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, - ); - expect(nonStreamingErrorSpan).toBeDefined(); - expect(nonStreamingErrorSpan!.name).toBe('chat error-model'); - expect(nonStreamingErrorSpan!.status).toBe('error'); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(nonStreamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(nonStreamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); - expect(nonStreamingErrorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: '[{"role":"user","content":"This will fail"}]', - }); + const allSpans = await spansPromise; + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(6); + const chatCompletionSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', + ); + expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(chatCompletionSpan!.status).toBe('ok'); + expect(chatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ + type: 'double', + value: 0.7, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"What is the capital of France?"}]', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toEqual({ + type: 'string', + value: '[{"type":"text","content":"You are a helpful assistant."}]', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-mock123', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ + type: 'string', + value: '["Hello from OpenAI mock!"]', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); - const streamingChatCompletionSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', - ); - expect(streamingChatCompletionSpan).toBeDefined(); - expect(streamingChatCompletionSpan!.name).toBe('chat gpt-4'); - expect(streamingChatCompletionSpan!.status).toBe('ok'); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ - type: 'double', - value: 0.8, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: '[{"role":"user","content":"Tell me about streaming"}]', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toEqual({ - type: 'string', - value: '[{"type":"text","content":"You are a helpful assistant."}]', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ - type: 'string', - value: 'Hello from OpenAI streaming!', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["stop"]', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'chatcmpl-stream-123', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 12, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 18, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 30, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ - type: 'boolean', - value: true, - }); + const responsesSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456'); + expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(responsesSpan!.status).toBe('ok'); + expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(responsesSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(responsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(responsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(responsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(responsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: 'Translate this to French: Hello', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ + type: 'string', + value: 'Response to: Translate this to French: Hello', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["completed"]', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'resp_mock456', + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 5, + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 8, + }); + expect(responsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 13, + }); - const streamingResponsesSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', - ); - expect(streamingResponsesSpan).toBeDefined(); - expect(streamingResponsesSpan!.name).toBe('chat gpt-4'); - expect(streamingResponsesSpan!.status).toBe('ok'); - expect(streamingResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingResponsesSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: 'Test streaming responses API', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ - type: 'string', - value: 'Streaming response to: Test streaming responses APITest streaming responses API', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["in_progress","completed"]', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ - type: 'string', - value: 'resp_stream_456', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-4', - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 6, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 16, - }); - expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ - type: 'boolean', - value: true, - }); + const nonStreamingErrorSpan = spans.find( + span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, + ); + expect(nonStreamingErrorSpan!.status).toBe('error'); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(nonStreamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(nonStreamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(nonStreamingErrorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"This will fail"}]', + }); - const streamingErrorSpan = container.items.find( - span => - span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, - ); - expect(streamingErrorSpan).toBeDefined(); - expect(streamingErrorSpan!.name).toBe('chat error-model'); - expect(streamingErrorSpan!.status).toBe('error'); - expect(streamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'chat', - }); - expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); - expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingErrorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ - type: 'string', - value: '[{"role":"user","content":"This will fail"}]', - }); - expect(streamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(streamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(streamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - }, - }) - .start() - .completed(); + const streamingChatCompletionSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', + ); + expect(streamingChatCompletionSpan!.name).toBe('chat gpt-4'); + expect(streamingChatCompletionSpan!.status).toBe('ok'); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingChatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingChatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ + type: 'double', + value: 0.8, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"Tell me about streaming"}]', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toEqual({ + type: 'string', + value: '[{"type":"text","content":"You are a helpful assistant."}]', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ + type: 'string', + value: 'Hello from OpenAI streaming!', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'chatcmpl-stream-123', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 12, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 18, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ + type: 'boolean', + value: true, + }); + + const streamingResponsesSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_stream_456', + ); + expect(streamingResponsesSpan!.name).toBe('chat gpt-4'); + expect(streamingResponsesSpan!.status).toBe('ok'); + expect(streamingResponsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingResponsesSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingResponsesSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: 'Test streaming responses API', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({ + type: 'string', + value: 'Streaming response to: Test streaming responses APITest streaming responses API', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["in_progress","completed"]', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_ID]).toEqual({ + type: 'string', + value: 'resp_stream_456', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-4', + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 6, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 16, + }); + expect(streamingResponsesSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ + type: 'boolean', + value: true, + }); + + const streamingErrorSpan = spans.find( + span => span.name === 'chat error-model' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, + ); + expect(streamingErrorSpan!.status).toBe('error'); + expect(streamingErrorSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'chat', + }); + expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(streamingErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingErrorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"This will fail"}]', + }); + expect(streamingErrorSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(streamingErrorSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(streamingErrorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); }); }, { @@ -702,53 +677,50 @@ describe('OpenAI integration (V6)', () => { 'instrument-with-options.mjs', (createRunner, test) => { test('creates openai related spans with custom options (v6)', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(6); - const chatCompletionSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', - ); - expect(chatCompletionSpan).toBeDefined(); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); - expect(chatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - expect(chatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); + const runner = createRunner(); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); - const streamingChatCompletionSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', - ); - expect(streamingChatCompletionSpan).toBeDefined(); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ - type: 'boolean', - value: true, - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ - type: 'string', - value: expect.any(String), - }); - }, - }) - .start() - .completed(); + await runner.start().completed(); + + const allSpans = await spansPromise; + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(6); + const chatCompletionSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', + ); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(chatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + expect(chatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + + const streamingChatCompletionSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', + ); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toEqual({ + type: 'boolean', + value: true, + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); + expect(streamingChatCompletionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toMatchObject({ + type: 'string', + value: expect.any(String), + }); }); }, { @@ -764,133 +736,129 @@ describe('OpenAI integration (V6)', () => { 'instrument.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording disabled (v6)', async () => { + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() - .expect({ - transaction: { - transaction: 'main', - }, - }) + .unordered() .expect({ span: container => { - expect(container.items).toHaveLength(3); - const singleEmbeddingSpan = container.items.find( - span => - span.name === 'embeddings text-embedding-3-small' && - span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] !== undefined, - ); - expect(singleEmbeddingSpan).toBeDefined(); - expect(singleEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); - expect(singleEmbeddingSpan!.status).toBe('ok'); - expect(singleEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(singleEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(singleEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]).toEqual({ - type: 'string', - value: 'float', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]).toEqual({ - type: 'integer', - value: 1536, - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - - const errorEmbeddingSpan = container.items.find(span => span.name === 'embeddings error-model'); - expect(errorEmbeddingSpan).toBeDefined(); - expect(errorEmbeddingSpan!.name).toBe('embeddings error-model'); - expect(errorEmbeddingSpan!.status).toBe('error'); - expect(errorEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(errorEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(errorEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(errorEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(errorEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); - - const multiEmbeddingSpan = container.items.find( - span => - span.name === 'embeddings text-embedding-3-small' && - span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] === undefined, - ); - expect(multiEmbeddingSpan).toBeDefined(); - expect(multiEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); - expect(multiEmbeddingSpan!.status).toBe('ok'); - expect(multiEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(multiEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(multiEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(3); + const singleEmbeddingSpan = spans.find( + span => + span.name === 'embeddings text-embedding-3-small' && + span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] !== undefined, + ); + expect(singleEmbeddingSpan!.status).toBe('ok'); + expect(singleEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(singleEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(singleEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'float', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1536, + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + + const errorEmbeddingSpan = spans.find(span => span.name === 'embeddings error-model'); + expect(errorEmbeddingSpan!.status).toBe('error'); + expect(errorEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(errorEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(errorEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(errorEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(errorEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); + + const multiEmbeddingSpan = spans.find( + span => + span.name === 'embeddings text-embedding-3-small' && + span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE] === undefined, + ); + expect(multiEmbeddingSpan!.status).toBe('ok'); + expect(multiEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(multiEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(multiEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); }); }, { @@ -906,145 +874,144 @@ describe('OpenAI integration (V6)', () => { 'instrument-with-pii.mjs', (createRunner, test) => { test('creates openai related spans with genAI recording enabled (v6)', async () => { + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() - .expect({ - transaction: { - transaction: 'main', - }, - }) + .unordered() .expect({ span: container => { - expect(container.items).toHaveLength(3); - const singleEmbeddingSpan = container.items.find( - span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', - ); - expect(singleEmbeddingSpan).toBeDefined(); - expect(singleEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); - expect(singleEmbeddingSpan!.status).toBe('ok'); - expect(singleEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(singleEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(singleEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]).toEqual({ - type: 'string', - value: 'float', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]).toEqual({ - type: 'integer', - value: 1536, - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ - type: 'string', - value: 'Embedding test!', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - - const errorEmbeddingSpan = container.items.find( - span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Error embedding test!', - ); - expect(errorEmbeddingSpan).toBeDefined(); - expect(errorEmbeddingSpan!.name).toBe('embeddings error-model'); - expect(errorEmbeddingSpan!.status).toBe('error'); - expect(errorEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(errorEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(errorEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(errorEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(errorEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'error-model', - }); - expect(errorEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ - type: 'string', - value: 'Error embedding test!', - }); - - const multiEmbeddingSpan = container.items.find( - span => - span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === - '["First input text","Second input text","Third input text"]', - ); - expect(multiEmbeddingSpan).toBeDefined(); - expect(multiEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); - expect(multiEmbeddingSpan!.status).toBe('ok'); - expect(multiEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'embeddings', - }); - expect(multiEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(multiEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ - type: 'string', - value: '["First input text","Second input text","Third input text"]', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'text-embedding-3-small', - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); - expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 10, - }); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(3); + const singleEmbeddingSpan = spans.find( + span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', + ); + expect(singleEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); + expect(singleEmbeddingSpan!.status).toBe('ok'); + expect(singleEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(singleEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(singleEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'float', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE]).toEqual({ + type: 'integer', + value: 1536, + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ + type: 'string', + value: 'Embedding test!', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + + const errorEmbeddingSpan = spans.find( + span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Error embedding test!', + ); + expect(errorEmbeddingSpan!.name).toBe('embeddings error-model'); + expect(errorEmbeddingSpan!.status).toBe('error'); + expect(errorEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(errorEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(errorEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(errorEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(errorEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'error-model', + }); + expect(errorEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ + type: 'string', + value: 'Error embedding test!', + }); + + const multiEmbeddingSpan = spans.find( + span => + span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === + '["First input text","Second input text","Third input text"]', + ); + expect(multiEmbeddingSpan!.name).toBe('embeddings text-embedding-3-small'); + expect(multiEmbeddingSpan!.status).toBe('ok'); + expect(multiEmbeddingSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'embeddings', + }); + expect(multiEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(multiEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toEqual({ + type: 'string', + value: '["First input text","Second input text","Third input text"]', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'text-embedding-3-small', + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(multiEmbeddingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); }); }, { @@ -1057,46 +1024,41 @@ describe('OpenAI integration (V6)', () => { createEsmAndCjsTests( __dirname, 'scenario-root-span.mjs', - 'instrument-root-span.mjs', + 'instrument.mjs', (createRunner, test) => { test('it works without a wrapping span (v6)', async () => { + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() - // First the span that our mock express server is emitting, unrelated to this test + .unordered() .expect({ - transaction: { - transaction: 'POST /openai/chat/completions', - }, - }) - .expect({ - transaction: { - transaction: 'chat gpt-3.5-turbo', - contexts: { - trace: { - span_id: expect.any(String), - trace_id: expect.any(String), - data: { - [GEN_AI_OPERATION_NAME]: 'chat', - [SENTRY_OP]: 'gen_ai.chat', - [SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_PROVIDER_NAME]: 'openai', - [GEN_AI_REQUEST_MODEL]: 'gpt-3.5-turbo', - [GEN_AI_REQUEST_TEMPERATURE]: 0.7, - [GEN_AI_RESPONSE_MODEL]: 'gpt-3.5-turbo', - [GEN_AI_RESPONSE_ID]: 'chatcmpl-mock123', - [GEN_AI_RESPONSE_FINISH_REASONS]: '["stop"]', - [GEN_AI_USAGE_INPUT_TOKENS]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS]: 25, - }, - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }, - }, + span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const segment = receivedSpans.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); + expect(segment?.span_id).toEqual(expect.any(String)); + expect(segment?.trace_id).toEqual(expect.any(String)); + expect(segment?.attributes[GEN_AI_OPERATION_NAME]).toEqual({ value: 'chat', type: 'string' }); + expect(segment?.attributes[SENTRY_OP]).toEqual({ value: 'gen_ai.chat', type: 'string' }); + expect(segment?.attributes[SENTRY_ORIGIN]).toEqual({ value: 'auto.ai.openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ value: 'openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ value: 0.7, type: 'double' }); + expect(segment?.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_ID]).toEqual({ value: 'chatcmpl-mock123', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + value: '["stop"]', + type: 'string', + }); + expect(segment?.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ value: 10, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ value: 15, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ value: 25, type: 'integer' }); + expect(segment?.status).toEqual('ok'); }); }, { @@ -1113,43 +1075,38 @@ describe('OpenAI integration (V6)', () => { 'instrument.mjs', (createRunner, test) => { test('it works with Azure OpenAI (v6)', async () => { + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() - // First the span that our mock express server is emitting, unrelated to this test + .unordered() .expect({ - transaction: { - transaction: 'POST /azureopenai/deployments/:model/chat/completions', - }, - }) - .expect({ - transaction: { - transaction: 'chat gpt-3.5-turbo', - contexts: { - trace: { - span_id: expect.any(String), - trace_id: expect.any(String), - data: { - [GEN_AI_OPERATION_NAME]: 'chat', - [SENTRY_OP]: 'gen_ai.chat', - [SENTRY_ORIGIN]: 'auto.ai.openai', - [GEN_AI_PROVIDER_NAME]: 'openai', - [GEN_AI_REQUEST_MODEL]: 'gpt-3.5-turbo', - [GEN_AI_REQUEST_TEMPERATURE]: 0.7, - [GEN_AI_RESPONSE_MODEL]: 'gpt-3.5-turbo', - [GEN_AI_RESPONSE_ID]: 'chatcmpl-mock123', - [GEN_AI_RESPONSE_FINISH_REASONS]: '["stop"]', - [GEN_AI_USAGE_INPUT_TOKENS]: 10, - [GEN_AI_USAGE_OUTPUT_TOKENS]: 15, - [GEN_AI_USAGE_TOTAL_TOKENS]: 25, - }, - op: 'gen_ai.chat', - origin: 'auto.ai.openai', - status: 'ok', - }, - }, + span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const segment = receivedSpans.find(span => span.is_segment && span.name === 'chat gpt-3.5-turbo'); + expect(segment?.span_id).toEqual(expect.any(String)); + expect(segment?.trace_id).toEqual(expect.any(String)); + expect(segment?.attributes[GEN_AI_OPERATION_NAME]).toEqual({ value: 'chat', type: 'string' }); + expect(segment?.attributes[SENTRY_OP]).toEqual({ value: 'gen_ai.chat', type: 'string' }); + expect(segment?.attributes[SENTRY_ORIGIN]).toEqual({ value: 'auto.ai.openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ value: 'openai', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_REQUEST_TEMPERATURE]).toEqual({ value: 0.7, type: 'double' }); + expect(segment?.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ value: 'gpt-3.5-turbo', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_ID]).toEqual({ value: 'chatcmpl-mock123', type: 'string' }); + expect(segment?.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + value: '["stop"]', + type: 'string', + }); + expect(segment?.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ value: 10, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ value: 15, type: 'integer' }); + expect(segment?.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ value: 25, type: 'integer' }); + expect(segment?.status).toEqual('ok'); }); }, { diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument-span-streaming.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument-span-streaming.mjs deleted file mode 100644 index 5e0b6fb5592f..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument-span-streaming.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: false, outputs: false } }, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs index 2097d76a4eff..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/instrument.mjs @@ -2,16 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, - beforeSendTransaction: event => { - if (event.transaction.includes('/openai/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts index fbea79d4f34c..34199f155036 100644 --- a/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts @@ -1,3 +1,4 @@ +import type { SerializedStreamedSpanContainer } from '@sentry/core'; import { afterAll, expect } from 'vitest'; import { GEN_AI_OPERATION_NAME, @@ -32,79 +33,69 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { 'instrument.mjs', (createRunner, test) => { test('instruments chat completions, the responses API and streaming on openai v7', async () => { - await createRunner() - .ignore('event') - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(6); + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); - const chatCompletionSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', - ); - expect(chatCompletionSpan).toBeDefined(); - expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(chatCompletionSpan!.status).toBe('ok'); - expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); - expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ - type: 'string', - value: '["stop"]', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ type: 'integer', value: 10 }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 15, - }); - expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 25 }); + await runner.start().completed(); - // The responses API is a separate instrumented resource file from chat completions. - const responsesSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456', - ); - expect(responsesSpan).toBeDefined(); - expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(responsesSpan!.status).toBe('ok'); - expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'chat' }); - expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ - type: 'string', - value: 'gpt-3.5-turbo', - }); + const allSpans = await spansPromise; + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + expect(spans).toHaveLength(6); - // Streaming goes through the patched async iterator rather than `beforeSpanEnd`, so it - // is the part most likely to break if the `Stream` shape changes across a major. - const streamingSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123', - ); - expect(streamingSpan).toBeDefined(); - expect(streamingSpan!.name).toBe('chat gpt-4'); - expect(streamingSpan!.status).toBe('ok'); - expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ type: 'boolean', value: true }); - expect(streamingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ type: 'integer', value: 12 }); - expect(streamingSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ type: 'integer', value: 18 }); - expect(streamingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 30 }); + const chatCompletionSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', + ); + expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(chatCompletionSpan!.status).toBe('ok'); + expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.chat', + }); + expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(chatCompletionSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({ + type: 'string', + value: '["stop"]', + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ type: 'integer', value: 10 }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(chatCompletionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 25 }); - const errorSpan = container.items.find(span => span.name === 'chat error-model' && span.status !== 'ok'); - expect(errorSpan).toBeDefined(); - expect(errorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - }, - }) - .start() - .completed(); + // The responses API is a separate instrumented resource file from chat completions. + const responsesSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'resp_mock456'); + expect(responsesSpan!.name).toBe('chat gpt-3.5-turbo'); + expect(responsesSpan!.status).toBe('ok'); + expect(responsesSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'chat' }); + expect(responsesSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'gpt-3.5-turbo', + }); + + // Streaming goes through the patched async iterator rather than `beforeSpanEnd`, so it + // is the part most likely to break if the `Stream` shape changes across a major. + const streamingSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-stream-123'); + expect(streamingSpan!.name).toBe('chat gpt-4'); + expect(streamingSpan!.status).toBe('ok'); + expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ type: 'boolean', value: true }); + expect(streamingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ type: 'integer', value: 12 }); + expect(streamingSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ type: 'integer', value: 18 }); + expect(streamingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 30 }); + + const errorSpan = spans.find(span => span.name === 'chat error-model' && span.status !== 'ok'); + expect(errorSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); }); }, { @@ -122,82 +113,43 @@ conditionalTest({ min: 22 })('OpenAI integration (V7)', () => { 'instrument.mjs', (createRunner, test) => { test('instruments the embeddings API on openai v7', async () => { - await createRunner() - .ignore('event') - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - const embeddingSpans = container.items.filter( - span => span.attributes[GEN_AI_OPERATION_NAME]?.value === 'embeddings', - ); - expect(embeddingSpans).toHaveLength(3); - - const singleEmbeddingSpan = embeddingSpans.find( - span => span.name === 'embeddings text-embedding-3-small' && span.status === 'ok', - ); - expect(singleEmbeddingSpan).toBeDefined(); - expect(singleEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.embeddings', - }); - expect(singleEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(singleEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ - type: 'string', - value: 'openai', - }); + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; - const errorEmbeddingSpan = embeddingSpans.find(span => span.name === 'embeddings error-model'); - expect(errorEmbeddingSpan).toBeDefined(); - expect(errorEmbeddingSpan!.status).not.toBe('ok'); - }, - }) - .start() - .completed(); - }); - }, - { - additionalDependencies: { - openai: '7.5.0', - }, - }, - ); - // Span streaming is the default trace lifecycle, so cover it too. The span buffer flushes on a 5s - // timer per trace, which splits this scenario's spans across envelopes under CI load, and the - // runner asserts against one envelope at a time — so this only asserts on the first call's span, - // which is always in the first flush. The exhaustive assertions above stay on the static lifecycle, - // where every span arrives in a single envelope. - createEsmAndCjsTests( - __dirname, - 'scenario-chat.mjs', - 'instrument-span-streaming.mjs', - (createRunner, test) => { - test('instruments chat completions on openai v7 with span streaming enabled', async () => { await createRunner() + .unordered() .ignore('event') .expect({ span: container => { - const chatCompletionSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'chatcmpl-mock123', - ); - expect(chatCompletionSpan).toBeDefined(); - expect(chatCompletionSpan!.name).toBe('chat gpt-3.5-turbo'); - expect(chatCompletionSpan!.status).toBe('ok'); - expect(chatCompletionSpan!.attributes[SENTRY_OP]).toEqual({ - type: 'string', - value: 'gen_ai.chat', - }); - expect(chatCompletionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ - type: 'string', - value: 'auto.ai.openai', - }); - expect(chatCompletionSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ type: 'string', value: 'openai' }); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.openai'); + const embeddingSpans = spans.filter(span => span.attributes[GEN_AI_OPERATION_NAME]?.value === 'embeddings'); + expect(embeddingSpans).toHaveLength(3); + + const singleEmbeddingSpan = embeddingSpans.find( + span => span.name === 'embeddings text-embedding-3-small' && span.status === 'ok', + ); + expect(singleEmbeddingSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.embeddings', + }); + expect(singleEmbeddingSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: 'auto.ai.openai', + }); + expect(singleEmbeddingSpan!.attributes[GEN_AI_PROVIDER_NAME]).toEqual({ + type: 'string', + value: 'openai', + }); + + const errorEmbeddingSpan = embeddingSpans.find(span => span.name === 'embeddings error-model'); + expect(errorEmbeddingSpan).toBeDefined(); + expect(errorEmbeddingSpan!.status).not.toBe('ok'); }); }, { From 96b56c5bb0da61494bd728e534a7710a28e497c4 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Wed, 7 Oct 2026 14:42:38 +0200 Subject: [PATCH 04/47] test(node): Port Bedrock tests to span streaming (#25086) Run the Bedrock Converse and InvokeModel integration test with default span streaming. Also replaces some consts with conventions equivalents. Part of #24138 --------- Co-authored-by: GPT-6 --- .../aws-serverless/bedrock/instrument.mjs | 1 - .../suites/aws-serverless/bedrock/test.ts | 69 +++++++++++-------- 2 files changed, 39 insertions(+), 31 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/instrument.mjs b/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/instrument.mjs index 14dbd52b3824..dcfc5fc614ed 100644 --- a/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/instrument.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/aws-serverless'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/test.ts b/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/test.ts index af7e80bd10b2..a7bc374499b4 100644 --- a/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/test.ts +++ b/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/test.ts @@ -1,3 +1,16 @@ +import { + GEN_AI_OPERATION_NAME, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MAX_TOKENS, + GEN_AI_REQUEST_MODEL, + GEN_AI_REQUEST_TEMPERATURE, + GEN_AI_REQUEST_TOP_P, + GEN_AI_RESPONSE_FINISH_REASONS, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + SENTRY_OP, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; import type { SerializedStreamedSpanContainer } from '@sentry/core'; import { afterAll, describe, expect } from 'vitest'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; @@ -5,37 +18,38 @@ import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runn const MODEL_ID = 'anthropic.claude-3-5-sonnet-20240620-v1:0'; function assertBedrockSpans(container: SerializedStreamedSpanContainer): void { + const segment = container.items.find(span => span.is_segment && span.name === 'Test Transaction'); + expect(segment).toBeDefined(); + // Converse (non-streaming) const converseSpan = container.items.find(span => span.name === `chat ${MODEL_ID}`); - expect(converseSpan).toBeDefined(); expect(converseSpan!.status).toBe('ok'); - expect(converseSpan!.attributes['sentry.origin'].value).toBe('auto.aws.aws_sdk'); - expect(converseSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(converseSpan!.attributes['gen_ai.provider.name'].value).toBe('aws.bedrock'); - expect(converseSpan!.attributes['gen_ai.operation.name'].value).toBe('chat'); - expect(converseSpan!.attributes['gen_ai.request.model'].value).toBe(MODEL_ID); - expect(converseSpan!.attributes['gen_ai.request.max_tokens'].value).toBe(100); - expect(converseSpan!.attributes['gen_ai.request.temperature'].value).toBe(0.5); - expect(converseSpan!.attributes['gen_ai.request.top_p'].value).toBe(0.9); - expect(converseSpan!.attributes['gen_ai.usage.input_tokens'].value).toBe(12); - expect(converseSpan!.attributes['gen_ai.usage.output_tokens'].value).toBe(8); - expect(converseSpan!.attributes['gen_ai.response.finish_reasons'].value).toEqual(['end_turn']); + expect(converseSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.aws.aws_sdk'); + expect(converseSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(converseSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('aws.bedrock'); + expect(converseSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(converseSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe(MODEL_ID); + expect(converseSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); + expect(converseSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.5); + expect(converseSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.9); + expect(converseSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(12); + expect(converseSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(8); + expect(converseSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS].value).toEqual(['end_turn']); // InvokeModel (non-streaming, anthropic.claude request/response body) const invokeModelSpan = container.items.find(span => span.name === `generate_content ${MODEL_ID}`); - expect(invokeModelSpan).toBeDefined(); expect(invokeModelSpan!.status).toBe('ok'); - expect(invokeModelSpan!.attributes['sentry.origin'].value).toBe('auto.aws.aws_sdk'); - expect(invokeModelSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(invokeModelSpan!.attributes['gen_ai.provider.name'].value).toBe('aws.bedrock'); - expect(invokeModelSpan!.attributes['gen_ai.operation.name'].value).toBe('generate_content'); - expect(invokeModelSpan!.attributes['gen_ai.request.model'].value).toBe(MODEL_ID); - expect(invokeModelSpan!.attributes['gen_ai.request.max_tokens'].value).toBe(100); - expect(invokeModelSpan!.attributes['gen_ai.request.temperature'].value).toBe(0.5); - expect(invokeModelSpan!.attributes['gen_ai.request.top_p'].value).toBe(0.9); - expect(invokeModelSpan!.attributes['gen_ai.usage.input_tokens'].value).toBe(15); - expect(invokeModelSpan!.attributes['gen_ai.usage.output_tokens'].value).toBe(9); - expect(invokeModelSpan!.attributes['gen_ai.response.finish_reasons'].value).toEqual(['end_turn']); + expect(invokeModelSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.aws.aws_sdk'); + expect(invokeModelSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); + expect(invokeModelSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('aws.bedrock'); + expect(invokeModelSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); + expect(invokeModelSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe(MODEL_ID); + expect(invokeModelSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); + expect(invokeModelSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.5); + expect(invokeModelSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.9); + expect(invokeModelSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(15); + expect(invokeModelSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(9); + expect(invokeModelSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS].value).toEqual(['end_turn']); } describe('awsIntegration - Bedrock', () => { @@ -49,12 +63,7 @@ describe('awsIntegration - Bedrock', () => { 'instrument.mjs', (createTestRunner, test) => { test('auto-instruments Bedrock Converse and InvokeModel', { timeout: 90_000 }, async () => { - await createTestRunner() - .ignore('event') - .expect({ transaction: transaction => expect(transaction.transaction).toBe('Test Transaction') }) - .expect({ span: assertBedrockSpans }) - .start() - .completed(); + await createTestRunner().ignore('event').expect({ span: assertBedrockSpans }).start().completed(); }); }, { additionalDependencies: { '@aws-sdk/client-bedrock-runtime': '^3.1046.0' } }, From 9400a38002c3cddfc78b085d7e6b054d34d4062a Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Wed, 7 Oct 2026 14:52:33 +0200 Subject: [PATCH 05/47] test(node): Port Anthropic integration tests to span streaming (#25082) Port the Anthropic integration suites to default span streaming. Remove the redundant streaming-only test and duplicate initialization files; no static coverage is added. Part of #24138 --------- Co-authored-by: GPT-6 --- .../anthropic/instrument-span-streaming.mjs | 11 - .../anthropic/instrument-with-options.mjs | 8 - .../tracing/anthropic/instrument-with-pii.mjs | 8 - .../suites/tracing/anthropic/instrument.mjs | 8 - .../suites/tracing/anthropic/test.ts | 406 ++++++++---------- .../v0.129/instrument-default-lifecycle.mjs | 10 - .../tracing/anthropic/v0.129/instrument.mjs | 8 - .../suites/tracing/anthropic/v0.129/test.ts | 11 +- 8 files changed, 197 insertions(+), 273 deletions(-) delete mode 100644 dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-span-streaming.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/instrument-default-lifecycle.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-span-streaming.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-span-streaming.mjs deleted file mode 100644 index 657bed0a3a8c..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-span-streaming.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: true, outputs: true } }, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-options.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-options.mjs index 1046a4b66191..286186a48620 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-options.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-options.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, @@ -14,11 +13,4 @@ Sentry.init({ recordOutputs: true, }), ], - beforeSendTransaction: event => { - // Filter out mock express server transactions - if (event.transaction.includes('/anthropic/v1/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-pii.mjs index fc8f4e266829..1c507ba84b9c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-pii.mjs @@ -2,17 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: true, outputs: true } }, transport: loggingTransport, - beforeSendTransaction: event => { - // Filter out mock express server transactions - if (event.transaction.includes('/anthropic/v1/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument.mjs index 9b15a0c635e8..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument.mjs @@ -2,17 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, - beforeSendTransaction: event => { - // Filter out mock express server transactions - if (event.transaction.includes('/anthropic/v1/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts index 937ae389ca30..ad9ee334d927 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts @@ -1,3 +1,4 @@ +import type { Event, SerializedStreamedSpanContainer } from '@sentry/core'; import { afterAll, describe, expect } from 'vitest'; import { GEN_AI_INPUT_MESSAGES, @@ -17,6 +18,8 @@ import { GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, + SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; import { GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; @@ -33,15 +36,12 @@ describe('Anthropic integration', () => { createEsmAndCjsTests(__dirname, 'scenario-with-response.mjs', 'instrument.mjs', (createRunner, test) => { test('preserves .withResponse() and .asResponse() for non-streaming and streaming', async () => { await createRunner() - .expect({ - transaction: { - transaction: 'main', - }, - }) .expect({ span: container => { - expect(container.items).toHaveLength(6); - const nonStreamingSpans = container.items.filter( + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(6); + const nonStreamingSpans = spans.filter( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_withresponse', ); expect(nonStreamingSpans).toHaveLength(1); @@ -52,15 +52,14 @@ describe('Anthropic integration', () => { expect(span.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); } - const streamingSpan = container.items.find( + const streamingSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_withresponse', ); - expect(streamingSpan).toBeDefined(); expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); expect(streamingSpan!.status).toBe('ok'); expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - const rawResponseSpans = container.items.filter(span => span.attributes[GEN_AI_RESPONSE_ID] === undefined); + const rawResponseSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID] === undefined); expect(rawResponseSpans).toHaveLength(4); for (const span of rawResponseSpans) { expect(span.name).toBe(`chat ${span.attributes[GEN_AI_REQUEST_MODEL].value}`); @@ -83,158 +82,166 @@ describe('Anthropic integration', () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { test('creates anthropic related spans with genAI recording disabled', async () => { - const runner = createRunner(); + let receivedEvent: Event | undefined; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; + + const runner = createRunner().unordered(); await runner - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - const completionSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123', - ); - expect(completionSpan).toBeDefined(); - expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(completionSpan!.status).toBe('ok'); - - const errorSpan = container.items.find(span => span.name === 'chat error-model'); - expect(errorSpan).toBeDefined(); - expect(errorSpan!.status).toBe('error'); - - const streamingSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123', - ); - expect(streamingSpan).toBeDefined(); - expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(streamingSpan!.status).toBe('ok'); - expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; + }, + }) + .expect({ + event: event => { + receivedEvent = event; }, }) - .expect({ event: EXPECTED_STREAM_EVENT_HANDLER_MESSAGE }) .start() .completed(); + + expect(receivedEvent).toMatchObject({ event_id: expect.any(String), ...EXPECTED_STREAM_EVENT_HANDLER_MESSAGE }); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(3); + const completionSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123'); + expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(completionSpan!.status).toBe('ok'); + + const errorSpan = spans.find(span => span.name === 'chat error-model'); + expect(errorSpan!.status).toBe('error'); + + const streamingSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123'); + expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(streamingSpan!.status).toBe('ok'); + expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); }); }); createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates anthropic related spans with genAI recording enabled', async () => { - const runner = createRunner(); + let receivedEvent: Event | undefined; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; + + const runner = createRunner().unordered(); await runner - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - const completionSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123', - ); - expect(completionSpan).toBeDefined(); - expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(completionSpan!.status).toBe('ok'); - expect(completionSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(completionSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); - expect(completionSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); - expect(completionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); - expect(completionSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('claude-3-haiku-20240307'); - expect(completionSpan!.attributes[GEN_AI_RESPONSE_ID].value).toBe('msg_mock123'); - expect(completionSpan!.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('Hello from Anthropic mock!'); - expect(completionSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"What is the capital of France?"}]', - ); - expect(completionSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(completionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(completionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); - expect(completionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); - expect(completionSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(completionSpan!.attributes['sentry.origin'].value).toBe('auto.ai.anthropic'); - - const errorSpan = container.items.find( - span => span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"This will fail"}]', - ); - expect(errorSpan).toBeDefined(); - expect(errorSpan!.name).toBe('chat error-model'); - expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); - - // TODO: messages.stream() should produce its own distinct gen_ai span, but it - // currently does not (pre-existing bug). Once fixed, add an additional indexed span assertion. - const streamingSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123', - ); - expect(streamingSpan).toBeDefined(); - expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(streamingSpan!.status).toBe('ok'); - expect(streamingSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(streamingSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); - expect(streamingSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); - expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - expect(streamingSpan!.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('Hello from stream!'); - expect(streamingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(streamingSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); - expect(streamingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; + }, + }) + .expect({ + event: event => { + receivedEvent = event; }, }) - .expect({ event: EXPECTED_STREAM_EVENT_HANDLER_MESSAGE }) .start() .completed(); + + expect(receivedEvent).toMatchObject({ event_id: expect.any(String), ...EXPECTED_STREAM_EVENT_HANDLER_MESSAGE }); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(3); + const completionSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123'); + expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(completionSpan!.status).toBe('ok'); + expect(completionSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(completionSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); + expect(completionSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); + expect(completionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); + expect(completionSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('claude-3-haiku-20240307'); + expect(completionSpan!.attributes[GEN_AI_RESPONSE_ID].value).toBe('msg_mock123'); + expect(completionSpan!.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('Hello from Anthropic mock!'); + expect(completionSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( + '[{"role":"user","content":"What is the capital of France?"}]', + ); + expect(completionSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); + expect(completionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); + expect(completionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); + expect(completionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); + expect(completionSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(completionSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.anthropic'); + + const errorSpan = spans.find( + span => span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"This will fail"}]', + ); + expect(errorSpan!.name).toBe('chat error-model'); + expect(errorSpan!.status).toBe('error'); + expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); + + // TODO: messages.stream() should produce its own distinct gen_ai span, but it + // currently does not (pre-existing bug). Once fixed, add an additional indexed span assertion. + const streamingSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123'); + expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(streamingSpan!.status).toBe('ok'); + expect(streamingSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(streamingSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); + expect(streamingSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); + expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(streamingSpan!.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('Hello from stream!'); + expect(streamingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); + expect(streamingSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); + expect(streamingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); }); }); createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-options.mjs', (createRunner, test) => { test('creates anthropic related spans with custom options', async () => { - const runner = createRunner(); + let receivedEvent: Event | undefined; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; + + const runner = createRunner().unordered(); await runner - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - const completionSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123', - ); - expect(completionSpan).toBeDefined(); - expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(completionSpan!.status).toBe('ok'); - expect(completionSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(completionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); - expect(completionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toBeDefined(); - - const errorSpan = container.items.find(span => span.name === 'chat error-model'); - expect(errorSpan).toBeDefined(); - expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - - const streamingSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123', - ); - expect(streamingSpan).toBeDefined(); - expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(streamingSpan!.status).toBe('ok'); - expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; + }, + }) + .expect({ + event: event => { + receivedEvent = event; }, }) - .expect({ event: EXPECTED_STREAM_EVENT_HANDLER_MESSAGE }) .start() .completed(); + + expect(receivedEvent).toMatchObject({ event_id: expect.any(String), ...EXPECTED_STREAM_EVENT_HANDLER_MESSAGE }); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(3); + const completionSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123'); + expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(completionSpan!.status).toBe('ok'); + expect(completionSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(completionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); + expect(completionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toBeDefined(); + + const errorSpan = spans.find(span => span.name === 'chat error-model'); + expect(errorSpan!.status).toBe('error'); + expect(errorSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + + const streamingSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123'); + expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(streamingSpan!.status).toBe('ok'); + expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); }); }); - const EXPECTED_STREAM_SPANS_PII_FALSE = { - transaction: 'main', - }; - - const EXPECTED_STREAM_SPANS_PII_TRUE = { - transaction: 'main', - }; - createEsmAndCjsTests(__dirname, 'scenario-stream.mjs', 'instrument.mjs', (createRunner, test) => { test('streams produce spans with token usage and metadata (PII false)', async () => { await createRunner() - .expect({ transaction: EXPECTED_STREAM_SPANS_PII_FALSE }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - const requestStreamSpans = container.items.filter( + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(3); + const requestStreamSpans = spans.filter( span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, ); expect(requestStreamSpans).toHaveLength(2); @@ -249,7 +256,6 @@ describe('Anthropic integration', () => { const detailedStreamSpan = requestStreamSpans.find( span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["end_turn"]', ); - expect(detailedStreamSpan).toBeDefined(); expect(detailedStreamSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); expect(detailedStreamSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); expect(detailedStreamSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); @@ -258,10 +264,9 @@ describe('Anthropic integration', () => { expect(detailedStreamSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); expect(detailedStreamSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); - const messagesStreamSpan = container.items.find( + const messagesStreamSpan = spans.find( span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, ); - expect(messagesStreamSpan).toBeDefined(); expect(messagesStreamSpan!.name).toBe('chat claude-3-haiku-20240307'); expect(messagesStreamSpan!.status).toBe('ok'); expect(messagesStreamSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); @@ -283,11 +288,12 @@ describe('Anthropic integration', () => { createEsmAndCjsTests(__dirname, 'scenario-stream.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('streams record response text when PII true', async () => { await createRunner() - .expect({ transaction: EXPECTED_STREAM_SPANS_PII_TRUE }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - const requestStreamSpans = container.items.filter( + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(3); + const requestStreamSpans = spans.filter( span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, ); expect(requestStreamSpans).toHaveLength(2); @@ -299,10 +305,9 @@ describe('Anthropic integration', () => { expect(span.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('Hello from stream!'); } - const messagesStreamSpan = container.items.find( + const messagesStreamSpan = spans.find( span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, ); - expect(messagesStreamSpan).toBeDefined(); expect(messagesStreamSpan!.name).toBe('chat claude-3-haiku-20240307'); expect(messagesStreamSpan!.status).toBe('ok'); expect(messagesStreamSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); @@ -333,20 +338,16 @@ describe('Anthropic integration', () => { createEsmAndCjsTests(__dirname, 'scenario-stream-nested-create.mjs', 'instrument.mjs', (createRunner, test) => { test('traces a create() invoked from a stream event handler (dedup does not over-suppress)', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const nestedSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_nested', - ); - expect(nestedSpan).toBeDefined(); - expect(nestedSpan.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + const nestedSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_nested'); + expect(nestedSpan.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); // The helper's own internal `create` delegation must be deduped: exactly one span // for the streamed response, not a duplicate child span. - const streamingSpans = container.items.filter( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_1', - ); + const streamingSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_1'); expect(streamingSpans).toHaveLength(1); }, }) @@ -363,18 +364,17 @@ describe('Anthropic integration', () => { const EXPECTED_TOOL_CALLS_JSON = '[{"type":"tool_use","id":"tool_weather_1","name":"weather","input":{"city":"Paris"}}]'; await createRunner() - .expect({ - transaction: {}, - }) .expect({ span: container => { - expect(container.items).toHaveLength(1); - const [firstSpan] = container.items; + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(1); + const [firstSpan] = spans; // [0] messages.create with tools — available tools + tool calls recorded with PII expect(firstSpan!.name).toBe('chat claude-3-haiku-20240307'); expect(firstSpan!.status).toBe('ok'); - expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); expect(firstSpan!.attributes[GEN_AI_TOOL_DEFINITIONS].value).toBe(EXPECTED_TOOLS_JSON); expect(firstSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS].value).toBe(EXPECTED_TOOL_CALLS_JSON); }, @@ -392,16 +392,15 @@ describe('Anthropic integration', () => { const EXPECTED_TOOL_CALLS_JSON = '[{"type":"tool_use","id":"tool_weather_2","name":"weather","input":{"city":"Paris"}}]'; await createRunner() - .expect({ - transaction: {}, - }) .expect({ span: container => { - expect(container.items).toHaveLength(2); - for (const span of container.items) { + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(2); + for (const span of spans) { expect(span.name).toBe('chat claude-3-haiku-20240307'); expect(span.status).toBe('ok'); - expect(span.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(span.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); expect(span.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); expect(span.attributes[GEN_AI_RESPONSE_FINISH_REASONS].value).toBe('["tool_use"]'); expect(span.attributes[GEN_AI_TOOL_DEFINITIONS].value).toBe(EXPECTED_TOOLS_JSON); @@ -409,11 +408,11 @@ describe('Anthropic integration', () => { } // messages.create({ stream: true }) carries the request stream param; messages.stream() does not. - const createStreamSpan = container.items.find( + const createStreamSpan = spans.find( span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, ); expect(createStreamSpan).toBeDefined(); - const messagesStreamSpan = container.items.find( + const messagesStreamSpan = spans.find( span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, ); expect(messagesStreamSpan).toBeDefined(); @@ -425,10 +424,6 @@ describe('Anthropic integration', () => { }); // Additional error scenarios - Streaming errors - const EXPECTED_STREAM_ERROR_SPANS = { - transaction: 'main', - }; - createEsmAndCjsTests(__dirname, 'scenario-stream-errors.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('handles streaming errors correctly', async () => { await createRunner() @@ -436,41 +431,38 @@ describe('Anthropic integration', () => { // being raised as an unhandled rejection, so the instrumentation captures it. This test only // asserts the spans. .ignore('event') - .expect({ transaction: EXPECTED_STREAM_ERROR_SPANS }) .expect({ span: container => { - expect(container.items).toHaveLength(4); - const createInitErrorSpan = container.items.find( + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(4); + const createInitErrorSpan = spans.find( span => span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-init' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, ); - expect(createInitErrorSpan).toBeDefined(); expect(createInitErrorSpan!.name).toBe('chat error-stream-init'); expect(createInitErrorSpan!.status).toBe('error'); - const streamInitErrorSpan = container.items.find( + const streamInitErrorSpan = spans.find( span => span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-init' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, ); - expect(streamInitErrorSpan).toBeDefined(); expect(streamInitErrorSpan!.name).toBe('chat error-stream-init'); expect(streamInitErrorSpan!.status).toBe('error'); - const createMidwayErrorSpan = container.items.find( + const createMidwayErrorSpan = spans.find( span => span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-midway' && span.status === 'ok', ); - expect(createMidwayErrorSpan).toBeDefined(); expect(createMidwayErrorSpan!.name).toBe('chat error-stream-midway'); expect(createMidwayErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); expect(createMidwayErrorSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); expect(createMidwayErrorSpan!.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('This stream will '); - const streamMidwayErrorSpan = container.items.find( + const streamMidwayErrorSpan = spans.find( span => span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-midway' && span.status === 'error', ); - expect(streamMidwayErrorSpan).toBeDefined(); expect(streamMidwayErrorSpan!.name).toBe('chat error-stream-midway'); expect(streamMidwayErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); }, @@ -483,18 +475,17 @@ describe('Anthropic integration', () => { createEsmAndCjsTests(__dirname, 'scenario-errors.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('handles tool errors correctly', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(2); - const invalidFormatSpan = container.items.find(span => span.name === 'chat invalid-format'); - expect(invalidFormatSpan).toBeDefined(); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(2); + const invalidFormatSpan = spans.find(span => span.name === 'chat invalid-format'); expect(invalidFormatSpan!.status).toBe('error'); expect(invalidFormatSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('invalid-format'); - expect(invalidFormatSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(invalidFormatSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); - const toolSuccessSpan = container.items.find(span => span.name === 'chat claude-3-haiku-20240307'); - expect(toolSuccessSpan).toBeDefined(); + const toolSuccessSpan = spans.find(span => span.name === 'chat claude-3-haiku-20240307'); expect(toolSuccessSpan!.status).toBe('ok'); expect(toolSuccessSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS].value).toContain('tool_ok_1'); }, @@ -512,15 +503,14 @@ describe('Anthropic integration', () => { test('extracts system instructions from messages', async () => { const expectedInstructions = JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]); await createRunner() - .expect({ - transaction: { - transaction: 'main', - }, - }) .expect({ span: container => { - expect(container.items).toHaveLength(1); - const [firstSpan] = container.items; + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic', + ); + expect(spans).toHaveLength(1); + const [firstSpan] = spans; // [0] messages.create — system instructions extracted into dedicated attribute expect(firstSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS].value).toBe(expectedInstructions); @@ -532,53 +522,39 @@ describe('Anthropic integration', () => { }, ); - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-span-streaming.mjs', (createRunner, test) => { - test('creates anthropic related spans with span streaming enabled', async () => { - await createRunner() - .expect({ - span: container => { - const completionSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123', - ); - expect(completionSpan).toBeDefined(); - expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(completionSpan!.status).toBe('ok'); - expect(completionSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(completionSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); - expect(completionSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"What is the capital of France?"}]', - ); - expect(completionSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(completionSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(completionSpan!.attributes['sentry.origin'].value).toBe('auto.ai.anthropic'); - }, - }) - .start() - .completed(); - }); - }); - createEsmAndCjsTests(__dirname, 'scenario-response-error.mjs', 'instrument.mjs', (createRunner, test) => { test('captures error-shaped responses returned as data', async () => { + let receivedEvent: Event | undefined; + await createRunner() // The API returns the error as data on a 200 response, never as a thrown error to the caller, // so the instrumentation intentionally captures it as an event. .unordered() .expect({ - event: { - exception: { - values: [ - { - value: 'Overloaded', - mechanism: { type: 'auto.ai.anthropic.anthropic_error', handled: false }, - }, - ], - }, + event: event => { + receivedEvent = event; + }, + }) + .expect({ + span: container => { + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); }, }) - .expect({ transaction: { transaction: 'main' } }) .start() .completed(); + + expect(receivedEvent).toMatchObject({ + event_id: expect.any(String), + exception: { + values: [ + { + value: 'Overloaded', + mechanism: { type: 'auto.ai.anthropic.anthropic_error', handled: false }, + }, + ], + }, + }); }); }); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/instrument-default-lifecycle.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/instrument-default-lifecycle.mjs deleted file mode 100644 index cc192fb89834..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/instrument-default-lifecycle.mjs +++ /dev/null @@ -1,10 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: false, outputs: false } }, - transport: loggingTransport, -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/instrument.mjs index 9b15a0c635e8..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/instrument.mjs @@ -2,17 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, - beforeSendTransaction: event => { - // Filter out mock express server transactions - if (event.transaction.includes('/anthropic/v1/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts index 04f9cf75d98a..05eac304d0e1 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts @@ -1,4 +1,4 @@ -import { GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_STREAMING } from '@sentry/conventions/attributes'; +import { GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_STREAMING, SENTRY_OP } from '@sentry/conventions/attributes'; import { afterAll, describe, expect } from 'vitest'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; @@ -18,14 +18,14 @@ describe('Anthropic integration (0.129)', () => { (createRunner, test) => { test('emits one span for messages.stream(), not a second one for its internal create', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); const streamingSpans = container.items.filter( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_1', ); expect(streamingSpans).toHaveLength(1); - expect(streamingSpans[0]!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' }); + expect(streamingSpans[0]!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); expect(streamingSpans[0]!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ type: 'boolean', value: true, @@ -48,16 +48,17 @@ describe('Anthropic integration (0.129)', () => { createEsmAndCjsTests( __dirname, 'scenario-beta-stream-helpers.mjs', - 'instrument-default-lifecycle.mjs', + 'instrument.mjs', (createRunner, test) => { test('emits one span each for beta.messages.stream() and the eager streaming tool runner', async () => { await createRunner() .expect({ span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); for (const id of ['msg_beta_stream', 'msg_tool_runner_eager']) { const spans = container.items.filter(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === id); expect(spans, id).toHaveLength(1); - expect(spans[0]!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' }); + expect(spans[0]!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); } }, }) From edb8113f737a9fb782f44a284fdc73ac996c213c Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Wed, 7 Oct 2026 15:07:22 +0200 Subject: [PATCH 06/47] test(node): Port LangGraph tests to span streaming (#25085) Port LangGraph integration tests to default span streaming and remove the redundant streaming-only case. Part of #24138 --------- Co-authored-by: GPT-6 --- .../tracing/langgraph/instrument-agent.mjs | 8 - .../langgraph/instrument-span-streaming.mjs | 11 -- .../tracing/langgraph/instrument-with-pii.mjs | 1 - .../suites/tracing/langgraph/instrument.mjs | 1 - .../suites/tracing/langgraph/test.ts | 146 +++++++----------- 5 files changed, 53 insertions(+), 114 deletions(-) delete mode 100644 dev-packages/node-integration-tests/suites/tracing/langgraph/instrument-span-streaming.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/instrument-agent.mjs b/dev-packages/node-integration-tests/suites/tracing/langgraph/instrument-agent.mjs index b080c8d5ec03..1c507ba84b9c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langgraph/instrument-agent.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/instrument-agent.mjs @@ -2,17 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: true, outputs: true } }, transport: loggingTransport, - beforeSendTransaction: event => { - // Filter out mock express server transactions - if (event.transaction && event.transaction.includes('/v1/messages')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/instrument-span-streaming.mjs b/dev-packages/node-integration-tests/suites/tracing/langgraph/instrument-span-streaming.mjs deleted file mode 100644 index 657bed0a3a8c..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/langgraph/instrument-span-streaming.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: true, outputs: true } }, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/langgraph/instrument-with-pii.mjs index bdf24d6bbd0e..1c507ba84b9c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langgraph/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/instrument-with-pii.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/langgraph/instrument.mjs index 42052d281304..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langgraph/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/instrument.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts b/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts index 3ec4961fb10f..6954a7467886 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts @@ -28,21 +28,23 @@ describe('LangGraph integration', () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { test('should instrument LangGraph with default PII settings', async () => { await createRunner() - .expect({ transaction: { transaction: 'langgraph-test' } }) .expect({ span: container => { - expect(container.items).toHaveLength(2); - expect(container.items.map(span => span.name).sort()).toEqual([ + const segment = container.items.find(span => span.is_segment && span.name === 'langgraph-test'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langgraph'); + expect(spans).toHaveLength(2); + expect(spans.map(span => span.name).sort()).toEqual([ 'invoke_agent weather_assistant', 'invoke_agent weather_assistant', ]); - const invokeAgentSpans = container.items.filter(span => span.name === 'invoke_agent weather_assistant'); + const invokeAgentSpans = spans.filter(span => span.name === 'invoke_agent weather_assistant'); expect(invokeAgentSpans).toHaveLength(2); for (const span of invokeAgentSpans) { expect(span.status).toBe('ok'); - expect(span.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(span.attributes['sentry.origin'].value).toBe('auto.ai.langgraph'); + expect(span.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(span.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langgraph'); expect(span.attributes[GEN_AI_OPERATION_NAME].value).toBe('invoke_agent'); expect(span.attributes[GEN_AI_AGENT_NAME].value).toBe('weather_assistant'); expect(span.attributes[GEN_AI_PIPELINE_NAME].value).toBe('weather_assistant'); @@ -57,31 +59,31 @@ describe('LangGraph integration', () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('should instrument LangGraph with genAI recording enabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'langgraph-test' } }) .expect({ span: container => { - expect(container.items).toHaveLength(2); + const segment = container.items.find(span => span.is_segment && span.name === 'langgraph-test'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langgraph'); + expect(spans).toHaveLength(2); - const weatherTodaySpan = container.items.find(span => + const weatherTodaySpan = spans.find(span => getStringAttributeValue(span.attributes[GEN_AI_INPUT_MESSAGES]?.value)?.includes( 'What is the weather today?', ), ); - expect(weatherTodaySpan).toBeDefined(); expect(weatherTodaySpan!.name).toBe('invoke_agent weather_assistant'); expect(weatherTodaySpan!.status).toBe('ok'); - expect(weatherTodaySpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(weatherTodaySpan!.attributes['sentry.origin'].value).toBe('auto.ai.langgraph'); + expect(weatherTodaySpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(weatherTodaySpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langgraph'); - const weatherDetailsSpan = container.items.find(span => + const weatherDetailsSpan = spans.find(span => getStringAttributeValue(span.attributes[GEN_AI_INPUT_MESSAGES]?.value)?.includes( 'Tell me about the weather', ), ); - expect(weatherDetailsSpan).toBeDefined(); expect(weatherDetailsSpan!.name).toBe('invoke_agent weather_assistant'); expect(weatherDetailsSpan!.status).toBe('ok'); - expect(weatherDetailsSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(weatherDetailsSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); }, }) .start() @@ -92,19 +94,20 @@ describe('LangGraph integration', () => { createEsmAndCjsTests(__dirname, 'scenario-tools.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('should capture tools from LangGraph agent', { timeout: 30000 }, async () => { await createRunner() - .expect({ transaction: { transaction: 'langgraph-tools-test' } }) .expect({ span: container => { - expect(container.items).toHaveLength(2); - expect(container.items.map(span => span.name).sort()).toEqual([ + const segment = container.items.find(span => span.is_segment && span.name === 'langgraph-tools-test'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langgraph'); + expect(spans).toHaveLength(2); + expect(spans.map(span => span.name).sort()).toEqual([ 'invoke_agent tool_agent', 'invoke_agent tool_calling_agent', ]); - const toolAgentInvokeSpan = container.items.find(span => span.name === 'invoke_agent tool_agent'); - expect(toolAgentInvokeSpan).toBeDefined(); + const toolAgentInvokeSpan = spans.find(span => span.name === 'invoke_agent tool_agent'); expect(toolAgentInvokeSpan!.status).toBe('ok'); - expect(toolAgentInvokeSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(toolAgentInvokeSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(toolAgentInvokeSpan!.attributes[GEN_AI_TOOL_DEFINITIONS].value).toContain('get_weather'); expect(toolAgentInvokeSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toContain('What is the weather?'); expect(toolAgentInvokeSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('gpt-4-0613'); @@ -115,10 +118,9 @@ describe('LangGraph integration', () => { expect(toolAgentInvokeSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); expect(toolAgentInvokeSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(40); - const toolCallingInvokeSpan = container.items.find(span => span.name === 'invoke_agent tool_calling_agent'); - expect(toolCallingInvokeSpan).toBeDefined(); + const toolCallingInvokeSpan = spans.find(span => span.name === 'invoke_agent tool_calling_agent'); expect(toolCallingInvokeSpan!.status).toBe('ok'); - expect(toolCallingInvokeSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(toolCallingInvokeSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(toolCallingInvokeSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toContain('San Francisco'); expect(toolCallingInvokeSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('gpt-4-0613'); expect(toolCallingInvokeSpan!.attributes[GEN_AI_RESPONSE_TEXT].value).toMatch(/"role":"tool"/); @@ -137,31 +139,30 @@ describe('LangGraph integration', () => { createEsmAndCjsTests(__dirname, 'scenario-thread-id.mjs', 'instrument.mjs', (createRunner, test) => { test('should capture thread_id as gen_ai.conversation.id', async () => { await createRunner() - .expect({ transaction: { transaction: 'langgraph-thread-id-test' } }) .expect({ span: container => { - expect(container.items).toHaveLength(3); + const segment = container.items.find(span => span.is_segment && span.name === 'langgraph-thread-id-test'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langgraph'); + expect(spans).toHaveLength(3); - const firstThreadSpan = container.items.find( + const firstThreadSpan = spans.find( span => span.attributes[GEN_AI_CONVERSATION_ID]?.value === 'thread_abc123_session_1', ); - expect(firstThreadSpan).toBeDefined(); expect(firstThreadSpan!.name).toBe('invoke_agent thread_test_agent'); expect(firstThreadSpan!.status).toBe('ok'); - expect(firstThreadSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstThreadSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); - const secondThreadSpan = container.items.find( + const secondThreadSpan = spans.find( span => span.attributes[GEN_AI_CONVERSATION_ID]?.value === 'thread_xyz789_session_2', ); - expect(secondThreadSpan).toBeDefined(); expect(secondThreadSpan!.name).toBe('invoke_agent thread_test_agent'); expect(secondThreadSpan!.status).toBe('ok'); - const noThreadSpan = container.items.find( + const noThreadSpan = spans.find( span => span.name === 'invoke_agent thread_test_agent' && span.attributes[GEN_AI_CONVERSATION_ID] === undefined, ); - expect(noThreadSpan).toBeDefined(); expect(noThreadSpan!.status).toBe('ok'); }, }) @@ -177,13 +178,16 @@ describe('LangGraph integration', () => { (createRunner, test) => { test('extracts system instructions from messages', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(1); - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent test-agent'); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langgraph', + ); + expect(spans).toHaveLength(1); + const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent test-agent'); - expect(invokeAgentSpan).toBeDefined(); expect(invokeAgentSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS].value).toBe( JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]), ); @@ -199,28 +203,20 @@ describe('LangGraph integration', () => { createEsmAndCjsTests(__dirname, 'scenario-resume.mjs', 'instrument.mjs', (createRunner, test) => { test('should not throw when invoke is called with null input (resume scenario)', async () => { await createRunner() - .expect({ - transaction: { - transaction: 'langgraph-resume-test', - contexts: { - trace: expect.objectContaining({ - status: 'ok', - }), - }, - }, - }) .expect({ span: container => { - expect(container.items).toHaveLength(2); + const segment = container.items.find(span => span.is_segment && span.name === 'langgraph-resume-test'); + expect(segment?.status).toBe('ok'); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langgraph'); + expect(spans).toHaveLength(2); - const invokeAgentSpan = container.items.find( + const invokeAgentSpan = spans.find( span => span.attributes[GEN_AI_CONVERSATION_ID]?.value === 'resume-thread-1', ); - expect(invokeAgentSpan).toBeDefined(); expect(invokeAgentSpan!.name).toBe('invoke_agent resume_agent'); expect(invokeAgentSpan!.status).toBe('ok'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(invokeAgentSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langgraph'); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langgraph'); expect(invokeAgentSpan!.attributes[GEN_AI_AGENT_NAME].value).toBe('resume_agent'); expect(invokeAgentSpan!.attributes[GEN_AI_PIPELINE_NAME].value).toBe('resume_agent'); }, @@ -230,28 +226,6 @@ describe('LangGraph integration', () => { }); }); - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-span-streaming.mjs', (createRunner, test) => { - test('creates langgraph related spans with span streaming enabled', async () => { - await createRunner() - .expect({ - span: container => { - const weatherTodaySpan = container.items.find(span => - getStringAttributeValue(span.attributes[GEN_AI_INPUT_MESSAGES]?.value)?.includes( - 'What is the weather today?', - ), - ); - expect(weatherTodaySpan).toBeDefined(); - expect(weatherTodaySpan!.name).toBe('invoke_agent weather_assistant'); - expect(weatherTodaySpan!.status).toBe('ok'); - expect(weatherTodaySpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(weatherTodaySpan!.attributes['sentry.origin'].value).toBe('auto.ai.langgraph'); - }, - }) - .start() - .completed(); - }); - }); - // createReactAgent tests. // Spans are asserted order-independently: the span-array order is not a protocol guarantee (Sentry // rebuilds the tree from `parent_span_id`), and the provider emits tree order while the OTel exporter @@ -259,15 +233,10 @@ describe('LangGraph integration', () => { createEsmAndCjsTests(__dirname, 'agent-scenario.mjs', 'instrument-agent.mjs', (createRunner, test) => { test('should instrument createReactAgent with agent and chat spans', { timeout: 30000 }, async () => { await createRunner() - .expect({ - transaction: event => { - const spans = event.spans ?? []; - expect(event.transaction).toBe('main'); - expect(spans).toContainEqual(expect.objectContaining({ op: 'http.client' })); - }, - }) .expect({ span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + expect(container.items.find(span => span.attributes[SENTRY_OP]?.value === 'http.client')).toBeDefined(); const spans = container.items; expect(spans).toContainEqual( expect.objectContaining({ @@ -301,15 +270,10 @@ describe('LangGraph integration', () => { createEsmAndCjsTests(__dirname, 'agent-tools-scenario.mjs', 'instrument-agent.mjs', (createRunner, test) => { test('should create tool execution spans for createReactAgent with tools', { timeout: 30000 }, async () => { await createRunner() - .expect({ - transaction: event => { - const spans = event.spans ?? []; - expect(event.transaction).toBe('main'); - expect(spans.filter(span => span.op === 'http.client')).toHaveLength(3); - }, - }) .expect({ span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + expect(container.items.filter(span => span.attributes[SENTRY_OP]?.value === 'http.client')).toHaveLength(3); const spans = container.items; expect(spans).toContainEqual( expect.objectContaining({ @@ -361,13 +325,9 @@ describe('LangGraph integration', () => { createEsmAndCjsTests(__dirname, 'scenario-stategraph-chat.mjs', 'instrument-agent.mjs', (createRunner, test) => { test('auto-injects langchain handler for plain StateGraph and emits chat spans', { timeout: 30000 }, async () => { await createRunner() - .expect({ - transaction: event => { - expect(event.transaction).toBe('main'); - }, - }) .expect({ span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); const chatSpans = container.items.filter(s => s.attributes[SENTRY_OP]?.value === 'gen_ai.chat'); expect(chatSpans).toHaveLength(1); expect(chatSpans[0]?.attributes[GEN_AI_AGENT_NAME]?.value).toBe('plain_assistant'); From 145e48b0029b6e5fdd282a2b151d2fb7f0966c58 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Wed, 7 Oct 2026 15:24:28 +0200 Subject: [PATCH 07/47] test(node): Port Google GenAI tests to span streaming (#25083) Port Google GenAI v1 and v2 integration tests to default span streaming. Fold the redundant streaming-only case into the PII test and remove its initialization file. No static coverage is added. Part of #24138 --------- Co-authored-by: GPT-6 --- .../tracing/google-genai-v2/instrument.mjs | 8 - .../suites/tracing/google-genai-v2/test.ts | 148 ++++--- .../instrument-span-streaming.mjs | 11 - .../google-genai/instrument-with-options.mjs | 8 - .../google-genai/instrument-with-pii.mjs | 8 - .../tracing/google-genai/instrument.mjs | 8 - .../suites/tracing/google-genai/test.ts | 394 +++++++++--------- 7 files changed, 271 insertions(+), 314 deletions(-) delete mode 100644 dev-packages/node-integration-tests/suites/tracing/google-genai/instrument-span-streaming.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/instrument.mjs index c83310d623f6..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/instrument.mjs @@ -2,17 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, - beforeSendTransaction: event => { - // Filter out mock express server transactions - if (event.transaction.includes('/v1beta')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/test.ts b/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/test.ts index 97f65eeb0680..1025569ac5d4 100644 --- a/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/test.ts @@ -9,6 +9,8 @@ import { GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, + SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; @@ -30,54 +32,51 @@ describe('Google GenAI integration (v2)', () => { 'instrument.mjs', (createRunner, test) => { test('auto-instruments chat and generateContent on @google/genai v2', async () => { - await createRunner() - .ignore('event') - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(3); - expect(container.items.map(span => span.name).sort()).toEqual([ - 'chat gemini-1.5-pro', - 'generate_content error-model', - 'generate_content gemini-1.5-flash', - ]); + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); - const chatSpan = container.items.find(span => span.name === 'chat gemini-1.5-pro'); - expect(chatSpan!.status).toBe('ok'); - expect(chatSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(chatSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(chatSpan!.attributes['sentry.origin'].value).toBe(EXPECTED_ORIGIN); - expect(chatSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('google_genai'); - expect(chatSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gemini-1.5-pro'); - expect(chatSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(8); - expect(chatSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); - expect(chatSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); + await runner.start().completed(); - const generateContentSpan = container.items.find( - span => span.name === 'generate_content gemini-1.5-flash', - ); - expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); - expect(generateContentSpan!.attributes['sentry.origin'].value).toBe(EXPECTED_ORIGIN); - expect(generateContentSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('google_genai'); - expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gemini-1.5-flash'); - expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); - expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.9); - expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); - expect(generateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(8); - expect(generateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); - expect(generateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); + const spans = (await spansPromise).filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.google_genai', + ); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual([ + 'chat gemini-1.5-pro', + 'generate_content error-model', + 'generate_content gemini-1.5-flash', + ]); - const errorSpan = container.items.find(span => span.name === 'generate_content error-model'); - expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(errorSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); - expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); - }, - }) - .start() - .completed(); + const chatSpan = spans.find(span => span.name === 'chat gemini-1.5-pro'); + expect(chatSpan!.status).toBe('ok'); + expect(chatSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(chatSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(chatSpan!.attributes[SENTRY_ORIGIN].value).toBe(EXPECTED_ORIGIN); + expect(chatSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('google_genai'); + expect(chatSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gemini-1.5-pro'); + expect(chatSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(8); + expect(chatSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); + expect(chatSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); + + const generateContentSpan = spans.find(span => span.name === 'generate_content gemini-1.5-flash'); + expect(generateContentSpan!.status).toBe('ok'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); + expect(generateContentSpan!.attributes[SENTRY_ORIGIN].value).toBe(EXPECTED_ORIGIN); + expect(generateContentSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('google_genai'); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gemini-1.5-flash'); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.9); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(8); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); + + const errorSpan = spans.find(span => span.name === 'generate_content error-model'); + expect(errorSpan!.status).toBe('error'); + expect(errorSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); + expect(errorSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); + expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); }); }, { additionalDependencies: { '@google/genai': '^2' } }, @@ -91,38 +90,37 @@ describe('Google GenAI integration (v2)', () => { // `embedContent` is the member that changed shape in v2; asserting its span proves the // `className`/`methodName` selector still matches the constructor-assigned arrow. test('auto-instruments embedContent on @google/genai v2', async () => { - await createRunner() - .ignore('event') - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(3); - expect(container.items.map(span => span.name).sort()).toEqual([ - 'embeddings error-model', - 'embeddings text-embedding-004', - 'embeddings text-embedding-004', - ]); + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = (await spansPromise).filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.google_genai', + ); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual([ + 'embeddings error-model', + 'embeddings text-embedding-004', + 'embeddings text-embedding-004', + ]); - const successfulSpans = container.items.filter( - span => span.name === 'embeddings text-embedding-004' && span.status === 'ok', - ); - expect(successfulSpans).toHaveLength(2); - for (const span of successfulSpans) { - expect(span.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); - expect(span.attributes[GEN_AI_OPERATION_NAME].value).toBe('embeddings'); - expect(span.attributes['sentry.origin'].value).toBe(EXPECTED_ORIGIN); - expect(span.attributes[GEN_AI_PROVIDER_NAME].value).toBe('google_genai'); - expect(span.attributes[GEN_AI_REQUEST_MODEL].value).toBe('text-embedding-004'); - } + const successfulSpans = spans.filter( + span => span.name === 'embeddings text-embedding-004' && span.status === 'ok', + ); + expect(successfulSpans).toHaveLength(2); + for (const span of successfulSpans) { + expect(span.attributes[SENTRY_OP].value).toBe('gen_ai.embeddings'); + expect(span.attributes[GEN_AI_OPERATION_NAME].value).toBe('embeddings'); + expect(span.attributes[SENTRY_ORIGIN].value).toBe(EXPECTED_ORIGIN); + expect(span.attributes[GEN_AI_PROVIDER_NAME].value).toBe('google_genai'); + expect(span.attributes[GEN_AI_REQUEST_MODEL].value).toBe('text-embedding-004'); + } - const errorSpan = container.items.find(span => span.name === 'embeddings error-model'); - expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); - expect(errorSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('embeddings'); - }, - }) - .start() - .completed(); + const errorSpan = spans.find(span => span.name === 'embeddings error-model'); + expect(errorSpan!.status).toBe('error'); + expect(errorSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.embeddings'); + expect(errorSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('embeddings'); }); }, { additionalDependencies: { '@google/genai': '^2' } }, diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai/instrument-span-streaming.mjs b/dev-packages/node-integration-tests/suites/tracing/google-genai/instrument-span-streaming.mjs deleted file mode 100644 index 657bed0a3a8c..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/google-genai/instrument-span-streaming.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: true, outputs: true } }, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai/instrument-with-options.mjs b/dev-packages/node-integration-tests/suites/tracing/google-genai/instrument-with-options.mjs index f1299d9c3542..8edd2db39bc3 100644 --- a/dev-packages/node-integration-tests/suites/tracing/google-genai/instrument-with-options.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai/instrument-with-options.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, @@ -14,11 +13,4 @@ Sentry.init({ recordOutputs: true, }), ], - beforeSendTransaction: event => { - // Filter out mock express server transactions - if (event.transaction.includes('/v1beta/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/google-genai/instrument-with-pii.mjs index e5ddcf227d1f..1c507ba84b9c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/google-genai/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai/instrument-with-pii.mjs @@ -2,17 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: true, outputs: true } }, transport: loggingTransport, - beforeSendTransaction: event => { - // Filter out mock express server transactions - if (event.transaction.includes('/v1beta/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/google-genai/instrument.mjs index c83310d623f6..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/google-genai/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai/instrument.mjs @@ -2,17 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, - beforeSendTransaction: event => { - // Filter out mock express server transactions - if (event.transaction.includes('/v1beta')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts b/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts index 6f51960996fa..710d167bea2e 100644 --- a/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts @@ -1,3 +1,4 @@ +import type { Event, SerializedStreamedSpanContainer } from '@sentry/core'; import { afterAll, describe, expect } from 'vitest'; import { GEN_AI_EMBEDDINGS_INPUT, @@ -20,6 +21,8 @@ import { GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, + SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; @@ -33,22 +36,24 @@ describe('Google GenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { test('creates google genai related spans with genAI recording disabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - expect(container.items.map(span => span.name).sort()).toEqual([ + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.google_genai', + ); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual([ 'chat gemini-1.5-pro', 'generate_content error-model', 'generate_content gemini-1.5-flash', ]); - const chatSpan = container.items.find(span => span.name === 'chat gemini-1.5-pro'); - expect(chatSpan).toBeDefined(); + const chatSpan = spans.find(span => span.name === 'chat gemini-1.5-pro'); expect(chatSpan!.status).toBe('ok'); - expect(chatSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(chatSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); expect(chatSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(chatSpan!.attributes['sentry.origin'].value).toBe(EXPECTED_ORIGIN); + expect(chatSpan!.attributes[SENTRY_ORIGIN].value).toBe(EXPECTED_ORIGIN); expect(chatSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('google_genai'); expect(chatSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gemini-1.5-pro'); // Given once to `chats.create()` and reused for every message the chat sends. @@ -60,10 +65,9 @@ describe('Google GenAI integration', () => { expect(chatSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); expect(chatSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); - const generateContentSpan = container.items.find(span => span.name === 'generate_content gemini-1.5-flash'); - expect(generateContentSpan).toBeDefined(); + const generateContentSpan = spans.find(span => span.name === 'generate_content gemini-1.5-flash'); expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); expect(generateContentSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('google_genai'); expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gemini-1.5-flash'); @@ -74,10 +78,9 @@ describe('Google GenAI integration', () => { expect(generateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); expect(generateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); - const errorSpan = container.items.find(span => span.name === 'generate_content error-model'); - expect(errorSpan).toBeDefined(); + const errorSpan = spans.find(span => span.name === 'generate_content error-model'); expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(errorSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(errorSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); expect(errorSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('google_genai'); expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); @@ -91,20 +94,22 @@ describe('Google GenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates google genai related spans with genAI recording enabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - expect(container.items.map(span => span.name).sort()).toEqual([ + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.google_genai', + ); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual([ 'chat gemini-1.5-pro', 'generate_content error-model', 'generate_content gemini-1.5-flash', ]); - const chatSpan = container.items.find(span => span.name === 'chat gemini-1.5-pro'); - expect(chatSpan).toBeDefined(); + const chatSpan = spans.find(span => span.name === 'chat gemini-1.5-pro'); expect(chatSpan!.status).toBe('ok'); - expect(chatSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(chatSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); expect(chatSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); expect(chatSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('google_genai'); expect(chatSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gemini-1.5-pro'); @@ -121,21 +126,21 @@ describe('Google GenAI integration', () => { expect(chatSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); expect(chatSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); - const generateContentSpan = container.items.find(span => span.name === 'generate_content gemini-1.5-flash'); - expect(generateContentSpan).toBeDefined(); + const generateContentSpan = spans.find(span => span.name === 'generate_content gemini-1.5-flash'); expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); + expect(generateContentSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('google_genai'); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gemini-1.5-flash'); expect(generateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); expect(generateContentSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toBeDefined(); expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.9); expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); - const errorSpan = container.items.find(span => span.name === 'generate_content error-model'); - expect(errorSpan).toBeDefined(); + const errorSpan = spans.find(span => span.name === 'generate_content error-model'); expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(errorSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(errorSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); expect(errorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); }, @@ -148,28 +153,28 @@ describe('Google GenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-options.mjs', (createRunner, test) => { test('creates google genai related spans with custom options', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - expect(container.items.map(span => span.name).sort()).toEqual([ + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.google_genai', + ); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual([ 'chat gemini-1.5-pro', 'generate_content error-model', 'generate_content gemini-1.5-flash', ]); - const chatSpan = container.items.find(span => span.name === 'chat gemini-1.5-pro'); - expect(chatSpan).toBeDefined(); + const chatSpan = spans.find(span => span.name === 'chat gemini-1.5-pro'); expect(chatSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); expect(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); expect(chatSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toBeDefined(); - const generateContentSpan = container.items.find(span => span.name === 'generate_content gemini-1.5-flash'); - expect(generateContentSpan).toBeDefined(); + const generateContentSpan = spans.find(span => span.name === 'generate_content gemini-1.5-flash'); expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); - const errorSpan = container.items.find(span => span.name === 'generate_content error-model'); - expect(errorSpan).toBeDefined(); + const errorSpan = spans.find(span => span.name === 'generate_content error-model'); expect(errorSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); }, }) @@ -184,16 +189,18 @@ describe('Google GenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-tools.mjs', 'instrument-with-options.mjs', (createRunner, test) => { test('creates google genai related spans with tool calls', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - const nonStreamingToolsSpan = container.items.find( + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.google_genai', + ); + expect(spans).toHaveLength(3); + const nonStreamingToolsSpan = spans.find( span => span.attributes[GEN_AI_TOOL_DEFINITIONS]?.value === EXPECTED_AVAILABLE_TOOLS_JSON && span.attributes[GEN_AI_RESPONSE_STREAMING] === undefined, ); - expect(nonStreamingToolsSpan).toBeDefined(); expect(nonStreamingToolsSpan!.name).toBe('generate_content gemini-2.0-flash-001'); expect(nonStreamingToolsSpan!.status).toBe('ok'); expect(nonStreamingToolsSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); @@ -226,12 +233,11 @@ describe('Google GenAI integration', () => { expect(nonStreamingToolsSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(8); expect(nonStreamingToolsSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(23); - const streamingToolsSpan = container.items.find( + const streamingToolsSpan = spans.find( span => span.attributes[GEN_AI_TOOL_DEFINITIONS]?.value === EXPECTED_AVAILABLE_TOOLS_JSON && span.attributes[GEN_AI_RESPONSE_STREAMING]?.value === true, ); - expect(streamingToolsSpan).toBeDefined(); expect(streamingToolsSpan!.name).toBe('generate_content gemini-2.0-flash-001'); expect(streamingToolsSpan!.status).toBe('ok'); expect(streamingToolsSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); @@ -268,8 +274,7 @@ describe('Google GenAI integration', () => { expect(streamingToolsSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(10); expect(streamingToolsSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(22); - const noToolsSpan = container.items.find(span => span.attributes[GEN_AI_TOOL_DEFINITIONS] === undefined); - expect(noToolsSpan).toBeDefined(); + const noToolsSpan = spans.find(span => span.attributes[GEN_AI_TOOL_DEFINITIONS] === undefined); expect(noToolsSpan!.name).toBe('generate_content gemini-2.0-flash-001'); expect(noToolsSpan!.status).toBe('ok'); expect(noToolsSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); @@ -287,148 +292,162 @@ describe('Google GenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-streaming.mjs', 'instrument.mjs', (createRunner, test) => { test('creates google genai streaming spans with genAI recording disabled', async () => { + let receivedEvent: Event | undefined; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() // The provider surfaces blocked content within the stream and never returns it to the caller as // a thrown error, so the instrumentation intentionally captures it as an event. .unordered() .expect({ - event: { - exception: { - values: [ - { - value: 'Content blocked: The prompt was blocked due to safety concerns', - mechanism: { type: 'auto.ai.google_genai', handled: false }, - }, - ], - }, + event: event => { + receivedEvent = event; }, }) - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(4); - expect(container.items.map(span => span.name).sort()).toEqual([ - 'chat gemini-1.5-pro', - 'generate_content blocked-model', - 'generate_content error-model', - 'generate_content gemini-1.5-flash', - ]); - - const generateContentSpan = container.items.find(span => span.name === 'generate_content gemini-1.5-flash'); - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); - expect(generateContentSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); - expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.9); - expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); - expect(generateContentSpan!.attributes[GEN_AI_RESPONSE_ID].value).toBe('mock-response-streaming-id'); - expect(generateContentSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('gemini-1.5-pro'); - expect(generateContentSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS].value).toBe('["STOP"]'); - expect(generateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(generateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); - expect(generateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(22); - - const chatSpan = container.items.find(span => span.name === 'chat gemini-1.5-pro'); - expect(chatSpan).toBeDefined(); - expect(chatSpan!.status).toBe('ok'); - expect(chatSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(chatSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - // Given once to `chats.create()` and reused for every message the chat sends. - expect(chatSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.8); - expect(chatSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.9); - expect(chatSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(150); - expect(chatSpan!.attributes[GEN_AI_RESPONSE_ID].value).toBe('mock-response-streaming-id'); - expect(chatSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('gemini-1.5-pro'); - - const blockedSpan = container.items.find(span => span.name === 'generate_content blocked-model'); - expect(blockedSpan).toBeDefined(); - expect(blockedSpan!.status).toBe('error'); - expect(blockedSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(blockedSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); - - const errorSpan = container.items.find(span => span.name === 'generate_content error-model'); - expect(errorSpan).toBeDefined(); - expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(errorSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + expect(receivedEvent).toMatchObject({ + event_id: expect.any(String), + exception: { + values: [ + { + value: 'Content blocked: The prompt was blocked due to safety concerns', + mechanism: { type: 'auto.ai.google_genai', handled: false }, + }, + ], + }, + }); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.google_genai'); + expect(spans).toHaveLength(4); + expect(spans.map(span => span.name).sort()).toEqual([ + 'chat gemini-1.5-pro', + 'generate_content blocked-model', + 'generate_content error-model', + 'generate_content gemini-1.5-flash', + ]); + + const generateContentSpan = spans.find(span => span.name === 'generate_content gemini-1.5-flash'); + expect(generateContentSpan!.status).toBe('ok'); + expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); + expect(generateContentSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.9); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); + expect(generateContentSpan!.attributes[GEN_AI_RESPONSE_ID].value).toBe('mock-response-streaming-id'); + expect(generateContentSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('gemini-1.5-pro'); + expect(generateContentSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS].value).toBe('["STOP"]'); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(22); + + const chatSpan = spans.find(span => span.name === 'chat gemini-1.5-pro'); + expect(chatSpan!.status).toBe('ok'); + expect(chatSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(chatSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + // Given once to `chats.create()` and reused for every message the chat sends. + expect(chatSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.8); + expect(chatSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.9); + expect(chatSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(150); + expect(chatSpan!.attributes[GEN_AI_RESPONSE_ID].value).toBe('mock-response-streaming-id'); + expect(chatSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('gemini-1.5-pro'); + + const blockedSpan = spans.find(span => span.name === 'generate_content blocked-model'); + expect(blockedSpan!.status).toBe('error'); + expect(blockedSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); + expect(blockedSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); + + const errorSpan = spans.find(span => span.name === 'generate_content error-model'); + expect(errorSpan!.status).toBe('error'); + expect(errorSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); + expect(errorSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); }); }); createEsmAndCjsTests(__dirname, 'scenario-streaming.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates google genai streaming spans with genAI recording enabled', async () => { + let receivedEvent: Event | undefined; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() // The provider surfaces blocked content within the stream and never returns it to the caller as // a thrown error, so the instrumentation intentionally captures it as an event. .unordered() .expect({ - event: { - exception: { - values: [ - { - value: 'Content blocked: The prompt was blocked due to safety concerns', - mechanism: { type: 'auto.ai.google_genai', handled: false }, - }, - ], - }, + event: event => { + receivedEvent = event; }, }) - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(4); - expect(container.items.map(span => span.name).sort()).toEqual([ - 'chat gemini-1.5-pro', - 'generate_content blocked-model', - 'generate_content error-model', - 'generate_content gemini-1.5-flash', - ]); - - const generateContentSpan = container.items.find(span => span.name === 'generate_content gemini-1.5-flash'); - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); - expect(generateContentSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - expect(generateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); - expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); - expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.9); - expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); - expect(generateContentSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS].value).toBe('["STOP"]'); - - const chatSpan = container.items.find(span => span.name === 'chat gemini-1.5-pro'); - expect(chatSpan).toBeDefined(); - expect(chatSpan!.status).toBe('ok'); - expect(chatSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(chatSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - expect(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); - // Given once to `chats.create()` and reused for every message the chat sends. - expect(chatSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.8); - expect(chatSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.9); - expect(chatSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(150); - expect(chatSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS].value).toBe('["STOP"]'); - - const blockedSpan = container.items.find(span => span.name === 'generate_content blocked-model'); - expect(blockedSpan).toBeDefined(); - expect(blockedSpan!.status).toBe('error'); - expect(blockedSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); - expect(blockedSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - expect(blockedSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); - expect(blockedSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); - - const errorSpan = container.items.find(span => span.name === 'generate_content error-model'); - expect(errorSpan).toBeDefined(); - expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); - expect(errorSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); - expect(errorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + expect(receivedEvent).toMatchObject({ + event_id: expect.any(String), + exception: { + values: [ + { + value: 'Content blocked: The prompt was blocked due to safety concerns', + mechanism: { type: 'auto.ai.google_genai', handled: false }, + }, + ], + }, + }); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.google_genai'); + expect(spans).toHaveLength(4); + expect(spans.map(span => span.name).sort()).toEqual([ + 'chat gemini-1.5-pro', + 'generate_content blocked-model', + 'generate_content error-model', + 'generate_content gemini-1.5-flash', + ]); + + const generateContentSpan = spans.find(span => span.name === 'generate_content gemini-1.5-flash'); + expect(generateContentSpan!.status).toBe('ok'); + expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); + expect(generateContentSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(generateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.9); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); + expect(generateContentSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS].value).toBe('["STOP"]'); + + const chatSpan = spans.find(span => span.name === 'chat gemini-1.5-pro'); + expect(chatSpan!.status).toBe('ok'); + expect(chatSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(chatSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); + // Given once to `chats.create()` and reused for every message the chat sends. + expect(chatSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.8); + expect(chatSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.9); + expect(chatSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(150); + expect(chatSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS].value).toBe('["STOP"]'); + + const blockedSpan = spans.find(span => span.name === 'generate_content blocked-model'); + expect(blockedSpan!.status).toBe('error'); + expect(blockedSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); + expect(blockedSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(blockedSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); + expect(blockedSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); + + const errorSpan = spans.find(span => span.name === 'generate_content error-model'); + expect(errorSpan!.status).toBe('error'); + expect(errorSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); + expect(errorSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); + expect(errorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); }); }); @@ -439,11 +458,14 @@ describe('Google GenAI integration', () => { (createRunner, test) => { test('extracts system instructions and normalizes messages', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(2); - const [firstSpan, secondSpan] = container.items; + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.google_genai', + ); + expect(spans).toHaveLength(2); + const [firstSpan, secondSpan] = spans; // [0] generate_content with a string system instruction expect(firstSpan!.name).toBe('generate_content gemini-1.5-flash'); @@ -483,33 +505,35 @@ describe('Google GenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument.mjs', (createRunner, test) => { test('creates google genai embeddings spans with genAI recording disabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - expect(container.items.map(span => span.name).sort()).toEqual([ + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.google_genai', + ); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual([ 'embeddings error-model', 'embeddings text-embedding-004', 'embeddings text-embedding-004', ]); - const successfulSpans = container.items.filter( + const successfulSpans = spans.filter( span => span.name === 'embeddings text-embedding-004' && span.status === 'ok', ); expect(successfulSpans).toHaveLength(2); for (const span of successfulSpans) { - expect(span.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); + expect(span.attributes[SENTRY_OP].value).toBe('gen_ai.embeddings'); expect(span.attributes[GEN_AI_OPERATION_NAME].value).toBe('embeddings'); - expect(span.attributes['sentry.origin'].value).toBe(EXPECTED_ORIGIN); + expect(span.attributes[SENTRY_ORIGIN].value).toBe(EXPECTED_ORIGIN); expect(span.attributes[GEN_AI_PROVIDER_NAME].value).toBe('google_genai'); expect(span.attributes[GEN_AI_REQUEST_MODEL].value).toBe('text-embedding-004'); expect(span.attributes[GEN_AI_EMBEDDINGS_INPUT]).toBeUndefined(); } - const errorSpan = container.items.find(span => span.name === 'embeddings error-model'); - expect(errorSpan).toBeDefined(); + const errorSpan = spans.find(span => span.name === 'embeddings error-model'); expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); + expect(errorSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.embeddings'); expect(errorSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('embeddings'); }, }) @@ -521,39 +545,37 @@ describe('Google GenAI integration', () => { createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates google genai embeddings spans with genAI recording enabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - expect(container.items.map(span => span.name).sort()).toEqual([ + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.google_genai', + ); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual([ 'embeddings error-model', 'embeddings text-embedding-004', 'embeddings text-embedding-004', ]); - const stringInputSpan = container.items.find( + const stringInputSpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'What is the capital of France?', ); - expect(stringInputSpan).toBeDefined(); expect(stringInputSpan!.name).toBe('embeddings text-embedding-004'); expect(stringInputSpan!.status).toBe('ok'); expect(stringInputSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('embeddings'); expect(stringInputSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('google_genai'); - const errorSpan = container.items.find( - span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'This will fail', - ); - expect(errorSpan).toBeDefined(); + const errorSpan = spans.find(span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'This will fail'); expect(errorSpan!.name).toBe('embeddings error-model'); expect(errorSpan!.status).toBe('error'); expect(errorSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('embeddings'); - const arrayInputSpan = container.items.find( + const arrayInputSpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === '[{"role":"user","parts":[{"text":"First input text"}]},{"role":"user","parts":[{"text":"Second input text"}]}]', ); - expect(arrayInputSpan).toBeDefined(); expect(arrayInputSpan!.name).toBe('embeddings text-embedding-004'); expect(arrayInputSpan!.status).toBe('ok'); expect(arrayInputSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('embeddings'); @@ -563,24 +585,4 @@ describe('Google GenAI integration', () => { .completed(); }); }); - - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-span-streaming.mjs', (createRunner, test) => { - test('creates google genai related spans with span streaming enabled', async () => { - await createRunner() - .expect({ - span: container => { - const generateContentSpan = container.items.find(span => span.name === 'generate_content gemini-1.5-flash'); - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); - expect(generateContentSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('google_genai'); - expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gemini-1.5-flash'); - expect(generateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); - }, - }) - .start() - .completed(); - }); - }); }); From 02892e65e042e2bae442043bd1d8b7143f2c4af6 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Wed, 7 Oct 2026 15:31:51 +0200 Subject: [PATCH 08/47] test(node): Port LangChain tests to span streaming (#25084) Port the LangChain integration suites to default span streaming. Fold the streaming-only and duplicate count-only cases into the retained tests, without adding static coverage. Part of #24138 --------- Co-authored-by: GPT-6 --- .../langchain/instrument-span-streaming.mjs | 11 - .../tracing/langchain/instrument-with-pii.mjs | 8 - .../suites/tracing/langchain/instrument.mjs | 8 - .../suites/tracing/langchain/test.ts | 298 +++++-------- .../langchain/v1/instrument-with-pii.mjs | 8 - .../tracing/langchain/v1/instrument.mjs | 8 - .../suites/tracing/langchain/v1/test.ts | 404 ++++++++---------- 7 files changed, 290 insertions(+), 455 deletions(-) delete mode 100644 dev-packages/node-integration-tests/suites/tracing/langchain/instrument-span-streaming.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-span-streaming.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-span-streaming.mjs deleted file mode 100644 index 657bed0a3a8c..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-span-streaming.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: true, outputs: true } }, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-pii.mjs index e60b44f864d7..1c507ba84b9c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-pii.mjs @@ -2,17 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: true, outputs: true } }, transport: loggingTransport, - beforeSendTransaction: event => { - // Filter out mock express server transactions - if (event.transaction.includes('/v1/messages') || event.transaction.includes('/v1/embeddings')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/instrument.mjs index c81bb4dc1322..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/instrument.mjs @@ -2,17 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, - beforeSendTransaction: event => { - // Filter out mock express server transactions - if (event.transaction.includes('/v1/messages') || event.transaction.includes('/v1/embeddings')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts index 290cf38bcba6..48767041abf9 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts @@ -18,6 +18,9 @@ import { GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + LANGCHAIN_CHAIN_NAME, + SENTRY_OP, + SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; import { GEN_AI_EMBEDDINGS } from '@sentry/conventions/op'; import { @@ -35,21 +38,21 @@ describe('LangChain integration', () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { test('creates langchain related spans with genAI recording disabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - expect(container.items.map(span => span.name).sort()).toEqual([ + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain'); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual([ 'chat claude-3-5-sonnet-20241022', 'chat claude-3-opus-20240229', 'chat error-model', ]); - const sonnetSpan = container.items.find(span => span.name === 'chat claude-3-5-sonnet-20241022'); - expect(sonnetSpan).toBeDefined(); + const sonnetSpan = spans.find(span => span.name === 'chat claude-3-5-sonnet-20241022'); expect(sonnetSpan!.status).toBe('ok'); - expect(sonnetSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(sonnetSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(sonnetSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(sonnetSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); expect(sonnetSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); expect(sonnetSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); expect(sonnetSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-5-sonnet-20241022'); @@ -62,11 +65,10 @@ describe('LangChain integration', () => { expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toBeDefined(); expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]).toBeDefined(); - const opusSpan = container.items.find(span => span.name === 'chat claude-3-opus-20240229'); - expect(opusSpan).toBeDefined(); + const opusSpan = spans.find(span => span.name === 'chat claude-3-opus-20240229'); expect(opusSpan!.status).toBe('ok'); - expect(opusSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(opusSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(opusSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(opusSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); expect(opusSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); expect(opusSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-opus-20240229'); expect(opusSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.9); @@ -76,11 +78,10 @@ describe('LangChain integration', () => { expect(opusSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); expect(opusSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); - const errorSpan = container.items.find(span => span.name === 'chat error-model'); - expect(errorSpan).toBeDefined(); + const errorSpan = spans.find(span => span.name === 'chat error-model'); expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(errorSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(errorSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(errorSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); expect(errorSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); }, @@ -88,44 +89,26 @@ describe('LangChain integration', () => { .start() .completed(); }); - - test('does not create duplicate spans from double module patching', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - // The scenario makes 3 LangChain calls (2 successful + 1 error). - // Without the dedup guard, the file-level and module-level hooks - // both patch the same prototype, producing 6 spans instead of 3. - expect(container.items).toHaveLength(3); - for (const span of container.items) { - expect(span.attributes['sentry.op'].value).toBe('gen_ai.chat'); - } - }, - }) - .start() - .completed(); - }); }); createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates langchain related spans with genAI recording enabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - expect(container.items.map(span => span.name).sort()).toEqual([ + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain'); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual([ 'chat claude-3-5-sonnet-20241022', 'chat claude-3-opus-20240229', 'chat error-model', ]); - const sonnetSpan = container.items.find(span => span.name === 'chat claude-3-5-sonnet-20241022'); - expect(sonnetSpan).toBeDefined(); + const sonnetSpan = spans.find(span => span.name === 'chat claude-3-5-sonnet-20241022'); expect(sonnetSpan!.status).toBe('ok'); - expect(sonnetSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(sonnetSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(sonnetSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(sonnetSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); expect(sonnetSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); expect(sonnetSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-5-sonnet-20241022'); expect(sonnetSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); @@ -139,8 +122,7 @@ describe('LangChain integration', () => { expect(sonnetSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); expect(sonnetSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); - const opusSpan = container.items.find(span => span.name === 'chat claude-3-opus-20240229'); - expect(opusSpan).toBeDefined(); + const opusSpan = spans.find(span => span.name === 'chat claude-3-opus-20240229'); expect(opusSpan!.status).toBe('ok'); expect(opusSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); expect(opusSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-opus-20240229'); @@ -153,8 +135,7 @@ describe('LangChain integration', () => { expect(opusSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); expect(opusSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); - const errorSpan = container.items.find(span => span.name === 'chat error-model'); - expect(errorSpan).toBeDefined(); + const errorSpan = spans.find(span => span.name === 'chat error-model'); expect(errorSpan!.status).toBe('error'); expect(errorSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); @@ -169,17 +150,18 @@ describe('LangChain integration', () => { createEsmAndCjsTests(__dirname, 'scenario-tools.mjs', 'instrument.mjs', (createRunner, test) => { test('creates langchain spans with tool calls', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(1); - const [firstSpan] = container.items; + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain'); + expect(spans).toHaveLength(1); + const [firstSpan] = spans; // [0] chat with tool_use stop reason expect(firstSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); expect(firstSpan!.status).toBe('ok'); - expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(firstSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); expect(firstSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-5-sonnet-20241022'); expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); @@ -198,31 +180,23 @@ describe('LangChain integration', () => { createEsmTests(__dirname, 'scenario-openai-before-langchain.mjs', 'instrument.mjs', (createRunner, test) => { test('demonstrates timing issue with duplicate spans', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(2); - const anthropicSpan = container.items.find( - span => span.attributes['sentry.origin'].value === 'auto.ai.anthropic', - ); - expect(anthropicSpan).toBeDefined(); - expect(anthropicSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); - - // LangChain call is instrumented by LangChain. - const langchainSpan = container.items.find( - span => span.attributes['sentry.origin'].value === 'auto.ai.langchain', - ); - expect(langchainSpan).toBeDefined(); - expect(langchainSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); - - // Third call (not present): Direct Anthropic call made AFTER LangChain import - // is NOT instrumented, which demonstrates the skip mechanism works for NEW - // clients. We should only have ONE Anthropic span (the first one), not two. - }, - }) - .start() - .completed(); + const runner = createRunner(); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = (await spansPromise).filter( + span => + span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain' || + span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic', + ); + expect(spans).toHaveLength(2); + const anthropicSpan = spans.find(span => span.attributes[SENTRY_ORIGIN].value === 'auto.ai.anthropic'); + expect(anthropicSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + + // LangChain call is instrumented by LangChain. + const langchainSpan = spans.find(span => span.attributes[SENTRY_ORIGIN].value === 'auto.ai.langchain'); + expect(langchainSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); }); }); @@ -233,15 +207,18 @@ describe('LangChain integration', () => { (createRunner, test) => { test('extracts system instructions from messages', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(1); - const [firstSpan] = container.items; + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain', + ); + expect(spans).toHaveLength(1); + const [firstSpan] = spans; // [0] chat with extracted system instructions expect(firstSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); - expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); expect(firstSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS].value).toBe( JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]), ); @@ -256,42 +233,41 @@ describe('LangChain integration', () => { createEsmAndCjsTests(__dirname, 'scenario-chain.mjs', 'instrument.mjs', (createRunner, test) => { test('uses runName for chain spans instead of unknown_chain', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(4); - expect(container.items.map(span => span.name).sort()).toEqual([ - 'chain format_prompt', - 'chain parse_output', - 'chain unknown_chain', + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain'); + expect(spans).toHaveLength(4); + expect(spans.map(span => span.name).sort()).toEqual([ 'chat claude-3-5-sonnet-20241022', + 'invoke_agent', + 'invoke_agent format_prompt', + 'invoke_agent parse_output', ]); - const formatPromptSpan = container.items.find(span => span.name === 'chain format_prompt'); - expect(formatPromptSpan).toBeDefined(); - expect(formatPromptSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(formatPromptSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + const formatPromptSpan = spans.find(span => span.name === 'invoke_agent format_prompt'); + expect(formatPromptSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(formatPromptSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); expect(formatPromptSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('invoke_agent'); expect(formatPromptSpan!.attributes[GEN_AI_PIPELINE_NAME].value).toBe('format_prompt'); - expect(formatPromptSpan!.attributes['langchain.chain.name']).toBeUndefined(); + expect(formatPromptSpan!.attributes[LANGCHAIN_CHAIN_NAME]).toBeUndefined(); - const chatSpan = container.items.find(span => span.name === 'chat claude-3-5-sonnet-20241022'); - expect(chatSpan).toBeDefined(); - expect(chatSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(chatSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + const chatSpan = spans.find(span => span.name === 'chat claude-3-5-sonnet-20241022'); + expect(chatSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(chatSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); - const parseOutputSpan = container.items.find(span => span.name === 'chain parse_output'); - expect(parseOutputSpan).toBeDefined(); - expect(parseOutputSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(parseOutputSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + const parseOutputSpan = spans.find(span => span.name === 'invoke_agent parse_output'); + expect(parseOutputSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(parseOutputSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(parseOutputSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('invoke_agent'); expect(parseOutputSpan!.attributes[GEN_AI_PIPELINE_NAME].value).toBe('parse_output'); - expect(parseOutputSpan!.attributes['langchain.chain.name']).toBeUndefined(); + expect(parseOutputSpan!.attributes[LANGCHAIN_CHAIN_NAME]).toBeUndefined(); - const unknownChainSpan = container.items.find(span => span.name === 'chain unknown_chain'); - expect(unknownChainSpan).toBeDefined(); - expect(unknownChainSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + const unknownChainSpan = spans.find(span => span.name === 'invoke_agent'); + expect(unknownChainSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(unknownChainSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('invoke_agent'); expect(unknownChainSpan!.attributes[GEN_AI_PIPELINE_NAME]).toBeUndefined(); - expect(unknownChainSpan!.attributes['langchain.chain.name']).toBeUndefined(); + expect(unknownChainSpan!.attributes[LANGCHAIN_CHAIN_NAME]).toBeUndefined(); }, }) .start() @@ -306,89 +282,69 @@ describe('LangChain integration', () => { createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument.mjs', (createRunner, test) => { test('creates embedding spans with genAI recording disabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - expect(container.items.map(span => span.name).sort()).toEqual([ + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain'); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual([ 'embeddings error-model', 'embeddings text-embedding-3-small', 'embeddings text-embedding-3-small', ]); - const successfulSpans = container.items.filter( + const successfulSpans = spans.filter( span => span.name === 'embeddings text-embedding-3-small' && span.status === 'ok', ); expect(successfulSpans).toHaveLength(2); for (const span of successfulSpans) { - expect(span.attributes['sentry.op'].value).toBe(GEN_AI_EMBEDDINGS); - expect(span.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(span.attributes[SENTRY_OP].value).toBe(GEN_AI_EMBEDDINGS); + expect(span.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); expect(span.attributes[GEN_AI_OPERATION_NAME].value).toBe('embeddings'); expect(span.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); expect(span.attributes[GEN_AI_REQUEST_MODEL].value).toBe('text-embedding-3-small'); expect(span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE].value).toBe(1536); } - const errorSpan = container.items.find(span => span.name === 'embeddings error-model'); - expect(errorSpan).toBeDefined(); + const errorSpan = spans.find(span => span.name === 'embeddings error-model'); expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes['sentry.op'].value).toBe(GEN_AI_EMBEDDINGS); + expect(errorSpan!.attributes[SENTRY_OP].value).toBe(GEN_AI_EMBEDDINGS); expect(errorSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); }, }) .start() .completed(); }); - - test('does not create duplicate embedding spans from double module patching', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - // The scenario makes 3 embedding calls (2 successful + 1 error). - expect(container.items).toHaveLength(3); - for (const span of container.items) { - expect(span.attributes['sentry.op'].value).toBe(GEN_AI_EMBEDDINGS); - } - }, - }) - .start() - .completed(); - }); }); createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates embedding spans with genAI recording enabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - expect(container.items.map(span => span.name).sort()).toEqual([ + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain'); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual([ 'embeddings error-model', 'embeddings text-embedding-3-small', 'embeddings text-embedding-3-small', ]); - const querySpan = container.items.find( - span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Hello world', - ); - expect(querySpan).toBeDefined(); + const querySpan = spans.find(span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Hello world'); expect(querySpan!.name).toBe('embeddings text-embedding-3-small'); expect(querySpan!.status).toBe('ok'); expect(querySpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE].value).toBe(1536); - const documentsSpan = container.items.find( + const documentsSpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === JSON.stringify(['First document', 'Second document']), ); - expect(documentsSpan).toBeDefined(); expect(documentsSpan!.name).toBe('embeddings text-embedding-3-small'); expect(documentsSpan!.status).toBe('ok'); - const errorSpan = container.items.find(span => span.name === 'embeddings error-model'); - expect(errorSpan).toBeDefined(); + const errorSpan = spans.find(span => span.name === 'embeddings error-model'); expect(errorSpan!.status).toBe('error'); }, }) @@ -397,44 +353,25 @@ describe('LangChain integration', () => { }); }); - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-span-streaming.mjs', (createRunner, test) => { - test('creates langchain related spans with span streaming enabled', async () => { - await createRunner() - .expect({ - span: container => { - const sonnetSpan = container.items.find(span => span.name === 'chat claude-3-5-sonnet-20241022'); - expect(sonnetSpan).toBeDefined(); - expect(sonnetSpan!.status).toBe('ok'); - expect(sonnetSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(sonnetSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); - expect(sonnetSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(sonnetSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-5-sonnet-20241022'); - expect(sonnetSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); - }, - }) - .start() - .completed(); - }); - }); - createEsmAndCjsTests(__dirname, 'scenario-conversation-id.mjs', 'instrument.mjs', (createRunner, test) => { test('derives gen_ai.conversation.id from the invoke config and lets a scope id win', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { // Four invocations, each producing the outer chain span, the lambda step span and the chat span. - expect(container.items).toHaveLength(12); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain'); + expect(spans).toHaveLength(12); const idsByName = (name: string): (string | undefined)[] => - container.items + spans .filter(span => span.name === name) .sort((a, b) => a.start_timestamp - b.start_timestamp) .map(span => span.attributes[GEN_AI_CONVERSATION_ID]?.value as string | undefined); // The id reaches the chain span, the child lambda span and the child chat span alike. const expected = ['thread_from_config', 'session_from_config', 'conversation_from_scope', undefined]; - expect(idsByName('chain weather_chain')).toEqual(expected); - expect(idsByName('chain format_prompt')).toEqual(expected); + expect(idsByName('invoke_agent weather_chain')).toEqual(expected); + expect(idsByName('invoke_agent format_prompt')).toEqual(expected); expect(idsByName('chat claude-3-5-sonnet-20241022')).toEqual(expected); }, }) @@ -442,39 +379,4 @@ describe('LangChain integration', () => { .completed(); }); }); - - createEsmAndCjsTests(__dirname, 'scenario-chain.mjs', 'instrument-span-streaming.mjs', (createRunner, test) => { - test('leads chain span names with the operation when span streaming is enabled', async () => { - await createRunner() - .ignore('event') - .expect({ - span: container => { - const chainSpans = container.items.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - ); - // The `unknown_chain` sentinel is dropped, so that span falls back to the bare operation. - expect(chainSpans.map(span => span.name).sort()).toEqual([ - 'invoke_agent', - 'invoke_agent format_prompt', - 'invoke_agent parse_output', - ]); - for (const span of chainSpans) { - expect(span.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('invoke_agent'); - } - expect( - chainSpans - .map(span => span.attributes[GEN_AI_PIPELINE_NAME]?.value) - .filter(Boolean) - .sort(), - ).toEqual(['format_prompt', 'parse_output']); - const unnamedChainSpan = chainSpans.find(span => span.name === 'invoke_agent'); - expect(unnamedChainSpan).toBeDefined(); - expect(unnamedChainSpan!.attributes[GEN_AI_PIPELINE_NAME]).toBeUndefined(); - expect(unnamedChainSpan!.attributes['langchain.chain.name']).toBeUndefined(); - }, - }) - .start() - .completed(); - }); - }); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument-with-pii.mjs index a220d64b148a..1c507ba84b9c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument-with-pii.mjs @@ -2,17 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: true, outputs: true } }, transport: loggingTransport, - beforeSendTransaction: event => { - // Filter out mock express server transactions - if (event.transaction.includes('/v1/messages') || event.transaction.includes('/v1/chat/completions')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument.mjs index 8a2ba440b758..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument.mjs @@ -2,17 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, - beforeSendTransaction: event => { - // Filter out mock express server transactions - if (event.transaction.includes('/v1/messages') || event.transaction.includes('/v1/chat/completions')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts index 669ac68afa34..41158cb85db1 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts @@ -15,6 +15,8 @@ import { GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, + SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; import { GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; @@ -31,60 +33,56 @@ describe('LangChain integration (v1)', () => { 'instrument.mjs', (createRunner, test) => { test('creates langchain related spans with genAI recording disabled', async () => { - await createRunner() - .ignore('event') - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(3); - expect(container.items.map(span => span.name).sort()).toEqual([ - 'chat claude-3-5-sonnet-20241022', - 'chat claude-3-opus-20240229', - 'chat error-model', - ]); + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); - const sonnetSpan = container.items.find(span => span.name === 'chat claude-3-5-sonnet-20241022'); - expect(sonnetSpan).toBeDefined(); - expect(sonnetSpan!.status).toBe('ok'); - expect(sonnetSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(sonnetSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); - expect(sonnetSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(sonnetSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(sonnetSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-5-sonnet-20241022'); - expect(sonnetSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); - expect(sonnetSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); - expect(sonnetSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(sonnetSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); - expect(sonnetSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); - expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_ID]).toBeDefined(); - expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toBeDefined(); - expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]).toBeDefined(); + await runner.start().completed(); - const opusSpan = container.items.find(span => span.name === 'chat claude-3-opus-20240229'); - expect(opusSpan).toBeDefined(); - expect(opusSpan!.status).toBe('ok'); - expect(opusSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(opusSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); - expect(opusSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(opusSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-opus-20240229'); - expect(opusSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.9); - expect(opusSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.95); - expect(opusSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(200); - expect(opusSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(opusSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); - expect(opusSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); + const spans = (await spansPromise).filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain', + ); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual([ + 'chat claude-3-5-sonnet-20241022', + 'chat claude-3-opus-20240229', + 'chat error-model', + ]); - const errorSpan = container.items.find(span => span.name === 'chat error-model'); - expect(errorSpan).toBeDefined(); - expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(errorSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); - expect(errorSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); - }, - }) - .start() - .completed(); + const sonnetSpan = spans.find(span => span.name === 'chat claude-3-5-sonnet-20241022'); + expect(sonnetSpan!.status).toBe('ok'); + expect(sonnetSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(sonnetSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(sonnetSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(sonnetSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); + expect(sonnetSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-5-sonnet-20241022'); + expect(sonnetSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); + expect(sonnetSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); + expect(sonnetSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); + expect(sonnetSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); + expect(sonnetSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); + expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_ID]).toBeDefined(); + expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toBeDefined(); + expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]).toBeDefined(); + + const opusSpan = spans.find(span => span.name === 'chat claude-3-opus-20240229'); + expect(opusSpan!.status).toBe('ok'); + expect(opusSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(opusSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(opusSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); + expect(opusSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-opus-20240229'); + expect(opusSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.9); + expect(opusSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.95); + expect(opusSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(200); + expect(opusSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); + expect(opusSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); + expect(opusSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); + + const errorSpan = spans.find(span => span.name === 'chat error-model'); + expect(errorSpan!.status).toBe('error'); + expect(errorSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(errorSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(errorSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); + expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); }); }, { @@ -102,60 +100,56 @@ describe('LangChain integration (v1)', () => { 'instrument-with-pii.mjs', (createRunner, test) => { test('creates langchain related spans with genAI recording enabled', async () => { - await createRunner() - .ignore('event') - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(3); - expect(container.items.map(span => span.name).sort()).toEqual([ - 'chat claude-3-5-sonnet-20241022', - 'chat claude-3-opus-20240229', - 'chat error-model', - ]); + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); - const sonnetSpan = container.items.find(span => span.name === 'chat claude-3-5-sonnet-20241022'); - expect(sonnetSpan).toBeDefined(); - expect(sonnetSpan!.status).toBe('ok'); - expect(sonnetSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(sonnetSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); - expect(sonnetSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(sonnetSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-5-sonnet-20241022'); - expect(sonnetSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); - expect(sonnetSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); - expect(sonnetSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); - expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toBeDefined(); - expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_ID]).toBeDefined(); - expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toBeDefined(); - expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]).toBeDefined(); - expect(sonnetSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(sonnetSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); - expect(sonnetSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); + const spans = (await spansPromise).filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain', + ); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual([ + 'chat claude-3-5-sonnet-20241022', + 'chat claude-3-opus-20240229', + 'chat error-model', + ]); - const opusSpan = container.items.find(span => span.name === 'chat claude-3-opus-20240229'); - expect(opusSpan).toBeDefined(); - expect(opusSpan!.status).toBe('ok'); - expect(opusSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(opusSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-opus-20240229'); - expect(opusSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.9); - expect(opusSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.95); - expect(opusSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(200); - expect(opusSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); - expect(opusSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toBeDefined(); - expect(opusSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(opusSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); - expect(opusSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); + const sonnetSpan = spans.find(span => span.name === 'chat claude-3-5-sonnet-20241022'); + expect(sonnetSpan!.status).toBe('ok'); + expect(sonnetSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(sonnetSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(sonnetSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); + expect(sonnetSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-5-sonnet-20241022'); + expect(sonnetSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); + expect(sonnetSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); + expect(sonnetSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); + expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toBeDefined(); + expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_ID]).toBeDefined(); + expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toBeDefined(); + expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]).toBeDefined(); + expect(sonnetSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); + expect(sonnetSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); + expect(sonnetSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); - const errorSpan = container.items.find(span => span.name === 'chat error-model'); - expect(errorSpan).toBeDefined(); - expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); - expect(errorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); - }, - }) - .start() - .completed(); + const opusSpan = spans.find(span => span.name === 'chat claude-3-opus-20240229'); + expect(opusSpan!.status).toBe('ok'); + expect(opusSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); + expect(opusSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-opus-20240229'); + expect(opusSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.9); + expect(opusSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.95); + expect(opusSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(200); + expect(opusSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); + expect(opusSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toBeDefined(); + expect(opusSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); + expect(opusSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); + expect(opusSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); + + const errorSpan = spans.find(span => span.name === 'chat error-model'); + expect(errorSpan!.status).toBe('error'); + expect(errorSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); + expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); + expect(errorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); }); }, { @@ -173,32 +167,31 @@ describe('LangChain integration (v1)', () => { 'instrument.mjs', (createRunner, test) => { test('creates langchain spans with tool calls', async () => { - await createRunner() - .ignore('event') - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(1); - const [firstSpan] = container.items; + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = (await spansPromise).filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain', + ); + expect(spans).toHaveLength(1); + const [firstSpan] = spans; - // [0] chat with tool_use stop reason - expect(firstSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); - expect(firstSpan!.status).toBe('ok'); - expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); - expect(firstSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-5-sonnet-20241022'); - expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); - expect(firstSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(150); - expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(20); - expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(30); - expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(50); - expect(firstSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE].value).toBe('tool_use'); - expect(firstSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS]).toBeDefined(); - }, - }) - .start() - .completed(); + // [0] chat with tool_use stop reason + expect(firstSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(firstSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-5-sonnet-20241022'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(150); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(20); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(30); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(50); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE].value).toBe('tool_use'); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS]).toBeDefined(); }); }, { @@ -216,20 +209,19 @@ describe('LangChain integration (v1)', () => { 'instrument.mjs', (createRunner, test) => { test('derives gen_ai.conversation.id from the invoke config and lets a scope id win', async () => { - await createRunner() - .ignore('event') - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(4); - const ids = container.items - .sort((a, b) => a.start_timestamp - b.start_timestamp) - .map(span => span.attributes[GEN_AI_CONVERSATION_ID]?.value); - expect(ids).toEqual(['thread_from_config', 'session_from_config', 'conversation_from_scope', undefined]); - }, - }) - .start() - .completed(); + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = (await spansPromise).filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain', + ); + expect(spans).toHaveLength(4); + const ids = spans + .sort((a, b) => a.start_timestamp - b.start_timestamp) + .map(span => span.attributes[GEN_AI_CONVERSATION_ID]?.value); + expect(ids).toEqual(['thread_from_config', 'session_from_config', 'conversation_from_scope', undefined]); }); }, { @@ -247,30 +239,22 @@ describe('LangChain integration (v1)', () => { 'instrument.mjs', (createRunner, test) => { test('demonstrates timing issue with duplicate spans', async () => { - await createRunner() - .ignore('event') - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(2); - const anthropicSpan = container.items.find( - span => span.attributes['sentry.origin'].value === 'auto.ai.anthropic', - ); - expect(anthropicSpan).toBeDefined(); - expect(anthropicSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); - const langchainSpan = container.items.find( - span => span.attributes['sentry.origin'].value === 'auto.ai.langchain', - ); - expect(langchainSpan).toBeDefined(); - expect(langchainSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + const spans = (await spansPromise).filter( + span => + span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain' || + span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic', + ); + expect(spans).toHaveLength(2); + const anthropicSpan = spans.find(span => span.attributes[SENTRY_ORIGIN].value === 'auto.ai.anthropic'); + expect(anthropicSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); - // Third call (not present): Direct Anthropic call made AFTER LangChain import - // is NOT instrumented, demonstrating the skip mechanism works for NEW clients. - }, - }) - .start() - .completed(); + const langchainSpan = spans.find(span => span.attributes[SENTRY_ORIGIN].value === 'auto.ai.langchain'); + expect(langchainSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); }); }, { @@ -288,60 +272,52 @@ describe('LangChain integration (v1)', () => { 'instrument.mjs', (createRunner, test) => { test('creates langchain spans using initChatModel with OpenAI', async () => { - await createRunner() - .ignore('event') - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(3); - expect(container.items.map(span => span.name).sort()).toEqual([ - 'chat error-model', - 'chat gpt-3.5-turbo', - 'chat gpt-4o', - ]); + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = (await spansPromise).filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain', + ); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual(['chat error-model', 'chat gpt-3.5-turbo', 'chat gpt-4o']); - const gpt4oSpan = container.items.find(span => span.name === 'chat gpt-4o'); - expect(gpt4oSpan).toBeDefined(); - expect(gpt4oSpan!.status).toBe('ok'); - expect(gpt4oSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(gpt4oSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); - expect(gpt4oSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(gpt4oSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); - expect(gpt4oSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gpt-4o'); - expect(gpt4oSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); - expect(gpt4oSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); - expect(gpt4oSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(8); - expect(gpt4oSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); - expect(gpt4oSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); - expect(gpt4oSpan!.attributes[GEN_AI_RESPONSE_ID]).toBeDefined(); - expect(gpt4oSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('gpt-4o'); - expect(gpt4oSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE].value).toBe('stop'); + const gpt4oSpan = spans.find(span => span.name === 'chat gpt-4o'); + expect(gpt4oSpan!.status).toBe('ok'); + expect(gpt4oSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(gpt4oSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(gpt4oSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(gpt4oSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); + expect(gpt4oSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gpt-4o'); + expect(gpt4oSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); + expect(gpt4oSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); + expect(gpt4oSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(8); + expect(gpt4oSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); + expect(gpt4oSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); + expect(gpt4oSpan!.attributes[GEN_AI_RESPONSE_ID]).toBeDefined(); + expect(gpt4oSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('gpt-4o'); + expect(gpt4oSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE].value).toBe('stop'); - const gpt35Span = container.items.find(span => span.name === 'chat gpt-3.5-turbo'); - expect(gpt35Span).toBeDefined(); - expect(gpt35Span!.status).toBe('ok'); - expect(gpt35Span!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(gpt35Span!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); - expect(gpt35Span!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); - expect(gpt35Span!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gpt-3.5-turbo'); - expect(gpt35Span!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.5); - expect(gpt35Span!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(8); - expect(gpt35Span!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); - expect(gpt35Span!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); - expect(gpt35Span!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('gpt-3.5-turbo'); - expect(gpt35Span!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE].value).toBe('stop'); + const gpt35Span = spans.find(span => span.name === 'chat gpt-3.5-turbo'); + expect(gpt35Span!.status).toBe('ok'); + expect(gpt35Span!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(gpt35Span!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(gpt35Span!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); + expect(gpt35Span!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gpt-3.5-turbo'); + expect(gpt35Span!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.5); + expect(gpt35Span!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(8); + expect(gpt35Span!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); + expect(gpt35Span!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); + expect(gpt35Span!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('gpt-3.5-turbo'); + expect(gpt35Span!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE].value).toBe('stop'); - const errorSpan = container.items.find(span => span.name === 'chat error-model'); - expect(errorSpan).toBeDefined(); - expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(errorSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); - expect(errorSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); - expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); - }, - }) - .start() - .completed(); + const errorSpan = spans.find(span => span.name === 'chat error-model'); + expect(errorSpan!.status).toBe('error'); + expect(errorSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(errorSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(errorSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); + expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); }); }, { From d3e5b7ba75df45691679a1f6b67e08c4aeef342c Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Wed, 7 Oct 2026 15:32:09 +0200 Subject: [PATCH 09/47] test(node): Port Mastra tests to span streaming (#25087) Port the Mastra integration suites to default span streaming. Part of #24138 --------- Co-authored-by: GPT-6 --- .../mastra/instrument-no-bootstrap.mjs | 5 - .../tracing/mastra/instrument-with-pii.mjs | 5 - .../suites/tracing/mastra/instrument.mjs | 5 - .../suites/tracing/mastra/test.ts | 235 +++++++++--------- 4 files changed, 112 insertions(+), 138 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/instrument-no-bootstrap.mjs b/dev-packages/node-integration-tests/suites/tracing/mastra/instrument-no-bootstrap.mjs index f4ebcd4a74cc..377693b0a7f1 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mastra/instrument-no-bootstrap.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/instrument-no-bootstrap.mjs @@ -2,15 +2,10 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: false, outputs: false } }, integrations: [Sentry.mastraIntegration({ bootstrapObservability: false })], transport: loggingTransport, - beforeSendTransaction: event => { - // Drop the in-process mock provider's own transactions. - return event.transaction?.includes('/v1/chat/completions') ? null : event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/mastra/instrument-with-pii.mjs index 6cd8bfaa55bc..1c507ba84b9c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mastra/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/instrument-with-pii.mjs @@ -2,14 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: true, outputs: true } }, transport: loggingTransport, - beforeSendTransaction: event => { - // Drop the in-process mock provider's own transactions. - return event.transaction?.includes('/v1/chat/completions') ? null : event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/mastra/instrument.mjs index ec49598a39c9..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mastra/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/instrument.mjs @@ -2,14 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, - beforeSendTransaction: event => { - // Drop the in-process mock provider's own transactions. - return event.transaction?.includes('/v1/chat/completions') ? null : event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts index 728a12762da6..281362c074f4 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts @@ -1,4 +1,5 @@ import { + DB_OPERATION_NAME, GEN_AI_AGENT_NAME, GEN_AI_INPUT_MESSAGES, GEN_AI_OPERATION_NAME, @@ -15,6 +16,9 @@ import { GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, + SENTRY_ORIGIN, + URL_FULL, } from '@sentry/conventions/attributes'; import { afterAll, expect } from 'vitest'; import { conditionalTest } from '../../../utils'; @@ -51,18 +55,16 @@ conditionalTest({ min: 22 })('Mastra integration', () => { (createRunner, test) => { test('creates invoke_agent and chat spans following the gen_ai conventions', async () => { await createRunner() - .expect({ transaction: { transaction: 'mastra-test' } }) .expect({ span: container => { - expect(container.items.map(span => span.name).sort()).toEqual([ - 'chat gpt-4o-mini', - 'invoke_agent weather_agent', - ]); + expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra'); + expect(spans.map(span => span.name).sort()).toEqual(['chat gpt-4o-mini', 'invoke_agent weather_agent']); - const agentSpan = container.items.find(span => span.name === 'invoke_agent weather_agent')!; + const agentSpan = spans.find(span => span.name === 'invoke_agent weather_agent')!; expect(agentSpan.status).toBe('ok'); - expect(agentSpan.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(agentSpan.attributes['sentry.origin'].value).toBe('auto.ai.mastra'); + expect(agentSpan.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(agentSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra'); expect(agentSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('invoke_agent'); expect(agentSpan.attributes[GEN_AI_AGENT_NAME].value).toBe('weather_agent'); expect(agentSpan.attributes[GEN_AI_PIPELINE_NAME].value).toBe('weather_agent'); @@ -71,10 +73,10 @@ conditionalTest({ min: 22 })('Mastra integration', () => { expect(agentSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(19); expect(agentSpan.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('gpt-4o-mini'); - const chatSpan = container.items.find(span => span.name === 'chat gpt-4o-mini')!; + const chatSpan = spans.find(span => span.name === 'chat gpt-4o-mini')!; expect(chatSpan.status).toBe('ok'); - expect(chatSpan.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(chatSpan.attributes['sentry.origin'].value).toBe('auto.ai.mastra'); + expect(chatSpan.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(chatSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra'); expect(chatSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); expect(chatSpan.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gpt-4o-mini'); expect(chatSpan.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); @@ -89,10 +91,12 @@ conditionalTest({ min: 22 })('Mastra integration', () => { test('omits prompts and responses when genAI recording is off', async () => { await createRunner() - .expect({ transaction: { transaction: 'mastra-test' } }) .expect({ span: container => { - for (const span of container.items) { + expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra'); + expect(spans).toHaveLength(2); + for (const span of spans) { expect(span.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); expect(span.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); expect(span.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]).toBeUndefined(); @@ -113,15 +117,16 @@ conditionalTest({ min: 22 })('Mastra integration', () => { (createRunner, test) => { test('records prompts and responses when genAI recording is on', async () => { await createRunner() - .expect({ transaction: { transaction: 'mastra-test' } }) .expect({ span: container => { - const agentSpan = container.items.find(span => span.name === 'invoke_agent weather_agent')!; + expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra'); + const agentSpan = spans.find(span => span.name === 'invoke_agent weather_agent')!; expect(agentSpan.attributes[GEN_AI_SYSTEM_INSTRUCTIONS].value).toBe('You report the weather.'); expect(agentSpan.attributes[GEN_AI_INPUT_MESSAGES].value).toContain('What is the weather in Berlin?'); expect(agentSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value).toContain('It is 22C in Berlin.'); - const chatSpan = container.items.find(span => span.name === 'chat gpt-4o-mini')!; + const chatSpan = spans.find(span => span.name === 'chat gpt-4o-mini')!; expect(chatSpan.attributes[GEN_AI_INPUT_MESSAGES].value).toContain('What is the weather in Berlin?'); expect(chatSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value).toContain('It is 22C in Berlin.'); }, @@ -140,28 +145,29 @@ conditionalTest({ min: 22 })('Mastra integration', () => { (createRunner, test) => { test('creates an execute_tool span parented onto the generation', async () => { await createRunner() - .expect({ transaction: { transaction: 'mastra-test' } }) .expect({ span: container => { - expect(container.items.map(span => span.name).sort()).toEqual([ + expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra'); + expect(spans.map(span => span.name).sort()).toEqual([ 'chat gpt-4o-mini', 'execute_tool get_weather', 'invoke_agent weather_agent', ]); - const toolSpan = container.items.find(span => span.name === 'execute_tool get_weather')!; + const toolSpan = spans.find(span => span.name === 'execute_tool get_weather')!; expect(toolSpan.status).toBe('ok'); - expect(toolSpan.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); - expect(toolSpan.attributes['sentry.origin'].value).toBe('auto.ai.mastra'); + expect(toolSpan.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); + expect(toolSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra'); expect(toolSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('execute_tool'); expect(toolSpan.attributes[GEN_AI_TOOL_NAME].value).toBe('get_weather'); expect(toolSpan.attributes[GEN_AI_TOOL_CALL_ARGUMENTS].value).toContain('Berlin'); expect(toolSpan.attributes[GEN_AI_TOOL_CALL_RESULT].value).toContain('22'); - const chatSpan = container.items.find(span => span.name === 'chat gpt-4o-mini')!; + const chatSpan = spans.find(span => span.name === 'chat gpt-4o-mini')!; expect(toolSpan.parent_span_id).toBe(chatSpan.span_id); - const agentSpan = container.items.find(span => span.name === 'invoke_agent weather_agent')!; + const agentSpan = spans.find(span => span.name === 'invoke_agent weather_agent')!; expect(agentSpan.attributes[GEN_AI_TOOL_DEFINITIONS].value).toContain('get_weather'); expect(agentSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(50); expect(agentSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(13); @@ -182,13 +188,15 @@ conditionalTest({ min: 22 })('Mastra integration', () => { (createRunner, test) => { test('exports the workflow run but drops workflow steps, which have no conventional op', async () => { await createRunner() - .expect({ transaction: { transaction: 'mastra-test' } }) .expect({ span: container => { - expect(container.items.map(span => span.name)).toEqual(['invoke_agent math_workflow']); + const segment = container.items.find(span => span.is_segment && span.name === 'mastra-test'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra'); + expect(spans.map(span => span.name)).toEqual(['invoke_agent math_workflow']); - const workflowSpan = container.items[0]!; - expect(workflowSpan.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + const workflowSpan = spans[0]!; + expect(workflowSpan.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(workflowSpan.attributes[GEN_AI_PIPELINE_NAME].value).toBe('math_workflow'); }, }) @@ -206,15 +214,13 @@ conditionalTest({ min: 22 })('Mastra integration', () => { (createRunner, test) => { test('auto-instruments a Mastra app that configured no observability at all', async () => { await createRunner() - .expect({ transaction: { transaction: 'mastra-test' } }) .expect({ span: container => { - expect(container.items.map(span => span.name).sort()).toEqual([ - 'chat gpt-4o-mini', - 'invoke_agent weather_agent', - ]); - for (const span of container.items) { - expect(span.attributes['sentry.origin'].value).toBe('auto.ai.mastra'); + expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra'); + expect(spans.map(span => span.name).sort()).toEqual(['chat gpt-4o-mini', 'invoke_agent weather_agent']); + for (const span of spans) { + expect(span.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra'); } }, }) @@ -232,13 +238,11 @@ conditionalTest({ min: 22 })('Mastra integration', () => { (createRunner, test) => { test('still exports its own spans when the community @mastra/sentry exporter is present', async () => { await createRunner() - .expect({ transaction: { transaction: 'mastra-test' } }) .expect({ span: container => { - expect(container.items.map(span => span.name).sort()).toEqual([ - 'chat gpt-4o-mini', - 'invoke_agent weather_agent', - ]); + expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra'); + expect(spans.map(span => span.name).sort()).toEqual(['chat gpt-4o-mini', 'invoke_agent weather_agent']); }, }) .start() @@ -255,9 +259,11 @@ conditionalTest({ min: 22 })('Mastra integration', () => { test('leaves an unconfigured app alone when bootstrapObservability is false', async () => { await createRunner() .expect({ - transaction: event => { - expect(event.transaction).toBe('mastra-test'); - const genAiSpans = (event.spans ?? []).filter(span => span.op?.startsWith('gen_ai.')); + span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); + const genAiSpans = container.items.filter(span => + String(span.attributes[SENTRY_OP]?.value ?? '').startsWith('gen_ai.'), + ); expect(genAiSpans).toEqual([]); }, }) @@ -274,56 +280,48 @@ conditionalTest({ min: 22 })('Mastra integration', () => { 'instrument.mjs', (createRunner, test) => { test('nests the model fetch and tool work under the exporter spans', async () => { - // The exporter's gen_ai spans stream as their own span items; the `dataloader` and model - // `http.client` spans ride in the transaction event. Cross-reference once both have arrived. - let chatSpanId: string | undefined; - let executeToolSpanId: string | undefined; - let modelFetchParentIds: (string | undefined)[] = []; - let cacheGetParentIds: (string | undefined)[] = []; - await createRunner() .expect({ - transaction: event => { - expect(event.transaction).toBe('mastra-test'); + span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); // The model-provider requests Mastra makes for each step. - const modelFetches = (event.spans ?? []).filter( - span => span.op === 'http.client' && (span.description ?? '').includes('/v1/chat/completions'), + const modelFetches = container.items.filter( + span => + span.attributes[SENTRY_OP]?.value === 'http.client' && + String(span.attributes[URL_FULL]?.value ?? '').includes('/v1/chat/completions'), ); expect(modelFetches.length).toBeGreaterThan(0); - modelFetchParentIds = modelFetches.map(span => span.parent_span_id); + const modelFetchParentIds = modelFetches.map(span => span.parent_span_id); // The per-key `dataloader.load` spans opened while the tool runs. (dataloader also emits a // `dataloader.batch` span nested under a load — its own internal structure.) - const loadSpans = (event.spans ?? []).filter( - span => span.op === 'cache.get' && span.description === 'dataloader.load', + const loadSpans = container.items.filter( + span => + span.attributes[SENTRY_OP]?.value === 'cache.get' && + span.attributes[DB_OPERATION_NAME]?.value === 'load', ); expect(loadSpans.length).toBeGreaterThan(0); - cacheGetParentIds = loadSpans.map(span => span.parent_span_id); - }, - }) - .expect({ - span: container => { - const chat = container.items.find(span => span.attributes['sentry.op']?.value === 'gen_ai.chat')!; + const cacheGetParentIds = loadSpans.map(span => span.parent_span_id); + const chat = container.items.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.chat')!; expect(chat).toBeDefined(); - chatSpanId = chat.span_id; + const chatSpanId = chat.span_id; const executeTool = container.items.find( span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'count_items', )!; - expect(executeTool).toBeDefined(); - expect(executeTool.attributes['sentry.op']?.value).toBe('gen_ai.execute_tool'); - executeToolSpanId = executeTool.span_id; + expect(executeTool.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); + const executeToolSpanId = executeTool.span_id; + + // The `executeWithContext` bridge makes the exporter spans active during the real work, so the + // model fetch parents onto the `chat` span and the dataloader loads onto `execute_tool`. + expect(chatSpanId).toBeDefined(); + expect(executeToolSpanId).toBeDefined(); + expect(new Set(modelFetchParentIds)).toEqual(new Set([chatSpanId])); + expect(new Set(cacheGetParentIds)).toEqual(new Set([executeToolSpanId])); }, }) .start() .completed(); - - // The `executeWithContext` bridge makes the exporter spans active during the real work, so the - // model fetch parents onto the `chat` span and the dataloader loads onto `execute_tool`. - expect(chatSpanId).toBeDefined(); - expect(executeToolSpanId).toBeDefined(); - expect(new Set(modelFetchParentIds)).toEqual(new Set([chatSpanId])); - expect(new Set(cacheGetParentIds)).toEqual(new Set([executeToolSpanId])); }); }, MASTRA_NESTING_DEPENDENCIES, @@ -337,49 +335,44 @@ conditionalTest({ min: 22 })('Mastra integration', () => { test('nests the model fetch and tool work under the exporter spans when streaming', async () => { // Same nesting guarantee as `generate`, but the agent is driven via `stream()` — the model // request is a real streaming (SSE) call, and the tool still runs inside `executeWithContext`. - let chatSpanId: string | undefined; - let executeToolSpanId: string | undefined; - let modelFetchParentIds: (string | undefined)[] = []; - let cacheGetParentIds: (string | undefined)[] = []; await createRunner() .expect({ - transaction: event => { - expect(event.transaction).toBe('mastra-test'); - const modelFetches = (event.spans ?? []).filter( - span => span.op === 'http.client' && (span.description ?? '').includes('/v1/chat/completions'), + span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); + const modelFetches = container.items.filter( + span => + span.attributes[SENTRY_OP]?.value === 'http.client' && + String(span.attributes[URL_FULL]?.value ?? '').includes('/v1/chat/completions'), ); expect(modelFetches.length).toBeGreaterThan(0); - modelFetchParentIds = modelFetches.map(span => span.parent_span_id); + const modelFetchParentIds = modelFetches.map(span => span.parent_span_id); - const loadSpans = (event.spans ?? []).filter( - span => span.op === 'cache.get' && span.description === 'dataloader.load', + const loadSpans = container.items.filter( + span => + span.attributes[SENTRY_OP]?.value === 'cache.get' && + span.attributes[DB_OPERATION_NAME]?.value === 'load', ); expect(loadSpans.length).toBeGreaterThan(0); - cacheGetParentIds = loadSpans.map(span => span.parent_span_id); - }, - }) - .expect({ - span: container => { - const chat = container.items.find(span => span.attributes['sentry.op']?.value === 'gen_ai.chat')!; + const cacheGetParentIds = loadSpans.map(span => span.parent_span_id); + const chat = container.items.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.chat')!; expect(chat).toBeDefined(); - chatSpanId = chat.span_id; + const chatSpanId = chat.span_id; const executeTool = container.items.find( span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'count_items', )!; - expect(executeTool).toBeDefined(); - expect(executeTool.attributes['sentry.op']?.value).toBe('gen_ai.execute_tool'); - executeToolSpanId = executeTool.span_id; + expect(executeTool.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); + const executeToolSpanId = executeTool.span_id; + + expect(chatSpanId).toBeDefined(); + expect(executeToolSpanId).toBeDefined(); + expect(new Set(modelFetchParentIds)).toEqual(new Set([chatSpanId])); + expect(new Set(cacheGetParentIds)).toEqual(new Set([executeToolSpanId])); }, }) .start() .completed(); - - expect(chatSpanId).toBeDefined(); - expect(executeToolSpanId).toBeDefined(); - expect(new Set(modelFetchParentIds)).toEqual(new Set([chatSpanId])); - expect(new Set(cacheGetParentIds)).toEqual(new Set([executeToolSpanId])); }); }, MASTRA_NESTING_DEPENDENCIES, @@ -394,40 +387,37 @@ conditionalTest({ min: 22 })('Mastra integration', () => { // Two instances run concurrently in one trace. Each instance's `dataloader.load` must nest under // that instance's own `execute_tool` span — if the registry cross-talked, one tool span would // have no dataloader child. - const toolSpanIds: Record = {}; - let loadParentIds: (string | undefined)[] = []; await createRunner() - .expect({ - transaction: event => { - expect(event.transaction).toBe('mastra-multi'); - const loadSpans = (event.spans ?? []).filter( - span => span.op === 'cache.get' && span.description === 'dataloader.load', - ); - loadParentIds = loadSpans.map(span => span.parent_span_id); - }, - }) .expect({ span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'mastra-multi')).toBeDefined(); + const loadSpans = container.items.filter( + span => + span.attributes[SENTRY_OP]?.value === 'cache.get' && + span.attributes[DB_OPERATION_NAME]?.value === 'load', + ); + const loadParentIds = loadSpans.map(span => span.parent_span_id); + const toolSpanIds: Record = {}; for (const toolName of ['count_a', 'count_b']) { const toolSpan = container.items.find(span => span.attributes[GEN_AI_TOOL_NAME]?.value === toolName)!; expect(toolSpan).toBeDefined(); toolSpanIds[toolName] = toolSpan.span_id; } + + // Each instance's tool span parents at least one dataloader load — neither is starved. + for (const toolName of ['count_a', 'count_b']) { + expect(loadParentIds).toContain(toolSpanIds[toolName]); + } + // And every load parents onto one of the two tool spans (no stray reparenting). + const toolIds = new Set(Object.values(toolSpanIds)); + for (const parent of loadParentIds) { + expect(toolIds.has(parent!)).toBe(true); + } }, }) .start() .completed(); - - // Each instance's tool span parents at least one dataloader load — neither is starved. - for (const toolName of ['count_a', 'count_b']) { - expect(loadParentIds).toContain(toolSpanIds[toolName]); - } - // And every load parents onto one of the two tool spans (no stray reparenting). - const toolIds = new Set(Object.values(toolSpanIds)); - for (const parent of loadParentIds) { - expect(toolIds.has(parent!)).toBe(true); - } }); }, MASTRA_NESTING_DEPENDENCIES, @@ -439,10 +429,9 @@ conditionalTest({ min: 22 })('Mastra integration', () => { 'instrument.mjs', (createRunner, test) => { test('captures an error thrown in a Mastra tool as a Sentry issue', async () => { - // The scenario also emits the `mastra-test` transaction and its streamed gen_ai spans; their - // flush order relative to the error event races, so ignore them and match only the issue. + // Span containers can arrive before the error event, so match only the issue. await createRunner() - .ignore('transaction', 'span') + .ignore('span') .expect({ event: event => { const exception = event.exception?.values?.[0]; From f8abf005a5c1c929f766e732740620293ce78f2b Mon Sep 17 00:00:00 2001 From: Asuran Date: Wed, 7 Oct 2026 19:58:48 +0530 Subject: [PATCH 10/47] fix(server-utils): Record LangGraph span I/O for non-MessagesAnnotation state (#23315) `instrumentLangGraph` recorded span input and output only when the graph state uses `MessagesAnnotation`. The input read did `args[0].messages ?? []` and the output helper returned early when `result.messages` was not an array, so a graph on a custom state annotation produced an empty `gen_ai.input.messages` and no `gen_ai.response.text`. No error, just an empty `invoke_agent` span in the AI Agents view. This keeps the `MessagesAnnotation` path unchanged. When there is no `messages` array, it serializes the whole input and output state and records it, wrapped as a single `{ role, content }` message so the attribute stays a valid chat array (the convention `extractLLMRequestAttributes` already uses in this package). A null input (the resume case) records nothing rather than a misleading empty array. Serialization uses core's circular-safe `stringify` so an unusual state object cannot throw inside the span callback. Fixes getsentry/sentry-javascript#19628 AI assistance (Claude, Anthropic) was used in developing this change. The design, review and verification were done by the author. Verified locally: the `@sentry/server-utils` vitest suite (335 to 339 passing), the new tests failing before and passing after the fix, `oxlint --type-aware`, `oxfmt --check`, `tsc` on the changed files, plus a real Google Gemini LangGraph run showing the span input and output empty before and populated after. --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: Abdelrahman Awad Co-authored-by: Andrei Borza --- .../langgraph/scenario-custom-state.mjs | 30 +++++ .../suites/tracing/langgraph/test.ts | 28 ++++ .../server-utils/src/ai/langgraph/index.ts | 38 +++--- .../server-utils/src/ai/langgraph/utils.ts | 11 +- .../test/ai/lib/tracing/langgraph.test.ts | 121 +++++++++++++++++- 5 files changed, 211 insertions(+), 17 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/langgraph/scenario-custom-state.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/scenario-custom-state.mjs b/dev-packages/node-integration-tests/suites/tracing/langgraph/scenario-custom-state.mjs new file mode 100644 index 000000000000..91537c9e33f0 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/scenario-custom-state.mjs @@ -0,0 +1,30 @@ +import { Annotation, END, START, StateGraph } from '@langchain/langgraph'; +import * as Sentry from '@sentry/node'; + +// A graph built on a custom `Annotation.Root` state (no `messages` channel). The invoke_agent span +// records the whole state object on both the input and the output side, unlike the MessagesAnnotation +// graphs the other scenarios use. +const CustomState = Annotation.Root({ + topic: Annotation(), + summary: Annotation(), +}); + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const summarize = state => { + return { summary: `Summary of ${state.topic}` }; + }; + + const graph = new StateGraph(CustomState) + .addNode('summarize', summarize) + .addEdge(START, 'summarize') + .addEdge('summarize', END) + .compile({ name: 'custom_state_agent' }); + + await graph.invoke({ topic: 'weather' }); + }); + + await Sentry.flush(2000); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts b/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts index 6954a7467886..53dc3f6846af 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts @@ -4,6 +4,7 @@ import { GEN_AI_CONVERSATION_ID, GEN_AI_INPUT_MESSAGES, GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, GEN_AI_PIPELINE_NAME, GEN_AI_RESPONSE_MODEL, GEN_AI_RESPONSE_TEXT, @@ -226,6 +227,33 @@ describe('LangGraph integration', () => { }); }); + // Custom `Annotation.Root` state has no `messages` channel, so the whole state object is recorded on + // both the input and the output side of the invoke_agent span. + createEsmAndCjsTests(__dirname, 'scenario-custom-state.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { + test('records custom Annotation.Root state as input and output on the invoke_agent span', async () => { + await createRunner() + .expect({ + span: container => { + const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent custom_state_agent'); + expect(invokeAgentSpan).toBeDefined(); + expect(invokeAgentSpan!.status).toBe('ok'); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langgraph'); + + const inputMessages = getStringAttributeValue(invokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES]?.value); + expect(inputMessages).toContain('"role":"user"'); + expect(inputMessages).toContain('weather'); + + const outputMessages = getStringAttributeValue(invokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]?.value); + expect(outputMessages).toContain('"role":"assistant"'); + expect(outputMessages).toContain('Summary of weather'); + }, + }) + .start() + .completed(); + }); + }); + // createReactAgent tests. // Spans are asserted order-independently: the span-array order is not a protocol guarantee (Sentry // rebuilds the tree from `parent_span_id`), and the provider emits tree order while the OTel exporter diff --git a/packages/server-utils/src/ai/langgraph/index.ts b/packages/server-utils/src/ai/langgraph/index.ts index c5b2da14ffc1..0b1dc9a566d6 100644 --- a/packages/server-utils/src/ai/langgraph/index.ts +++ b/packages/server-utils/src/ai/langgraph/index.ts @@ -153,28 +153,36 @@ export function instrumentCompiledGraphInvoke( span.setAttribute(GEN_AI_TOOL_DEFINITIONS, JSON.stringify(tools)); } - // Parse input messages - const inputMessages = - args.length > 0 ? ((args[0] as { messages?: LangChainMessage[] } | null)?.messages ?? []) : []; - - if (inputMessages && recordInputs) { - const normalizedMessages = normalizeLangChainMessages(inputMessages); - const { systemInstructions, filteredMessages } = extractSystemInstructions(normalizedMessages); - - if (systemInstructions) { - span.setAttribute(GEN_AI_SYSTEM_INSTRUCTIONS, systemInstructions); + // Custom state annotations have no `messages` array, the whole state is recorded instead. + const inputState = args[0] as { messages?: LangChainMessage[]; lg_name?: string } | null | undefined; + const inputMessages = Array.isArray(inputState?.messages) ? inputState.messages : null; + // `new Command({ resume })` resumes an interrupted run and carries no user turn (LangGraph + // tags it `lg_name: 'Command'`), so skip it like a `null` resume rather than recording the + // control object as a message nobody wrote. + const isResumeCommand = inputState?.lg_name === 'Command'; + + if (recordInputs) { + if (inputMessages) { + const normalizedMessages = normalizeLangChainMessages(inputMessages); + const { systemInstructions, filteredMessages } = extractSystemInstructions(normalizedMessages); + + if (systemInstructions) { + span.setAttribute(GEN_AI_SYSTEM_INSTRUCTIONS, systemInstructions); + } + + span.setAttributes({ + [GEN_AI_INPUT_MESSAGES]: stringify(filteredMessages), + }); + } else if (inputState && typeof inputState === 'object' && !isResumeCommand) { + span.setAttribute(GEN_AI_INPUT_MESSAGES, stringify([{ role: 'user', content: stringify(inputState) }])); } - - span.setAttributes({ - [GEN_AI_INPUT_MESSAGES]: stringify(filteredMessages), - }); } // Call original invoke const result = await Reflect.apply(target, thisArg, args); if (recordOutputs) { - setResponseAttributes(span, inputMessages ?? null, result); + setResponseAttributes(span, inputMessages, result); } return result; diff --git a/packages/server-utils/src/ai/langgraph/utils.ts b/packages/server-utils/src/ai/langgraph/utils.ts index dd6d7e61358e..1b261472554b 100644 --- a/packages/server-utils/src/ai/langgraph/utils.ts +++ b/packages/server-utils/src/ai/langgraph/utils.ts @@ -1,5 +1,5 @@ /* eslint-disable typescript-eslint/no-deprecated */ -import { SPAN_STATUS_ERROR, startSpan } from '@sentry/core'; +import { SPAN_STATUS_ERROR, startSpan, stringify } from '@sentry/core'; import type { Span, SpanAttributes } from '@sentry/core'; import { GEN_AI_AGENT_NAME, @@ -22,6 +22,7 @@ import { } from '@sentry/conventions/attributes'; import { GEN_AI_EXECUTE_TOOL } from '@sentry/conventions/op'; import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../core/gen-ai-attributes'; +import { setOutputMessagesAttribute } from '../core/utils'; import type { BaseChatModel, LangChainMessage } from '../langchain/types'; import { extractMessageTokenUsageAttributes, @@ -256,6 +257,14 @@ export function setResponseAttributes(span: Span, inputMessages: LangChainMessag const outputMessages = resultObj?.messages; if (!outputMessages || !Array.isArray(outputMessages)) { + // Custom state annotations have no `messages` array, the whole state is recorded instead. + if (result && typeof result === 'object') { + const serializedState = stringify(result); + // `gen_ai.output.messages` is what the product reads first; `gen_ai.response.text` is kept for + // back-compat (Relay still migrates it). + setOutputMessagesAttribute(span, { responseText: serializedState }); + span.setAttribute(GEN_AI_RESPONSE_TEXT, stringify([{ role: 'assistant', content: serializedState }])); + } return; } diff --git a/packages/server-utils/test/ai/lib/tracing/langgraph.test.ts b/packages/server-utils/test/ai/lib/tracing/langgraph.test.ts index b1f79c9d184c..786fd45ff6df 100644 --- a/packages/server-utils/test/ai/lib/tracing/langgraph.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/langgraph.test.ts @@ -1,9 +1,13 @@ -import { describe, expect, it } from 'vitest'; +import { GEN_AI_INPUT_MESSAGES, GEN_AI_OUTPUT_MESSAGES, GEN_AI_RESPONSE_TEXT } from '@sentry/conventions/attributes'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { getMainCarrier, setCurrentClient, spanToJSON } from '@sentry/core'; +import type { Span } from '@sentry/core'; import { instrumentCreateReactAgent, instrumentStateGraph, instrumentStateGraphCompile, } from '../../../../src/ai/langgraph'; +import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; describe('langgraph double-patch guard', () => { it('instrumentStateGraphCompile returns the same wrapper when applied twice', () => { @@ -32,3 +36,118 @@ describe('instrumentStateGraph', () => { expect(stateGraph.compile).not.toBe(originalCompile); }); }); + +describe('invoke_agent input/output recording', () => { + beforeEach(() => { + getMainCarrier().__SENTRY__ = undefined; + }); + + afterEach(() => { + getMainCarrier().__SENTRY__ = undefined; + }); + + function setupClient(): Span[] { + const client = new TestClient( + getDefaultTestClientOptions({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + tracesSampleRate: 1, + }), + ); + setCurrentClient(client); + client.init(); + + const endedSpans: Span[] = []; + client.on('spanEnd', span => endedSpans.push(span)); + return endedSpans; + } + + async function getInvokeAttributes(invoke: (input: T) => Promise, input: T) { + const endedSpans = setupClient(); + const stateGraph = { compile: () => ({ invoke }) }; + + instrumentStateGraph(stateGraph, { recordInputs: true, recordOutputs: true }); + await stateGraph.compile().invoke(input); + + expect(endedSpans).toHaveLength(1); + return spanToJSON(endedSpans[0]!).attributes; + } + + it('records the full state for a graph that does not use MessagesAnnotation', async () => { + const attributes = await getInvokeAttributes( + async (input: Record) => ({ ...input, expanded: 'expanded idea', validated: true }), + { idea: 'test idea' }, + ); + + expect(JSON.parse(attributes[GEN_AI_INPUT_MESSAGES] as string)).toEqual([ + { role: 'user', content: JSON.stringify({ idea: 'test idea' }) }, + ]); + expect(JSON.parse(attributes[GEN_AI_RESPONSE_TEXT] as string)).toEqual([ + { + role: 'assistant', + content: JSON.stringify({ idea: 'test idea', expanded: 'expanded idea', validated: true }), + }, + ]); + }); + + it('still records chat messages for a MessagesAnnotation graph', async () => { + const attributes = await getInvokeAttributes( + async (input: { messages: Array<{ role: string; content: string }> }) => ({ + messages: [...input.messages, { role: 'assistant', content: 'The weather is sunny' }], + }), + { messages: [{ role: 'user', content: 'What is the weather today?' }] }, + ); + + expect(JSON.parse(attributes[GEN_AI_INPUT_MESSAGES] as string)).toEqual([ + { role: 'user', content: 'What is the weather today?' }, + ]); + expect(attributes[GEN_AI_RESPONSE_TEXT]).toContain('The weather is sunny'); + }); + + it('records an empty messages array as an empty chat array', async () => { + const attributes = await getInvokeAttributes( + async (_input: { messages: unknown[] }) => ({ messages: [{ role: 'assistant', content: 'Hello' }] }), + { messages: [] }, + ); + + expect(attributes[GEN_AI_INPUT_MESSAGES]).toBe('[]'); + expect(attributes[GEN_AI_RESPONSE_TEXT]).toContain('Hello'); + }); + + it('does not record input messages when invoked with null input', async () => { + const attributes = await getInvokeAttributes( + async (_input: null) => ({ messages: [{ role: 'assistant', content: 'resumed' }] }), + null, + ); + + expect(attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); + }); + + it('does not record a Command resume as input (skips it like a null resume)', async () => { + const attributes = await getInvokeAttributes( + async (_input: { lg_name: string; resume: string; goto: unknown[] }) => ({ + messages: [{ role: 'assistant', content: 'resumed' }], + }), + { lg_name: 'Command', resume: 'approved', goto: [] }, + ); + + expect(attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); + expect(attributes[GEN_AI_RESPONSE_TEXT]).toContain('resumed'); + }); + + it('records custom state output as gen_ai.output.messages alongside the deprecated response.text', async () => { + const attributes = await getInvokeAttributes( + async (input: Record) => ({ ...input, summary: 'done' }), + { topic: 'weather' }, + ); + + expect(JSON.parse(attributes[GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([ + { + role: 'assistant', + parts: [{ type: 'text', content: JSON.stringify({ topic: 'weather', summary: 'done' }) }], + }, + ]); + expect(JSON.parse(attributes[GEN_AI_RESPONSE_TEXT] as string)).toEqual([ + { role: 'assistant', content: JSON.stringify({ topic: 'weather', summary: 'done' }) }, + ]); + }); +}); From 49dee2652423a8ee71961f50dbde9e5470c1f392 Mon Sep 17 00:00:00 2001 From: Francesco Gringl-Novy Date: Wed, 7 Oct 2026 16:35:14 +0200 Subject: [PATCH 11/47] test(cloudflare): Widen the wrangler readiness retry window in integration tests (#24925) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `cache-client > cacheClient: true - dedupe drops the same error across invocations` failed on CI on its very first request: wrangler had reported "Ready", but the request still got 500s for the whole ~5s retry budget in `fetchWithRetry`. wrangler printed no error for it, so it's the known "ready but not serving yet" startup race (#20994, #22083), not a failure in the worker. This widens the budget to ~15s. Requests that are expected to fail already disable retries, so the only cost is that a genuinely unexpected 500 takes longer to report, which is fine within the 60s test timeout. The "expected to succeed" error now also includes the response body: CI never showed what the 500 actually was, and this makes the next occurrence diagnosable instead of us widening the window again blindly. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- dev-packages/cloudflare-integration-tests/runner.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dev-packages/cloudflare-integration-tests/runner.ts b/dev-packages/cloudflare-integration-tests/runner.ts index 1ce7493466bd..69ab9e20fc2e 100644 --- a/dev-packages/cloudflare-integration-tests/runner.ts +++ b/dev-packages/cloudflare-integration-tests/runner.ts @@ -85,7 +85,7 @@ type RetryOptions = { maxRetries?: number; retryDelayMs?: number }; async function fetchWithRetry( url: string, init: RequestInit, - { maxRetries = 25, retryDelayMs = 200 }: RetryOptions = {}, + { maxRetries = 75, retryDelayMs = 200 }: RetryOptions = {}, ): Promise { for (let attempt = 0; attempt < maxRetries; attempt++) { try { @@ -569,7 +569,12 @@ export function createRunner(...paths: string[]) { if (!res.ok) { if (!expectError) { - reject(new Error(`Expected request to "${path}" to succeed, but got a ${res.status} response`)); + const responseBody = await res.text().catch(() => ''); + reject( + new Error( + `Expected request to "${path}" to succeed, but got a ${res.status} response: ${responseBody.slice(0, 500)}`, + ), + ); } return; From fcb6cdfd8a69852b3d5f51b01c9936f4b1036daa Mon Sep 17 00:00:00 2001 From: "javascript-sdk-gitflow[bot]" <255134079+javascript-sdk-gitflow[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:57:55 +0200 Subject: [PATCH 12/47] chore: Add external contributor to CHANGELOG.md (#25136) This PR adds the external contributor to the CHANGELOG.md file, so that they are credited for their contribution. See #23315 Co-authored-by: andreiborza <168741329+andreiborza@users.noreply.github.com> --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ca19496299f..a693a101773e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ - "You miss 100 percent of the chances you don't take. — Wayne Gretzky" — Michael Scott +Work in this release was contributed by @zkasuran. Thank you for your contribution! + ## 11.5.0 ### Important Changes From 0201779bd1836f2c06dea158dd1b0c8842e033a2 Mon Sep 17 00:00:00 2001 From: Lukas Stracke Date: Wed, 7 Oct 2026 17:02:12 +0200 Subject: [PATCH 13/47] fix(core): Correct sleep clock drift on telemetry timestamps (#23054) This PR fixes* clock drift that occurs when devices go to sleep. After devices sleep for a few seconds, the browser's monotonic clock (accessed via `performance.now()`) stops counting. Once the sleep stops, the monotonic clock resumes right where it left off before the sleep, causing significant discrepancies between the actual time (wall clock) and the browser's monotonic clock time. Or in other words, there are two ways to get an absolute timestamp: - Using the monotonic clock via `peformance.timeOrigin + performance.now()` - has high, sub-millisecond precision and a guarantee that there are no time jumps or adjustments. But suffers from the sleep problem described above - Using the wall clock via `Date.now()` - lower millisecond precision, is known to be corrected (NTP or via users) and can even cause back jumps. But no sleep problem. With this fix, we make the following adjustments, to kinda get the best of both worlds: - Every `timestampInSecondsCall` checks the monotonic against the wall clock. If a threshold of difference is enocuntered, it corrects the `timeOrigin` so that we can keep using the precise monotonic clock, but anchor its relative time to a corrected time origin. - On every time origin correction, we remember the previous origin (up to 30 corrections). Needed for performance entries - Makes `browserPerformanceTimeOrigin` a time origin corrected helper function, where you pass in a relative time that comes from monotonic clock relative timestamps and it returns the time origin that was most accurate at that relative time point. - All spans and other telemetry we create from browsers `PerformanceEntry` objects carry relative times which are completely sleep-drift uncorrected. We use `browserPerformanceTimeOrigin` to return a corrected time stamp so that we can create an absolute timestamp and bring the performance entries to their respective actual time. ### FAQ If you think this sounds complicated, I agree. So let me answer the most obvious questions, because I asked myself these a lot, too: **Why not just always use `Date.now()` and avoid the complicated click drift detection and correction logic?** The main issue with this is that we still need to rely on relative monotonic clock timestamps for performance entries. We cannot correct them just with `Date.now()` alone but we still need to anchor them. So either we rely on the original `window.timeOrigin` and therefore have all performance entry telemetry happen much "earlier" than its surrounding telemetry relying on `Date.now()`, or we keep the time origin correction logic. But even in the second case (where we already pay the tax for drift detection and correction), telemetry would still have two different anchors: `Date.now()` for regular telemetry and the corrected time origin + monotonic time for performance entry telemetry. Another reason is that the monotonic clock gives us sun-millisecond precision while the wall clock stops at a millisecond resolution. **Why the reduction from a drift detection threshold of 5 minutes to just one second?** Because we keep everything centered around detection and origin offset correction, these timestamps need to be as accurate as possible. On phones, short frequent sleeps are very likely to happen. A lot of sleeps can accumulate until that 5 minutes threshold is reached. So it's really important we make these timestamps as accurate as possible. Fwiw, I reproduced this locally and the drift is already noticeable after just a few seconds of sleep. The 5 minutes threshold was arguably far too big beforehand. **Is there precedence for all of this stuff?** Somewhat, but I'd argue we're doing a bit more: OTel uses a mixture of `Date.now()` and `performance.now`: - Spans start at `Date.now()` and at start time, they record a `performance.now()` timestamp. On span end, they also take `performance.now()`, compute the diff and convert that to an end timestamp based on the start timestamp + diff. This is neat because it guarantees monotony in span durations. I stole this in #24903 In other places, OTel relies purely on Date.now(), and for performance entries, they simply take uncorrected timestamps. So our fix is more complete. \* **So... we're good now?** Well, not perfectly and we never will. With this choice we make another commitment (which we already did previously in less obvious cases) to `Date.now()`. This value can drift as well, just not for sleeps: - NTP adjustments: Can make hard correction of the wall clock, or make the clock tick just a bit faster or slower until the device wall clock synced with the network time. - User adjustments: Users can adjust the device time at any time into any direction. I think we'll have to live with both and I'm not particularly worried about them. My main objective is getting rid of the sleep drift. Fixes #2590 Supersedes #22488, #22585, #23067, #23068 --------- Co-authored-by: Claude Opus 5 (1M context) --- .../browser-utils/src/performance/entries.ts | 95 ++-- .../src/performance/interactions.ts | 7 +- .../src/performance/resourceTiming.ts | 20 +- .../src/performance/userTiming.ts | 54 +- .../browser-utils/src/web-vitals/spans.ts | 25 +- .../performance/addPerformanceEntries.test.ts | 83 ++++ .../test/performance/browserMetrics.test.ts | 47 +- .../test/performance/resourceTiming.test.ts | 115 +---- .../test/performance/userTiming.test.ts | 66 ++- .../test/web-vitals/spans.test.ts | 48 ++ packages/browser/src/client.ts | 6 +- packages/browser/src/profiling/utils.ts | 12 +- packages/browser/test/client.test.ts | 27 +- packages/core/src/index.ts | 7 +- packages/core/src/utils/time.ts | 170 +++++-- packages/core/test/lib/utils/time.test.ts | 468 +++++++++++++++++- .../ember/src/utils/instrumentEmberGlobals.ts | 7 +- .../src/util/createPerformanceEntries.ts | 16 +- .../unit/util/createPerformanceEntry.test.ts | 68 ++- 19 files changed, 1015 insertions(+), 326 deletions(-) create mode 100644 packages/browser-utils/test/performance/addPerformanceEntries.test.ts diff --git a/packages/browser-utils/src/performance/entries.ts b/packages/browser-utils/src/performance/entries.ts index ce03b8405d7c..668a0fb3a077 100644 --- a/packages/browser-utils/src/performance/entries.ts +++ b/packages/browser-utils/src/performance/entries.ts @@ -2,9 +2,9 @@ import type { Span, SpanAttributes } from '@sentry/core'; import { BROWSER_NAVIGATION_TIMING_SPAN_NAMES, - browserPerformanceTimeOrigin, getActiveSpan, parseUrl, + _INTERNAL_performanceTimeToSeconds, RESOURCE_SPAN_NAME_FALLBACK, setMeasurement, spanToJSON, @@ -108,7 +108,10 @@ export function startTrackingLongTasks(): void { const { attributes: parentAttributes, start_timestamp: parentStartTimestamp } = spanToJSON(parent); for (const entry of entries) { - const startTime = msToSec((browserPerformanceTimeOrigin() as number) + entry.startTime); + const startTime = _INTERNAL_performanceTimeToSeconds(entry.startTime); + if (!startTime) { + continue; + } const duration = msToSec(entry.duration); if (parentAttributes[SENTRY_OP] === 'navigation' && parentStartTimestamp && startTime < parentStartTimestamp) { @@ -143,12 +146,11 @@ export function startTrackingLongAnimationFrames(): void { return; } for (const entry of list.getEntries() as PerformanceLongAnimationFrameTiming[]) { - if (!entry.scripts[0]) { + const startTime = _INTERNAL_performanceTimeToSeconds(entry.startTime); + if (!startTime || !entry.scripts[0]) { continue; } - const startTime = msToSec((browserPerformanceTimeOrigin() as number) + entry.startTime); - const { start_timestamp: parentStartTimestamp, attributes: { [SENTRY_OP]: parentOp }, @@ -209,22 +211,22 @@ interface AddPerformanceEntriesOptions { /** Add performance related spans to a transaction */ export function addPerformanceEntries(span: Span, options: AddPerformanceEntriesOptions): void { const performance = getBrowserPerformanceAPI(); - const origin = browserPerformanceTimeOrigin(); - if (!performance?.getEntries || !origin) { + if (!performance?.getEntries) { // Gatekeeper if performance API not available return; } const { spanStreamingEnabled, ignoreResourceSpans } = options; - const timeOrigin = msToSec(origin); - const performanceEntries = performance.getEntries(); const { attributes, start_timestamp: transactionStartTime } = spanToJSON(span); performanceEntries.slice(_performanceCursor).forEach(entry => { - const startTime = msToSec(entry.startTime); + const startTimestamp = _INTERNAL_performanceTimeToSeconds(entry.startTime); + if (!startTimestamp) { + return; + } const duration = msToSec( // Inexplicably, Chrome sometimes emits a negative duration. We need to work around this. // There is a SO post attempting to explain this, but it leaves one with open questions: https://stackoverflow.com/questions/23191918/peformance-getentries-and-negative-duration-display @@ -233,21 +235,17 @@ export function addPerformanceEntries(span: Span, options: AddPerformanceEntries Math.max(0, entry.duration), ); - if ( - attributes[SENTRY_OP] === 'navigation' && - transactionStartTime && - timeOrigin + startTime < transactionStartTime - ) { + if (attributes[SENTRY_OP] === 'navigation' && transactionStartTime && startTimestamp < transactionStartTime) { return; } switch (entry.entryType) { case 'navigation': { - _addNavigationSpans(span, entry as PerformanceNavigationTiming, timeOrigin, spanStreamingEnabled); + _addNavigationSpans(span, entry as PerformanceNavigationTiming, spanStreamingEnabled); break; } case 'paint': { - _addPaintSpan(span, entry, startTime, duration, timeOrigin); + _addPaintSpan(span, entry, startTimestamp, duration); break; } case 'resource': { @@ -255,9 +253,8 @@ export function addPerformanceEntries(span: Span, options: AddPerformanceEntries span, entry as PerformanceResourceTiming, entry.name, - startTime, + startTimestamp, duration, - timeOrigin, ignoreResourceSpans, spanStreamingEnabled, ); @@ -276,15 +273,7 @@ export function addPerformanceEntries(span: Span, options: AddPerformanceEntries * Create a span for a browser paint performance entry. * Exported only for tests. */ -export function _addPaintSpan( - span: Span, - entry: PerformanceEntry, - startTime: number, - duration: number, - timeOrigin: number, -): void { - const startTimestamp = timeOrigin + startTime; - +export function _addPaintSpan(span: Span, entry: PerformanceEntry, startTimestamp: number, duration: number): void { startAndEndSpan(span, startTimestamp, startTimestamp + duration, { // The entry name (`first-paint`, `first-contentful-paint`) is already the low-cardinality name // the conventions ask for, so only the attribute backing it has to be added. @@ -304,19 +293,27 @@ export function _addPaintSpan( export function _addNavigationSpans( span: Span, entry: PerformanceNavigationTiming, - timeOrigin: number, spanStreamingEnabled?: boolean, ): void { - _addPerformanceNavigationTiming(span, entry, 'unloadEvent', timeOrigin, spanStreamingEnabled); - _addPerformanceNavigationTiming(span, entry, 'redirect', timeOrigin, spanStreamingEnabled); - _addPerformanceNavigationTiming(span, entry, 'domContentLoadedEvent', timeOrigin, spanStreamingEnabled); - _addPerformanceNavigationTiming(span, entry, 'loadEvent', timeOrigin, spanStreamingEnabled); - _addPerformanceNavigationTiming(span, entry, 'connect', timeOrigin, spanStreamingEnabled); - _addPerformanceNavigationTiming(span, entry, 'secureConnection', timeOrigin, spanStreamingEnabled); - _addPerformanceNavigationTiming(span, entry, 'fetch', timeOrigin, spanStreamingEnabled); - _addPerformanceNavigationTiming(span, entry, 'domainLookup', timeOrigin, spanStreamingEnabled); - - _addRequest(span, entry, timeOrigin, spanStreamingEnabled); + _addPerformanceNavigationTiming(span, entry, 'unloadEvent', spanStreamingEnabled); + _addPerformanceNavigationTiming(span, entry, 'redirect', spanStreamingEnabled); + _addPerformanceNavigationTiming(span, entry, 'domContentLoadedEvent', spanStreamingEnabled); + _addPerformanceNavigationTiming(span, entry, 'loadEvent', spanStreamingEnabled); + _addPerformanceNavigationTiming(span, entry, 'connect', spanStreamingEnabled); + _addPerformanceNavigationTiming(span, entry, 'secureConnection', spanStreamingEnabled); + _addPerformanceNavigationTiming(span, entry, 'fetch', spanStreamingEnabled); + _addPerformanceNavigationTiming(span, entry, 'domainLookup', spanStreamingEnabled); + + _addRequest(span, entry, spanStreamingEnabled); +} + +/** + * Navigations can happen long after page load, after the time origin was corrected for drift. We use the origin from + * the entry's start for all its timings, so its duration stays correct. + */ +function _navigationTimeToSeconds(entry: PerformanceNavigationTiming, time: number): number { + // The cast is safe: `addPerformanceEntries` only adds navigation spans if the time origin is available. + return _INTERNAL_performanceTimeToSeconds(time, entry.startTime) as number; } type StartEventName = @@ -354,7 +351,6 @@ function _addPerformanceNavigationTiming( span: Span, entry: PerformanceNavigationTiming, event: StartEventName, - timeOrigin: number, spanStreamingEnabled: boolean | undefined, ): void { const eventEnd = _getEndPropertyNameForNavigationTiming(event) satisfies keyof PerformanceNavigationTiming; @@ -364,7 +360,7 @@ function _addPerformanceNavigationTiming( return; } const op = NAVIGATION_TIMING_SPAN_OPS[event]; - startAndEndSpan(span, timeOrigin + msToSec(start), timeOrigin + msToSec(end), { + startAndEndSpan(span, _navigationTimeToSeconds(entry, start), _navigationTimeToSeconds(entry, end), { // With span streaming, span names have to be low cardinality, so we can't fall back to the // document URL. `url.full` keeps it, and is what Relay derives the description from. name: spanStreamingEnabled ? BROWSER_NAVIGATION_TIMING_SPAN_NAMES[op] : entry.name, @@ -388,15 +384,10 @@ function _getEndPropertyNameForNavigationTiming(event: StartEventName): EndEvent } /** Create request and response related spans */ -function _addRequest( - span: Span, - entry: PerformanceNavigationTiming, - timeOrigin: number, - spanStreamingEnabled: boolean | undefined, -): void { - const requestStartTimestamp = timeOrigin + msToSec(entry.requestStart); - const responseEndTimestamp = timeOrigin + msToSec(entry.responseEnd); - const responseStartTimestamp = timeOrigin + msToSec(entry.responseStart); +function _addRequest(span: Span, entry: PerformanceNavigationTiming, spanStreamingEnabled: boolean | undefined): void { + const requestStartTimestamp = _navigationTimeToSeconds(entry, entry.requestStart); + const responseEndTimestamp = _navigationTimeToSeconds(entry, entry.responseEnd); + const responseStartTimestamp = _navigationTimeToSeconds(entry, entry.responseStart); if (entry.responseEnd) { // It is possible that we are collecting these metrics when the page hasn't finished loading yet, for example when the HTML slowly streams in. // In this case, ie. when the document request hasn't finished yet, `entry.responseEnd` will be 0. @@ -435,9 +426,8 @@ export function _addResourceSpans( span: Span, entry: PerformanceResourceTiming, resourceUrl: string, - startTime: number, + startTimestamp: number, duration: number, - timeOrigin: number, ignoredResourceSpanOps?: Array, spanStreamingEnabled?: boolean, ): void { @@ -497,7 +487,6 @@ export function _addResourceSpans( const attributesWithResourceTiming: SpanAttributes = { ...attributes, ...resourceTimingToSpanAttributes(entry) }; - const startTimestamp = timeOrigin + startTime; const endTimestamp = startTimestamp + duration; startAndEndSpan(span, startTimestamp, endTimestamp, { diff --git a/packages/browser-utils/src/performance/interactions.ts b/packages/browser-utils/src/performance/interactions.ts index 7a3cc89a981d..33913a40c619 100644 --- a/packages/browser-utils/src/performance/interactions.ts +++ b/packages/browser-utils/src/performance/interactions.ts @@ -16,13 +16,13 @@ import { import { UI_ACTION_CLICK, UI_INTERACTION_CLICK } from '@sentry/conventions/op'; import type { Client, IntegrationFn, Span, StartSpanOptions, TransactionSource } from '@sentry/core'; import { - browserPerformanceTimeOrigin, debug, defineIntegration, filterCollectedUrl, getActiveSpan, getRootSpan, hasSpanStreamingEnabled, + _INTERNAL_performanceTimeToSeconds, spanToJSON, UI_ACTION_CLICK_SPAN_NAME_FALLBACK, UI_INTERACTION_CLICK_SPAN_NAME_FALLBACK, @@ -242,7 +242,10 @@ function trackInteractionsAsSpans(client: Client): void { } for (const entry of entries) { if (entry.name === 'click') { - const startTime = msToSec((browserPerformanceTimeOrigin() as number) + entry.startTime); + const startTime = _INTERNAL_performanceTimeToSeconds(entry.startTime); + if (!startTime) { + continue; + } const duration = msToSec(entry.duration); const selector = htmlTreeAsString(entry.target); diff --git a/packages/browser-utils/src/performance/resourceTiming.ts b/packages/browser-utils/src/performance/resourceTiming.ts index 5a711d307cf3..172a454f31da 100644 --- a/packages/browser-utils/src/performance/resourceTiming.ts +++ b/packages/browser-utils/src/performance/resourceTiming.ts @@ -1,12 +1,6 @@ import type { SpanAttributes } from '@sentry/core'; -import { browserPerformanceTimeOrigin } from '@sentry/core'; -import { extractNetworkProtocol, getBrowserPerformanceAPI } from './utils'; - -function getAbsoluteTime(time: number | undefined): number | undefined { - // falsy values should be preserved so that we can later on drop undefined values and - // preserve 0 vals for cross-origin resources without proper `Timing-Allow-Origin` header. - return time ? ((browserPerformanceTimeOrigin() || performance.timeOrigin) + time) / 1000 : time; -} +import { _INTERNAL_performanceTimeToSeconds } from '@sentry/core'; +import { extractNetworkProtocol, msToSec } from './utils'; /** * Converts a PerformanceResourceTiming entry to span data for the resource span. Most importantly, @@ -28,10 +22,16 @@ export function resourceTimingToSpanAttributes(resourceTiming: PerformanceResour timingSpanData['network.protocol.name'] = name; } - if (!(browserPerformanceTimeOrigin() || getBrowserPerformanceAPI()?.timeOrigin)) { + if (_INTERNAL_performanceTimeToSeconds(resourceTiming.startTime) === undefined) { return timingSpanData; } + // Use the origin from the request start for all timings, so the durations between them stay correct. + const getAbsoluteTime = (time: number | undefined): number | undefined => + // falsy values should be preserved so that we can later on drop undefined values and + // preserve 0 vals for cross-origin resources without proper `Timing-Allow-Origin` header. + time ? _INTERNAL_performanceTimeToSeconds(time, resourceTiming.startTime) : time; + return dropUndefinedKeysFromObject({ ...timingSpanData, @@ -58,7 +58,7 @@ export function resourceTimingToSpanAttributes(resourceTiming: PerformanceResour // This way, TTFB always measures the "first page load" experience. // see: https://web.dev/articles/ttfb#measure-resource-requests 'http.request.time_to_first_byte': - resourceTiming.responseStart != null ? resourceTiming.responseStart / 1000 : undefined, + resourceTiming.responseStart != null ? msToSec(resourceTiming.responseStart) : undefined, }); } diff --git a/packages/browser-utils/src/performance/userTiming.ts b/packages/browser-utils/src/performance/userTiming.ts index 02d6fc25694d..9c970f0dea78 100644 --- a/packages/browser-utils/src/performance/userTiming.ts +++ b/packages/browser-utils/src/performance/userTiming.ts @@ -1,13 +1,13 @@ import { SENTRY_OP, SENTRY_ORIGIN } from '@sentry/conventions/attributes'; import type { IntegrationFn, Span, SpanAttributes, SpanAttributeValue } from '@sentry/core'; import { - browserPerformanceTimeOrigin, defineIntegration, isPrimitive, + _INTERNAL_performanceTimeToSeconds, spanToJSON, stringMatchesSomePattern, } from '@sentry/core'; -import { getBrowserPerformanceAPI, msToSec, startAndEndSpan } from './utils'; +import { getBrowserPerformanceAPI, startAndEndSpan } from './utils'; import { getNavigationEntry } from '../web-vitals/utils'; interface UserTimingOptions { @@ -26,11 +26,9 @@ const _userTimingIntegration = ((options: UserTimingOptions = {}) => { name: INTEGRATION_NAME, setup(client) { const performance = getBrowserPerformanceAPI(); - const timeOrigin = browserPerformanceTimeOrigin(); - if (!performance?.getEntries || !timeOrigin) { + if (!performance?.getEntries) { return; } - const timeOriginInSeconds = msToSec(timeOrigin); let performanceCursor = 0; client.on('beforeIdleSpanEnd', idleSpan => { @@ -41,7 +39,7 @@ const _userTimingIntegration = ((options: UserTimingOptions = {}) => { return; } - const requestTime = msToSec(getNavigationEntry(false)?.requestStart ?? 0); + const requestStart = getNavigationEntry(false)?.requestStart ?? 0; const performanceEntries = performance.getEntries(); for (const entry of performanceEntries.slice(performanceCursor)) { @@ -49,22 +47,25 @@ const _userTimingIntegration = ((options: UserTimingOptions = {}) => { continue; } - const startTime = msToSec(entry.startTime); - const absoluteStartTime = timeOriginInSeconds + startTime; + const startTimestamp = _INTERNAL_performanceTimeToSeconds(entry.startTime); + if (!startTimestamp) { + continue; + } - if (parentOp === 'navigation' && parentStartTimestamp && absoluteStartTime < parentStartTimestamp) { + if (parentOp === 'navigation' && parentStartTimestamp && startTimestamp < parentStartTimestamp) { continue; } - _addUserTimingSpan( - idleSpan, - entry, - startTime, - msToSec(Math.max(0, entry.duration)), - timeOriginInSeconds, - requestTime, - options.ignore ?? [], - ); + const endTimestamp = _INTERNAL_performanceTimeToSeconds( + entry.startTime + Math.max(0, entry.duration), + entry.startTime, + ) as number; + + // We convert the request start with the entry's time origin, so that the clamp in `_addUserTimingSpan` compares + // timestamps on the same timeline, even if the time origin was corrected between the request and the entry. + const requestTimestamp = _INTERNAL_performanceTimeToSeconds(requestStart, entry.startTime) as number; + + _addUserTimingSpan(idleSpan, entry, startTimestamp, endTimestamp, requestTimestamp, options.ignore ?? []); } performanceCursor = performanceEntries.length; @@ -100,10 +101,9 @@ export const userTimingIntegration = defineIntegration(_userTimingIntegration); export function _addUserTimingSpan( parentSpan: Span, entry: PerformanceEntry, - startTime: number, - duration: number, - timeOrigin: number, - requestTime: number, + startTimestamp: number, + endTimestamp: number, + requestTimestamp: number, ignore: Array, ): void { if (isReact19MeasureEntry(entry) || stringMatchesSomePattern(entry.name, ignore)) { @@ -111,15 +111,13 @@ export function _addUserTimingSpan( } // Measures can reference arbitrary timestamps, including timestamps before the page request started. - const spanStartTimestamp = timeOrigin + Math.max(startTime, requestTime); - const originalStartTimestamp = timeOrigin + startTime; - const spanEndTimestamp = originalStartTimestamp + duration; + const spanStartTimestamp = Math.max(startTimestamp, requestTimestamp); const attributes: SpanAttributes = { [SENTRY_ORIGIN]: `auto.browser.user_timing.${entry.entryType}`, }; - if (spanStartTimestamp !== originalStartTimestamp) { + if (spanStartTimestamp !== startTimestamp) { attributes['sentry.browser.measure_happened_before_request'] = true; attributes['sentry.browser.measure_start_time'] = spanStartTimestamp; } @@ -127,8 +125,8 @@ export function _addUserTimingSpan( addDetailToSpanAttributes(attributes, entry as PerformanceMeasure); // Third-party measurements can contain timestamps which would produce invalid spans. - if (spanStartTimestamp <= spanEndTimestamp) { - startAndEndSpan(parentSpan, spanStartTimestamp, spanEndTimestamp, { + if (spanStartTimestamp <= endTimestamp) { + startAndEndSpan(parentSpan, spanStartTimestamp, endTimestamp, { name: entry.name, op: entry.entryType, attributes, diff --git a/packages/browser-utils/src/web-vitals/spans.ts b/packages/browser-utils/src/web-vitals/spans.ts index 0e60188b456f..5c44ea8d1078 100644 --- a/packages/browser-utils/src/web-vitals/spans.ts +++ b/packages/browser-utils/src/web-vitals/spans.ts @@ -7,6 +7,7 @@ import { getClient, getRootSpan, hasSpanStreamingEnabled, + _INTERNAL_performanceTimeToSeconds, spanToJSON, timestampInSeconds, UI_INTERACTION_CLICK_SPAN_NAME_FALLBACK, @@ -190,14 +191,17 @@ export function _sendLcpSpan( DEBUG_BUILD && debug.log(`Sending LCP span (${lcpValue})`); - const performanceTimeOrigin = browserPerformanceTimeOrigin() || 0; // A soft navigation's LCP is measured from the triggering interaction, not the document time // origin. Starting the span there too keeps it inside the navigation span it is parented to and - // keeps its duration equal to the value it reports. - const startTime = msToSec(performanceTimeOrigin + (navigationStartTime || 0)); + // keeps its duration equal to the reported value. The span's end uses the same origin, even if + // the time origin was corrected in between. + const navigationStart = navigationStartTime || 0; + const startTime = _INTERNAL_performanceTimeToSeconds(navigationStart) ?? msToSec(navigationStart); // Without an entry there is no render time to end at, so the span lasts the value it reports, // like an entry-less INP does. Ending at the time origin instead would invert the span. - const endTime = entry ? msToSec(performanceTimeOrigin + entry.startTime) : startTime + msToSec(lcpValue); + const endTime = entry + ? (_INTERNAL_performanceTimeToSeconds(entry.startTime, navigationStart) ?? msToSec(entry.startTime)) + : startTime + msToSec(lcpValue); const selector = entry ? htmlTreeAsString(entry.element) : undefined; const componentName = entry?.element ? getComponentName(entry.element) : null; const client = getClient(); @@ -294,12 +298,12 @@ export function _sendClsSpan( ): void { DEBUG_BUILD && debug.log(`Sending CLS span (${clsValue})`); - const performanceTimeOrigin = browserPerformanceTimeOrigin(); // A CLS of 0 has no shift to place the span at. It is reported when the navigation it was // measured on is already over - the next soft navigation, or pagehide - so the current time would // land it outside that navigation, on the route that follows it. const offset = entry?.startTime ?? navigationStartTime ?? 0; - const startTime = performanceTimeOrigin ? msToSec(performanceTimeOrigin + offset) : timestampInSeconds(); + // CLS is only reported on pagehide, so we use the time origin from when the layout shift happened. + const startTime = _INTERNAL_performanceTimeToSeconds(offset) ?? timestampInSeconds(); const firstSourceNode = entry?.sources[0]?.node; const selector = entry ? htmlTreeAsString(firstSourceNode) : undefined; const componentName = firstSourceNode ? getComponentName(firstSourceNode) : null; @@ -406,9 +410,12 @@ export function _sendInpSpan( // A web vital span carries the metric, not a real interaction timing, so an INP without an entry // is still worth reporting. It just has no element or interaction type to describe, and is placed // at the start of the navigation it belongs to rather than at the interaction. - const startTime = msToSec( - (browserPerformanceTimeOrigin() as number) + (entry?.startTime ?? metric?.navigationStartTime ?? 0), - ); + // INP is reported on pagehide, often long after the interaction, so we use the time origin from when the + // interaction happened. + const startTime = _INTERNAL_performanceTimeToSeconds(entry?.startTime ?? metric?.navigationStartTime ?? 0); + if (!startTime) { + return; + } const duration = msToSec(inpValue); // An INP without an entry has no interaction type to report. It still has to land inside the // `ui.interaction.*` family, because falling outside it would hide exactly the fast navigations diff --git a/packages/browser-utils/test/performance/addPerformanceEntries.test.ts b/packages/browser-utils/test/performance/addPerformanceEntries.test.ts new file mode 100644 index 000000000000..8d37db8b992b --- /dev/null +++ b/packages/browser-utils/test/performance/addPerformanceEntries.test.ts @@ -0,0 +1,83 @@ +import { HTTP_REQUEST_RESPONSE_END, SENTRY_OP } from '@sentry/conventions/attributes'; +import type { Span } from '@sentry/core'; +import * as SentryCore from '@sentry/core'; +import { getClient, getMainCarrier, SentrySpan, setCurrentClient, spanToJSON } from '@sentry/core'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { addPerformanceEntries } from '../../src/performance/entries'; +import { WINDOW } from '../../src/types'; +import { getDefaultClientOptions, TestClient } from '../utils/TestClient'; + +vi.mock('@sentry/core', async () => { + const actual = await vi.importActual('@sentry/core'); + const browserPerformanceTimeOrigin = vi.fn(); + return { + ...actual, + browserPerformanceTimeOrigin, + _INTERNAL_performanceTimeToSeconds: (time: number, entryStartTime = time) => + (browserPerformanceTimeOrigin(entryStartTime) + time) / 1000, + }; +}); + +const pageloadOriginMs = 1_000_000; +const sleepDurationMs = 3_600_000; +// The `performance.now()` time from which the corrected time origin applies. +const correctionFromMs = 10_000; + +describe('addPerformanceEntries', () => { + beforeEach(() => { + getMainCarrier().__SENTRY__ = undefined; + const client = new TestClient(getDefaultClientOptions({ tracesSampleRate: 1 })); + setCurrentClient(client); + client.init(); + + vi.mocked(SentryCore.browserPerformanceTimeOrigin).mockImplementation((monotonicTimeInMs = 0) => + monotonicTimeInMs < correctionFromMs ? pageloadOriginMs : pageloadOriginMs + sleepDurationMs, + ); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + }); + + it('keeps the resource spans of a navigation that happens after a time origin correction', () => { + const resourceStartTime = 12_000; + vi.stubGlobal('addEventListener', vi.fn()); + vi.stubGlobal('location', { origin: 'https://example.com' }); + vi.spyOn(WINDOW.performance, 'getEntries').mockReturnValue([ + { + entryType: 'resource', + name: 'https://example.com/app.js', + initiatorType: 'script', + startTime: resourceStartTime, + duration: 100, + responseEnd: resourceStartTime + 100, + toJSON: () => ({}), + } as PerformanceResourceTiming, + ]); + + const navigationStartTimestamp = (pageloadOriginMs + sleepDurationMs + resourceStartTime - 50) / 1000; + const span = new SentrySpan({ + op: 'navigation', + name: '/next', + sampled: true, + startTimestamp: navigationStartTimestamp, + }); + + const endedSpans: Span[] = []; + getClient()?.on('spanEnd', endedSpan => void endedSpans.push(endedSpan)); + + addPerformanceEntries(span, { ignoreResourceSpans: [] }); + + expect(endedSpans.map(spanToJSON)).toEqual([ + expect.objectContaining({ + start_timestamp: (pageloadOriginMs + sleepDurationMs + resourceStartTime) / 1000, + end_timestamp: (pageloadOriginMs + sleepDurationMs + resourceStartTime + 100) / 1000, + attributes: expect.objectContaining({ + [SENTRY_OP]: 'resource.script', + [HTTP_REQUEST_RESPONSE_END]: (pageloadOriginMs + sleepDurationMs + resourceStartTime + 100) / 1000, + }), + }), + ]); + }); +}); diff --git a/packages/browser-utils/test/performance/browserMetrics.test.ts b/packages/browser-utils/test/performance/browserMetrics.test.ts index f9dcccabb0b4..7a5455aa491d 100644 --- a/packages/browser-utils/test/performance/browserMetrics.test.ts +++ b/packages/browser-utils/test/performance/browserMetrics.test.ts @@ -173,7 +173,7 @@ describe('_addResourceSpans', () => { renderBlockingStatus: 'non-blocking', nextHopProtocol: 'http/1.1', }); - _addResourceSpans(span, entry, resourceEntryName, 123, 456, 100); + _addResourceSpans(span, entry, resourceEntryName, 123, 456); expect(spans).toHaveLength(0); }); @@ -193,7 +193,7 @@ describe('_addResourceSpans', () => { renderBlockingStatus: 'non-blocking', nextHopProtocol: 'http/1.1', }); - _addResourceSpans(span, entry, 'https://example.com/assets/to/me', 123, 456, 100); + _addResourceSpans(span, entry, 'https://example.com/assets/to/me', 123, 456); expect(spans).toHaveLength(0); }); @@ -226,18 +226,17 @@ describe('_addResourceSpans', () => { workerStart: 1006, }); - const timeOrigin = 100; - const startTime = 23; + const startTimestamp = 123; const duration = 356; - _addResourceSpans(span, entry, resourceEntryName, startTime, duration, timeOrigin); + _addResourceSpans(span, entry, resourceEntryName, startTimestamp, duration); expect(spans).toHaveLength(1); expect(spanToJSON(spans[0]!)).toEqual( expect.objectContaining({ name: '/assets/to/css', - start_timestamp: timeOrigin + startTime, - end_timestamp: timeOrigin + startTime + duration, + start_timestamp: startTimestamp, + end_timestamp: startTimestamp + duration, attributes: { [SENTRY_OP]: 'resource.css', [SENTRY_ORIGIN]: 'auto.resource.browser.metrics', @@ -282,7 +281,7 @@ describe('_addResourceSpans', () => { nextHopProtocol: 'http/1.1', }); - _addResourceSpans(span, entry, 'https://example.com/assets/app.js?v=42#main', 100, 23, 345); + _addResourceSpans(span, entry, 'https://example.com/assets/app.js?v=42#main', 100, 23); expect(spans).toHaveLength(1); const json = spanToJSON(spans[0]!); @@ -302,7 +301,7 @@ describe('_addResourceSpans', () => { nextHopProtocol: 'http/1.1', }); - _addResourceSpans(span, entry, 'https://cdn.example.org/static/logo.png', 100, 23, 345); + _addResourceSpans(span, entry, 'https://cdn.example.org/static/logo.png', 100, 23); expect(spans).toHaveLength(1); const json = spanToJSON(spans[0]!); @@ -346,7 +345,7 @@ describe('_addResourceSpans', () => { initiatorType, nextHopProtocol: 'http/1.1', }); - _addResourceSpans(span, entry, 'https://example.com/assets/to/me', 123, 234, 465); + _addResourceSpans(span, entry, 'https://example.com/assets/to/me', 123, 234); expect(spans).toHaveLength(i + 1); expect(spanToJSON(spans[i]!).attributes).toEqual(expect.objectContaining({ [SENTRY_OP]: op })); @@ -389,7 +388,7 @@ describe('_addResourceSpans', () => { initiatorType, nextHopProtocol: 'http/1.1', }); - _addResourceSpans(span, entry, 'https://example.com/assets/to/me', 123, 234, 465, ignoredResourceSpans); + _addResourceSpans(span, entry, 'https://example.com/assets/to/me', 123, 234, ignoredResourceSpans); } expect(spans).toHaveLength(table.length - ignoredResourceSpans.length); const spanOps = new Set( @@ -416,7 +415,7 @@ describe('_addResourceSpans', () => { nextHopProtocol: 'h2', }); - _addResourceSpans(span, entry, resourceEntryName, 100, 23, 345); + _addResourceSpans(span, entry, resourceEntryName, 100, 23); expect(spans).toHaveLength(1); expect(spanToJSON(spans[0]!)).toEqual( @@ -455,7 +454,7 @@ describe('_addResourceSpans', () => { nextHopProtocol: 'h3', }); - _addResourceSpans(span, entry, resourceEntryName, 100, 23, 345); + _addResourceSpans(span, entry, resourceEntryName, 100, 23); expect(spans).toHaveLength(1); expect(spanToJSON(spans[0]!)).toEqual( @@ -472,8 +471,8 @@ describe('_addResourceSpans', () => { ['network.protocol.version']: '3', }), name: '/assets/to/css', - end_timestamp: 468, - start_timestamp: 445, + end_timestamp: 123, + start_timestamp: 100, }), ); }); @@ -507,7 +506,7 @@ describe('_addResourceSpans', () => { workerStart: 1006, } as unknown as PerformanceResourceTiming; - _addResourceSpans(span, entry, resourceEntryName, 100, 23, 345); + _addResourceSpans(span, entry, resourceEntryName, 100, 23); expect(spans).toHaveLength(1); expect(spanToJSON(spans[0]!)).toEqual( @@ -537,8 +536,8 @@ describe('_addResourceSpans', () => { 'http.request.worker_start': expect.any(Number), }, name: '/assets/to/css', - end_timestamp: 468, - start_timestamp: 445, + end_timestamp: 123, + start_timestamp: 100, }), ); }); @@ -563,7 +562,7 @@ describe('_addResourceSpans', () => { nextHopProtocol: 'h3', }); - _addResourceSpans(span, entry, resourceEntryName, 100, 23, 345); + _addResourceSpans(span, entry, resourceEntryName, 100, 23); expect(spans).toHaveLength(1); expect(spanToJSON(spans[0]!).attributes).toMatchObject({ @@ -586,7 +585,7 @@ describe('_addResourceSpans', () => { const entry = mockPerformanceResourceTiming({ initiatorType: 'css', nextHopProtocol: 'h2' }); - _addResourceSpans(span, entry, url, 100, 23, 345, undefined, true); + _addResourceSpans(span, entry, url, 100, 23, undefined, true); expect(spans).toHaveLength(1); expect(spanToJSON(spans[0]!)).toEqual( @@ -609,7 +608,7 @@ describe('_addResourceSpans', () => { const entry = mockPerformanceResourceTiming({ initiatorType: 'script', nextHopProtocol: 'h2' }); - _addResourceSpans(span, entry, 'blob:0f6b3f0a-1e2d-4d1a-9c3f-2a5c1d7b8e90', 100, 23, 345, undefined, true); + _addResourceSpans(span, entry, 'blob:0f6b3f0a-1e2d-4d1a-9c3f-2a5c1d7b8e90', 100, 23, undefined, true); expect(spans).toHaveLength(1); const spanJson = spanToJSON(spans[0]!); @@ -688,7 +687,7 @@ describe('_addNavigationSpans', () => { spans.push(span); }); - _addNavigationSpans(pageloadSpan, entry, 999); + _addNavigationSpans(pageloadSpan, entry); const trace_id = pageloadSpan.spanContext().traceId; const parent_span_id = pageloadSpan.spanContext().spanId; @@ -811,7 +810,7 @@ describe('_addNavigationSpans', () => { }); // `unloadEventStart`/`End` are 0 in the shared entry, so that span is never created. - _addNavigationSpans(pageloadSpan, { ...entry, unloadEventStart: 1, unloadEventEnd: 2 }, 999, true); + _addNavigationSpans(pageloadSpan, { ...entry, unloadEventStart: 1, unloadEventEnd: 2 }, true); const spanJson = spans.map(spanToJSON).find(span => span.attributes[SENTRY_OP] === op); @@ -846,7 +845,7 @@ describe('_addPaintSpan', () => { duration: 0, } as PerformanceEntry; - _addPaintSpan(pageloadSpan, entry, 12, 0, 999); + _addPaintSpan(pageloadSpan, entry, 12, 0); expect(spans).toHaveLength(1); const spanJson = spanToJSON(spans[0]!); diff --git a/packages/browser-utils/test/performance/resourceTiming.test.ts b/packages/browser-utils/test/performance/resourceTiming.test.ts index c6749c6455aa..d1e6426e837a 100644 --- a/packages/browser-utils/test/performance/resourceTiming.test.ts +++ b/packages/browser-utils/test/performance/resourceTiming.test.ts @@ -11,6 +11,10 @@ describe('resourceTimingToSpanAttributes', () => { beforeEach(() => { vi.clearAllMocks(); browserPerformanceTimeOriginSpy = vi.spyOn(utils, 'browserPerformanceTimeOrigin'); + vi.spyOn(utils, '_INTERNAL_performanceTimeToSeconds').mockImplementation((time, entryStartTime = time) => { + const origin = utils.browserPerformanceTimeOrigin(entryStartTime); + return origin ? (origin + time) / 1000 : undefined; + }); extractNetworkProtocolSpy = vi.spyOn(browserMetricsUtils, 'extractNetworkProtocol'); }); @@ -367,117 +371,6 @@ describe('resourceTimingToSpanAttributes', () => { }); }); - describe('fallback to performance.timeOrigin', () => { - it('uses performance.timeOrigin when browserPerformanceTimeOrigin returns null', () => { - // Mock browserPerformanceTimeOrigin to return null for the main check - browserPerformanceTimeOriginSpy.mockReturnValue(null); - - extractNetworkProtocolSpy.mockReturnValue({ - name: '', - version: 'unknown', - }); - - const mockResourceTiming = createMockResourceTiming({ - nextHopProtocol: '', - redirectStart: 20, - fetchStart: 40, - domainLookupStart: 60, - domainLookupEnd: 80, - connectStart: 100, - secureConnectionStart: 120, - connectEnd: 140, - requestStart: 160, - responseStart: 300, - responseEnd: 400, - }); - - const result = resourceTimingToSpanAttributes(mockResourceTiming); - - // When browserPerformanceTimeOrigin returns null, function returns early with only network protocol attributes - expect(result).toEqual({ - 'network.protocol.version': 'unknown', - 'network.protocol.name': '', - }); - }); - - it('uses performance.timeOrigin fallback in getAbsoluteTime when available', () => { - // Mock browserPerformanceTimeOrigin to return 500000 for the main check - browserPerformanceTimeOriginSpy.mockReturnValue(500000); - - extractNetworkProtocolSpy.mockReturnValue({ - name: '', - version: 'unknown', - }); - - const mockResourceTiming = createMockResourceTiming({ - nextHopProtocol: '', - redirectStart: 20, - redirectEnd: 30, - workerStart: 35, - fetchStart: 40, - domainLookupStart: 60, - domainLookupEnd: 80, - connectStart: 100, - secureConnectionStart: 120, - connectEnd: 140, - requestStart: 160, - responseStart: 300, - responseEnd: 400, - }); - - const result = resourceTimingToSpanAttributes(mockResourceTiming); - - expect(result).toEqual({ - 'network.protocol.version': 'unknown', - 'network.protocol.name': '', - 'http.request.redirect_start': 500.02, // (500000 + 20) / 1000 - 'http.request.redirect_end': 500.03, // (500000 + 30) / 1000 - 'http.request.worker_start': 500.035, // (500000 + 35) / 1000 - 'http.request.fetch_start': 500.04, // (500000 + 40) / 1000 - 'http.request.domain_lookup_start': 500.06, // (500000 + 60) / 1000 - 'http.request.domain_lookup_end': 500.08, // (500000 + 80) / 1000 - 'http.request.connect_start': 500.1, // (500000 + 100) / 1000 - 'http.request.secure_connection_start': 500.12, // (500000 + 120) / 1000 - 'http.request.connection_end': 500.14, // (500000 + 140) / 1000 - 'http.request.request_start': 500.16, // (500000 + 160) / 1000 - 'http.request.response_start': 500.3, // (500000 + 300) / 1000 - 'http.request.response_end': 500.4, // (500000 + 400) / 1000 - 'http.request.time_to_first_byte': 0.3, // 300 / 1000 - }); - }); - - it('handles case when neither browserPerformanceTimeOrigin nor performance.timeOrigin is available', () => { - browserPerformanceTimeOriginSpy.mockReturnValue(null); - - extractNetworkProtocolSpy.mockReturnValue({ - name: '', - version: 'unknown', - }); - - // Mock performance.timeOrigin as undefined - const originalPerformance = global.performance; - global.performance = { - ...originalPerformance, - timeOrigin: undefined, - } as any; - - const mockResourceTiming = createMockResourceTiming({ - nextHopProtocol: '', - }); - - const result = resourceTimingToSpanAttributes(mockResourceTiming); - - // When neither timing source is available, should return network protocol attributes for empty string - expect(result).toEqual({ - 'network.protocol.version': 'unknown', - 'network.protocol.name': '', - }); - - // Restore global performance - global.performance = originalPerformance; - }); - }); - describe('edge cases', () => { it("doesn't include undefined timing values", () => { browserPerformanceTimeOriginSpy.mockReturnValue(1000000); diff --git a/packages/browser-utils/test/performance/userTiming.test.ts b/packages/browser-utils/test/performance/userTiming.test.ts index 4fb0ffd204e5..4ac90fbdd816 100644 --- a/packages/browser-utils/test/performance/userTiming.test.ts +++ b/packages/browser-utils/test/performance/userTiming.test.ts @@ -1,10 +1,23 @@ +import type * as SentryCore from '@sentry/core'; import type { Span } from '@sentry/core'; -import { getMainCarrier, SentrySpan, setCurrentClient, spanToJSON } from '@sentry/core'; +import { + getMainCarrier, + _INTERNAL_performanceTimeToSeconds, + SentrySpan, + setCurrentClient, + spanToJSON, +} from '@sentry/core'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { _addUserTimingSpan, userTimingIntegration } from '../../src/performance/userTiming'; import * as utils from '../../src/performance/utils'; +import * as webVitalsUtils from '../../src/web-vitals/utils'; import { getDefaultClientOptions, TestClient } from '../utils/TestClient'; +vi.mock('@sentry/core', async importOriginal => { + const actual = await importOriginal(); + return { ...actual, _INTERNAL_performanceTimeToSeconds: vi.fn(actual._INTERNAL_performanceTimeToSeconds) }; +}); + describe('userTimingIntegration', () => { let client: TestClient; let performanceEntries: PerformanceEntry[]; @@ -12,6 +25,7 @@ describe('userTimingIntegration', () => { beforeEach(() => { vi.restoreAllMocks(); + vi.mocked(_INTERNAL_performanceTimeToSeconds).mockReset(); getMainCarrier().__SENTRY__ = undefined; client = new TestClient(getDefaultClientOptions({ tracesSampleRate: 1 })); @@ -124,6 +138,45 @@ describe('userTimingIntegration', () => { expect(spans).toHaveLength(1); expect(spanToJSON(spans[0]!).name).toBe('current-route'); }); + + it('gives measures starting before the request the exact request span start timestamp', () => { + userTimingIntegration().setup?.(client); + + // Many values, because only some of them round differently with the old conversion. + for (let i = 0; i < 50; i++) { + const requestStart = 10.1 + i * 0.37; + vi.spyOn(webVitalsUtils, 'getNavigationEntry').mockReturnValue({ + startTime: 0, + requestStart, + } as PerformanceNavigationTiming); + performanceEntries.push(createPerformanceEntry('measure', `before-request-${i}`, 0, 1000)); + + client.emit('beforeIdleSpanEnd', new SentrySpan({ op: 'pageload', name: '/', sampled: true })); + + expect(spanToJSON(spans[i]!).start_timestamp).toBe(_INTERNAL_performanceTimeToSeconds(requestStart, 0)); + } + }); + + it('keeps measures recorded after the wall clock was corrected backwards', () => { + const timeOrigin = performance.timeOrigin; + // The time origin was corrected 60s backwards at 1000ms, e.g. by NTP. + vi.mocked(_INTERNAL_performanceTimeToSeconds).mockImplementation( + (monotonicTimeInMs, entryStartTimeInMs = monotonicTimeInMs) => + ((entryStartTimeInMs < 1000 ? timeOrigin : timeOrigin - 60_000) + monotonicTimeInMs) / 1000, + ); + vi.spyOn(webVitalsUtils, 'getNavigationEntry').mockReturnValue({ + startTime: 0, + requestStart: 10, + } as PerformanceNavigationTiming); + + userTimingIntegration().setup?.(client); + performanceEntries.push(createPerformanceEntry('measure', 'after-correction', 2000, 10)); + client.emit('beforeIdleSpanEnd', new SentrySpan({ op: 'pageload', name: '/', sampled: true })); + + expect(spans).toHaveLength(1); + expect(spanToJSON(spans[0]!).start_timestamp).toBe((timeOrigin - 60_000 + 2000) / 1000); + expect(spanToJSON(spans[0]!).attributes['sentry.browser.measure_happened_before_request']).toBeUndefined(); + }); }); describe('_addUserTimingSpan', () => { @@ -154,7 +207,7 @@ describe('_addUserTimingSpan', () => { }, } as PerformanceMeasure; - _addUserTimingSpan(parentSpan, entry, 0.012, 0.01, 100, 0, []); + _addUserTimingSpan(parentSpan, entry, 100.012, 100.022, 0, []); expect(spans).toHaveLength(1); expect(spanToJSON(spans[0]!).attributes).toEqual({ @@ -175,7 +228,7 @@ describe('_addUserTimingSpan', () => { }, } as PerformanceMeasure; - _addUserTimingSpan(parentSpan, entry, 0.012, 0.01, 100, 0, []); + _addUserTimingSpan(parentSpan, entry, 100.012, 100.022, 0, []); expect(spans).toHaveLength(0); }); @@ -184,10 +237,9 @@ describe('_addUserTimingSpan', () => { _addUserTimingSpan( parentSpan, createPerformanceEntry('measure', 'before-request', 10, 10), - 0.01, - 0.01, - 100, - 0.05, + 100.01, + 100.02, + 100.05, [], ); diff --git a/packages/browser-utils/test/web-vitals/spans.test.ts b/packages/browser-utils/test/web-vitals/spans.test.ts index f9182b52d153..05a20d5f2d2d 100644 --- a/packages/browser-utils/test/web-vitals/spans.test.ts +++ b/packages/browser-utils/test/web-vitals/spans.test.ts @@ -22,6 +22,7 @@ vi.mock('@sentry/core', async () => { return { ...actual, browserPerformanceTimeOrigin: vi.fn(), + _INTERNAL_performanceTimeToSeconds: vi.fn(), timestampInSeconds: vi.fn(), getCurrentScope: vi.fn(), getClient: vi.fn(), @@ -378,6 +379,9 @@ describe('_sendLcpSpan', () => { beforeEach(() => { vi.mocked(SentryCore.getCurrentScope).mockReturnValue(mockScope as any); vi.mocked(SentryCore.browserPerformanceTimeOrigin).mockReturnValue(1000); + vi.mocked(SentryCore._INTERNAL_performanceTimeToSeconds).mockImplementation( + (time, entryStartTime = time) => (SentryCore.browserPerformanceTimeOrigin(entryStartTime)! + time) / 1000, + ); vi.mocked(htmlTreeAsString).mockImplementation((node: any) => `<${node?.tagName || 'div'}>`); vi.mocked(SentryCoreBrowser.startInactiveSpan).mockReturnValue(mockSpan as any); vi.mocked(SentryCore.spanToJSON).mockReturnValue({ @@ -475,6 +479,22 @@ describe('_sendLcpSpan', () => { expect(mockSpan.end).toHaveBeenCalledWith(3.25); }); + it('uses the time origin from the start of a soft navigation for LCP', () => { + // The soft navigation happens after a time origin correction. + const sleepDurationMs = 3_600_000; + vi.mocked(SentryCore.browserPerformanceTimeOrigin).mockImplementation((monotonicTimeInMs = 0) => + monotonicTimeInMs < 1500 ? 1000 : 1000 + sleepDurationMs, + ); + const entry = { element: { tagName: 'img' } as Element, startTime: 2250 } as LargestContentfulPaint; + + _sendLcpSpan(250, entry, undefined, 2, 'soft-navigation', 2000); + + expect(SentryCoreBrowser.startInactiveSpan).toHaveBeenCalledWith( + expect.objectContaining({ startTime: (1000 + sleepDurationMs + 2000) / 1000 }), + ); + expect(mockSpan.end).toHaveBeenCalledWith((1000 + sleepDurationMs + 2250) / 1000); + }); + it('drops implausible LCP values', () => { _sendLcpSpan(0, undefined); _sendLcpSpan(MAX_PLAUSIBLE_LCP_DURATION + 1, undefined); @@ -497,6 +517,7 @@ describe('_sendClsSpan', () => { beforeEach(() => { vi.mocked(SentryCore.getCurrentScope).mockReturnValue(mockScope as any); vi.mocked(SentryCore.browserPerformanceTimeOrigin).mockReturnValue(1000); + vi.mocked(SentryCore._INTERNAL_performanceTimeToSeconds).mockImplementation(time => (1000 + time) / 1000); vi.mocked(SentryCore.timestampInSeconds).mockReturnValue(1.5); vi.mocked(htmlTreeAsString).mockImplementation((node: any) => `<${node?.tagName || 'div'}>`); vi.mocked(SentryCoreBrowser.startInactiveSpan).mockReturnValue(mockSpan as any); @@ -590,6 +611,7 @@ describe('_sendClsSpan', () => { it('falls back to the current time when there is no performance time origin', () => { vi.mocked(SentryCore.browserPerformanceTimeOrigin).mockReturnValue(undefined); + vi.mocked(SentryCore._INTERNAL_performanceTimeToSeconds).mockReturnValue(undefined); _sendClsSpan(0, undefined); @@ -612,6 +634,7 @@ describe('_sendInpSpan', () => { beforeEach(() => { vi.mocked(SentryCore.getCurrentScope).mockReturnValue(mockScope as any); vi.mocked(SentryCore.browserPerformanceTimeOrigin).mockReturnValue(1000); + vi.mocked(SentryCore._INTERNAL_performanceTimeToSeconds).mockImplementation(time => (1000 + time) / 1000); vi.mocked(htmlTreeAsString).mockReturnValue(' diff --git a/dev-packages/e2e-tests/test-applications/sveltekit-3-sourcemaps/src/routes/server-error/+page.server.ts b/dev-packages/e2e-tests/test-applications/sveltekit-3-sourcemaps/src/routes/server-error/+page.server.ts new file mode 100644 index 000000000000..131d44a2f2a0 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/sveltekit-3-sourcemaps/src/routes/server-error/+page.server.ts @@ -0,0 +1,4 @@ +// SOURCEMAP_MARKER_SERVER +export const load = async () => { + throw new Error('Server load error'); +}; diff --git a/dev-packages/e2e-tests/test-applications/sveltekit-3-sourcemaps/src/routes/server-error/+page.svelte b/dev-packages/e2e-tests/test-applications/sveltekit-3-sourcemaps/src/routes/server-error/+page.svelte new file mode 100644 index 000000000000..089965d0419f --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/sveltekit-3-sourcemaps/src/routes/server-error/+page.svelte @@ -0,0 +1 @@ +

Server error

diff --git a/dev-packages/e2e-tests/test-applications/sveltekit-3-sourcemaps/start-mock-sentry-server.mjs b/dev-packages/e2e-tests/test-applications/sveltekit-3-sourcemaps/start-mock-sentry-server.mjs new file mode 100644 index 000000000000..7d2ade5cb226 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/sveltekit-3-sourcemaps/start-mock-sentry-server.mjs @@ -0,0 +1,4 @@ +import { startMockSentryServer } from '@sentry-internal/test-utils'; + +// Not a hidden dir, which the SDK's default `./.*/**/*.map` cleanup would empty after the upload +startMockSentryServer({ org: 'test-org', outputDir: 'tmp_mock_chunks' }); diff --git a/dev-packages/e2e-tests/test-applications/sveltekit-3-sourcemaps/static/favicon.png b/dev-packages/e2e-tests/test-applications/sveltekit-3-sourcemaps/static/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..825b9e65af7c104cfb07089bb28659393b4f2097 GIT binary patch literal 1571 zcmV+;2Hg3HP)Px)-AP12RCwC$UE6KzI1p6{F2N z1VK2vi|pOpn{~#djwYcWXTI_im_u^TJgMZ4JMOsSj!0ma>B?-(Hr@X&W@|R-$}W@Z zgj#$x=!~7LGqHW?IO8+*oE1MyDp!G=L0#^lUx?;!fXv@l^6SvTnf^ac{5OurzC#ZMYc20lI%HhX816AYVs1T3heS1*WaWH z%;x>)-J}YB5#CLzU@GBR6sXYrD>Vw(Fmt#|JP;+}<#6b63Ike{Fuo!?M{yEffez;| zp!PfsuaC)>h>-AdbnwN13g*1LowNjT5?+lFVd#9$!8Z9HA|$*6dQ8EHLu}U|obW6f z2%uGv?vr=KNq7YYa2Roj;|zooo<)lf=&2yxM@e`kM$CmCR#x>gI>I|*Ubr({5Y^rb zghxQU22N}F51}^yfDSt786oMTc!W&V;d?76)9KXX1 z+6Okem(d}YXmmOiZq$!IPk5t8nnS{%?+vDFz3BevmFNgpIod~R{>@#@5x9zJKEHLHv!gHeK~n)Ld!M8DB|Kfe%~123&Hz1Z(86nU7*G5chmyDe ziV7$pB7pJ=96hpxHv9rCR29%bLOXlKU<_13_M8x)6;P8E1Kz6G<&P?$P^%c!M5`2` zfY2zg;VK5~^>TJGQzc+33-n~gKt{{of8GzUkWmU110IgI0DLxRIM>0US|TsM=L|@F z0Bun8U!cRB7-2apz=y-7*UxOxz@Z0)@QM)9wSGki1AZ38ceG7Q72z5`i;i=J`ILzL z@iUO?SBBG-0cQuo+an4TsLy-g-x;8P4UVwk|D8{W@U1Zi z!M)+jqy@nQ$p?5tsHp-6J304Q={v-B>66$P0IDx&YT(`IcZ~bZfmn11#rXd7<5s}y zBi9eim&zQc0Dk|2>$bs0PnLmDfMP5lcXRY&cvJ=zKxI^f0%-d$tD!`LBf9^jMSYUA zI8U?CWdY@}cRq6{5~y+)#h1!*-HcGW@+gZ4B};0OnC~`xQOyH19z*TA!!BJ%9s0V3F?CAJ{hTd#*tf+ur-W9MOURF-@B77_-OshsY}6 zOXRY=5%C^*26z?l)1=$bz30!so5tfABdSYzO+H=CpV~aaUefmjvfZ3Ttu9W&W3Iu6 zROlh0MFA5h;my}8lB0tAV-Rvc2Zs_CCSJnx@d`**$idgy-iMob4dJWWw|21b4NB=LfsYp0Aeh{Ov)yztQi;eL4y5 zMi>8^SzKqk8~k?UiQK^^-5d8c%bV?$F8%X~czyiaKCI2=UH { + sourcemapSettingAlreadySet = typeof config.build?.sourcemap !== 'undefined'; return { ...config, build: { @@ -125,18 +129,9 @@ export async function makeCustomSentryVitePlugins( }, }; - // Whether `build.sourcemap` is already set when this `config` hook runs. Read here rather than in - // `configResolved`, where it is always set - but note that nothing here may *await* the SvelteKit - // config from a `config` hook, see the note in `kitConfig.ts`. - let sourcemapSettingAlreadySet = false; - const filesToDeleteAfterUploadConfigPlugin: Plugin = { name: 'sentry-sveltekit-files-to-delete-after-upload-setting-plugin', apply: 'build', // only apply this plugin at build time - config: (config: UserConfig) => { - sourcemapSettingAlreadySet = typeof config.build?.sourcemap !== 'undefined'; - return config; - }, configResolved: async () => { // Resolved here (not in `closeBundle`) so the adapter is invoked before the build writes its // output - see the note on the adapter output dir in `sentrySvelteKit()`. @@ -165,6 +160,86 @@ export async function makeCustomSentryVitePlugins( }, }; + let sourceMapsUploaded = false; + + // We need to start uploading source maps later than in the original plugin + // because SvelteKit is invoking the adapter after its builds. + // This means that we need to wait until the adapter is done before we start uploading. + const uploadSourceMaps = async (): Promise => { + sourceMapsUploaded = true; + + const outDir = path.resolve(process.cwd(), await getAdapterOutputDir()); + // eslint-disable-next-line no-console + debug && console.log('[Source Maps Plugin] Looking up source maps in', outDir); + + const jsFiles = getFiles(outDir).filter(file => file.endsWith('.js')); + // eslint-disable-next-line no-console + debug && console.log('[Source Maps Plugin] Flattening source maps'); + + // @ts-expect-error - we're using dynamic import here and TS complains about that. It works though. + const sorcery = await import('sorcery'); + + for (const file of jsFiles) { + try { + await (sorcery as Sorcery).load(file).then(async chain => { + if (!chain) { + // We end up here, if we don't have a source map for the file. + // This is fine, as we're not interested in files w/o source maps. + return; + } + // This flattens the source map + await chain.apply(); + // Write it back to the original file + await chain.write(); + }); + } catch (e) { + // Sometimes sorcery fails to flatten the source map. While this isn't ideal, it seems to be mostly + // happening in Kit-internal files which is fine as they're not in-app. + // This mostly happens when sorcery tries to resolve a source map while flattening that doesn't exist. + const isKnownError = e instanceof Error && e.message.includes('ENOENT: no such file or directory, open'); + if (debug && !isKnownError) { + // eslint-disable-next-line no-console + console.error('[Source Maps Plugin] error while flattening', file, e); + } + } + + // We need to remove the query string from the source map files that our auto-instrument plugin added + // to proxy the load functions during building. + const mapFile = `${file}.map`; + try { + const mapContent = (await fs.promises.readFile(mapFile, 'utf-8')).toString(); + const cleanedMapContent = mapContent.replace( + // oxlint-disable-next-line sdk/no-regexp-constructor -- no user input + escaped anyway + new RegExp(escapeStringForRegex(WRAPPED_MODULE_SUFFIX), 'gm'), + '', + ); + await fs.promises.writeFile(mapFile, cleanedMapContent); + } catch { + // Map file doesn't exist, nothing to clean + } + } + + try { + // Call the original plugin's writeBundle to upload source maps, manage releases, and delete files. + // We pass in the `outDir` we determined earlier as output options. + if (typeof originalWriteBundle === 'function') { + // @ts-expect-error - calling writeBundle with only outputOptions, without the full bundle arg + await originalWriteBundle({ dir: outDir }); + } + } catch { + // eslint-disable-next-line no-console + console.warn('[Source Maps Plugin] Failed to upload source maps!'); + // eslint-disable-next-line no-console + console.log( + '[Source Maps Plugin] Please make sure you specified a valid Sentry auth token, as well as your org and project slugs.', + ); + // eslint-disable-next-line no-console + console.log( + '[Source Maps Plugin] Further information: https://github.com/getsentry/sentry-javascript/blob/develop/packages/sveltekit/README.md#uploading-source-maps', + ); + } + }; + const customDebugIdUploadPlugin: Plugin = { name: 'sentry-sveltekit-debug-id-upload-plugin', apply: 'build', // only apply this plugin at build time @@ -179,84 +254,24 @@ export async function makeCustomSentryVitePlugins( } }, - // We need to start uploading source maps later than in the original plugin - // because SvelteKit is invoking the adapter at closeBundle. - // This means that we need to wait until the adapter is done before we start uploading. + // SvelteKit 2 invokes the adapter in `closeBundle` of the SSR build. closeBundle: async () => { if (!isSSRBuild) { return; } + await uploadSourceMaps(); + }, - const outDir = path.resolve(process.cwd(), await getAdapterOutputDir()); - // eslint-disable-next-line no-console - debug && console.log('[Source Maps Plugin] Looking up source maps in', outDir); - - const jsFiles = getFiles(outDir).filter(file => file.endsWith('.js')); - // eslint-disable-next-line no-console - debug && console.log('[Source Maps Plugin] Flattening source maps'); - - // @ts-expect-error - we're using dynamic import here and TS complains about that. It works though. - const sorcery = await import('sorcery'); - - for (const file of jsFiles) { - try { - await (sorcery as Sorcery).load(file).then(async chain => { - if (!chain) { - // We end up here, if we don't have a source map for the file. - // This is fine, as we're not interested in files w/o source maps. - return; - } - // This flattens the source map - await chain.apply(); - // Write it back to the original file - await chain.write(); - }); - } catch (e) { - // Sometimes sorcery fails to flatten the source map. While this isn't ideal, it seems to be mostly - // happening in Kit-internal files which is fine as they're not in-app. - // This mostly happens when sorcery tries to resolve a source map while flattening that doesn't exist. - const isKnownError = e instanceof Error && e.message.includes('ENOENT: no such file or directory, open'); - if (debug && !isKnownError) { - // eslint-disable-next-line no-console - console.error('[Source Maps Plugin] error while flattening', file, e); - } - } - - // We need to remove the query string from the source map files that our auto-instrument plugin added - // to proxy the load functions during building. - const mapFile = `${file}.map`; - try { - const mapContent = (await fs.promises.readFile(mapFile, 'utf-8')).toString(); - const cleanedMapContent = mapContent.replace( - // oxlint-disable-next-line sdk/no-regexp-constructor -- no user input + escaped anyway - new RegExp(escapeStringForRegex(WRAPPED_MODULE_SUFFIX), 'gm'), - '', - ); - await fs.promises.writeFile(mapFile, cleanedMapContent); - } catch { - // Map file doesn't exist, nothing to clean - } - } - - try { - // Call the original plugin's writeBundle to upload source maps, manage releases, and delete files. - // We pass in the `outDir` we determined earlier as output options. - if (typeof originalWriteBundle === 'function') { - // @ts-expect-error - calling writeBundle with only outputOptions, without the full bundle arg - await originalWriteBundle({ dir: outDir }); + // SvelteKit 3 invokes the adapter in its post `buildApp` hook, which runs before ours (`enforce: 'post'`). + // With SvelteKit 2, nothing is built yet when this runs, so `closeBundle` uploads instead. + buildApp: { + order: 'post', + handler: async builder => { + if (sourceMapsUploaded || !Object.values(builder.environments).some(environment => environment.isBuilt)) { + return; } - } catch { - // eslint-disable-next-line no-console - console.warn('[Source Maps Plugin] Failed to upload source maps!'); - // eslint-disable-next-line no-console - console.log( - '[Source Maps Plugin] Please make sure you specified a valid Sentry auth token, as well as your org and project slugs.', - ); - // eslint-disable-next-line no-console - console.log( - '[Source Maps Plugin] Further information: https://github.com/getsentry/sentry-javascript/blob/develop/packages/sveltekit/README.md#uploading-source-maps', - ); - } + await uploadSourceMaps(); + }, }, }; diff --git a/packages/sveltekit/test/vite/injectGlobalValues.test.ts b/packages/sveltekit/test/vite/injectGlobalValues.test.ts index 10b81dadf3fc..4e4226e5368b 100644 --- a/packages/sveltekit/test/vite/injectGlobalValues.test.ts +++ b/packages/sveltekit/test/vite/injectGlobalValues.test.ts @@ -145,6 +145,9 @@ describe('adapter output dir resolution', () => { ], }); + const sourceMapSettingPlugin = plugins.find( + plugin => plugin.name === 'sentry-sveltekit-update-source-map-setting-plugin', + )!; const filesToDeletePlugin = plugins.find( plugin => plugin.name === 'sentry-sveltekit-files-to-delete-after-upload-setting-plugin', )!; @@ -153,7 +156,7 @@ describe('adapter output dir resolution', () => { // This takes the branch that leaves `filesToDeleteAfterUpload` untouched - the adapter still // has to be resolved by `configResolved`, not later in `closeBundle` // @ts-expect-error these hooks exist and are callable - filesToDeletePlugin.config({ build: { sourcemap: true } }); + await sourceMapSettingPlugin.config({ build: { sourcemap: true } }); // @ts-expect-error these hooks exist and are callable await filesToDeletePlugin.configResolved(); diff --git a/packages/sveltekit/test/vite/sourceMaps.test.ts b/packages/sveltekit/test/vite/sourceMaps.test.ts index d12ee0fdf8f5..d4f2d3c06db8 100644 --- a/packages/sveltekit/test/vite/sourceMaps.test.ts +++ b/packages/sveltekit/test/vite/sourceMaps.test.ts @@ -188,6 +188,61 @@ describe('makeCustomSentryVitePlugins()', () => { await plugin.closeBundle(); expect(mockedMainPlugin.writeBundle).not.toHaveBeenCalled(); }); + + it('uploads source maps in the post `buildApp` hook when there is no separate SSR build (SvelteKit 3)', async () => { + const plugin = await getSentryViteSubPlugin('sentry-sveltekit-debug-id-upload-plugin'); + + // @ts-expect-error this function exists! + plugin.configResolved({ build: { ssr: false } }); + // @ts-expect-error this function exists! + await plugin.closeBundle(); + expect(mockedMainPlugin.writeBundle).not.toHaveBeenCalled(); + + expect(plugin?.buildApp).toEqual({ order: 'post', handler: expect.any(Function) }); + // @ts-expect-error this function exists! + await plugin.buildApp.handler({ environments: { ssr: { isBuilt: true }, client: { isBuilt: true } } }); + expect(mockedMainPlugin.writeBundle).toHaveBeenCalledTimes(1); + }); + + it("doesn't upload source maps in the post `buildApp` hook before any environment was built (SvelteKit 2 on Vite 7+)", async () => { + const plugin = await getSentryViteSubPlugin('sentry-sveltekit-debug-id-upload-plugin'); + + // @ts-expect-error this function exists! + await plugin.buildApp.handler({ environments: { ssr: { isBuilt: false } } }); + expect(mockedMainPlugin.writeBundle).not.toHaveBeenCalled(); + + // @ts-expect-error this function exists! + plugin.configResolved({ build: { ssr: true } }); + // @ts-expect-error this function exists! + await plugin.closeBundle(); + expect(mockedMainPlugin.writeBundle).toHaveBeenCalledTimes(1); + }); + + it('uploads source maps only once when both `closeBundle` and `buildApp` run (SvelteKit 2 on Vite 7+)', async () => { + const plugin = await getSentryViteSubPlugin('sentry-sveltekit-debug-id-upload-plugin'); + + // @ts-expect-error this function exists! + plugin.configResolved({ build: { ssr: true } }); + // @ts-expect-error this function exists! + await plugin.closeBundle(); + // @ts-expect-error this function exists! + await plugin.buildApp.handler({ environments: { ssr: { isBuilt: true } } }); + + expect(mockedMainPlugin.writeBundle).toHaveBeenCalledTimes(1); + }); + + it('uploads source maps on every SSR rebuild in watch mode', async () => { + const plugin = await getSentryViteSubPlugin('sentry-sveltekit-debug-id-upload-plugin'); + + // @ts-expect-error this function exists! + plugin.configResolved({ build: { ssr: true } }); + // @ts-expect-error this function exists! + await plugin.closeBundle(); + // @ts-expect-error this function exists! + await plugin.closeBundle(); + + expect(mockedMainPlugin.writeBundle).toHaveBeenCalledTimes(2); + }); }); it('catches errors while uploading source maps', async () => { @@ -200,13 +255,10 @@ describe('makeCustomSentryVitePlugins()', () => { const plugin = await getSentryViteSubPlugin('sentry-sveltekit-debug-id-upload-plugin'); - // @ts-expect-error this function exists! - expect(plugin.closeBundle).not.toThrow(); - // @ts-expect-error this function exists! plugin.configResolved({ build: { ssr: true } }); // @ts-expect-error this function exists! - await plugin.closeBundle(); + await expect(plugin.closeBundle()).resolves.toBeUndefined(); expect(consoleWarnSpy).toHaveBeenCalledWith(expect.stringContaining('Failed to upload source maps')); expect(consoleLogSpy).toHaveBeenCalled(); @@ -348,11 +400,7 @@ describe('deleteFilesAfterUpload', () => { plugin => plugin.name === 'sentry-sveltekit-files-to-delete-after-upload-setting-plugin', )!; - // `config` records whether the user set `build.sourcemap`; `configResolved` resolves the - // setting (it can't be awaited from `config` - see the note in `kitConfig.ts`) - // @ts-expect-error these functions exist! - filesToDeleteAfterUploadSettingPlugin.config(viteConfig); - // @ts-expect-error these functions exist! + // @ts-expect-error this function exists! await filesToDeleteAfterUploadSettingPlugin.configResolved(); await expect(mergedOptions.sourcemaps.filesToDeleteAfterUpload).resolves.toEqual([ @@ -430,11 +478,7 @@ describe('deleteFilesAfterUpload', () => { plugin => plugin.name === 'sentry-sveltekit-files-to-delete-after-upload-setting-plugin', )!; - // `config` records whether the user set `build.sourcemap`; `configResolved` resolves the - // setting (it can't be awaited from `config` - see the note in `kitConfig.ts`) - // @ts-expect-error these functions exist! - filesToDeleteAfterUploadSettingPlugin.config(viteConfig); - // @ts-expect-error these functions exist! + // @ts-expect-error this function exists! await filesToDeleteAfterUploadSettingPlugin.configResolved(); await expect(mergedOptions.sourcemaps.filesToDeleteAfterUpload).resolves.toEqual( From cd0f3dca88eb522cee888a1cf3881b802e1458a1 Mon Sep 17 00:00:00 2001 From: Charly Gomez Date: Thu, 8 Oct 2026 13:16:43 +0200 Subject: [PATCH 24/47] fix(deps): Bump @graphql-tools/utils to 12.0.3 (#25162) Re-resolves three `@graphql-tools` lockfile entries to replace the vulnerable `@graphql-tools/utils` copy: | Package | Before | After | | --- | --- | --- | | `@graphql-tools/schema` | 10.0.31 | 10.1.3 | | `@graphql-tools/merge` | 9.1.7 | 9.2.6 | | `@graphql-tools/utils` | 11.0.0 | 12.0.3 | Fixes GHSA-7mx3-vvmw-hjmv (high), prototype pollution in `mergeDeep`: https://github.com/getsentry/sentry-javascript/security/dependabot/2696 The first patched release is 12.0.1, so the fix has to cross a major. It still needs no `package.json` change: `@apollo/server` already declares `@graphql-tools/schema@^10.0.0`, and schema 10.1.3 pulls `utils` into 12.x. Lockfile-only diff, dev-only dependency path. Verified with the affected suites, 52 passing: yarn vitest run suites/tracing/apollo-graphql suites/tracing/graphql-tracing-channel Co-authored-by: Claude Opus 5 --- yarn.lock | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/yarn.lock b/yarn.lock index c07d28325713..43f25e6bf314 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4590,30 +4590,30 @@ google-auth-library "^9.14.2" ws "^8.18.0" -"@graphql-tools/merge@^9.1.7": - version "9.1.7" - resolved "https://registry.yarnpkg.com/@graphql-tools/merge/-/merge-9.1.7.tgz#8040553c19aeadf6df41060b811e9c5df8716be1" - integrity sha512-Y5E1vTbTabvcXbkakdFUt4zUIzB1fyaEnVmIWN0l0GMed2gdD01TpZWLUm4RNAxpturvolrb24oGLQrBbPLSoQ== +"@graphql-tools/merge@^9.2.6": + version "9.2.6" + resolved "https://registry.yarnpkg.com/@graphql-tools/merge/-/merge-9.2.6.tgz#2cf3cd419a7ace9be3381873160882d84512c48b" + integrity sha512-M7iI/qoCnks6iITkimtdZgI2SrWOInr+QRWuhEsInaumzQGPd0pnWbRAVTRVcFN2Gu9hM2aS5kYzASC93d9epw== dependencies: - "@graphql-tools/utils" "^11.0.0" + "@graphql-tools/utils" "^12.0.3" tslib "^2.4.0" "@graphql-tools/schema@^10.0.0": - version "10.0.31" - resolved "https://registry.yarnpkg.com/@graphql-tools/schema/-/schema-10.0.31.tgz#dea275db4d7423f293826cff2aa11e7b3027ddce" - integrity sha512-ZewRgWhXef6weZ0WiP7/MV47HXiuFbFpiDUVLQl6mgXsWSsGELKFxQsyUCBos60Qqy1JEFAIu3Ns6GGYjGkqkQ== + version "10.1.3" + resolved "https://registry.yarnpkg.com/@graphql-tools/schema/-/schema-10.1.3.tgz#1dbbb63e7c82054f62371f9d818bc0d1bb226f3b" + integrity sha512-N1ZpzguxzqTy5fpurqWQ5BXyjirlrtvVLeSafsu+SCDNjDeRnVxf9n5PVbdgzwFPAmP/7NK9sq9UB/kyR8oFiw== dependencies: - "@graphql-tools/merge" "^9.1.7" - "@graphql-tools/utils" "^11.0.0" + "@graphql-tools/merge" "^9.2.6" + "@graphql-tools/utils" "^12.0.3" tslib "^2.4.0" -"@graphql-tools/utils@^11.0.0": - version "11.0.0" - resolved "https://registry.yarnpkg.com/@graphql-tools/utils/-/utils-11.0.0.tgz#f54a09372e96c32416fcbd0eb8e9e0f1338900bd" - integrity sha512-bM1HeZdXA2C3LSIeLOnH/bcqSgbQgKEDrjxODjqi3y58xai2TkNrtYcQSoWzGbt9VMN1dORGjR7Vem8SPnUFQA== +"@graphql-tools/utils@^12.0.3": + version "12.0.3" + resolved "https://registry.yarnpkg.com/@graphql-tools/utils/-/utils-12.0.3.tgz#8cae349fe99ff6dd142e497485e4a031b6d32248" + integrity sha512-M+04bHvI1SKoug7W2EkwfKjig2mdtgIIlFY8p2jLHmxPNVs5bC/3R+58JrD+7ZXWgPILWTwuqJ9AqTjPlahroA== dependencies: "@graphql-typed-document-node/core" "^3.1.1" - "@whatwg-node/promise-helpers" "^1.0.0" + "@whatwg-node/promise-helpers" "^2.0.0" cross-inspect "1.0.1" tslib "^2.4.0" @@ -10727,10 +10727,10 @@ "@webassemblyjs/ast" "1.14.1" "@xtuc/long" "4.2.2" -"@whatwg-node/promise-helpers@^1.0.0": - version "1.3.2" - resolved "https://registry.yarnpkg.com/@whatwg-node/promise-helpers/-/promise-helpers-1.3.2.tgz#3b54987ad6517ef6db5920c66a6f0dada606587d" - integrity sha512-Nst5JdK47VIl9UcGwtv2Rcgyn5lWtZ0/mhRQ4G8NN2isxpq2TO30iqHzmwoJycjWuyUfg3GFXqP/gFHXeV57IA== +"@whatwg-node/promise-helpers@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@whatwg-node/promise-helpers/-/promise-helpers-2.0.0.tgz#93a89655355bbd5d7e2b0d64fc93a88224d7030e" + integrity sha512-/i9CS0Z96o+cPwaYoAyVxAThDiUX1wV3tDmNWPJbn+vtdR+JftuCDg1NGCXBrzIYVDGkJ8uT6ctwmrBOUzOpKQ== dependencies: tslib "^2.6.3" From 2882a29b55dac7f9a61eb0df6cbba5df606cb29b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Peer=20St=C3=B6cklmair?= Date: Thu, 8 Oct 2026 14:28:40 +0300 Subject: [PATCH 25/47] feat(nextjs): Add `@sentry/nextjs/cloudflare` (#24998) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (disclaimer, most of the additions are from `withSentry.test.ts` and other tests) Adds `withSentry` from `@sentry/nextjs/cloudflare` for the Worker entry of a Next.js app on Cloudflare Workers, for example `.open-next/worker.js` of OpenNext or the fetch handler of vinext. It is `withSentry` of `@sentry/cloudflare` with the Next.js handling of the server `init` added, so a Next.js app on Workers gets the spans it gets on Node.js from one wrapper, without new options. This is the only new public API. Decisions: - It installs the OpenTelemetry async context strategy at module load. The AsyncLocalStorage strategy of `@sentry/cloudflare` loses the OpenTelemetry context of the Next.js spans, so their parents and context values break. - The request spans of Next.js (`BaseServer.handleRequest`) are ignored, so the request span of `withSentry` is the only `http.server` span. The other Next.js spans become its children; it gets the route from their `next.route` and its status from the response. When the middleware answers the request or throws, the span is named `middleware GET`, like the middleware segment on Node.js. - From `compatibility_date` 2026-02-19, `process.cwd()` is `/bundle` during a request. Next.js then misses its router server context and extracts the incoming trace again. The propagator keeps the request span as parent in that case, else each continued request has two segments. - `sentry.server.config.ts` and `sentry.edge.config.ts` still run in the Worker but create no client there. Their options do not apply, and a debug log says so. On Workers they are optional: they only set the `turbopack` tag and hand over the build release of `withSentryConfig`, which only code that Next.js compiles can read. Without them, the release comes from the `withSentry` options, `SENTRY_RELEASE` or `CF_VERSION_METADATA`. Apps keep them for `next dev`, which runs on Node.js and does not use the Worker entry. `instrumentation.ts` stays required for `onRequestError`. - The Next.js handling is added by a `Nextjs` integration. `withSentry` of `@sentry/cloudflare` creates the client itself (once per isolate), and the options callback only returns options. The `setup` of an integration is the only place that gets this client before the request span starts, without a new option in `@sentry/cloudflare`. It registers the span hooks of the server `init`, the propagator, the tunnel route sampling and the event processor that drops the control flow errors of React and Next.js, so a Worker without `instrumentation.ts` drops them too. The server `init` checks for this integration, so it does not add the processor to the global scope again. - `@sentry/cloudflare` becomes a dependency of `@sentry/nextjs`. Its dependencies are already in the dependency tree of `@sentry/nextjs`. It also aligns with other SDKs, such as `@sentry/sveltekit` Known difference: on Workers, a release from the `withSentry` options, `SENTRY_RELEASE` or `CF_VERSION_METADATA` wins over the build release. On Node.js, the build release wins over the environment. The `nextjs-16-cf-workers` e2e app now uses it, which un-skips its server tests and adds tests for D1 spans, the OpenTelemetry context and trace continuation. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .size-limit.js | 21 + .../nextjs-16-cf-workers/app/api/d1/route.ts | 12 + .../app/api/otel-context/route.ts | 22 + .../nextjs-16-cf-workers/package.json | 3 +- .../sentry.edge.config.ts | 15 +- .../sentry.server.config.ts | 12 +- .../tests/cloudflare-runtime.test.ts | 1 + .../tests/d1-binding.test.ts | 21 + .../tests/metrics.test.ts | 7 +- .../tests/middleware.test.ts | 73 ++- .../tests/nested-rsc-error.test.ts | 3 +- .../tests/otel-context.test.ts | 19 + .../tests/pageload-tracing.test.ts | 6 +- .../tests/route-handler.test.ts | 10 +- .../tests/server-action-redirect.test.ts | 26 +- .../tests/server-components.test.ts | 24 +- .../tests/streaming-rsc-error.test.ts | 5 +- .../tests/trace-continuation.test.ts | 38 ++ .../nextjs-16-cf-workers/worker.ts | 17 + .../nextjs-16-cf-workers/wrangler.jsonc | 13 +- packages/core/src/utils/clientIPAddress.ts | 4 +- .../test/lib/utils/clientIPAddress.test.ts | 4 + packages/nextjs/package.json | 6 + packages/nextjs/rollup.npm.config.mjs | 1 + packages/nextjs/src/cloudflare/index.ts | 96 +++ .../utils/dropMiddlewareTunnelRequests.ts | 2 +- .../utils/dropReactControlFlowErrors.ts | 43 ++ .../nextjs/src/common/utils/responseEnd.ts | 56 +- packages/nextjs/src/edge/index.ts | 4 +- .../server/enhanceHandleRequestRootSpan.ts | 7 +- packages/nextjs/src/server/index.ts | 60 +- packages/nextjs/src/server/serverSpanHooks.ts | 125 ++-- .../nextjs/test/cloudflare/withSentry.test.ts | 567 ++++++++++++++++++ .../utils/dropReactControlFlowErrors.test.ts | 48 ++ packages/nextjs/test/edgeSdk.test.ts | 11 + .../test/server/serverSpanHooks.test.ts | 173 ++++++ packages/nextjs/test/serverSdk.test.ts | 43 ++ 37 files changed, 1379 insertions(+), 219 deletions(-) create mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/d1/route.ts create mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/otel-context/route.ts create mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/d1-binding.test.ts create mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/otel-context.test.ts create mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/trace-continuation.test.ts create mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/worker.ts create mode 100644 packages/nextjs/src/cloudflare/index.ts create mode 100644 packages/nextjs/src/common/utils/dropReactControlFlowErrors.ts create mode 100644 packages/nextjs/test/cloudflare/withSentry.test.ts create mode 100644 packages/nextjs/test/common/utils/dropReactControlFlowErrors.test.ts create mode 100644 packages/nextjs/test/server/serverSpanHooks.test.ts diff --git a/.size-limit.js b/.size-limit.js index 220fa9c349e8..fcb14bb45828 100644 --- a/.size-limit.js +++ b/.size-limit.js @@ -533,6 +533,27 @@ module.exports = [ return config; }, }, + // Next.js on Cloudflare Workers (ESM) - minified to match `wrangler deploy --dry-run --minify` output. An import of + // `@sentry/node` in this entry more than doubles its size. + { + name: '@sentry/nextjs/cloudflare (withSentry) - minified', + path: 'packages/nextjs/build/esm/cloudflare/index.js', + import: createImport('withSentry'), + ignore: [...builtinModules, ...nodePrefixedBuiltinModules], + gzip: false, + brotli: false, + limit: '223 KiB', + disablePlugins: ['@size-limit/webpack'], + webpack: false, + modifyEsbuildConfig: function (config) { + config.keepNames = true; + // Match wrangler's build settings + config.conditions = ['workerd', 'worker', 'browser']; + config.platform = 'browser'; + config.format = 'esm'; + return config; + }, + }, ]; function createImport(...args) { diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/d1/route.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/d1/route.ts new file mode 100644 index 000000000000..244a4d21942b --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/d1/route.ts @@ -0,0 +1,12 @@ +import { getCloudflareContext } from '@opennextjs/cloudflare'; +import { NextResponse } from 'next/server'; + +export const dynamic = 'force-dynamic'; + +type Env = { DB: { prepare(query: string): { all(): Promise<{ results: unknown[] }> } } }; + +export async function GET() { + const { env } = await getCloudflareContext({ async: true }); + const { results } = await (env as unknown as Env).DB.prepare('SELECT 1 AS one').all(); + return NextResponse.json(results); +} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/otel-context/route.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/otel-context/route.ts new file mode 100644 index 000000000000..1ca661b60b63 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/otel-context/route.ts @@ -0,0 +1,22 @@ +import { context, createContextKey, trace } from '@opentelemetry/api'; +import { NextResponse } from 'next/server'; + +export const dynamic = 'force-dynamic'; + +const E2E_CONTEXT_KEY = createContextKey('e2e.context.key'); + +export async function GET() { + const tracer = trace.getTracer('e2e'); + + context.with(context.active().setValue(E2E_CONTEXT_KEY, 'e2e-value'), () => { + tracer.startActiveSpan('otel-context-outer', outer => { + // An explicit context, as OpenTelemetry instrumentations pass it. + const inner = tracer.startSpan('otel-context-inner', {}, context.active()); + inner.setAttribute('e2e.context.value', String(context.active().getValue(E2E_CONTEXT_KEY))); + inner.end(); + outer.end(); + }); + }); + + return NextResponse.json({ ok: true }); +} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/package.json b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/package.json index 2bd1b3f77857..f0b40e15fb07 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/package.json +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/package.json @@ -5,7 +5,7 @@ "scripts": { "dev": "next dev", "cf:build": "opennextjs-cloudflare build", - "cf:preview": "opennextjs-cloudflare preview", + "cf:preview": "opennextjs-cloudflare preview --var \"E2E_TEST_DSN:$E2E_TEST_DSN\"", "build": "next build", "clean": "npx rimraf node_modules pnpm-lock.yaml .tmp_dev_server_logs", "start": "pnpm cf:preview", @@ -17,6 +17,7 @@ "test:assert": "pnpm test:prod" }, "dependencies": { + "@opentelemetry/api": "^1.9.1", "@opennextjs/cloudflare": "^1.14.9", "@sentry/nextjs": "file:../../packed/sentry-nextjs-packed.tgz", "@sentry/core": "file:../../packed/sentry-core-packed.tgz", diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/sentry.edge.config.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/sentry.edge.config.ts index d6aeeac195c8..eceebb29d45d 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/sentry.edge.config.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/sentry.edge.config.ts @@ -1,18 +1,7 @@ import * as Sentry from '@sentry/nextjs'; +// Only hands over the build release: `withSentry` from `@sentry/nextjs/cloudflare` in worker.ts creates the client and +// holds the options. Sentry.init({ - environment: 'qa', // dynamic sampling bias to keep transactions dsn: process.env.NEXT_PUBLIC_E2E_TEST_DSN, - tunnel: `http://localhost:3031/`, // proxy server - tracesSampler: samplingContext => { - if (samplingContext.attributes?.['next.span_type'] === 'Middleware.execute') { - // Only keep the middleware transaction when `normalizedRequest` was available at sampling time. - // Test times out and fails when transaction is dropped - const { normalizedRequest } = samplingContext; - return Boolean(normalizedRequest?.url && normalizedRequest?.method); - } - - return 1.0; - }, - // debug: true, }); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/sentry.server.config.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/sentry.server.config.ts index f55707dea51a..1009ee9e1f39 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/sentry.server.config.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/sentry.server.config.ts @@ -1,15 +1,7 @@ import * as Sentry from '@sentry/nextjs'; -import { Log } from '@sentry/nextjs'; +// Only hands over the build release and adds the global Next.js event processors and the turbopack tag: `withSentry` +// from `@sentry/nextjs/cloudflare` in worker.ts creates the client and holds the options. Sentry.init({ - environment: 'qa', // dynamic sampling bias to keep transactions dsn: process.env.NEXT_PUBLIC_E2E_TEST_DSN, - tunnel: `http://localhost:3031/`, // proxy server - tracesSampleRate: 1.0, - // debug: true, - integrations: [Sentry.vercelAIIntegration()], - // Verify Log type is available - beforeSendLog(log: Log) { - return log; - }, }); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/cloudflare-runtime.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/cloudflare-runtime.test.ts index cba53fa1970d..d088edd32dcb 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/cloudflare-runtime.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/cloudflare-runtime.test.ts @@ -18,6 +18,7 @@ test.describe('Cloudflare Runtime', () => { expect(errorEvent.contexts?.runtime).toEqual({ name: 'cloudflare', }); + expect(errorEvent.tags?.turbopack).toBe(true); // The SDK info should include cloudflare in the packages expect(errorEvent.sdk?.packages).toEqual( diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/d1-binding.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/d1-binding.test.ts new file mode 100644 index 000000000000..43c31b94084d --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/d1-binding.test.ts @@ -0,0 +1,21 @@ +import { expect, test } from '@playwright/test'; +import { collectStreamedSpansUntilSegment, getSpanOp } from '@sentry-internal/test-utils'; + +// `withSentry` passes an instrumented `env` to the OpenNext handler, which `getCloudflareContext()` returns. +test('Should create a span for a D1 query through the `env` of `getCloudflareContext`', async ({ request }) => { + const spansPromise = collectStreamedSpansUntilSegment('nextjs-16-cf-workers', 'GET /api/d1'); + + const response = await request.get('/api/d1'); + expect(await response.json()).toStrictEqual([{ one: 1 }]); + + const spans = await spansPromise; + const d1Span = spans.find(span => getSpanOp(span) === 'db.query')!; + + expect(spans.filter(span => getSpanOp(span) === 'db.query')).toEqual([d1Span]); + expect(d1Span.name).toBe('SELECT'); + expect(d1Span.is_segment).toBe(false); + expect(d1Span.status).toBe('ok'); + expect(d1Span.attributes['db.system.name']?.value).toBe('cloudflare-d1'); + expect(d1Span.attributes['db.query.text']?.value).toBe('SELECT ? AS one'); + expect(d1Span.attributes['sentry.origin']?.value).toBe('auto.db.cloudflare.d1'); +}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/metrics.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/metrics.test.ts index 6569c3d21890..1ccbe00953de 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/metrics.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/metrics.test.ts @@ -1,9 +1,7 @@ import { expect, test } from '@playwright/test'; import { waitForMetric } from '@sentry-internal/test-utils'; -// Metrics are not currently supported on Cloudflare Workers -// TODO: Investigate and enable when metrics support is added for CF Workers -test.skip('Should emit metrics from server and client', async ({ request, page }) => { +test('Should emit metrics from server and client', async ({ request, page }) => { const clientCountPromise = waitForMetric('nextjs-16-cf-workers', async metric => { return metric.name === 'test.page.count'; }); @@ -92,7 +90,6 @@ test.skip('Should emit metrics from server and client', async ({ request, page } type: 'counter', value: 1, attributes: { - 'server.address': { value: expect.any(String), type: 'string' }, 'random.attribute': { value: 'Potatoes', type: 'string' }, endpoint: { value: '/metrics/route-handler', type: 'string' }, 'sentry.environment': { value: 'qa', type: 'string' }, @@ -108,7 +105,6 @@ test.skip('Should emit metrics from server and client', async ({ request, page } type: 'distribution', value: 100, attributes: { - 'server.address': { value: expect.any(String), type: 'string' }, 'random.attribute': { value: 'Patatas', type: 'string' }, endpoint: { value: '/metrics/route-handler', type: 'string' }, 'sentry.environment': { value: 'qa', type: 'string' }, @@ -124,7 +120,6 @@ test.skip('Should emit metrics from server and client', async ({ request, page } type: 'gauge', value: 200, attributes: { - 'server.address': { value: expect.any(String), type: 'string' }, 'random.attribute': { value: 'Patate', type: 'string' }, endpoint: { value: '/metrics/route-handler', type: 'string' }, 'sentry.environment': { value: 'qa', type: 'string' }, diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/middleware.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/middleware.test.ts index 78b27c8427d9..32d797f6d836 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/middleware.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/middleware.test.ts @@ -1,61 +1,56 @@ import { expect, test } from '@playwright/test'; -import { collectStreamedSpans, getSpanOp, waitForStreamedSpan } from '@sentry-internal/test-utils'; +import { collectStreamedSpans, collectStreamedSpansUntilSegment, getSpanOp } from '@sentry-internal/test-utils'; import { isDevMode } from './isDevMode'; -// TODO: Skipped until the Cloudflare Workers edge middleware setup emits middleware spans reliably. -test.skip('tracesSampler receives normalizedRequest for edge middleware', async ({ request }) => { - const middlewareSpanPromise = waitForStreamedSpan('nextjs-16-cf-workers', span => { - return span.name === 'middleware GET' && span.is_segment; - }); - - await request.get('/api/endpoint-behind-middleware'); - - const middlewareSpan = await middlewareSpanPromise; - - expect(String(middlewareSpan.attributes['http.target']?.value)).toContain('/api/endpoint-behind-middleware'); - expect(middlewareSpan.attributes['http.request.method']?.value).toBe('GET'); -}); - -// TODO: Middleware tests need SDK adjustments for Cloudflare Workers edge runtime -test.skip('Should create a span for middleware', async ({ request }) => { - const middlewareSpanPromise = waitForStreamedSpan('nextjs-16-cf-workers', span => { - return span.name === 'middleware GET' && span.is_segment; - }); +// `withSentry` from `@sentry/nextjs/cloudflare` wraps the Worker entry, so middleware runs inside the request's +// `http.server` segment and its span is a child of it rather than a segment of its own. +test('Should create a span for middleware inside the request segment', async ({ request }) => { + const spansPromise = collectStreamedSpansUntilSegment('nextjs-16-cf-workers', 'GET /api/endpoint-behind-middleware'); const response = await request.get('/api/endpoint-behind-middleware'); expect(await response.json()).toStrictEqual({ name: 'John Doe' }); - const middlewareSpan = await middlewareSpanPromise; - + const spans = await spansPromise; + const segmentSpan = spans.find(span => span.is_segment)!; + const middlewareSpan = spans.find(span => span.name === 'middleware GET')!; + + expect(spans.filter(span => getSpanOp(span) === 'middleware')).toEqual([middlewareSpan]); + expect(segmentSpan.attributes['url.path']?.value).toBe('/api/endpoint-behind-middleware'); + expect(segmentSpan.attributes['sentry.segment.name.source']?.value).toBe('route'); + expect(segmentSpan.attributes['isolation_scope.is_default']).toEqual({ value: false, type: 'boolean' }); + expect(middlewareSpan.is_segment).toBe(false); + expect(middlewareSpan.parent_span_id).toBe(segmentSpan.span_id); expect(middlewareSpan.status).toBe('ok'); expect(getSpanOp(middlewareSpan)).toBe('middleware'); - expect(middlewareSpan.attributes['sentry.segment.name.source']?.value).toBe('route'); - - // Assert that isolation scope works properly - // expect(middlewareSpan.attributes['isolation_scope.is_default']).toEqual({ value: false, type: 'boolean' }); }); -// TODO: Middleware tests need SDK adjustments for Cloudflare Workers edge runtime -test.skip('Faulty middlewares', async ({ request }) => { +test('Faulty middlewares', async ({ request }) => { test.skip(isDevMode, 'Throwing crashes the dev server atm'); // https://github.com/vercel/next.js/issues/85261 - const middlewareSpanPromise = waitForStreamedSpan('nextjs-16-cf-workers', span => { - return span.name === 'middleware GET' && span.is_segment; - }); + // The middleware throws, so the request span of `withSentry` keeps the middleware name, like the middleware segment + // of Node.js. + const spansPromise = collectStreamedSpans( + 'nextjs-16-cf-workers', + spans => + spans.some(span => span.is_segment && span.name === 'middleware GET') && + spans.some(span => !span.is_segment && span.name === 'middleware GET'), + ); request.get('/api/endpoint-behind-middleware', { headers: { 'x-should-throw': '1' } }).catch(() => { // Noop }); - await test.step('should record spans', async () => { - const middlewareSpan = await middlewareSpanPromise; - expect(middlewareSpan.status).toBe('error'); - expect(getSpanOp(middlewareSpan)).toBe('middleware'); - expect(middlewareSpan.attributes['sentry.segment.name.source']?.value).toBe('route'); - }); + const spans = await spansPromise; + const segmentSpan = spans.find(span => span.is_segment)!; + const middlewareSpan = spans.find(span => !span.is_segment && span.name === 'middleware GET')!; + + expect(segmentSpan.status).toBe('error'); + expect(segmentSpan.attributes['sentry.segment.name.source']?.value).toBe('route'); + expect(middlewareSpan.parent_span_id).toBe(segmentSpan.span_id); + expect(middlewareSpan.status).toBe('error'); + expect(getSpanOp(middlewareSpan)).toBe('middleware'); }); -// TODO: Middleware tests need SDK adjustments for Cloudflare Workers edge runtime -test.skip('Should trace outgoing fetch requests inside middleware', async ({ request }) => { +test('Should trace outgoing fetch requests inside middleware', async ({ request }) => { test.skip(isDevMode, 'The fetch requests ends up in a separate tx in dev atm'); // `http.client` span names are low cardinality under span streaming, hence `GET localhost` rather diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/nested-rsc-error.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/nested-rsc-error.test.ts index 509dc49cf364..03f7d224e466 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/nested-rsc-error.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/nested-rsc-error.test.ts @@ -1,8 +1,7 @@ import { expect, test } from '@playwright/test'; import { waitForError, waitForStreamedSpan } from '@sentry-internal/test-utils'; -// TODO: Flakey on CI -test.skip('Should capture errors from nested server components when `Sentry.captureRequestError` is added to the `onRequestError` hook', async ({ +test('Should capture errors from nested server components when `Sentry.captureRequestError` is added to the `onRequestError` hook', async ({ page, }) => { const errorEventPromise = waitForError('nextjs-16-cf-workers', errorEvent => { diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/otel-context.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/otel-context.test.ts new file mode 100644 index 000000000000..5303c62a984f --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/otel-context.test.ts @@ -0,0 +1,19 @@ +import { expect, test } from '@playwright/test'; +import { collectStreamedSpansUntilSegment } from '@sentry-internal/test-utils'; + +test('Keeps OpenTelemetry context values and parents', async ({ request }) => { + const spansPromise = collectStreamedSpansUntilSegment('nextjs-16-cf-workers', 'GET /api/otel-context'); + + const response = await request.get('/api/otel-context'); + expect(await response.json()).toStrictEqual({ ok: true }); + + const spans = await spansPromise; + const segmentSpan = spans.find(span => span.is_segment)!; + const outerSpan = spans.find(span => span.name === 'otel-context-outer')!; + const innerSpan = spans.find(span => span.name === 'otel-context-inner')!; + + expect(spans.some(span => span.span_id === outerSpan.parent_span_id)).toBe(true); + expect(innerSpan.parent_span_id).toBe(outerSpan.span_id); + expect(outerSpan.attributes['sentry.segment.id']?.value).toBe(segmentSpan.span_id); + expect(innerSpan.attributes['e2e.context.value']?.value).toBe('e2e-value'); +}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/pageload-tracing.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/pageload-tracing.test.ts index 19879c4e6853..d6165244752e 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/pageload-tracing.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/pageload-tracing.test.ts @@ -1,8 +1,7 @@ import { expect, test } from '@playwright/test'; import { getSpanOp, waitForStreamedSpan } from '@sentry-internal/test-utils'; -// TODO: Flakey on CI -test.skip('App router spans should be attached to the pageload request span', async ({ page }) => { +test('App router spans should be attached to the pageload request span', async ({ page }) => { const serverSpanPromise = waitForStreamedSpan('nextjs-16-cf-workers', span => { return span.name === 'GET /pageload-tracing' && span.is_segment; }); @@ -19,8 +18,7 @@ test.skip('App router spans should be attached to the pageload request span', as expect(serverSpan.trace_id).toBe(pageloadSpan.trace_id); }); -// TODO: HTTP request headers are not extracted as span attributes on Cloudflare Workers -test.skip('extracts HTTP request headers as span attributes', async ({ baseURL }) => { +test('extracts HTTP request headers as span attributes', async ({ baseURL }) => { const serverSpanPromise = waitForStreamedSpan('nextjs-16-cf-workers', span => { const requestId = span.attributes['http.request.header.x-request-id']; return ( diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/route-handler.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/route-handler.test.ts index 7623d1c29e31..f8a8420f3694 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/route-handler.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/route-handler.test.ts @@ -1,7 +1,7 @@ import test, { expect } from '@playwright/test'; import { getSpanOp, waitForStreamedSpan } from '@sentry-internal/test-utils'; -test.skip('Should create a span for node route handlers', async ({ request }) => { +test('Should create a span for node route handlers', async ({ request }) => { const routehandlerSpanPromise = waitForStreamedSpan('nextjs-16-cf-workers', span => { return span.name === 'GET /route-handler/[xoxo]/node' && span.is_segment; }); @@ -13,15 +13,11 @@ test.skip('Should create a span for node route handlers', async ({ request }) => expect(routehandlerSpan.status).toBe('ok'); expect(getSpanOp(routehandlerSpan)).toBe('http.server'); - - // Custom headers are not captured on Cloudflare Workers - // This assertion is skipped for CF Workers environment + expect(routehandlerSpan.attributes['http.request.header.x-charly']?.value).toEqual(['gomez']); }); test('Should create a span for edge route handlers', async ({ request }) => { - // This test only works for webpack builds on non-async param extraction - // todo: check if we can set request headers for edge on sdkProcessingMetadata - test.skip(); + test.skip(true, 'OpenNext does not support the Edge runtime'); const routehandlerSpanPromise = waitForStreamedSpan('nextjs-16-cf-workers', span => { return span.name === 'GET /route-handler/[xoxo]/edge' && span.is_segment; }); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/server-action-redirect.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/server-action-redirect.test.ts index 7d7f75e5eb7b..a5961ae41274 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/server-action-redirect.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/server-action-redirect.test.ts @@ -1,7 +1,7 @@ import { expect, test } from '@playwright/test'; import { waitForError, waitForStreamedSpan } from '@sentry-internal/test-utils'; -test.skip('Should handle server action redirect without capturing errors', async ({ page }) => { +test('Should handle server action redirect without capturing errors', async ({ page, request }) => { // Wait for the initial pageload span const pageLoadSpanPromise = waitForStreamedSpan('nextjs-16-cf-workers', span => { return span.name === '/redirect/origin' && span.is_segment; @@ -18,10 +18,9 @@ test.skip('Should handle server action redirect without capturing errors', async return span.name === 'GET /redirect/destination' && span.is_segment; }); - // No error should be captured - const redirectErrorPromise = waitForError('nextjs-16-cf-workers', async errorEvent => { - return !!errorEvent; - }); + // The error of `/api/test-error`, requested after the redirect, is the sentinel: an error of the redirect would + // arrive before it. + const firstErrorPromise = waitForError('nextjs-16-cf-workers', () => true); // Click the redirect button await page.click('button[type="submit"]'); @@ -31,17 +30,8 @@ test.skip('Should handle server action redirect without capturing errors', async // Verify we got redirected to the destination page await expect(page).toHaveURL('/redirect/destination'); - // Wait for potential errors with a 2 second timeout - const errorTimeout = new Promise((_, reject) => - setTimeout(() => reject(new Error('No error captured (timeout)')), 2000), - ); - - // We expect this to timeout since no error should be captured during the redirect - try { - await Promise.race([redirectErrorPromise, errorTimeout]); - throw new Error('Expected no error to be captured, but an error was found'); - } catch (e) { - // If we get a timeout error (as expected), no error was captured - expect((e as Error).message).toBe('No error captured (timeout)'); - } + await request.get('/api/test-error'); + + const firstError = await firstErrorPromise; + expect(firstError.exception?.values?.[0]?.value).toBe('This is a test error from an API route'); }); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/server-components.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/server-components.test.ts index 7a34986062dc..344a62127543 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/server-components.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/server-components.test.ts @@ -5,13 +5,14 @@ import { collectStreamedSpansUntilSegment, getSpanOp } from '@sentry-internal/te // them, so asserting `undefined` pins that they stay untouched. const nextjsSpan = { op: undefined, description: undefined, codeFunctionName: undefined }; -// TODO: Server component tests need SDK adjustments for Cloudflare Workers -test.skip('Sends a span for a request to app router with URL', async ({ page }) => { +// `withSentry` from `@sentry/nextjs/cloudflare` wraps the Worker entry, so the segment is its `http.server` +// span, named after the route Next.js resolved. +test('Sends a span for a request to app router with URL', async ({ page }) => { const spansPromise = collectStreamedSpansUntilSegment( 'nextjs-16-cf-workers', span => span.name === 'GET /parameterized/[one]/beep/[two]' && - String(span.attributes['http.target']?.value).startsWith('/parameterized/1337/beep/42'), + span.attributes['url.path']?.value === '/parameterized/1337/beep/42', ); await page.goto('/parameterized/1337/beep/42'); @@ -21,7 +22,7 @@ test.skip('Sends a span for a request to app router with URL', async ({ page }) span => span.name === 'GET /parameterized/[one]/beep/[two]' && span.is_segment && - String(span.attributes['http.target']?.value).startsWith('/parameterized/1337/beep/42'), + span.attributes['url.path']?.value === '/parameterized/1337/beep/42', )!; expect(segmentSpan.span_id).toEqual(expect.stringMatching(/[a-f0-9]{16}/)); @@ -29,24 +30,22 @@ test.skip('Sends a span for a request to app router with URL', async ({ page }) expect(segmentSpan.status).toBe('ok'); expect(segmentSpan.attributes).toMatchObject({ 'sentry.op': { value: 'http.server', type: 'string' }, - 'sentry.origin': { value: 'auto', type: 'string' }, + 'sentry.origin': { value: 'auto.http.cloudflare', type: 'string' }, 'sentry.sample_rate': { value: 1, type: 'integer' }, 'sentry.segment.name.source': { value: 'route', type: 'string' }, - 'http.method': { value: 'GET', type: 'string' }, + 'http.request.method': { value: 'GET', type: 'string' }, 'http.response.status_code': { value: 200, type: 'integer' }, 'http.route': { value: '/parameterized/[one]/beep/[two]', type: 'string' }, - 'http.status_code': { value: 200, type: 'integer' }, - 'http.target': { value: '/parameterized/1337/beep/42', type: 'string' }, - 'sentry.kind': { value: 'server', type: 'string' }, + 'url.path': { value: '/parameterized/1337/beep/42', type: 'string' }, 'next.route': { value: '/parameterized/[one]/beep/[two]', type: 'string' }, }); // No child span should share the segment span's name expect(spans.filter(span => !span.is_segment && span.name === segmentSpan.name)).toHaveLength(0); + expect(spans.filter(span => getSpanOp(span) === 'http.server')).toEqual([segmentSpan]); }); -// TODO: Server component span tests need SDK adjustments for Cloudflare Workers -test.skip('Will create spans for every server component and metadata generation functions when visiting a page', async ({ +test('Will create spans for every server component and metadata generation functions when visiting a page', async ({ page, }) => { const spansPromise = collectStreamedSpansUntilSegment('nextjs-16-cf-workers', 'GET /nested-layout'); @@ -93,8 +92,7 @@ test.skip('Will create spans for every server component and metadata generation expect(spans).toContainEqual({ ...nextjsSpan, name: 'start response' }); }); -// TODO: Server component span tests need SDK adjustments for Cloudflare Workers -test.skip('Will create spans for every server component and metadata generation functions when visiting a dynamic page', async ({ +test('Will create spans for every server component and metadata generation functions when visiting a dynamic page', async ({ page, }) => { const spansPromise = collectStreamedSpansUntilSegment('nextjs-16-cf-workers', 'GET /nested-layout/[dynamic]'); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/streaming-rsc-error.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/streaming-rsc-error.test.ts index 296d485e8644..1b991f1a9fe4 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/streaming-rsc-error.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/streaming-rsc-error.test.ts @@ -1,10 +1,7 @@ import { expect, test } from '@playwright/test'; import { waitForError, waitForStreamedSpan } from '@sentry-internal/test-utils'; -// TODO: Under span streaming the Workers runtime emits no server segment span for a request -// interrupted by a streaming RSC error, so the correlation below cannot be satisfied. -// See https://github.com/getsentry/sentry-javascript/issues/23932 -test.skip('Should capture errors for crashing streaming promises in server components when `Sentry.captureRequestError` is added to the `onRequestError` hook', async ({ +test('Should capture errors for crashing streaming promises in server components when `Sentry.captureRequestError` is added to the `onRequestError` hook', async ({ page, }) => { const errorEventPromise = waitForError('nextjs-16-cf-workers', errorEvent => { diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/trace-continuation.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/trace-continuation.test.ts new file mode 100644 index 000000000000..447aa7df0cb6 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/trace-continuation.test.ts @@ -0,0 +1,38 @@ +import { expect, test } from '@playwright/test'; +import { collectStreamedSpans } from '@sentry-internal/test-utils'; + +test('Continues an incoming trace in the segment of the Worker request', async ({ request }) => { + const traceId = crypto.randomUUID().replace(/-/g, ''); + const parentSpanId = traceId.slice(0, 16); + + const spansPromise = collectStreamedSpans( + 'nextjs-16-cf-workers', + spans => + spans.some( + span => + span.trace_id === traceId && + span.is_segment && + span.attributes['sentry.origin']?.value === 'auto.http.cloudflare', + ) && + spans.some( + span => span.trace_id === traceId && span.name === 'render route (app) /parameterized/[one]/beep/[two]', + ), + ); + + await request.get('/parameterized/1337/beep/42', { + headers: { + 'sentry-trace': `${traceId}-${parentSpanId}-1`, + baggage: `sentry-trace_id=${traceId},sentry-sample_rate=1,sentry-sampled=true`, + }, + }); + + const spans = await spansPromise; + const segmentSpan = spans.find( + span => span.is_segment && span.attributes['sentry.origin']?.value === 'auto.http.cloudflare', + )!; + const renderSpan = spans.find(span => span.name === 'render route (app) /parameterized/[one]/beep/[two]')!; + + expect(segmentSpan.parent_span_id).toBe(parentSpanId); + expect(segmentSpan.name).toBe('GET /parameterized/[one]/beep/[two]'); + expect(renderSpan.attributes['sentry.segment.id']?.value).toBe(segmentSpan.span_id); +}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/worker.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/worker.ts new file mode 100644 index 000000000000..6718e015a32c --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/worker.ts @@ -0,0 +1,17 @@ +import * as Sentry from '@sentry/nextjs/cloudflare'; +// @ts-expect-error generated by `opennextjs-cloudflare build` +import handler from './.open-next/worker.js'; + +// @ts-expect-error generated by `opennextjs-cloudflare build` +export { DOQueueHandler, DOShardedTagCache, BucketCachePurge } from './.open-next/worker.js'; + +export default Sentry.withSentry( + (env: { E2E_TEST_DSN: string }) => ({ + environment: 'qa', // dynamic sampling bias to keep transactions + dsn: env.E2E_TEST_DSN, + tunnel: `http://localhost:3031/`, // proxy server + tracesSampleRate: 1.0, + integrations: [Sentry.vercelAIIntegration()], + }), + handler, +); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/wrangler.jsonc b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/wrangler.jsonc index 4bf131c387e9..16b0271383b1 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/wrangler.jsonc +++ b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/wrangler.jsonc @@ -9,8 +9,10 @@ { "$schema": "node_modules/wrangler/config-schema.json", "name": "next-cf", - "main": ".open-next/worker.js", - "compatibility_date": "2025-12-01", + "main": "./worker.ts", + // From 2026-02-19 on, wrangler uses the native `process` of workerd, whose `cwd()` is `/bundle` during a request. + // Next.js then misses its router server context and extracts the incoming trace again, see trace-continuation.test.ts. + "compatibility_date": "2026-02-19", "compatibility_flags": ["nodejs_compat", "global_fetch_strictly_public"], "assets": { "binding": "ASSETS", @@ -29,6 +31,13 @@ "service": "next-cf", }, ], + "d1_databases": [ + { + "binding": "DB", + "database_name": "e2e", + "database_id": "00000000-0000-0000-0000-000000000000", + }, + ], "observability": { "enabled": true, }, diff --git a/packages/core/src/utils/clientIPAddress.ts b/packages/core/src/utils/clientIPAddress.ts index eb16facb032f..6727aa41017b 100644 --- a/packages/core/src/utils/clientIPAddress.ts +++ b/packages/core/src/utils/clientIPAddress.ts @@ -9,12 +9,12 @@ const IP_HEADER_NAMES = new Set(ipHeaderNames.map(name => name.toLowerCase())); * normalized first. Some proxies (for example Azure App Service) add the client port, * and RFC 7239 allows a quoted `Forwarded` value and any case for `for=`. */ -export function getClientIPAddress(headers: { [key: string]: string | string[] | undefined }): string | null { +export function getClientIPAddress(headers: { [key: string]: string | string[] | null | undefined }): string | null { const normalized: { [key: string]: string } = {}; for (const [key, value] of Object.entries(headers)) { const name = key.toLowerCase(); - if (value === undefined || !IP_HEADER_NAMES.has(name)) { + if (value == null || !IP_HEADER_NAMES.has(name)) { continue; } diff --git a/packages/core/test/lib/utils/clientIPAddress.test.ts b/packages/core/test/lib/utils/clientIPAddress.test.ts index c19c090752ec..09ec6a1da734 100644 --- a/packages/core/test/lib/utils/clientIPAddress.test.ts +++ b/packages/core/test/lib/utils/clientIPAddress.test.ts @@ -31,6 +31,10 @@ describe('getClientIPAddress', () => { expect(getClientIPAddress({ 'x-forwarded-for': ['unknown', '203.0.113.7:4711'] })).toBe('203.0.113.7'); }); + it('skips headers whose value is null', () => { + expect(getClientIPAddress({ 'x-forwarded-for': null, 'x-real-ip': '198.51.100.1' })).toBe('198.51.100.1'); + }); + it('keeps the header priority order', () => { expect(getClientIPAddress({ 'X-Real-IP': '198.51.100.1', 'X-Forwarded-For': '203.0.113.7:4711' })).toBe( '203.0.113.7', diff --git a/packages/nextjs/package.json b/packages/nextjs/package.json index 8ae2a6372cc5..97488a84ec01 100644 --- a/packages/nextjs/package.json +++ b/packages/nextjs/package.json @@ -46,6 +46,11 @@ "node": "./build/cjs/index.server.js", "import": "./build/esm/index.server.js" }, + "./cloudflare": { + "types": "./build/types/cloudflare/index.d.ts", + "import": "./build/esm/cloudflare/index.js", + "require": "./build/cjs/cloudflare/index.js" + }, "./config": { "types": "./build/types/config/index.d.ts", "default": "./build/cjs/config/index.js" @@ -76,6 +81,7 @@ "@rollup/plugin-commonjs": "28.0.1", "@sentry/browser-utils": "11.5.0", "@sentry/bundler-plugins": "11.5.0", + "@sentry/cloudflare": "11.5.0", "@sentry/conventions": "^0.26.0", "@sentry/core": "11.5.0", "@sentry/node": "11.5.0", diff --git a/packages/nextjs/rollup.npm.config.mjs b/packages/nextjs/rollup.npm.config.mjs index 0b5edc3fa0e3..bef1bd0f92d3 100644 --- a/packages/nextjs/rollup.npm.config.mjs +++ b/packages/nextjs/rollup.npm.config.mjs @@ -11,6 +11,7 @@ export default [ 'src/client/index.ts', 'src/server/index.ts', 'src/edge/index.ts', + 'src/cloudflare/index.ts', 'src/config/index.ts', ], diff --git a/packages/nextjs/src/cloudflare/index.ts b/packages/nextjs/src/cloudflare/index.ts new file mode 100644 index 000000000000..c97b7298a143 --- /dev/null +++ b/packages/nextjs/src/cloudflare/index.ts @@ -0,0 +1,96 @@ +/* eslint-disable import/export */ +import type { Context, TextMapGetter } from '@opentelemetry/api'; +import { context, propagation, trace } from '@opentelemetry/api'; +import type { CloudflareOptions } from '@sentry/cloudflare'; +import { withSentry as withSentryCloudflare } from '@sentry/cloudflare'; +import type { Integration } from '@sentry/core'; +import { applySdkMetadata, extractTraceparentData, getRootSpan, GLOBAL_OBJ, spanToJSON } from '@sentry/core'; +import { + registerPrepareSpanScope, + SentryPropagator, + setOpenTelemetryContextAsyncContextStrategy, +} from '@sentry/opentelemetry'; +import { ATTR_NEXT_SPAN_TYPE } from '../common/nextSpanAttributes'; +import { isTunnelRouteSpan } from '../common/utils/dropMiddlewareTunnelRequests'; +import { dropReactControlFlowErrorsEventProcessor } from '../common/utils/dropReactControlFlowErrors'; +import { markAsyncContextOwnedByNextjsCloudflare } from '../common/utils/responseEnd'; +import { addNextjsServerSpanHooks, NEXTJS_SERVER_IGNORE_SPANS } from '../server/serverSpanHooks'; +import { nextjsUseCacheIntegration } from '../server/useCacheInstrumentation'; + +export * from '@sentry/cloudflare'; + +class NextjsCloudflarePropagator extends SentryPropagator { + /** @inheritDoc */ + public extract(ctx: Context, carrier: unknown, getter: TextMapGetter): Context { + // Next.js extracts the request headers again from the root context when it misses its router server context, e.g. + // on Workers where `process.cwd()` is `/bundle`. When the root span of `withSentry` already continued that trace, + // this would start the Next.js request span as a second segment under the upstream span. + const activeSpan = trace.getSpan(context.active()); + const sentryTrace = getter.get(carrier, 'sentry-trace'); + const incoming = extractTraceparentData(Array.isArray(sentryTrace) ? sentryTrace[0] : sentryTrace); + if ( + activeSpan && + incoming?.parentSpanId && + incoming.parentSpanId === spanToJSON(getRootSpan(activeSpan)).parent_span_id + ) { + return ctx; + } + return super.extract(ctx, carrier, getter); + } +} + +const nextjsIntegration = (): Integration => ({ + name: 'Nextjs', + setup: client => { + if ((client.getOptions() as CloudflareOptions).enableOpenTelemetrySetup) { + propagation.setGlobalPropagator(new NextjsCloudflarePropagator()); + } + client.getOptions().release ??= (GLOBAL_OBJ as { _sentryRelease?: string })._sentryRelease; + addNextjsServerSpanHooks(client); + client.addEventProcessor(dropReactControlFlowErrorsEventProcessor); + registerPrepareSpanScope(client); + // `dropMiddlewareTunnelRequests` only marks spans of Next.js, which are children of the root span of `withSentry`. + client.on('beforeSampling', samplingData => { + if (isTunnelRouteSpan(samplingData.spanAttributes)) { + samplingData.parentSampled = false; + } + }); + }, +}); + +/** + * `withSentry` of `@sentry/cloudflare` for the Worker entry of a Next.js app, e.g. `.open-next/worker.js`. It adds the + * span handling of `@sentry/nextjs` and the OpenTelemetry context of Next.js. Call it at module scope of the Worker entry. + */ +export const withSentry: typeof withSentryCloudflare = (optionsCallback, handler) => { + setOpenTelemetryContextAsyncContextStrategy(); + markAsyncContextOwnedByNextjsCloudflare(); + + return withSentryCloudflare(env => { + const options: CloudflareOptions = { enableOpenTelemetrySetup: true, ...optionsCallback(env) }; + options.ignoreSpans = [ + ...(options.ignoreSpans || []), + ...NEXTJS_SERVER_IGNORE_SPANS, + // The request spans of Next.js repeat the request span of `withSentry`. An ignored span does not become active, + // so the other Next.js spans become children of the request span of `withSentry`, which then gets the route + // from their `next.route` and its status from the response. + { attributes: { [ATTR_NEXT_SPAN_TYPE]: 'BaseServer.handleRequest' } }, + ]; + const { tracesSampler } = options; + if (tracesSampler) { + // A `tracesSampler` can ignore the `parentSampled: false` that the Next.js integration sets for tunnel requests. + options.tracesSampler = samplingContext => + !isTunnelRouteSpan(samplingContext.attributes || {}) && tracesSampler(samplingContext); + } + const userIntegrations = options.integrations; + // A function, so the integrations are only created with a new client and not on every request. + options.integrations = defaults => { + const integrations = [...defaults, nextjsIntegration(), nextjsUseCacheIntegration()]; + return typeof userIntegrations === 'function' + ? userIntegrations(integrations) + : [...integrations, ...(userIntegrations || [])]; + }; + applySdkMetadata(options, 'nextjs', ['nextjs', 'cloudflare']); + return options as ReturnType; + }, handler); +}; diff --git a/packages/nextjs/src/common/utils/dropMiddlewareTunnelRequests.ts b/packages/nextjs/src/common/utils/dropMiddlewareTunnelRequests.ts index 6cb11040038f..31064377e4ba 100644 --- a/packages/nextjs/src/common/utils/dropMiddlewareTunnelRequests.ts +++ b/packages/nextjs/src/common/utils/dropMiddlewareTunnelRequests.ts @@ -62,7 +62,7 @@ function isSentryRequestSpan(attrs: RawAttributes>): boo /** * Checks if a span's HTTP target matches the tunnel route. */ -function isTunnelRouteSpan(spanAttributes: Record): boolean { +export function isTunnelRouteSpan(spanAttributes: Record): boolean { const tunnelPath = globalWithInjectedValues._sentryRewritesTunnelPath || process.env._sentryRewritesTunnelPath; if (!tunnelPath) { return false; diff --git a/packages/nextjs/src/common/utils/dropReactControlFlowErrors.ts b/packages/nextjs/src/common/utils/dropReactControlFlowErrors.ts new file mode 100644 index 000000000000..032ae1714bd2 --- /dev/null +++ b/packages/nextjs/src/common/utils/dropReactControlFlowErrors.ts @@ -0,0 +1,43 @@ +import type { EventProcessor } from '@sentry/core'; +import { isPrerenderControlFlowError } from '../nextNavigationErrorUtils'; + +/** Drops the errors that React and Next.js throw for control flow: postpones, prerender aborts and Suspense exceptions. */ +export const dropReactControlFlowErrorsEventProcessor: EventProcessor = Object.assign( + ((event, hint) => { + if (event.type !== undefined) { + return event; + } + + const originalException = hint.originalException; + + const isPostponeError = + typeof originalException === 'object' && + originalException !== null && + '$$typeof' in originalException && + originalException.$$typeof === Symbol.for('react.postpone'); + + if (isPostponeError) { + // Postpone errors are used for partial-pre-rendering (PPR) + return null; + } + + if (isPrerenderControlFlowError(originalException)) { + // Next.js aborts prerenders by rejecting the promises it handed out (e.g. `fetch()` under Cache + // Components) and throws to bail out of static rendering. These never reach the user, so drop them + // here as well - the wrappers cannot cover every path they escape through. + return null; + } + + // We don't want to capture suspense errors as they are simply used by React/Next.js for control flow + const exceptionMessage = event.exception?.values?.[0]?.value; + if ( + exceptionMessage?.includes('Suspense Exception: This is not a real error!') || + exceptionMessage?.includes('Suspense Exception: This is not a real error, and should not leak') + ) { + return null; + } + + return event; + }) satisfies EventProcessor, + { id: 'DropReactControlFlowErrors' }, +); diff --git a/packages/nextjs/src/common/utils/responseEnd.ts b/packages/nextjs/src/common/utils/responseEnd.ts index 4ec79d9cabdf..52d07f46cae6 100644 --- a/packages/nextjs/src/common/utils/responseEnd.ts +++ b/packages/nextjs/src/common/utils/responseEnd.ts @@ -1,5 +1,14 @@ -import type { Span } from '@sentry/core'; -import { debug, fill, flush, getAsyncContextStrategy, getMainCarrier, GLOBAL_OBJ, setHttpStatus } from '@sentry/core'; +import type { AsyncContextStrategy, Span } from '@sentry/core'; +import { + debug, + fill, + flush, + getAsyncContextStrategy, + getClient, + getMainCarrier, + GLOBAL_OBJ, + setHttpStatus, +} from '@sentry/core'; import { vercelWaitUntil } from '@sentry/core/server'; import type { ServerResponse } from 'http'; import { DEBUG_BUILD } from '../debug-build'; @@ -106,9 +115,29 @@ export function isCloudflareWaitUntilAvailable(): boolean { return typeof _getOpenNextCloudflareContext()?.waitUntil === 'function'; } +type MarkedAsyncContextStrategy = AsyncContextStrategy & { _sentryNextjsCloudflare?: boolean }; + +/** + * Marks the async context strategy that `withSentry` of `@sentry/nextjs/cloudflare` installed. The mark is on the + * strategy in the global carrier, because the Worker entry and the server build of Next.js each bundle their own copy + * of `@sentry/nextjs`. + */ +export function markAsyncContextOwnedByNextjsCloudflare(): void { + (getAsyncContextStrategy(getMainCarrier()) as MarkedAsyncContextStrategy)._sentryNextjsCloudflare = true; +} + +/** + * Whether `withSentry` of `@sentry/nextjs/cloudflare` set up the async context of this Worker, also outside a request. + */ +export function isAsyncContextOwnedByNextjsCloudflare(): boolean { + const strategy: MarkedAsyncContextStrategy = getAsyncContextStrategy(getMainCarrier()); + return !!strategy._sentryNextjsCloudflare; +} + /** - * Whether a request of `withSentry` from `@sentry/cloudflare` runs. A client of `init` would then replace its async - * context strategy while the request runs (#24603). + * Whether `withSentry` of `@sentry/nextjs/cloudflare` set up the async context of this Worker, or a request of + * `withSentry` from `@sentry/cloudflare` runs. A client of `init` would then replace its async context strategy while + * a request runs (#24603). */ export function isAsyncContextOwnedByCloudflare(): boolean { const strategy = getAsyncContextStrategy(getMainCarrier()); @@ -117,8 +146,21 @@ export function isAsyncContextOwnedByCloudflare(): boolean { | { getStore(): unknown } | undefined; return ( - !strategy.withActiveSpan && - asyncLocalStorage?.getStore() !== undefined && - GLOBAL_OBJ.navigator?.userAgent === 'Cloudflare-Workers' + isAsyncContextOwnedByNextjsCloudflare() || + (!strategy.withActiveSpan && + asyncLocalStorage?.getStore() !== undefined && + GLOBAL_OBJ.navigator?.userAgent === 'Cloudflare-Workers') ); } + +/** + * Sets the build release of `withSentryConfig` on the current client if it has none, and stores it for the clients + * that `withSentry` of `@sentry/nextjs/cloudflare` creates later. Only code that Next.js compiles can read it. + */ +export function setCloudflareWorkerRelease(release: string | undefined): void { + (GLOBAL_OBJ as { _sentryRelease?: string })._sentryRelease ??= release; + const options = getClient()?.getOptions(); + if (options && !options.release) { + options.release = release; + } +} diff --git a/packages/nextjs/src/edge/index.ts b/packages/nextjs/src/edge/index.ts index e582927fd6e8..5e8fd6d3826b 100644 --- a/packages/nextjs/src/edge/index.ts +++ b/packages/nextjs/src/edge/index.ts @@ -33,6 +33,7 @@ import { flushSafelyWithTimeout, isAsyncContextOwnedByCloudflare, isCloudflareWaitUntilAvailable, + setCloudflareWorkerRelease, waitUntil, } from '../common/utils/responseEnd'; import { setUrlProcessingMetadata } from '../common/utils/setUrlProcessingMetadata'; @@ -124,9 +125,10 @@ export function init(options: VercelEdgeOptions = {}): void { } if (isAsyncContextOwnedByCloudflare()) { + setCloudflareWorkerRelease(opts.release); DEBUG_BUILD && debug.log( - 'The client of `withSentry` handles this Worker, so `init` creates no client. Set the options in `withSentry`.', + 'The client of `withSentry` handles this Worker, so `init` creates no client and only applies `release`. Set the other options in `withSentry`.', ); return; } diff --git a/packages/nextjs/src/server/enhanceHandleRequestRootSpan.ts b/packages/nextjs/src/server/enhanceHandleRequestRootSpan.ts index 49551290316c..85f134515e82 100644 --- a/packages/nextjs/src/server/enhanceHandleRequestRootSpan.ts +++ b/packages/nextjs/src/server/enhanceHandleRequestRootSpan.ts @@ -4,6 +4,7 @@ import { HTTP_ROUTE, HTTP_TARGET, SENTRY_OP, + SENTRY_ORIGIN, SENTRY_SEGMENT_NAME_SOURCE, URL_PATH, } from '@sentry/conventions/attributes'; @@ -34,7 +35,11 @@ export interface MutableRootSpan { export function enhanceHandleRequestRootSpan(span: MutableRootSpan): void { const { attributes } = span; - if (attributes[ATTR_NEXT_SPAN_TYPE] !== 'BaseServer.handleRequest') { + // On Cloudflare Workers, the `http.server` span of `withSentry` from `@sentry/cloudflare` is the request root span. + if ( + attributes[ATTR_NEXT_SPAN_TYPE] !== 'BaseServer.handleRequest' && + attributes[SENTRY_ORIGIN] !== 'auto.http.cloudflare' + ) { return; } diff --git a/packages/nextjs/src/server/index.ts b/packages/nextjs/src/server/index.ts index 3b5ac2d0bd72..856be94c7363 100644 --- a/packages/nextjs/src/server/index.ts +++ b/packages/nextjs/src/server/index.ts @@ -1,7 +1,7 @@ // import/export got a false positive, and affects most of our index barrel files // can be removed once following issue is fixed: https://github.com/import-js/eslint-plugin-import/issues/703 /* eslint-disable import/export */ -import type { EventProcessor, Scope } from '@sentry/core'; +import type { Scope } from '@sentry/core'; import { _INTERNAL_getActiveClient, applySdkMetadata, @@ -14,9 +14,14 @@ import type { NodeClient, NodeOptions } from '@sentry/node'; import { getDefaultIntegrations, httpIntegration, init as nodeInit } from '@sentry/node'; import { DEBUG_BUILD } from '../common/debug-build'; import { devErrorSymbolicationEventProcessor } from '../common/devErrorSymbolicationEventProcessor'; -import { isPrerenderControlFlowError } from '../common/nextNavigationErrorUtils'; +import { dropReactControlFlowErrorsEventProcessor } from '../common/utils/dropReactControlFlowErrors'; import { isBuild } from '../common/utils/isBuild'; -import { isAsyncContextOwnedByCloudflare, isCloudflareWaitUntilAvailable } from '../common/utils/responseEnd'; +import { + isAsyncContextOwnedByCloudflare, + isAsyncContextOwnedByNextjsCloudflare, + isCloudflareWaitUntilAvailable, + setCloudflareWorkerRelease, +} from '../common/utils/responseEnd'; import { distDirRewriteFramesIntegration } from './distDirRewriteFramesIntegration'; import { addNextjsServerSpanHooks, NEXTJS_SERVER_IGNORE_SPANS } from './serverSpanHooks'; import { prepareSafeIdGeneratorContext } from './prepareSafeIdGeneratorContext'; @@ -173,6 +178,8 @@ export function init(options: NodeOptions): NodeClient | undefined { if (client) { addNextjsServerSpanHooks(client); + } else if (isOwnedByCloudflare) { + setCloudflareWorkerRelease(opts.release); } // On the client, not the global scope, so a later `init()` after @@ -187,47 +194,10 @@ export function init(options: NodeOptions): NodeClient | undefined { .eventProcessors.some(processor => processor.id === 'DropReactControlFlowErrors'); eventProcessorTarget = isAdded ? undefined : globalScope; } - eventProcessorTarget?.addEventProcessor( - Object.assign( - ((event, hint) => { - if (event.type !== undefined) { - return event; - } - - const originalException = hint.originalException; - - const isPostponeError = - typeof originalException === 'object' && - originalException !== null && - '$$typeof' in originalException && - originalException.$$typeof === Symbol.for('react.postpone'); - - if (isPostponeError) { - // Postpone errors are used for partial-pre-rendering (PPR) - return null; - } - - if (isPrerenderControlFlowError(originalException)) { - // Next.js aborts prerenders by rejecting the promises it handed out (e.g. `fetch()` under Cache - // Components) and throws to bail out of static rendering. These never reach the user, so drop them - // here as well - the wrappers cannot cover every path they escape through. - return null; - } - - // We don't want to capture suspense errors as they are simply used by React/Next.js for control flow - const exceptionMessage = event.exception?.values?.[0]?.value; - if ( - exceptionMessage?.includes('Suspense Exception: This is not a real error!') || - exceptionMessage?.includes('Suspense Exception: This is not a real error, and should not leak') - ) { - return null; - } - - return event; - }) satisfies EventProcessor, - { id: 'DropReactControlFlowErrors' }, - ), - ); + // Each client of `withSentry` from `@sentry/nextjs/cloudflare` gets the processor from its `Nextjs` integration. + if (!isAsyncContextOwnedByNextjsCloudflare()) { + eventProcessorTarget?.addEventProcessor(dropReactControlFlowErrorsEventProcessor); + } if (process.env.NODE_ENV === 'development') { eventProcessorTarget?.addEventProcessor(devErrorSymbolicationEventProcessor); @@ -247,7 +217,7 @@ export function init(options: NodeOptions): NodeClient | undefined { if (isOwnedByCloudflare) { DEBUG_BUILD && debug.log( - 'The client of `withSentry` handles this Worker, so `init` creates no client. Set the options in `withSentry`.', + 'The client of `withSentry` handles this Worker, so `init` creates no client and only applies `release`. Set the other options in `withSentry`.', ); } else { DEBUG_BUILD && debug.log('SDK successfully initialized'); diff --git a/packages/nextjs/src/server/serverSpanHooks.ts b/packages/nextjs/src/server/serverSpanHooks.ts index fd6eb7fb2154..3d21bb094eae 100644 --- a/packages/nextjs/src/server/serverSpanHooks.ts +++ b/packages/nextjs/src/server/serverSpanHooks.ts @@ -4,9 +4,11 @@ import { HTTP_REQUEST_METHOD, HTTP_ROUTE, HTTP_TARGET, + SENTRY_OP, SENTRY_ORIGIN, URL_QUERY, } from '@sentry/conventions/attributes'; +import { MIDDLEWARE } from '@sentry/conventions/op'; import type { Client, Options, Span } from '@sentry/core'; import { getIsolationScope, @@ -33,6 +35,8 @@ import { maybeEnrichQueueProducerSpan, } from './vercelQueuesMonitoring'; +const NEXTJS_ERROR_PAGE_ROUTES = ['/500', '/_error']; + export const NEXTJS_SERVER_IGNORE_SPANS: NonNullable = [ // Static assets (matches `_next/static` anywhere in the name to handle custom basePath) /^GET (\/.*)?\/_next\/static\//, @@ -97,6 +101,82 @@ export function addNextjsServerSpanHooks(client: Client): void { }); } +type SpanAttributes = ReturnType['attributes']; + +/** + * Hoists the parameterized route that a Next.js span carries in `next.route` up to the root span, as its name and + * `http.route`. + */ +function maybeHoistRouteToRootSpan( + spanAttributes: SpanAttributes, + rootSpan: Span, + rootSpanAttributes: SpanAttributes, +): void { + if (typeof spanAttributes?.[ATTR_NEXT_ROUTE] !== 'string') { + return; + } + + const route = spanAttributes[ATTR_NEXT_ROUTE].replace(/\/route$/, ''); + // When middleware throws, Next.js renders its error page. A root span that the middleware named keeps that name, as + // on Node.js. A 404 page still names the root span. + const isErrorPageBehindMiddleware = + NEXTJS_ERROR_PAGE_ROUTES.includes(route) && rootSpanAttributes?.[SENTRY_SEGMENT_NAME_SOURCE] === 'route'; + // eslint-disable-next-line typescript/no-deprecated + const method = rootSpanAttributes?.[HTTP_REQUEST_METHOD] || rootSpanAttributes?.[HTTP_METHOD]; + + // Only hoist the http.route attribute if the transaction doesn't already have it + if (!method || rootSpanAttributes?.[HTTP_ROUTE] || isErrorPageBehindMiddleware) { + return; + } + + const name = typeof method === 'string' ? `${method} ${route}` : route; + rootSpan.updateName(name); + rootSpan.setAttributes({ + [HTTP_ROUTE]: route, + // Preserving the original attribute despite internally not depending on it + [ATTR_NEXT_ROUTE]: route, + [SENTRY_SEGMENT_NAME_SOURCE]: 'route', + }); + + // Update the isolation scope's transaction name so that non-transaction events + // (e.g. captureMessage, captureException) also get the parameterized route. + getIsolationScope().setTransactionName(name); + + // Check if this is a Vercel cron request and start a check-in + maybeStartCronCheckIn(rootSpan, route); + + // Enrich queue consumer spans (Vercel Queue push delivery via CloudEvent) + maybeEnrichQueueConsumerSpan(rootSpan); +} + +/** Names the root span after the middleware. When Next.js did not start the root span, also sets the op of the middleware span. */ +function handleMiddlewareSpanStart( + span: Span, + spanAttributes: SpanAttributes, + rootSpan: Span, + rootSpanAttributes: SpanAttributes, +): void { + const middlewareName = spanAttributes?.[ATTR_NEXT_SPAN_NAME]; + if (rootSpanAttributes?.[ATTR_NEXT_SPAN_TYPE] === undefined) { + // The middleware runs inside a request span that Next.js did not start, e.g. the one of `withSentry` from + // `@sentry/cloudflare`. That span gets the middleware name until a route span hoists its route, so a request + // that the middleware answers is named like the middleware segment on Node.js. + span.setAttribute(SENTRY_OP, MIDDLEWARE); + if (typeof middlewareName === 'string' && !rootSpanAttributes?.[HTTP_ROUTE]) { + rootSpan.updateName(middlewareName); + rootSpan.setAttribute(SENTRY_SEGMENT_NAME_SOURCE, 'route'); + } + } else if (typeof middlewareName === 'string') { + rootSpan.updateName(middlewareName); + rootSpan.setAttributes({ + [HTTP_ROUTE]: middlewareName, + [ATTR_NEXT_SPAN_NAME]: middlewareName, + [SENTRY_SEGMENT_NAME_SOURCE]: 'route', + }); + } + span.setAttribute(SENTRY_ORIGIN, 'auto'); +} + /** * Handles the on span start event for Next.js spans. * This function is used to enhance the span with additional information such as the route, the method, the headers, etc. @@ -127,51 +207,10 @@ export function handleOnSpanStart(span: Span, client: Client): void { ); } - // What we do in this glorious piece of code, is hoist any information about parameterized routes from spans emitted - // by Next.js via the `next.route` attribute, up to the transaction by setting the http.route attribute. - if (typeof spanAttributes?.[ATTR_NEXT_ROUTE] === 'string') { - // Only hoist the http.route attribute if the transaction doesn't already have it - if ( - // eslint-disable-next-line typescript/no-deprecated - (rootSpanAttributes?.[HTTP_REQUEST_METHOD] || rootSpanAttributes?.[HTTP_METHOD]) && - !rootSpanAttributes?.[HTTP_ROUTE] - ) { - const route = spanAttributes[ATTR_NEXT_ROUTE].replace(/\/route$/, ''); - // eslint-disable-next-line typescript/no-deprecated - const method = rootSpanAttributes?.[HTTP_REQUEST_METHOD] || rootSpanAttributes?.[HTTP_METHOD]; - - const name = typeof method === 'string' ? `${method} ${route}` : route; - rootSpan.updateName(name); - rootSpan.setAttributes({ - [HTTP_ROUTE]: route, - // Preserving the original attribute despite internally not depending on it - [ATTR_NEXT_ROUTE]: route, - [SENTRY_SEGMENT_NAME_SOURCE]: 'route', - }); - - // Update the isolation scope's transaction name so that non-transaction events - // (e.g. captureMessage, captureException) also get the parameterized route. - getIsolationScope().setTransactionName(name); - - // Check if this is a Vercel cron request and start a check-in - maybeStartCronCheckIn(rootSpan, route); - - // Enrich queue consumer spans (Vercel Queue push delivery via CloudEvent) - maybeEnrichQueueConsumerSpan(rootSpan); - } - } + maybeHoistRouteToRootSpan(spanAttributes, rootSpan, rootSpanAttributes); if (spanAttributes?.[ATTR_NEXT_SPAN_TYPE] === 'Middleware.execute') { - const middlewareName = spanAttributes[ATTR_NEXT_SPAN_NAME]; - if (typeof middlewareName === 'string') { - rootSpan.updateName(middlewareName); - rootSpan.setAttributes({ - [HTTP_ROUTE]: middlewareName, - [ATTR_NEXT_SPAN_NAME]: middlewareName, - [SENTRY_SEGMENT_NAME_SOURCE]: 'route', - }); - } - span.setAttribute(SENTRY_ORIGIN, 'auto'); + handleMiddlewareSpanStart(span, spanAttributes, rootSpan, rootSpanAttributes); } // We want to skip span data inference for any spans generated by Next.js. Reason being that Next.js emits spans diff --git a/packages/nextjs/test/cloudflare/withSentry.test.ts b/packages/nextjs/test/cloudflare/withSentry.test.ts new file mode 100644 index 000000000000..ff3395f62b4e --- /dev/null +++ b/packages/nextjs/test/cloudflare/withSentry.test.ts @@ -0,0 +1,567 @@ +import { context, createContextKey, propagation, ROOT_CONTEXT, trace } from '@opentelemetry/api'; +import { HTTP_ROUTE, SENTRY_SEGMENT_NAME_SOURCE } from '@sentry/conventions/attributes'; +import type { Client, Span } from '@sentry/core'; +import { + getActiveSpan, + getAsyncContextStrategy, + getClient, + getGlobalScope, + getMainCarrier, + getRootSpan, + GLOBAL_OBJ, + SDK_VERSION, + spanIsSampled, + spanToJSON, + startSpan, +} from '@sentry/core'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { withSentry } from '../../src/cloudflare'; +import { ATTR_NEXT_SPAN_TYPE } from '../../src/common/nextSpanAttributes'; +import { init as initEdge } from '../../src/edge'; +import { init as initServer } from '../../src/server'; +import { NEXTJS_SERVER_IGNORE_SPANS } from '../../src/server/serverSpanHooks'; + +const DSN = 'https://public@dsn.ingest.sentry.io/1337'; + +describe('withSentry', () => { + afterEach(() => { + getMainCarrier().__SENTRY__ = undefined; + (GLOBAL_OBJ as typeof GLOBAL_OBJ & { __SENTRY_CLOUDFLARE_CLIENT__?: unknown }).__SENTRY_CLOUDFLARE_CLIENT__ = + undefined; + delete (GLOBAL_OBJ as typeof GLOBAL_OBJ & { _sentryRewritesTunnelPath?: string })._sentryRewritesTunnelPath; + delete (GLOBAL_OBJ as typeof GLOBAL_OBJ & { _sentryRelease?: string })._sentryRelease; + vi.restoreAllMocks(); + trace.disable(); + context.disable(); + propagation.disable(); + }); + + it('creates the Cloudflare client with the Next.js integration, SDK metadata and span filters', async () => { + let client: Client | undefined; + const handler = withSentry(() => ({ dsn: DSN }), { + fetch: () => { + client = getClient(); + return new Response('ok'); + }, + }); + + await handler.fetch?.(new Request('https://example.com/') as never, {}, { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), + props: {}, + } as never); + + expect(client?.getIntegrationByName('Nextjs')).toBeDefined(); + expect( + client?.getEventProcessors().filter(processor => processor.id === 'DropReactControlFlowErrors'), + ).toHaveLength(1); + expect(client?.getOptions()).toHaveProperty('enableOpenTelemetrySetup', true); + expect(client?.getOptions()._metadata).toEqual({ + sdk: { + name: 'sentry.javascript.nextjs', + version: SDK_VERSION, + packages: [ + { name: 'npm:@sentry/nextjs', version: SDK_VERSION }, + { name: 'npm:@sentry/cloudflare', version: SDK_VERSION }, + ], + }, + }); + expect(client?.getOptions().ignoreSpans).toEqual([ + ...NEXTJS_SERVER_IGNORE_SPANS, + { attributes: { [ATTR_NEXT_SPAN_TYPE]: 'BaseServer.handleRequest' } }, + ]); + }); + + it('adds the Next.js defaults when the callback returns no options and the env holds the DSN', async () => { + let client: Client | undefined; + const handler = withSentry(() => undefined, { + fetch: () => { + client = getClient(); + return new Response('ok'); + }, + }); + + const response = await handler.fetch?.(new Request('https://example.com/') as never, { SENTRY_DSN: DSN }, { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), + props: {}, + } as never); + await response?.text(); + + expect(client?.getOptions().dsn).toBe(DSN); + expect(client?.getIntegrationByName('Nextjs')).toBeDefined(); + expect(client?.getOptions()._metadata?.sdk?.name).toBe('sentry.javascript.nextjs'); + }); + + it('keeps the options the callback returns', async () => { + const userIgnoreSpan = /^GET \/health$/; + let client: Client | undefined; + const handler = withSentry( + () => ({ dsn: DSN, enableOpenTelemetrySetup: false, tracesSampleRate: 0.5, ignoreSpans: [userIgnoreSpan] }), + { + fetch: () => { + client = getClient(); + return new Response('ok'); + }, + }, + ); + + await handler.fetch?.(new Request('https://example.com/') as never, {}, { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), + props: {}, + } as never); + + expect(client?.getOptions()).toHaveProperty('enableOpenTelemetrySetup', false); + expect(client?.getOptions().tracesSampleRate).toBe(0.5); + expect(client?.getOptions().ignoreSpans).toEqual([ + userIgnoreSpan, + ...NEXTJS_SERVER_IGNORE_SPANS, + { attributes: { [ATTR_NEXT_SPAN_TYPE]: 'BaseServer.handleRequest' } }, + ]); + }); + + it('adds the integrations the callback returns to the Next.js integrations', async () => { + const userIntegration = { name: 'UserIntegration' }; + let client: Client | undefined; + const handler = withSentry(() => ({ dsn: DSN, integrations: [userIntegration] }), { + fetch: () => { + client = getClient(); + return new Response('ok'); + }, + }); + + await handler.fetch?.(new Request('https://example.com/') as never, {}, { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), + props: {}, + } as never); + + expect(client?.getIntegrationByName('UserIntegration')).toBe(userIntegration); + expect(client?.getIntegrationByName('Nextjs')).toBeDefined(); + }); + + it('passes the Next.js integrations to an `integrations` function of the callback', async () => { + let client: Client | undefined; + const handler = withSentry( + () => ({ + dsn: DSN, + integrations: integrations => integrations.filter(integration => integration.name !== 'NextjsUseCache'), + }), + { + fetch: () => { + client = getClient(); + return new Response('ok'); + }, + }, + ); + + await handler.fetch?.(new Request('https://example.com/') as never, {}, { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), + props: {}, + } as never); + + expect(client?.getIntegrationByName('Nextjs')).toBeDefined(); + expect(client?.getIntegrationByName('NextjsUseCache')).toBeUndefined(); + }); + + it('adds the Next.js integrations when the callback sets `defaultIntegrations`', async () => { + let client: Client | undefined; + const handler = withSentry(() => ({ dsn: DSN, defaultIntegrations: false }), { + fetch: () => { + client = getClient(); + return new Response('ok'); + }, + }); + + await handler.fetch?.(new Request('https://example.com/') as never, {}, { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), + props: {}, + } as never); + + expect(client?.getIntegrationByName('Nextjs')).toBeDefined(); + expect(client?.getIntegrationByName('NextjsUseCache')).toBeDefined(); + }); + + it('does not sample the request span of a request to the tunnel route', async () => { + (GLOBAL_OBJ as typeof GLOBAL_OBJ & { _sentryRewritesTunnelPath?: string })._sentryRewritesTunnelPath = + '/monitoring'; + const sampledByPath: Record = {}; + const handler = withSentry(() => ({ dsn: DSN, tracesSampleRate: 1 }), { + fetch: (request: Request) => { + const activeSpan = getActiveSpan(); + sampledByPath[new URL(request.url).pathname] = !!activeSpan && spanIsSampled(activeSpan); + return new Response('ok'); + }, + }); + + await handler.fetch?.(new Request('https://example.com/monitoring?o=1&p=2') as never, {}, { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), + props: {}, + } as never); + await handler.fetch?.(new Request('https://example.com/page') as never, {}, { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), + props: {}, + } as never); + + expect(sampledByPath).toEqual({ '/monitoring': false, '/page': true }); + }); + + it('does not sample the request span of a request to the tunnel route when a `tracesSampler` samples every span', async () => { + (GLOBAL_OBJ as typeof GLOBAL_OBJ & { _sentryRewritesTunnelPath?: string })._sentryRewritesTunnelPath = + '/monitoring'; + const sampledByPath: Record = {}; + const handler = withSentry(() => ({ dsn: DSN, tracesSampler: () => 1 }), { + fetch: (request: Request) => { + const activeSpan = getActiveSpan(); + sampledByPath[new URL(request.url).pathname] = !!activeSpan && spanIsSampled(activeSpan); + return new Response('ok'); + }, + }); + + await handler.fetch?.(new Request('https://example.com/monitoring?o=1&p=2') as never, {}, { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), + props: {}, + } as never); + await handler.fetch?.(new Request('https://example.com/page') as never, {}, { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), + props: {}, + } as never); + + expect(sampledByPath).toEqual({ '/monitoring': false, '/page': true }); + }); + + it('injects the trace data of the active span with the global propagator', async () => { + const carrier: Record = {}; + let activeSpan: Span | undefined; + const handler = withSentry(() => ({ dsn: DSN, tracesSampleRate: 1 }), { + fetch: () => { + activeSpan = getActiveSpan(); + propagation.inject(context.active(), carrier); + return new Response('ok'); + }, + }); + + await handler.fetch?.(new Request('https://example.com/') as never, {}, { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), + props: {}, + } as never); + + expect(carrier['sentry-trace']).toBe(`${activeSpan?.spanContext().traceId}-${activeSpan?.spanContext().spanId}-1`); + }); + + it('does not set the global propagator when the callback turns off the OpenTelemetry setup', async () => { + const setGlobalPropagator = vi.spyOn(propagation, 'setGlobalPropagator'); + const handler = withSentry(() => ({ dsn: DSN, enableOpenTelemetrySetup: false }), { + fetch: () => new Response('ok'), + }); + + await handler.fetch?.(new Request('https://example.com/') as never, {}, { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), + props: {}, + } as never); + + expect(setGlobalPropagator).not.toHaveBeenCalled(); + }); + + it('starts a span under an extracted remote parent as a root span that continues its trace', async () => { + const traceId = '12312012123120121231201212312012'; + const parentSpanId = '1121201211212012'; + let consumerSpan: Span | undefined; + const handler = withSentry(() => ({ dsn: DSN, tracesSampleRate: 1 }), { + fetch: () => { + context.with(propagation.extract(ROOT_CONTEXT, { 'sentry-trace': `${traceId}-${parentSpanId}-1` }), () => { + trace.getTracer('test').startActiveSpan('consumer', span => { + consumerSpan = span as Span; + span.end(); + }); + }); + return new Response('ok'); + }, + }); + + await handler.fetch?.(new Request('https://example.com/') as never, {}, { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), + props: {}, + } as never); + + expect(getRootSpan(consumerSpan!)).toBe(consumerSpan); + expect(spanToJSON(consumerSpan!)).toMatchObject({ trace_id: traceId, parent_span_id: parentSpanId }); + }); + + it('keeps the request span as parent when Next.js extracts the incoming trace it continued again', async () => { + const traceId = '12312012123120121231201212312012'; + const parentSpanId = '1121201211212012'; + let rootSpan: Span | undefined; + let pageSpan: Span | undefined; + const handler = withSentry(() => ({ dsn: DSN, tracesSampleRate: 1 }), { + fetch: (request: Request) => { + rootSpan = getRootSpan(getActiveSpan()!); + const headers = Object.fromEntries(request.headers); + // `withPropagatedContext` of Next.js with `force`, which it uses when it misses its router server context. + const remoteContext = propagation.extract(ROOT_CONTEXT, headers); + const pageContext = trace.getSpanContext(remoteContext) + ? remoteContext + : propagation.extract(context.active(), headers); + context.with(pageContext, () => { + trace.getTracer('test').startActiveSpan('GET /about/page', span => { + pageSpan = span as Span; + span.end(); + }); + }); + return new Response('ok'); + }, + }); + + const response = await handler.fetch?.( + new Request('https://example.com/about', { + headers: { 'sentry-trace': `${traceId}-${parentSpanId}-1` }, + }) as never, + {}, + { waitUntil: vi.fn(), passThroughOnException: vi.fn(), props: {} } as never, + ); + await response?.text(); + + expect(spanToJSON(rootSpan!)).toMatchObject({ trace_id: traceId, parent_span_id: parentSpanId }); + expect(getRootSpan(pageSpan!)).toBe(rootSpan); + }); + + it('attaches the children of the Next.js request span to the request span of `withSentry`', async () => { + let rootSpan: Span | undefined; + let activeSpanInRequestSpan: Span | undefined; + let renderSpan: Span | undefined; + const handler = withSentry(() => ({ dsn: DSN, tracesSampleRate: 1 }), { + fetch: () => { + rootSpan = getRootSpan(getActiveSpan()!); + trace + .getTracer('test') + .startActiveSpan('GET /about', { attributes: { 'next.span_type': 'BaseServer.handleRequest' } }, span => { + activeSpanInRequestSpan = getActiveSpan(); + trace.getTracer('test').startActiveSpan('render route (app) /about', childSpan => { + renderSpan = childSpan as Span; + childSpan.end(); + }); + span.end(); + }); + return new Response('ok'); + }, + }); + + const response = await handler.fetch?.(new Request('https://example.com/about') as never, {}, { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), + props: {}, + } as never); + await response?.text(); + + expect(activeSpanInRequestSpan).toBe(rootSpan); + expect(spanToJSON(renderSpan!).parent_span_id).toBe(rootSpan!.spanContext().spanId); + }); + + it('starts a new trace for a span under a context extracted without trace headers', async () => { + let requestTraceId: string | undefined; + let spanTraceId: string | undefined; + const handler = withSentry(() => ({ dsn: DSN, tracesSampleRate: 1 }), { + fetch: () => { + requestTraceId = getActiveSpan()?.spanContext().traceId; + context.with(propagation.extract(ROOT_CONTEXT, {}), () => { + trace.getTracer('test').startActiveSpan('consumer', span => { + spanTraceId = span.spanContext().traceId; + span.end(); + }); + }); + return new Response('ok'); + }, + }); + + await handler.fetch?.(new Request('https://example.com/') as never, {}, { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), + props: {}, + } as never); + + expect(spanTraceId).toEqual(expect.stringMatching(/^[a-f0-9]{32}$/)); + expect(spanTraceId).not.toBe(requestTraceId); + }); + + it('names the request span after the `http.route` Next.js sets on it', async () => { + let rootSpan: Span | undefined; + const handler = withSentry(() => ({ dsn: DSN, tracesSampleRate: 1 }), { + fetch: () => { + rootSpan = getRootSpan(getActiveSpan()!); + rootSpan.setAttribute(HTTP_ROUTE, '/api/users/[id]'); + // With a length, the request span ends with the response, in the request. Node.js 26 ends the span of a + // streamed body in the async context of its reader, which has no client to run the `spanEnd` hooks. + return new Response('ok', { headers: { 'content-length': '2' } }); + }, + }); + + await handler.fetch?.(new Request('https://example.com/api/users/1') as never, {}, { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), + props: {}, + } as never); + + expect(spanToJSON(rootSpan!).name).toBe('GET /api/users/[id]'); + expect(spanToJSON(rootSpan!).attributes).toMatchObject({ + [HTTP_ROUTE]: '/api/users/[id]', + [SENTRY_SEGMENT_NAME_SOURCE]: 'route', + }); + }); + + it('names the request span after the route a Pages Router API route wrapper backfills', async () => { + let rootSpan: Span | undefined; + const handler = withSentry(() => ({ dsn: DSN, tracesSampleRate: 1 }), { + fetch: () => { + rootSpan = getRootSpan(getActiveSpan()!); + rootSpan.setAttribute('sentry.route_backfill', '/api/users/[id]'); + return new Response('ok', { headers: { 'content-length': '2' } }); + }, + }); + + await handler.fetch?.(new Request('https://example.com/api/users/1', { method: 'POST' }) as never, {}, { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), + props: {}, + } as never); + + expect(spanToJSON(rootSpan!).name).toBe('POST /api/users/[id]'); + expect(spanToJSON(rootSpan!).attributes).toMatchObject({ + [HTTP_ROUTE]: '/api/users/[id]', + [SENTRY_SEGMENT_NAME_SOURCE]: 'route', + }); + }); + + it('sets the release of `Sentry.init` on a client created after it ran', async () => { + let client: Client | undefined; + const handler = withSentry(() => ({ dsn: DSN }), { + fetch: () => { + client = getClient(); + return new Response('ok'); + }, + }); + initEdge({ release: 'build' }); + + await handler.fetch?.(new Request('https://example.com/') as never, {}, { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), + props: {}, + } as never); + + expect(client?.getOptions().release).toBe('build'); + }); + + it('keeps the release from `SENTRY_RELEASE` over the release of `Sentry.init`', async () => { + let client: Client | undefined; + const handler = withSentry(() => ({ dsn: DSN }), { + fetch: () => { + client = getClient(); + return new Response('ok'); + }, + }); + initEdge({ release: 'build' }); + + await handler.fetch?.(new Request('https://example.com/') as never, { SENTRY_RELEASE: 'env' }, { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), + props: {}, + } as never); + + expect(client?.getOptions().release).toBe('env'); + }); + + it('leaves the control flow error processor to its client when `Sentry.init` of the server build runs during a request', async () => { + const handler = withSentry(() => ({ dsn: DSN }), { + fetch: () => { + initServer({}); + return new Response('ok'); + }, + }); + + await handler.fetch?.(new Request('https://example.com/') as never, {}, { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), + props: {}, + } as never); + + expect( + getGlobalScope() + .getScopeData() + .eventProcessors.filter(processor => processor.id === 'DropReactControlFlowErrors'), + ).toHaveLength(0); + }); + + it('leaves the control flow error processor to its clients when `Sentry.init` of the server build runs outside a request', () => { + withSentry(() => ({ dsn: DSN }), { fetch: () => new Response('ok') }); + + initServer({}); + + expect( + getGlobalScope() + .getScopeData() + .eventProcessors.filter(processor => processor.id === 'DropReactControlFlowErrors'), + ).toHaveLength(0); + }); + + // https://github.com/getsentry/sentry-javascript/issues/24603 + it('keeps its async context strategy when `Sentry.init` of the edge build runs during a request', async () => { + const handler = withSentry(() => ({ dsn: DSN, tracesSampleRate: 1 }), { + fetch: () => { + initEdge({ dsn: DSN }); + startSpan({ name: 'after-init' }, () => undefined); + return new Response('ok'); + }, + }); + const strategy = getAsyncContextStrategy(getMainCarrier()); + + const response = await handler.fetch?.(new Request('https://example.com/') as never, {}, { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), + props: {}, + } as never); + + expect(response?.status).toBe(200); + expect(getAsyncContextStrategy(getMainCarrier())).toBe(strategy); + }); + + it('keeps OpenTelemetry context values and parents when `Sentry.init` of the edge build runs during a request', async () => { + const contextKey = createContextKey('sentry.test.tenant'); + let contextValue: unknown; + let outerSpanId: string | undefined; + let innerParentSpanId: string | undefined; + const handler = withSentry(() => ({ dsn: DSN, tracesSampleRate: 1 }), { + fetch: () => { + initEdge({ dsn: DSN }); + context.with(context.active().setValue(contextKey, 'tenant-42'), () => { + trace.getTracer('test').startActiveSpan('outer', outer => { + const inner = trace.getTracer('test').startSpan('inner', {}, context.active()); + contextValue = context.active().getValue(contextKey); + outerSpanId = outer.spanContext().spanId; + innerParentSpanId = spanToJSON(inner as never).parent_span_id; + inner.end(); + outer.end(); + }); + }); + return new Response('ok'); + }, + }); + + await handler.fetch?.(new Request('https://example.com/') as never, {}, { + waitUntil: vi.fn(), + passThroughOnException: vi.fn(), + props: {}, + } as never); + + expect(contextValue).toBe('tenant-42'); + expect(innerParentSpanId).toBe(outerSpanId); + }); +}); diff --git a/packages/nextjs/test/common/utils/dropReactControlFlowErrors.test.ts b/packages/nextjs/test/common/utils/dropReactControlFlowErrors.test.ts new file mode 100644 index 000000000000..976dab9cf5b8 --- /dev/null +++ b/packages/nextjs/test/common/utils/dropReactControlFlowErrors.test.ts @@ -0,0 +1,48 @@ +import type { Event } from '@sentry/core'; +import { describe, expect, it } from 'vitest'; +import { dropReactControlFlowErrorsEventProcessor } from '../../../src/common/utils/dropReactControlFlowErrors'; + +describe('dropReactControlFlowErrorsEventProcessor', () => { + it('drops a React postpone error', () => { + const event: Event = { exception: { values: [{ type: 'Error', value: 'postponed' }] } }; + + expect( + dropReactControlFlowErrorsEventProcessor(event, { + originalException: { $$typeof: Symbol.for('react.postpone') }, + }), + ).toBeNull(); + }); + + it('drops an error that Next.js throws to abort a prerender', () => { + const event: Event = { exception: { values: [{ type: 'Error', value: 'prerender aborted' }] } }; + const error = Object.assign(new Error('prerender aborted'), { digest: 'HANGING_PROMISE_REJECTION' }); + + expect(dropReactControlFlowErrorsEventProcessor(event, { originalException: error })).toBeNull(); + }); + + it.each([ + "Suspense Exception: This is not a real error! It's an implementation detail of `use` to interrupt the current render.", + "Suspense Exception: This is not a real error, and should not leak into userspace. If you're seeing this, it's likely a bug in React.", + ])('drops a Suspense exception: %s', value => { + const event: Event = { exception: { values: [{ type: 'Error', value }] } }; + + expect(dropReactControlFlowErrorsEventProcessor(event, {})).toBeNull(); + }); + + it('keeps an error that is not thrown for control flow', () => { + const error = new TypeError("Cannot read properties of undefined (reading 'id')"); + const event: Event = { exception: { values: [{ type: 'TypeError', value: error.message }] } }; + + expect(dropReactControlFlowErrorsEventProcessor(event, { originalException: error })).toBe(event); + }); + + it('keeps a transaction event whose hint holds a React postpone', () => { + const event: Event = { type: 'transaction', transaction: 'GET /dashboard' }; + + expect( + dropReactControlFlowErrorsEventProcessor(event, { + originalException: { $$typeof: Symbol.for('react.postpone') }, + }), + ).toBe(event); + }); +}); diff --git a/packages/nextjs/test/edgeSdk.test.ts b/packages/nextjs/test/edgeSdk.test.ts index 2cbd0ef0070b..a3fb9dacc7b8 100644 --- a/packages/nextjs/test/edgeSdk.test.ts +++ b/packages/nextjs/test/edgeSdk.test.ts @@ -28,6 +28,8 @@ describe('Edge init()', () => { getMainCarrier().__SENTRY__ = undefined; delete (process as { turbopack?: boolean }).turbopack; + + delete (GLOBAL_OBJ as typeof GLOBAL_OBJ & { _sentryRelease?: string })._sentryRelease; }); it('inits the Vercel Edge SDK', () => { @@ -68,6 +70,15 @@ describe('Edge init()', () => { expect(vercelEdgeInit).not.toHaveBeenCalled(); }); + it('stores the release on Cloudflare Workers in a request of `withSentry` from `@sentry/cloudflare`', () => { + vi.stubGlobal('navigator', { userAgent: 'Cloudflare-Workers' }); + setAsyncLocalStorageAsyncContextStrategy(); + + withIsolationScope(() => init({ release: '1.2.3' })); + + expect((GLOBAL_OBJ as typeof GLOBAL_OBJ & { _sentryRelease?: string })._sentryRelease).toBe('1.2.3'); + }); + it('sets the `turbopack` tag on Cloudflare Workers in a request of `withSentry` from `@sentry/cloudflare`', () => { vi.stubGlobal('navigator', { userAgent: 'Cloudflare-Workers' }); setAsyncLocalStorageAsyncContextStrategy(); diff --git a/packages/nextjs/test/server/serverSpanHooks.test.ts b/packages/nextjs/test/server/serverSpanHooks.test.ts new file mode 100644 index 000000000000..aef1a76dd6d2 --- /dev/null +++ b/packages/nextjs/test/server/serverSpanHooks.test.ts @@ -0,0 +1,173 @@ +import { + HTTP_REQUEST_METHOD, + HTTP_ROUTE, + SENTRY_OP, + SENTRY_ORIGIN, + SENTRY_SEGMENT_NAME_SOURCE, +} from '@sentry/conventions/attributes'; +import { MIDDLEWARE } from '@sentry/conventions/op'; +import type { Client } from '@sentry/core'; +import { addChildSpanToSpan, SentrySpan, spanToStaticSpanJSON } from '@sentry/core'; +import { describe, expect, it } from 'vitest'; +import { ATTR_NEXT_SPAN_NAME, ATTR_NEXT_SPAN_TYPE } from '../../src/common/nextSpanAttributes'; +import { handleOnSpanStart } from '../../src/server/serverSpanHooks'; + +const client = { getOptions: () => ({}) } as unknown as Client; + +describe('handleOnSpanStart', () => { + it('names the root span after the middleware when Next.js started the root span', () => { + const middlewareSpan = new SentrySpan({ + sampled: true, + name: 'middleware GET', + attributes: { 'next.span_type': 'Middleware.execute', 'next.span_name': 'middleware GET' }, + }); + + handleOnSpanStart(middlewareSpan, client); + + const { description, data } = spanToStaticSpanJSON(middlewareSpan); + expect(description).toBe('middleware GET'); + expect(data).toEqual({ + [SENTRY_ORIGIN]: 'auto', + [ATTR_NEXT_SPAN_TYPE]: 'Middleware.execute', + [ATTR_NEXT_SPAN_NAME]: 'middleware GET', + [SENTRY_SEGMENT_NAME_SOURCE]: 'route', + [HTTP_ROUTE]: 'middleware GET', + }); + }); + + it('names a root span that another SDK started after the middleware', () => { + const rootSpan = new SentrySpan({ sampled: true, name: 'GET', attributes: { 'http.request.method': 'GET' } }); + const middlewareSpan = new SentrySpan({ + sampled: true, + name: 'middleware GET', + attributes: { 'next.span_type': 'Middleware.execute', 'next.span_name': 'middleware GET' }, + }); + addChildSpanToSpan(rootSpan, middlewareSpan); + + handleOnSpanStart(middlewareSpan, client); + + const root = spanToStaticSpanJSON(rootSpan); + expect(root.description).toBe('middleware GET'); + expect(root.data).toEqual({ + [SENTRY_ORIGIN]: 'manual', + [HTTP_REQUEST_METHOD]: 'GET', + [SENTRY_SEGMENT_NAME_SOURCE]: 'route', + }); + }); + + it('sets the middleware op on the middleware span when another SDK started the root span', () => { + const rootSpan = new SentrySpan({ sampled: true, name: 'GET', attributes: { 'http.request.method': 'GET' } }); + const middlewareSpan = new SentrySpan({ + sampled: true, + name: 'middleware GET', + attributes: { 'next.span_type': 'Middleware.execute', 'next.span_name': 'middleware GET' }, + }); + addChildSpanToSpan(rootSpan, middlewareSpan); + + handleOnSpanStart(middlewareSpan, client); + + expect(spanToStaticSpanJSON(middlewareSpan).data[SENTRY_OP]).toBe(MIDDLEWARE); + }); + + it('names a root span that another SDK started after the route once a route span starts behind the middleware', () => { + const rootSpan = new SentrySpan({ sampled: true, name: 'GET', attributes: { 'http.request.method': 'GET' } }); + const middlewareSpan = new SentrySpan({ + sampled: true, + name: 'middleware GET', + attributes: { 'next.span_type': 'Middleware.execute', 'next.span_name': 'middleware GET' }, + }); + const routeSpan = new SentrySpan({ + sampled: true, + name: 'executing api route (app) /api/endpoint/route', + attributes: { 'next.span_type': 'AppRouteRouteHandlers.runHandler', 'next.route': '/api/endpoint/route' }, + }); + addChildSpanToSpan(rootSpan, middlewareSpan); + addChildSpanToSpan(rootSpan, routeSpan); + + handleOnSpanStart(middlewareSpan, client); + handleOnSpanStart(routeSpan, client); + + const root = spanToStaticSpanJSON(rootSpan); + expect(root.description).toBe('GET /api/endpoint'); + expect(root.data[HTTP_ROUTE]).toBe('/api/endpoint'); + expect(root.data[SENTRY_SEGMENT_NAME_SOURCE]).toBe('route'); + }); + + it('keeps the middleware name of a root span that another SDK started when Next.js renders an error page', () => { + const rootSpan = new SentrySpan({ sampled: true, name: 'GET', attributes: { 'http.request.method': 'GET' } }); + const middlewareSpan = new SentrySpan({ + sampled: true, + name: 'middleware GET', + attributes: { 'next.span_type': 'Middleware.execute', 'next.span_name': 'middleware GET' }, + }); + const errorPageSpan = new SentrySpan({ + sampled: true, + name: 'resolve page components', + attributes: { 'next.span_type': 'NextNodeServer.findPageComponents', 'next.route': '/500' }, + }); + addChildSpanToSpan(rootSpan, middlewareSpan); + addChildSpanToSpan(rootSpan, errorPageSpan); + + handleOnSpanStart(middlewareSpan, client); + handleOnSpanStart(errorPageSpan, client); + + const root = spanToStaticSpanJSON(rootSpan); + expect(root.description).toBe('middleware GET'); + expect(root.data[HTTP_ROUTE]).toBeUndefined(); + }); + + it('names a root span that another SDK started after an error page when no middleware ran', () => { + const rootSpan = new SentrySpan({ sampled: true, name: 'GET', attributes: { 'http.request.method': 'GET' } }); + const errorPageSpan = new SentrySpan({ + sampled: true, + name: 'resolve page components', + attributes: { 'next.span_type': 'NextNodeServer.findPageComponents', 'next.route': '/500' }, + }); + addChildSpanToSpan(rootSpan, errorPageSpan); + + handleOnSpanStart(errorPageSpan, client); + + expect(spanToStaticSpanJSON(rootSpan).description).toBe('GET /500'); + }); + + it('names a root span that another SDK started after the 404 page behind the middleware', () => { + const rootSpan = new SentrySpan({ sampled: true, name: 'GET', attributes: { 'http.request.method': 'GET' } }); + const middlewareSpan = new SentrySpan({ + sampled: true, + name: 'middleware GET', + attributes: { 'next.span_type': 'Middleware.execute', 'next.span_name': 'middleware GET' }, + }); + const notFoundPageSpan = new SentrySpan({ + sampled: true, + name: 'resolve page components', + attributes: { 'next.span_type': 'NextNodeServer.findPageComponents', 'next.route': '/_not-found' }, + }); + addChildSpanToSpan(rootSpan, middlewareSpan); + addChildSpanToSpan(rootSpan, notFoundPageSpan); + + handleOnSpanStart(middlewareSpan, client); + handleOnSpanStart(notFoundPageSpan, client); + + const root = spanToStaticSpanJSON(rootSpan); + expect(root.description).toBe('GET /_not-found'); + expect(root.data[HTTP_ROUTE]).toBe('/_not-found'); + }); + + it('keeps the route of a root span that another SDK started when the middleware starts', () => { + const rootSpan = new SentrySpan({ + sampled: true, + name: 'GET /api/endpoint', + attributes: { 'http.request.method': 'GET', [HTTP_ROUTE]: '/api/endpoint' }, + }); + const middlewareSpan = new SentrySpan({ + sampled: true, + name: 'middleware GET', + attributes: { 'next.span_type': 'Middleware.execute', 'next.span_name': 'middleware GET' }, + }); + addChildSpanToSpan(rootSpan, middlewareSpan); + + handleOnSpanStart(middlewareSpan, client); + + expect(spanToStaticSpanJSON(rootSpan).description).toBe('GET /api/endpoint'); + }); +}); diff --git a/packages/nextjs/test/serverSdk.test.ts b/packages/nextjs/test/serverSdk.test.ts index 3ac99be0037f..61d15736a3d0 100644 --- a/packages/nextjs/test/serverSdk.test.ts +++ b/packages/nextjs/test/serverSdk.test.ts @@ -25,6 +25,7 @@ describe('Server init()', () => { delete process.env.VERCEL; delete (process as { turbopack?: boolean }).turbopack; + delete (GLOBAL_OBJ as typeof GLOBAL_OBJ & { _sentryRelease?: string })._sentryRelease; }); it('inits the Node SDK', () => { @@ -146,6 +147,48 @@ describe('Server init()', () => { expect(nodeInit).toHaveBeenCalledTimes(1); }); + it('sets the release on the client of `@sentry/cloudflare` when it has none', () => { + vi.stubGlobal('navigator', { userAgent: 'Cloudflare-Workers' }); + const client = SentryNode.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + enableOpenTelemetrySetup: false, + }); + // `init` of `@sentry/node` takes a release from CI env vars like `GITHUB_SHA`. + client!.getOptions().release = undefined; + + withIsolationScope(() => init({ release: '1.2.3' })); + + expect(client!.getOptions().release).toBe('1.2.3'); + expect((GLOBAL_OBJ as typeof GLOBAL_OBJ & { _sentryRelease?: string })._sentryRelease).toBe('1.2.3'); + }); + + it('keeps the release of the client of `@sentry/cloudflare`', () => { + vi.stubGlobal('navigator', { userAgent: 'Cloudflare-Workers' }); + const client = SentryNode.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + enableOpenTelemetrySetup: false, + release: 'worker@2.0.0', + }); + + withIsolationScope(() => init({ release: '1.2.3' })); + + expect(client!.getOptions().release).toBe('worker@2.0.0'); + }); + + it('adds its event processors to the global scope when the client of `@sentry/cloudflare` exists', () => { + vi.stubGlobal('navigator', { userAgent: 'Cloudflare-Workers' }); + SentryNode.init({ dsn: 'https://public@dsn.ingest.sentry.io/1337', enableOpenTelemetrySetup: false }); + + withIsolationScope(() => init({})); + + // With `cacheClient: false`, each request has a new client, so only the global scope reaches all of them. + expect( + getGlobalScope() + .getScopeData() + .eventProcessors.filter(processor => processor.id === 'DropReactControlFlowErrors'), + ).toHaveLength(1); + }); + // TODO: test `vercel` tag when running on Vercel // Can't just add the test and set env variables, since the value in `index.server.ts` // is resolved when importing. From ba3da3321ff0fb770001200a2c24e31be4ffef81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Peer=20St=C3=B6cklmair?= Date: Thu, 8 Oct 2026 14:28:40 +0300 Subject: [PATCH 26/47] test(nextjs): Run nextjs-16 on Bun, Deno and Cloudflare (#24999) part of #24830 part of JS-3835 This folds `nextjs-16-bun` and `nextjs-16-cf-workers` into `nextjs-16` and adds a Deno variant, so one suite runs on Node.js, Bun, Deno and Cloudflare Workers (OpenNext). Bun and Cloudflare stay required. Deno and Cloudflare with the latest Next.js are optional. New tests on every runtime check logs, outgoing `baggage` and trace continuation. Most removed lines are the two deleted apps. ## Why The two copies of the app drifted apart, and Deno had no coverage. Now all runtimes run the same tests, and each runtime difference is a visible `getRuntime()` branch or skip. ## Known issues - Bun and Deno need the request isolation and `fetch` integrations of `@sentry/bun` and `@sentry/deno` in `sentry.server.config.ts`. - Deno needs `--allow-sys` (the app runs `deno run -A`). - No `pg` spans on Bun and Workers (no runtime module hook). Co-authored-by: Claude Opus 5.5 --- dev-packages/e2e-tests/README.md | 9 +- .../nextjs-16-bun/.gitignore | 44 ----- .../app/(nested-layout)/layout.tsx | 12 -- .../nested-layout/[dynamic]/layout.tsx | 12 -- .../nested-layout/[dynamic]/page.tsx | 15 -- .../(nested-layout)/nested-layout/layout.tsx | 12 -- .../(nested-layout)/nested-layout/page.tsx | 11 -- .../api/endpoint-behind-middleware/route.ts | 3 - .../nextjs-16-bun/app/global-error.tsx | 23 --- .../nextjs-16-bun/app/layout.tsx | 7 - .../app/nested-rsc-error/[param]/page.tsx | 17 -- .../nextjs-16-bun/app/page.tsx | 3 - .../app/pageload-tracing/layout.tsx | 8 - .../app/pageload-tracing/page.tsx | 14 -- .../parameterized/[one]/beep/[two]/page.tsx | 3 - .../app/parameterized/[one]/beep/page.tsx | 3 - .../app/parameterized/[one]/page.tsx | 3 - .../app/parameterized/static/page.tsx | 3 - .../app/route-handler/[xoxo]/edge/route.ts | 8 - .../app/route-handler/[xoxo]/node/route.ts | 7 - .../[param]/client-page.tsx | 8 - .../app/streaming-rsc-error/[param]/page.tsx | 18 --- .../app/third-party-filter/page.tsx | 24 --- .../nextjs-16-bun/instrumentation-client.ts | 16 -- .../nextjs-16-bun/instrumentation.ts | 13 -- .../nextjs-16-bun/next.config.ts | 9 -- .../nextjs-16-bun/package.json | 35 ---- .../nextjs-16-bun/playwright.config.mjs | 25 --- .../test-applications/nextjs-16-bun/proxy.ts | 33 ---- .../nextjs-16-bun/sentry.edge.config.ts | 8 - .../nextjs-16-bun/sentry.server.config.ts | 19 --- .../nextjs-16-bun/start-event-proxy.mjs | 14 -- .../nextjs-16-bun/tests/middleware.test.ts | 20 --- .../tests/nested-rsc-error.test.ts | 43 ----- .../tests/pageload-tracing.test.ts | 23 --- .../tests/parameterized-routes.test.ts | 83 ---------- .../nextjs-16-bun/tests/route-handler.test.ts | 16 -- .../tests/server-components.test.ts | 149 ----------------- .../tests/streaming-rsc-error.test.ts | 45 ------ .../tests/third-party-filter.test.ts | 28 ---- .../nextjs-16-bun/tsconfig.json | 27 ---- .../nextjs-16-cf-workers/.gitignore | 44 ----- .../app/(nested-layout)/layout.tsx | 12 -- .../nested-layout/[dynamic]/layout.tsx | 12 -- .../nested-layout/[dynamic]/page.tsx | 15 -- .../(nested-layout)/nested-layout/layout.tsx | 12 -- .../(nested-layout)/nested-layout/page.tsx | 11 -- .../api/endpoint-behind-middleware/route.ts | 3 - .../app/api/test-success/route.ts | 7 - .../nextjs-16-cf-workers/app/favicon.ico | Bin 25931 -> 0 bytes .../nextjs-16-cf-workers/app/global-error.tsx | 23 --- .../app/isr-test/[product]/page.tsx | 17 -- .../app/isr-test/static/page.tsx | 15 -- .../nextjs-16-cf-workers/app/layout.tsx | 7 - .../nextjs-16-cf-workers/app/metrics/page.tsx | 34 ---- .../app/metrics/route-handler/route.ts | 23 --- .../app/nested-rsc-error/[param]/page.tsx | 17 -- .../app/non-isr-test/[item]/page.tsx | 11 -- .../nextjs-16-cf-workers/app/page.tsx | 3 - .../app/pageload-tracing/layout.tsx | 8 - .../app/pageload-tracing/page.tsx | 14 -- .../parameterized/[one]/beep/[two]/page.tsx | 3 - .../app/parameterized/[one]/beep/page.tsx | 3 - .../app/parameterized/[one]/page.tsx | 3 - .../app/parameterized/static/page.tsx | 3 - .../app/prefetching/page.tsx | 9 -- .../app/prefetching/to-be-prefetched/page.tsx | 5 - .../app/redirect/destination/page.tsx | 7 - .../app/redirect/origin/page.tsx | 18 --- .../app/route-handler/[xoxo]/edge/route.ts | 8 - .../app/route-handler/[xoxo]/node/route.ts | 7 - .../[param]/client-page.tsx | 8 - .../app/streaming-rsc-error/[param]/page.tsx | 18 --- .../nextjs-16-cf-workers/eslint.config.mjs | 19 --- .../instrumentation-client.ts | 15 -- .../nextjs-16-cf-workers/instrumentation.ts | 13 -- .../nextjs-16-cf-workers/middleware.ts | 35 ---- .../nextjs-16-cf-workers/next.config.ts | 8 - .../nextjs-16-cf-workers/package.json | 51 ------ .../playwright.config.mjs | 21 --- .../nextjs-16-cf-workers/public/file.svg | 1 - .../nextjs-16-cf-workers/public/globe.svg | 1 - .../nextjs-16-cf-workers/public/next.svg | 1 - .../nextjs-16-cf-workers/public/vercel.svg | 1 - .../nextjs-16-cf-workers/public/window.svg | 1 - .../sentry.edge.config.ts | 7 - .../sentry.server.config.ts | 7 - .../start-event-proxy.mjs | 14 -- .../tests/async-params.test.ts | 14 -- .../nextjs-16-cf-workers/tests/isDevMode.ts | 1 - .../tests/isr-routes.test.ts | 83 ---------- .../tests/metrics.test.ts | 130 --------------- .../tests/middleware.test.ts | 75 --------- .../tests/nested-rsc-error.test.ts | 43 ----- .../tests/pageload-tracing.test.ts | 53 ------- .../tests/parameterized-routes.test.ts | 83 ---------- .../tests/prefetch-spans.test.ts | 26 --- .../tests/route-handler.test.ts | 33 ---- .../tests/server-action-redirect.test.ts | 37 ----- .../tests/server-components.test.ts | 150 ------------------ .../tests/streaming-rsc-error.test.ts | 45 ------ .../tests/trace-continuation.test.ts | 38 ----- .../nextjs-16-cf-workers/tsconfig.json | 27 ---- .../nextjs-16-cf-workers/wrangler.jsonc | 74 --------- .../nextjs-16/ai-mock-server.mjs | 15 +- .../app/api/d1/route.ts | 0 .../nextjs-16/app/api/logs/route.ts | 9 ++ .../nextjs-16/app/api/openai/route.ts | 5 +- .../app/api/otel-context/route.ts | 8 +- .../app/api/test-error/route.ts | 0 .../check/route.ts | 0 .../route.ts | 0 .../test-outgoing-fetch/check/route.ts | 0 .../propagation/test-outgoing-fetch/route.ts | 0 .../app/propagation/utils.ts | 0 .../nextjs-16/global-setup.mjs | 5 + .../nextjs-16/next.config.ts | 3 + .../open-next.config.ts | 0 .../test-applications/nextjs-16/package.json | 34 +++- .../nextjs-16/playwright.config.mjs | 11 +- .../nextjs-16/sentry.server.config.ts | 13 +- .../tests/cloudflare-runtime.test.ts | 6 +- .../tests/d1-binding.test.ts | 6 +- .../nextjs-16/tests/db-page.test.ts | 22 ++- .../nextjs-16/tests/logs.test.ts | 39 +++++ .../nextjs-16/tests/metrics.test.ts | 12 +- .../nextjs-16/tests/middleware.test.ts | 82 ++++++++-- .../tests/node-runtime-metrics.test.ts | 4 +- .../tests/otel-context.test.ts | 2 +- .../tests/propagation.test.ts | 22 +-- .../tests/server-action-redirect.test.ts | 26 +-- .../nextjs-16/tests/server-components.test.ts | 46 +++++- .../tests/trace-continuation.test.ts | 37 +++++ .../tests/worker-bundle.test.ts | 3 + .../worker.ts | 5 +- .../nextjs-16/wrangler.jsonc | 36 +++++ 136 files changed, 382 insertions(+), 2429 deletions(-) delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/.gitignore delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/(nested-layout)/layout.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/(nested-layout)/nested-layout/[dynamic]/layout.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/(nested-layout)/nested-layout/[dynamic]/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/(nested-layout)/nested-layout/layout.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/(nested-layout)/nested-layout/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/api/endpoint-behind-middleware/route.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/global-error.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/layout.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/nested-rsc-error/[param]/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/pageload-tracing/layout.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/pageload-tracing/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/parameterized/[one]/beep/[two]/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/parameterized/[one]/beep/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/parameterized/[one]/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/parameterized/static/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/route-handler/[xoxo]/edge/route.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/route-handler/[xoxo]/node/route.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/streaming-rsc-error/[param]/client-page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/streaming-rsc-error/[param]/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/third-party-filter/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/instrumentation-client.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/instrumentation.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/next.config.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/package.json delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/playwright.config.mjs delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/proxy.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/sentry.edge.config.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/sentry.server.config.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/start-event-proxy.mjs delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/middleware.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/nested-rsc-error.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/pageload-tracing.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/parameterized-routes.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/route-handler.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/server-components.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/streaming-rsc-error.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/third-party-filter.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-bun/tsconfig.json delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/.gitignore delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/(nested-layout)/layout.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/(nested-layout)/nested-layout/[dynamic]/layout.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/(nested-layout)/nested-layout/[dynamic]/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/(nested-layout)/nested-layout/layout.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/(nested-layout)/nested-layout/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/endpoint-behind-middleware/route.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/test-success/route.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/favicon.ico delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/global-error.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/isr-test/[product]/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/isr-test/static/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/layout.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/metrics/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/metrics/route-handler/route.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/nested-rsc-error/[param]/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/non-isr-test/[item]/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/pageload-tracing/layout.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/pageload-tracing/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/parameterized/[one]/beep/[two]/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/parameterized/[one]/beep/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/parameterized/[one]/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/parameterized/static/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/prefetching/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/prefetching/to-be-prefetched/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/redirect/destination/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/redirect/origin/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/route-handler/[xoxo]/edge/route.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/route-handler/[xoxo]/node/route.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/streaming-rsc-error/[param]/client-page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/streaming-rsc-error/[param]/page.tsx delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/eslint.config.mjs delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/instrumentation-client.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/instrumentation.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/middleware.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/next.config.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/package.json delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/playwright.config.mjs delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/public/file.svg delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/public/globe.svg delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/public/next.svg delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/public/vercel.svg delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/public/window.svg delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/sentry.edge.config.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/sentry.server.config.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/start-event-proxy.mjs delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/async-params.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/isDevMode.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/isr-routes.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/metrics.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/middleware.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/nested-rsc-error.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/pageload-tracing.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/parameterized-routes.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/prefetch-spans.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/route-handler.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/server-action-redirect.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/server-components.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/streaming-rsc-error.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/trace-continuation.test.ts delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tsconfig.json delete mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/wrangler.jsonc rename dev-packages/e2e-tests/test-applications/{nextjs-16-cf-workers => nextjs-16}/app/api/d1/route.ts (100%) create mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16/app/api/logs/route.ts rename dev-packages/e2e-tests/test-applications/{nextjs-16-cf-workers => nextjs-16}/app/api/otel-context/route.ts (54%) rename dev-packages/e2e-tests/test-applications/{nextjs-16-cf-workers => nextjs-16}/app/api/test-error/route.ts (100%) rename dev-packages/e2e-tests/test-applications/{nextjs-16-bun => nextjs-16}/app/propagation/test-outgoing-fetch-external-disallowed/check/route.ts (100%) rename dev-packages/e2e-tests/test-applications/{nextjs-16-bun => nextjs-16}/app/propagation/test-outgoing-fetch-external-disallowed/route.ts (100%) rename dev-packages/e2e-tests/test-applications/{nextjs-16-bun => nextjs-16}/app/propagation/test-outgoing-fetch/check/route.ts (100%) rename dev-packages/e2e-tests/test-applications/{nextjs-16-bun => nextjs-16}/app/propagation/test-outgoing-fetch/route.ts (100%) rename dev-packages/e2e-tests/test-applications/{nextjs-16-bun => nextjs-16}/app/propagation/utils.ts (100%) rename dev-packages/e2e-tests/test-applications/{nextjs-16-cf-workers => nextjs-16}/open-next.config.ts (100%) rename dev-packages/e2e-tests/test-applications/{nextjs-16-cf-workers => nextjs-16}/tests/cloudflare-runtime.test.ts (79%) rename dev-packages/e2e-tests/test-applications/{nextjs-16-cf-workers => nextjs-16}/tests/d1-binding.test.ts (82%) create mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16/tests/logs.test.ts rename dev-packages/e2e-tests/test-applications/{nextjs-16-cf-workers => nextjs-16}/tests/otel-context.test.ts (95%) rename dev-packages/e2e-tests/test-applications/{nextjs-16-bun => nextjs-16}/tests/propagation.test.ts (73%) create mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16/tests/trace-continuation.test.ts rename dev-packages/e2e-tests/test-applications/{nextjs-16-cf-workers => nextjs-16}/tests/worker-bundle.test.ts (95%) rename dev-packages/e2e-tests/test-applications/{nextjs-16-cf-workers => nextjs-16}/worker.ts (68%) create mode 100644 dev-packages/e2e-tests/test-applications/nextjs-16/wrangler.jsonc diff --git a/dev-packages/e2e-tests/README.md b/dev-packages/e2e-tests/README.md index f4fd271d5c74..096dfa4f1b91 100644 --- a/dev-packages/e2e-tests/README.md +++ b/dev-packages/e2e-tests/README.md @@ -153,7 +153,10 @@ is the reference setup. - **SDK per runtime**: each runtime inits its own SDK, the way a user of that runtime would: Node the framework SDK (`instrument.mjs`), Bun `@sentry/bun` (`instrument.bun.mjs`), Deno `@sentry/deno` (`instrument.deno.mjs`) and Cloudflare `@sentry/cloudflare`. The framework SDK only provides the framework wrappers on the other runtimes, so - values that come from its `init()` (`sdk.name`, default integrations) are Node-only. + values that come from its `init()` (`sdk.name`, default integrations) are Node-only. Next.js is different: it loads + `instrumentation.ts` on every runtime, so `nextjs-16` calls `init()` of `@sentry/nextjs` on Bun and Deno too, and + adds the integrations of `@sentry/bun` or `@sentry/deno` that replace the Node instrumentation those runtimes do not + trigger. - **Start commands**: `playwright.config.mjs` selects the start command with `getRuntime()`. Bun and Deno use the same build as Node and only change the start command, for example `bun --bun --preload ./instrument.bun.mjs ./node_modules/@react-router/serve/bin.cjs ./build/server/index.js` and @@ -168,7 +171,9 @@ is the reference setup. separate file (for example `workers/app.ts`) and exports a plain handler: the Sentry plugin wraps it with `withSentry` and reads the init options from `instrument.server.ts` next to the entry. The start command runs `wrangler dev` on the build output. Code at module scope must not do I/O (for example open a database connection), - because workerd does not allow it. + because workerd does not allow it. `nextjs-16` has no Vite build: its Cloudflare variant builds with + `opennextjs-cloudflare build` and wraps the generated Worker in `worker.ts` with `withSentry` from + `@sentry/nextjs/cloudflare`. - **Runtime-specific files**: a file that has the runtime as a part of its name replaces the existing file without that part, for example `app/entry.server.cloudflare.tsx` replaces `app/entry.server.tsx` and `vite.cloudflare.config.ts` replaces `vite.config.ts`. For a variant with `runtime`, the runner copies these files over the others in the diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/.gitignore b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/.gitignore deleted file mode 100644 index dd146b53d966..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/.gitignore +++ /dev/null @@ -1,44 +0,0 @@ -# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. - -# dependencies -/node_modules -/.pnp -.pnp.* -.yarn/* -!.yarn/patches -!.yarn/plugins -!.yarn/releases -!.yarn/versions - -# testing -/coverage - -# next.js -/.next/ -/out/ - -# production -/build - -# misc -.DS_Store -*.pem - -# debug -npm-debug.log* -yarn-debug.log* -yarn-error.log* -.pnpm-debug.log* - -# env files (can opt-in for committing if needed) -.env* - -# vercel -.vercel - -# typescript -*.tsbuildinfo -next-env.d.ts - -# Sentry Config File -.env.sentry-build-plugin diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/(nested-layout)/layout.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/(nested-layout)/layout.tsx deleted file mode 100644 index ace0c2f086b7..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/(nested-layout)/layout.tsx +++ /dev/null @@ -1,12 +0,0 @@ -import { PropsWithChildren } from 'react'; - -export const dynamic = 'force-dynamic'; - -export default function Layout({ children }: PropsWithChildren<{}>) { - return ( -
-

Layout

- {children} -
- ); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/(nested-layout)/nested-layout/[dynamic]/layout.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/(nested-layout)/nested-layout/[dynamic]/layout.tsx deleted file mode 100644 index dbdc60adadc2..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/(nested-layout)/nested-layout/[dynamic]/layout.tsx +++ /dev/null @@ -1,12 +0,0 @@ -import { PropsWithChildren } from 'react'; - -export const dynamic = 'force-dynamic'; - -export default function Layout({ children }: PropsWithChildren<{}>) { - return ( -
-

DynamicLayout

- {children} -
- ); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/(nested-layout)/nested-layout/[dynamic]/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/(nested-layout)/nested-layout/[dynamic]/page.tsx deleted file mode 100644 index 3eaddda2a1df..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/(nested-layout)/nested-layout/[dynamic]/page.tsx +++ /dev/null @@ -1,15 +0,0 @@ -export const dynamic = 'force-dynamic'; - -export default async function Page() { - return ( -
-

Dynamic Page

-
- ); -} - -export async function generateMetadata() { - return { - title: 'I am dynamic page generated metadata', - }; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/(nested-layout)/nested-layout/layout.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/(nested-layout)/nested-layout/layout.tsx deleted file mode 100644 index ace0c2f086b7..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/(nested-layout)/nested-layout/layout.tsx +++ /dev/null @@ -1,12 +0,0 @@ -import { PropsWithChildren } from 'react'; - -export const dynamic = 'force-dynamic'; - -export default function Layout({ children }: PropsWithChildren<{}>) { - return ( -
-

Layout

- {children} -
- ); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/(nested-layout)/nested-layout/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/(nested-layout)/nested-layout/page.tsx deleted file mode 100644 index 8077c14d23ca..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/(nested-layout)/nested-layout/page.tsx +++ /dev/null @@ -1,11 +0,0 @@ -export const dynamic = 'force-dynamic'; - -export default function Page() { - return

Hello World!

; -} - -export async function generateMetadata() { - return { - title: 'I am generated metadata', - }; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/api/endpoint-behind-middleware/route.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/api/endpoint-behind-middleware/route.ts deleted file mode 100644 index 2733cc918f44..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/api/endpoint-behind-middleware/route.ts +++ /dev/null @@ -1,3 +0,0 @@ -export function GET() { - return Response.json({ name: 'John Doe' }); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/global-error.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/global-error.tsx deleted file mode 100644 index 20c175015b03..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/global-error.tsx +++ /dev/null @@ -1,23 +0,0 @@ -'use client'; - -import * as Sentry from '@sentry/nextjs'; -import NextError from 'next/error'; -import { useEffect } from 'react'; - -export default function GlobalError({ error }: { error: Error & { digest?: string } }) { - useEffect(() => { - Sentry.captureException(error); - }, [error]); - - return ( - - - {/* `NextError` is the default Next.js error page component. Its type - definition requires a `statusCode` prop. However, since the App Router - does not expose status codes for errors, we simply pass 0 to render a - generic error message. */} - - - - ); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/layout.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/layout.tsx deleted file mode 100644 index c8f9cee0b787..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/layout.tsx +++ /dev/null @@ -1,7 +0,0 @@ -export default function Layout({ children }: { children: React.ReactNode }) { - return ( - - {children} - - ); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/nested-rsc-error/[param]/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/nested-rsc-error/[param]/page.tsx deleted file mode 100644 index 675b248026be..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/nested-rsc-error/[param]/page.tsx +++ /dev/null @@ -1,17 +0,0 @@ -import { Suspense } from 'react'; - -export const dynamic = 'force-dynamic'; - -export default async function Page() { - return ( - Loading...

}> - {/* @ts-ignore */} - ; - - ); -} - -async function Crash() { - throw new Error('I am technically uncatchable'); - return

unreachable

; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/page.tsx deleted file mode 100644 index 3204f4df69e1..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/page.tsx +++ /dev/null @@ -1,3 +0,0 @@ -export default function Page() { - return

Next 16 Bun test app

; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/pageload-tracing/layout.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/pageload-tracing/layout.tsx deleted file mode 100644 index 1f0cbe478f88..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/pageload-tracing/layout.tsx +++ /dev/null @@ -1,8 +0,0 @@ -import { PropsWithChildren } from 'react'; - -export const dynamic = 'force-dynamic'; - -export default async function Layout({ children }: PropsWithChildren) { - await new Promise(resolve => setTimeout(resolve, 500)); - return <>{children}; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/pageload-tracing/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/pageload-tracing/page.tsx deleted file mode 100644 index 689735d61ddf..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/pageload-tracing/page.tsx +++ /dev/null @@ -1,14 +0,0 @@ -export const dynamic = 'force-dynamic'; - -export default async function Page() { - await new Promise(resolve => setTimeout(resolve, 1000)); - return

I am page 2

; -} - -export async function generateMetadata() { - (await fetch('https://example.com/', { cache: 'no-store' })).text(); - - return { - title: 'my title', - }; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/parameterized/[one]/beep/[two]/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/parameterized/[one]/beep/[two]/page.tsx deleted file mode 100644 index f34461c2bb07..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/parameterized/[one]/beep/[two]/page.tsx +++ /dev/null @@ -1,3 +0,0 @@ -export default function ParameterizedPage() { - return
Dynamic page two
; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/parameterized/[one]/beep/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/parameterized/[one]/beep/page.tsx deleted file mode 100644 index a7d9164c8c03..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/parameterized/[one]/beep/page.tsx +++ /dev/null @@ -1,3 +0,0 @@ -export default function BeepPage() { - return
Beep
; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/parameterized/[one]/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/parameterized/[one]/page.tsx deleted file mode 100644 index 9fa617a22381..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/parameterized/[one]/page.tsx +++ /dev/null @@ -1,3 +0,0 @@ -export default function ParameterizedPage() { - return
Dynamic page one
; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/parameterized/static/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/parameterized/static/page.tsx deleted file mode 100644 index 16ef0482d53b..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/parameterized/static/page.tsx +++ /dev/null @@ -1,3 +0,0 @@ -export default function StaticPage() { - return
Static page
; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/route-handler/[xoxo]/edge/route.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/route-handler/[xoxo]/edge/route.ts deleted file mode 100644 index 7cd1fc7e332c..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/route-handler/[xoxo]/edge/route.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { NextResponse } from 'next/server'; - -export const runtime = 'edge'; -export const dynamic = 'force-dynamic'; - -export async function GET() { - return NextResponse.json({ message: 'Hello Edge Route Handler' }); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/route-handler/[xoxo]/node/route.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/route-handler/[xoxo]/node/route.ts deleted file mode 100644 index 5bc418f077aa..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/route-handler/[xoxo]/node/route.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { NextResponse } from 'next/server'; - -export const dynamic = 'force-dynamic'; - -export async function GET() { - return NextResponse.json({ message: 'Hello Node Route Handler' }); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/streaming-rsc-error/[param]/client-page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/streaming-rsc-error/[param]/client-page.tsx deleted file mode 100644 index 7b66c3fbdeef..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/streaming-rsc-error/[param]/client-page.tsx +++ /dev/null @@ -1,8 +0,0 @@ -'use client'; - -import { use } from 'react'; - -export function RenderPromise({ stringPromise }: { stringPromise: Promise }) { - const s = use(stringPromise); - return <>{s}; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/streaming-rsc-error/[param]/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/streaming-rsc-error/[param]/page.tsx deleted file mode 100644 index 9531f9a42139..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/streaming-rsc-error/[param]/page.tsx +++ /dev/null @@ -1,18 +0,0 @@ -import { Suspense } from 'react'; -import { RenderPromise } from './client-page'; - -export const dynamic = 'force-dynamic'; - -export default async function Page() { - const crashingPromise = new Promise((_, reject) => { - setTimeout(() => { - reject(new Error('I am a data streaming error')); - }, 100); - }); - - return ( - Loading...

}> - ; - - ); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/third-party-filter/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/third-party-filter/page.tsx deleted file mode 100644 index b6b4bea80def..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/third-party-filter/page.tsx +++ /dev/null @@ -1,24 +0,0 @@ -'use client'; - -import * as Sentry from '@sentry/nextjs'; - -function throwFirstPartyError(): void { - throw new Error('first-party-error'); -} - -export default function Page() { - return ( - - ); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/instrumentation-client.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/instrumentation-client.ts deleted file mode 100644 index 9c0366b0cea2..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/instrumentation-client.ts +++ /dev/null @@ -1,16 +0,0 @@ -import * as Sentry from '@sentry/nextjs'; - -Sentry.init({ - environment: 'qa', // dynamic sampling bias to keep transactions - dsn: process.env.NEXT_PUBLIC_E2E_TEST_DSN, - tunnel: `http://localhost:3031/`, // proxy server - tracesSampleRate: 1.0, - integrations: [ - Sentry.thirdPartyErrorFilterIntegration({ - filterKeys: ['nextjs-16-bun-e2e'], - behaviour: 'apply-tag-if-exclusively-contains-third-party-frames', - }), - ], -}); - -export const onRouterTransitionStart = Sentry.captureRouterTransitionStart; diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/instrumentation.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/instrumentation.ts deleted file mode 100644 index 964f937c439a..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/instrumentation.ts +++ /dev/null @@ -1,13 +0,0 @@ -import * as Sentry from '@sentry/nextjs'; - -export async function register() { - if (process.env.NEXT_RUNTIME === 'nodejs') { - await import('./sentry.server.config'); - } - - if (process.env.NEXT_RUNTIME === 'edge') { - await import('./sentry.edge.config'); - } -} - -export const onRequestError = Sentry.captureRequestError; diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/next.config.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/next.config.ts deleted file mode 100644 index 34d17391df28..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/next.config.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { withSentryConfig } from '@sentry/nextjs/config'; -import type { NextConfig } from 'next'; - -const nextConfig: NextConfig = {}; - -export default withSentryConfig(nextConfig, { - silent: true, - applicationKey: 'nextjs-16-bun-e2e', -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/package.json b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/package.json deleted file mode 100644 index 6b510830e98e..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/package.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "name": "nextjs-16-bun", - "version": "0.1.0", - "private": true, - "scripts": { - "dev": "bun --bun next dev", - "build": "bun --bun next build > .tmp_build_stdout 2> .tmp_build_stderr || (cat .tmp_build_stdout && cat .tmp_build_stderr && exit 1)", - "clean": "npx rimraf node_modules pnpm-lock.yaml .tmp_dev_server_logs", - "start": "bun --bun next start", - "test:prod": "TEST_ENV=production playwright test", - "test:build": "pnpm install && pnpm build", - "test:assert": "pnpm test:prod" - }, - "dependencies": { - "@sentry/nextjs": "file:../../packed/sentry-nextjs-packed.tgz", - "@sentry/bun": "file:../../packed/sentry-bun-packed.tgz", - "@sentry/core": "file:../../packed/sentry-core-packed.tgz", - "import-in-the-middle": "^2", - "next": "16.2.11", - "react": "19.1.0", - "react-dom": "19.1.0", - "require-in-the-middle": "^8" - }, - "devDependencies": { - "@playwright/test": "~1.63.0", - "@sentry-internal/test-utils": "link:../../../test-utils", - "@types/node": "^20", - "@types/react": "^19", - "@types/react-dom": "^19", - "typescript": "^5" - }, - "volta": { - "extends": "../../package.json" - } -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/playwright.config.mjs b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/playwright.config.mjs deleted file mode 100644 index 28662eab2d7b..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/playwright.config.mjs +++ /dev/null @@ -1,25 +0,0 @@ -import { getPlaywrightConfig } from '@sentry-internal/test-utils'; -const testEnv = process.env.TEST_ENV; - -if (!testEnv) { - throw new Error('No test env defined'); -} - -const getStartCommand = () => { - if (testEnv === 'development') { - return 'bun --bun next dev -p 3030 2>&1 | tee .tmp_dev_server_logs'; - } - - if (testEnv === 'production') { - return 'bun --bun next start -p 3030'; - } - - throw new Error(`Unknown test env: ${testEnv}`); -}; - -const config = getPlaywrightConfig({ - startCommand: getStartCommand(), - port: 3030, -}); - -export default config; diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/proxy.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/proxy.ts deleted file mode 100644 index e2d263c4d80c..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/proxy.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { getDefaultIsolationScope } from '@sentry/core'; -import * as Sentry from '@sentry/nextjs'; -import { NextResponse } from 'next/server'; -import type { NextRequest } from 'next/server'; - -export async function proxy(request: NextRequest) { - Sentry.setTag('my-isolated-tag', true); - Sentry.setTag('my-global-scope-isolated-tag', getDefaultIsolationScope().getScopeData().tags['my-isolated-tag']); // We set this tag to be able to assert that the previously set tag has not leaked into the global isolation scope - - // Streamed spans carry no scope tags, so the tests read the isolation state from this attribute instead - const activeSpan = Sentry.getActiveSpan(); - if (activeSpan) { - Sentry.getRootSpan(activeSpan).setAttribute( - 'isolation_scope.is_default', - Sentry.getIsolationScope() === getDefaultIsolationScope(), - ); - } - - if (request.headers.has('x-should-throw')) { - throw new Error('Middleware Error'); - } - - if (request.headers.has('x-should-make-request')) { - await fetch('http://localhost:3030/'); - } - - return NextResponse.next(); -} - -// See "Matching Paths" below to learn more -export const config = { - matcher: ['/api/endpoint-behind-middleware', '/api/endpoint-behind-faulty-middleware'], -}; diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/sentry.edge.config.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/sentry.edge.config.ts deleted file mode 100644 index ad780407a5b7..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/sentry.edge.config.ts +++ /dev/null @@ -1,8 +0,0 @@ -import * as Sentry from '@sentry/nextjs'; - -Sentry.init({ - environment: 'qa', // dynamic sampling bias to keep transactions - dsn: process.env.NEXT_PUBLIC_E2E_TEST_DSN, - tunnel: `http://localhost:3031/`, // proxy server - tracesSampleRate: 1.0, -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/sentry.server.config.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/sentry.server.config.ts deleted file mode 100644 index e683c5362818..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/sentry.server.config.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { bunHttpServerIntegration } from '@sentry/bun'; -import * as Sentry from '@sentry/nextjs'; -import { bunServerIntegration, fetchIntegration } from '@sentry/bun'; - -Sentry.init({ - environment: 'qa', // dynamic sampling bias to keep transactions - dsn: process.env.NEXT_PUBLIC_E2E_TEST_DSN, - tunnel: `http://localhost:3031/`, // proxy server - tracesSampleRate: 1.0, - tracePropagationTargets: ['http://localhost:3030/propagation/test-outgoing-fetch/check'], - integrations: [ - // Bun does not emit the `node:http` diagnostics channel the Node SDK uses to isolate incoming - // requests, so each request would otherwise share one trace. Next.js emits its own server spans, - // hence `spans: false` — this only isolates the request and resets its trace. - bunHttpServerIntegration({ spans: false }), - // Adding bun-specific integration here - fetchIntegration(), - ], -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/start-event-proxy.mjs b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/start-event-proxy.mjs deleted file mode 100644 index d56d5fb74e59..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/start-event-proxy.mjs +++ /dev/null @@ -1,14 +0,0 @@ -import * as fs from 'fs'; -import * as path from 'path'; -import { startEventProxyServer } from '@sentry-internal/test-utils'; - -const packageJson = JSON.parse(fs.readFileSync(path.join(process.cwd(), 'package.json'))); - -startEventProxyServer({ - port: 3031, - proxyServerName: 'nextjs-16-bun', - envelopeDumpPath: path.join( - process.cwd(), - `event-dumps/nextjs-16-bun-v${packageJson.dependencies.next}-${process.env.TEST_ENV}.dump`, - ), -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/middleware.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/middleware.test.ts deleted file mode 100644 index 2b56d36de0ce..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/middleware.test.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { getSpanOp, waitForStreamedSpan } from '@sentry-internal/test-utils'; - -test('Should create a span for middleware', async ({ request }) => { - const middlewareSpanPromise = waitForStreamedSpan('nextjs-16-bun', span => { - return span.name === 'middleware GET' && span.is_segment; - }); - - const response = await request.get('/api/endpoint-behind-middleware'); - expect(await response.json()).toStrictEqual({ name: 'John Doe' }); - - const middlewareSpan = await middlewareSpanPromise; - - expect(middlewareSpan.status).toBe('ok'); - expect(getSpanOp(middlewareSpan)).toBe('middleware'); - expect(middlewareSpan.attributes['sentry.segment.name.source']?.value).toBe('route'); - - // Assert that isolation scope works properly - expect(middlewareSpan.attributes['isolation_scope.is_default']).toEqual({ value: false, type: 'boolean' }); -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/nested-rsc-error.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/nested-rsc-error.test.ts deleted file mode 100644 index 7ca749c0b181..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/nested-rsc-error.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { waitForError, waitForStreamedSpan } from '@sentry-internal/test-utils'; - -test('Should capture errors from nested server components when `Sentry.captureRequestError` is added to the `onRequestError` hook', async ({ - page, -}) => { - const errorEventPromise = waitForError('nextjs-16-bun', errorEvent => { - return !!errorEvent?.exception?.values?.some(value => value.value === 'I am technically uncatchable'); - }); - - // Matched on the error's own trace so a span from an earlier spec cannot satisfy the correlation. - const serverSpanPromise = waitForStreamedSpan('nextjs-16-bun', async span => { - return ( - span.name === 'GET /nested-rsc-error/[param]' && - span.is_segment && - (await errorEventPromise).contexts?.trace?.trace_id === span.trace_id - ); - }); - - await page.goto(`/nested-rsc-error/123`); - const errorEvent = await errorEventPromise; - const serverSpan = await serverSpanPromise; - - // error event is part of the same trace as the server span - expect(errorEvent.contexts?.trace?.trace_id).toBe(serverSpan.trace_id); - - expect(errorEvent.request).toMatchObject({ - headers: expect.any(Object), - method: 'GET', - }); - - expect(errorEvent.contexts?.nextjs).toEqual({ - route_type: 'render', - router_kind: 'App Router', - router_path: '/nested-rsc-error/[param]', - request_path: '/nested-rsc-error/123', - }); - - expect(errorEvent.exception?.values?.[0]?.mechanism).toEqual({ - handled: false, - type: 'auto.function.nextjs.on_request_error', - }); -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/pageload-tracing.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/pageload-tracing.test.ts deleted file mode 100644 index 89ac76eadc1c..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/pageload-tracing.test.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { getSpanOp, waitForStreamedSpan } from '@sentry-internal/test-utils'; - -test('App router spans should be attached to the pageload request span', async ({ page }) => { - const serverSpanPromise = waitForStreamedSpan('nextjs-16-bun', span => { - return span.name === 'GET /pageload-tracing' && span.is_segment; - }); - - const pageloadSpanPromise = waitForStreamedSpan('nextjs-16-bun', span => { - return span.name === '/pageload-tracing' && getSpanOp(span) === 'pageload' && span.is_segment; - }); - - await page.goto(`/pageload-tracing`); - - const [serverSpan, pageloadSpan] = await Promise.all([serverSpanPromise, pageloadSpanPromise]); - - expect(pageloadSpan.trace_id).toBeTruthy(); - expect(serverSpan.trace_id).toBe(pageloadSpan.trace_id); -}); - -// Bun runtime does not populate HTTP request headers as span attributes -// because the OTel HTTP instrumentation does not extract headers when running on Bun. -// This is a known behavioral difference from Node.js. diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/parameterized-routes.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/parameterized-routes.test.ts deleted file mode 100644 index 27afcc6c3cf3..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/parameterized-routes.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { getSpanOp, waitForStreamedSpan } from '@sentry-internal/test-utils'; - -test('should create a parameterized pageload span when the `app` directory is used', async ({ page }) => { - const spanPromise = waitForStreamedSpan('nextjs-16-bun', span => { - return span.name === '/parameterized/:one' && getSpanOp(span) === 'pageload' && span.is_segment; - }); - - await page.goto(`/parameterized/cappuccino`); - - const span = await spanPromise; - - expect(span.span_id).toEqual(expect.stringMatching(/[a-f0-9]{16}/)); - expect(span.trace_id).toEqual(expect.stringMatching(/[a-f0-9]{32}/)); - expect(span.attributes).toMatchObject({ - 'sentry.op': { value: 'pageload', type: 'string' }, - 'sentry.origin': { value: 'auto.pageload.nextjs.app_router_instrumentation', type: 'string' }, - 'sentry.segment.name.source': { value: 'route', type: 'string' }, - 'sentry.environment': { value: 'qa', type: 'string' }, - 'react.version': { value: expect.any(String), type: 'string' }, - }); -}); - -test('should create a span named after the static route when the `app` directory is used', async ({ page }) => { - const spanPromise = waitForStreamedSpan('nextjs-16-bun', span => { - return span.name === '/parameterized/static' && getSpanOp(span) === 'pageload' && span.is_segment; - }); - - await page.goto(`/parameterized/static`); - - const span = await spanPromise; - - expect(span.span_id).toEqual(expect.stringMatching(/[a-f0-9]{16}/)); - expect(span.trace_id).toEqual(expect.stringMatching(/[a-f0-9]{32}/)); - expect(span.attributes).toMatchObject({ - 'sentry.op': { value: 'pageload', type: 'string' }, - 'sentry.origin': { value: 'auto.pageload.nextjs.app_router_instrumentation', type: 'string' }, - 'sentry.segment.name.source': { value: 'route', type: 'string' }, - 'sentry.environment': { value: 'qa', type: 'string' }, - 'react.version': { value: expect.any(String), type: 'string' }, - 'url.template': { value: '/parameterized/static', type: 'string' }, - }); -}); - -test('should create a partially parameterized pageload span when the `app` directory is used', async ({ page }) => { - const spanPromise = waitForStreamedSpan('nextjs-16-bun', span => { - return span.name === '/parameterized/:one/beep' && getSpanOp(span) === 'pageload' && span.is_segment; - }); - - await page.goto(`/parameterized/cappuccino/beep`); - - const span = await spanPromise; - - expect(span.span_id).toEqual(expect.stringMatching(/[a-f0-9]{16}/)); - expect(span.trace_id).toEqual(expect.stringMatching(/[a-f0-9]{32}/)); - expect(span.attributes).toMatchObject({ - 'sentry.op': { value: 'pageload', type: 'string' }, - 'sentry.origin': { value: 'auto.pageload.nextjs.app_router_instrumentation', type: 'string' }, - 'sentry.segment.name.source': { value: 'route', type: 'string' }, - 'sentry.environment': { value: 'qa', type: 'string' }, - 'react.version': { value: expect.any(String), type: 'string' }, - }); -}); - -test('should create a nested parameterized pageload span when the `app` directory is used', async ({ page }) => { - const spanPromise = waitForStreamedSpan('nextjs-16-bun', span => { - return span.name === '/parameterized/:one/beep/:two' && getSpanOp(span) === 'pageload' && span.is_segment; - }); - - await page.goto(`/parameterized/cappuccino/beep/espresso`); - - const span = await spanPromise; - - expect(span.span_id).toEqual(expect.stringMatching(/[a-f0-9]{16}/)); - expect(span.trace_id).toEqual(expect.stringMatching(/[a-f0-9]{32}/)); - expect(span.attributes).toMatchObject({ - 'sentry.op': { value: 'pageload', type: 'string' }, - 'sentry.origin': { value: 'auto.pageload.nextjs.app_router_instrumentation', type: 'string' }, - 'sentry.segment.name.source': { value: 'route', type: 'string' }, - 'sentry.environment': { value: 'qa', type: 'string' }, - 'react.version': { value: expect.any(String), type: 'string' }, - }); -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/route-handler.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/route-handler.test.ts deleted file mode 100644 index da51193cb345..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/route-handler.test.ts +++ /dev/null @@ -1,16 +0,0 @@ -import test, { expect } from '@playwright/test'; -import { getSpanOp, waitForStreamedSpan } from '@sentry-internal/test-utils'; - -test('Should create a span for node route handlers', async ({ request }) => { - const routehandlerSpanPromise = waitForStreamedSpan('nextjs-16-bun', span => { - return span.name === 'GET /route-handler/[xoxo]/node' && span.is_segment; - }); - - const response = await request.get('/route-handler/123/node', { headers: { 'x-charly': 'gomez' } }); - expect(await response.json()).toStrictEqual({ message: 'Hello Node Route Handler' }); - - const routehandlerSpan = await routehandlerSpanPromise; - - expect(routehandlerSpan.status).toBe('ok'); - expect(getSpanOp(routehandlerSpan)).toBe('http.server'); -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/server-components.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/server-components.test.ts deleted file mode 100644 index c674845c4234..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/server-components.test.ts +++ /dev/null @@ -1,149 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { collectStreamedSpansUntilSegment, getSpanOp } from '@sentry-internal/test-utils'; - -// Next.js emits these spans itself. The SDK attaches no op, description or function name to -// them, so asserting `undefined` pins that they stay untouched. -const nextjsSpan = { op: undefined, description: undefined, codeFunctionName: undefined }; - -test('Sends a span for a request to app router with URL', async ({ page }) => { - const spansPromise = collectStreamedSpansUntilSegment( - 'nextjs-16-bun', - span => - span.name === 'GET /parameterized/[one]/beep/[two]' && - String(span.attributes['http.target']?.value).startsWith('/parameterized/1337/beep/42'), - ); - - await page.goto('/parameterized/1337/beep/42'); - - const spans = await spansPromise; - const segmentSpan = spans.find( - span => - span.name === 'GET /parameterized/[one]/beep/[two]' && - span.is_segment && - String(span.attributes['http.target']?.value).startsWith('/parameterized/1337/beep/42'), - )!; - - expect(segmentSpan.span_id).toEqual(expect.stringMatching(/[a-f0-9]{16}/)); - expect(segmentSpan.trace_id).toEqual(expect.stringMatching(/[a-f0-9]{32}/)); - expect(segmentSpan.status).toBe('ok'); - expect(segmentSpan.attributes).toMatchObject({ - 'sentry.op': { value: 'http.server', type: 'string' }, - 'sentry.origin': { value: 'auto', type: 'string' }, - 'sentry.sample_rate': { value: 1, type: 'integer' }, - 'sentry.segment.name.source': { value: 'route', type: 'string' }, - 'http.method': { value: 'GET', type: 'string' }, - 'http.response.status_code': { value: 200, type: 'integer' }, - 'http.route': { value: '/parameterized/[one]/beep/[two]', type: 'string' }, - 'http.status_code': { value: 200, type: 'integer' }, - 'http.target': { value: '/parameterized/1337/beep/42', type: 'string' }, - 'sentry.kind': { value: 'server', type: 'string' }, - 'next.route': { value: '/parameterized/[one]/beep/[two]', type: 'string' }, - }); - - // No child span should share the segment span's name - expect(spans.filter(span => !span.is_segment && span.name === segmentSpan.name)).toHaveLength(0); -}); - -test('Will create spans for every server component and metadata generation functions when visiting a page', async ({ - page, -}) => { - const spansPromise = collectStreamedSpansUntilSegment('nextjs-16-bun', 'GET /nested-layout'); - - await page.goto('/nested-layout'); - - const fullSpans = await spansPromise; - const spans = fullSpans.map(span => ({ - name: span.name, - op: getSpanOp(span), - description: span.attributes['sentry.description']?.value, - codeFunctionName: span.attributes['code.function.name']?.value, - })); - - expect(spans).toContainEqual({ ...nextjsSpan, name: 'render route (app) /nested-layout' }); - expect(spans).toContainEqual({ ...nextjsSpan, name: 'build component tree' }); - // Server component spans: the name is the low-cardinality `code.function.name`, and the - // segment each one resolved for is on the description. - expect(spans).toContainEqual({ - name: 'Layout', - op: 'function', - description: 'resolve root layout server component', - codeFunctionName: 'Layout', - }); - expect(spans).toContainEqual({ - name: 'Layout', - op: 'function', - description: 'resolve layout server component "(nested-layout)"', - codeFunctionName: 'Layout', - }); - expect(spans).toContainEqual({ - name: 'Layout', - op: 'function', - description: 'resolve layout server component "nested-layout"', - codeFunctionName: 'Layout', - }); - expect(spans).toContainEqual({ - name: 'Page', - op: 'function', - description: 'resolve page server component "/nested-layout"', - codeFunctionName: 'Page', - }); - expect(spans).toContainEqual({ ...nextjsSpan, name: 'generateMetadata /(nested-layout)/nested-layout/page' }); - expect(spans).toContainEqual({ ...nextjsSpan, name: 'start response' }); -}); - -test('Will create spans for every server component and metadata generation functions when visiting a dynamic page', async ({ - page, -}) => { - const spansPromise = collectStreamedSpansUntilSegment('nextjs-16-bun', 'GET /nested-layout/[dynamic]'); - - await page.goto('/nested-layout/123'); - - const fullSpans = await spansPromise; - const spans = fullSpans.map(span => ({ - name: span.name, - op: getSpanOp(span), - description: span.attributes['sentry.description']?.value, - codeFunctionName: span.attributes['code.function.name']?.value, - })); - - expect(spans).toContainEqual({ ...nextjsSpan, name: 'resolve page components' }); - expect(spans).toContainEqual({ ...nextjsSpan, name: 'render route (app) /nested-layout/[dynamic]' }); - expect(spans).toContainEqual({ ...nextjsSpan, name: 'build component tree' }); - // Server component spans: the name is the low-cardinality `code.function.name`, and the - // segment each one resolved for is on the description. - expect(spans).toContainEqual({ - name: 'Layout', - op: 'function', - description: 'resolve root layout server component', - codeFunctionName: 'Layout', - }); - expect(spans).toContainEqual({ - name: 'Layout', - op: 'function', - description: 'resolve layout server component "(nested-layout)"', - codeFunctionName: 'Layout', - }); - expect(spans).toContainEqual({ - name: 'Layout', - op: 'function', - description: 'resolve layout server component "nested-layout"', - codeFunctionName: 'Layout', - }); - expect(spans).toContainEqual({ - name: 'Layout', - op: 'function', - description: 'resolve layout server component "[dynamic]"', - codeFunctionName: 'Layout', - }); - expect(spans).toContainEqual({ - name: 'Page', - op: 'function', - description: 'resolve page server component "/nested-layout/[dynamic]"', - codeFunctionName: 'Page', - }); - expect(spans).toContainEqual({ - ...nextjsSpan, - name: 'generateMetadata /(nested-layout)/nested-layout/[dynamic]/page', - }); - expect(spans).toContainEqual({ ...nextjsSpan, name: 'start response' }); -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/streaming-rsc-error.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/streaming-rsc-error.test.ts deleted file mode 100644 index 7b5482d4d8e2..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/streaming-rsc-error.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { waitForError, waitForStreamedSpan } from '@sentry-internal/test-utils'; - -test('Should capture errors for crashing streaming promises in server components when `Sentry.captureRequestError` is added to the `onRequestError` hook', async ({ - page, -}) => { - const errorEventPromise = waitForError('nextjs-16-bun', errorEvent => { - return !!errorEvent?.exception?.values?.some(value => value.value === 'I am a data streaming error'); - }); - - // Matched on the error's own trace so a span from an earlier spec cannot satisfy the correlation. - const serverSpanPromise = waitForStreamedSpan('nextjs-16-bun', async span => { - return ( - span.name === 'GET /streaming-rsc-error/[param]' && - span.is_segment && - (await errorEventPromise).contexts?.trace?.trace_id === span.trace_id - ); - }); - - // The streaming RSC error can interrupt the HTTP response, causing the navigation to reject - // (e.g. net::ERR_ABORTED) even though the error and transaction are still captured. - await page.goto(`/streaming-rsc-error/123`).catch(() => {}); - const errorEvent = await errorEventPromise; - const serverSpan = await serverSpanPromise; - - // error event is part of the same trace as the server span - expect(errorEvent.contexts?.trace?.trace_id).toBe(serverSpan.trace_id); - - expect(errorEvent.request).toMatchObject({ - headers: expect.any(Object), - method: 'GET', - }); - - expect(errorEvent.contexts?.nextjs).toEqual({ - route_type: 'render', - router_kind: 'App Router', - router_path: '/streaming-rsc-error/[param]', - request_path: '/streaming-rsc-error/123', - }); - - expect(errorEvent.exception?.values?.[0]?.mechanism).toEqual({ - handled: false, - type: 'auto.function.nextjs.on_request_error', - }); -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/third-party-filter.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/third-party-filter.test.ts deleted file mode 100644 index 4de1433597b5..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/third-party-filter.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import test, { expect } from '@playwright/test'; -import { waitForError } from '@sentry-internal/test-utils'; - -const isWebpackDev = process.env.TEST_ENV === 'development-webpack'; - -test('First-party error should not be tagged as third-party with exclusively-contains mode', async ({ page }) => { - test.skip(isWebpackDev, 'Only relevant for Turbopack builds'); - - const errorPromise = waitForError('nextjs-16-bun', errorEvent => { - return errorEvent?.exception?.values?.some(value => value.value === 'first-party-error') ?? false; - }); - - await page.goto('/third-party-filter'); - await page.locator('#first-party-error-btn').click(); - - const errorEvent = await errorPromise; - - expect(errorEvent.exception?.values?.[0]?.value).toBe('first-party-error'); - - // In production, TEST_ENV=production is shared by both turbopack and webpack variants. - // Only assert when the build is actually turbopack. - if (errorEvent.tags?.turbopack) { - // The integration uses `apply-tag-if-exclusively-contains-third-party-frames` which - // only tags errors if ALL frames are third-party. A first-party error with React frames - // should not be tagged because the first-party frames have metadata. - expect(errorEvent.tags?.third_party_code).toBeUndefined(); - } -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tsconfig.json b/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tsconfig.json deleted file mode 100644 index cc9ed39b5aa2..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tsconfig.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2017", - "lib": ["dom", "dom.iterable", "esnext"], - "allowJs": true, - "skipLibCheck": true, - "strict": true, - "noEmit": true, - "esModuleInterop": true, - "module": "esnext", - "moduleResolution": "bundler", - "resolveJsonModule": true, - "isolatedModules": true, - "jsx": "react-jsx", - "incremental": true, - "plugins": [ - { - "name": "next" - } - ], - "paths": { - "@/*": ["./*"] - } - }, - "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts", ".next/dev/types/**/*.ts", "**/*.mts"], - "exclude": ["node_modules"] -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/.gitignore b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/.gitignore deleted file mode 100644 index dd146b53d966..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/.gitignore +++ /dev/null @@ -1,44 +0,0 @@ -# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. - -# dependencies -/node_modules -/.pnp -.pnp.* -.yarn/* -!.yarn/patches -!.yarn/plugins -!.yarn/releases -!.yarn/versions - -# testing -/coverage - -# next.js -/.next/ -/out/ - -# production -/build - -# misc -.DS_Store -*.pem - -# debug -npm-debug.log* -yarn-debug.log* -yarn-error.log* -.pnpm-debug.log* - -# env files (can opt-in for committing if needed) -.env* - -# vercel -.vercel - -# typescript -*.tsbuildinfo -next-env.d.ts - -# Sentry Config File -.env.sentry-build-plugin diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/(nested-layout)/layout.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/(nested-layout)/layout.tsx deleted file mode 100644 index ace0c2f086b7..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/(nested-layout)/layout.tsx +++ /dev/null @@ -1,12 +0,0 @@ -import { PropsWithChildren } from 'react'; - -export const dynamic = 'force-dynamic'; - -export default function Layout({ children }: PropsWithChildren<{}>) { - return ( -
-

Layout

- {children} -
- ); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/(nested-layout)/nested-layout/[dynamic]/layout.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/(nested-layout)/nested-layout/[dynamic]/layout.tsx deleted file mode 100644 index dbdc60adadc2..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/(nested-layout)/nested-layout/[dynamic]/layout.tsx +++ /dev/null @@ -1,12 +0,0 @@ -import { PropsWithChildren } from 'react'; - -export const dynamic = 'force-dynamic'; - -export default function Layout({ children }: PropsWithChildren<{}>) { - return ( -
-

DynamicLayout

- {children} -
- ); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/(nested-layout)/nested-layout/[dynamic]/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/(nested-layout)/nested-layout/[dynamic]/page.tsx deleted file mode 100644 index 3eaddda2a1df..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/(nested-layout)/nested-layout/[dynamic]/page.tsx +++ /dev/null @@ -1,15 +0,0 @@ -export const dynamic = 'force-dynamic'; - -export default async function Page() { - return ( -
-

Dynamic Page

-
- ); -} - -export async function generateMetadata() { - return { - title: 'I am dynamic page generated metadata', - }; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/(nested-layout)/nested-layout/layout.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/(nested-layout)/nested-layout/layout.tsx deleted file mode 100644 index ace0c2f086b7..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/(nested-layout)/nested-layout/layout.tsx +++ /dev/null @@ -1,12 +0,0 @@ -import { PropsWithChildren } from 'react'; - -export const dynamic = 'force-dynamic'; - -export default function Layout({ children }: PropsWithChildren<{}>) { - return ( -
-

Layout

- {children} -
- ); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/(nested-layout)/nested-layout/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/(nested-layout)/nested-layout/page.tsx deleted file mode 100644 index 8077c14d23ca..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/(nested-layout)/nested-layout/page.tsx +++ /dev/null @@ -1,11 +0,0 @@ -export const dynamic = 'force-dynamic'; - -export default function Page() { - return

Hello World!

; -} - -export async function generateMetadata() { - return { - title: 'I am generated metadata', - }; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/endpoint-behind-middleware/route.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/endpoint-behind-middleware/route.ts deleted file mode 100644 index 2733cc918f44..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/endpoint-behind-middleware/route.ts +++ /dev/null @@ -1,3 +0,0 @@ -export function GET() { - return Response.json({ name: 'John Doe' }); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/test-success/route.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/test-success/route.ts deleted file mode 100644 index 26d85db4ac28..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/test-success/route.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { NextResponse } from 'next/server'; - -export const dynamic = 'force-dynamic'; - -export async function GET() { - return NextResponse.json({ success: true }); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/favicon.ico b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/favicon.ico deleted file mode 100644 index 718d6fea4835ec2d246af9800eddb7ffb276240c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 25931 zcmeHv30#a{`}aL_*G&7qml|y<+KVaDM2m#dVr!KsA!#An?kSQM(q<_dDNCpjEux83 zLb9Z^XxbDl(w>%i@8hT6>)&Gu{h#Oeyszu?xtw#Zb1mO{pgX9699l+Qppw7jXaYf~-84xW z)w4x8?=youko|}Vr~(D$UXIbiXABHh`p1?nn8Po~fxRJv}|0e(BPs|G`(TT%kKVJAdg5*Z|x0leQq0 zkdUBvb#>9F()jo|T~kx@OM8$9wzs~t2l;K=woNssA3l6|sx2r3+kdfVW@e^8e*E}v zA1y5{bRi+3Z`uD3{F7LgFJDdvm;nJilkzDku>BwXH(8ItVCXk*-lSJnR?-2UN%hJ){&rlvg`CDTj z)Bzo!3v7Ou#83zEDEFcKt(f1E0~=rqeEbTnMvWR#{+9pg%7G8y>u1OVRUSoox-ovF z2Ydma(;=YuBY(eI|04{hXzZD6_f(v~H;C~y5=DhAC{MMS>2fm~1H_t2$56pc$NH8( z5bH|<)71dV-_oCHIrzrT`2s-5w_+2CM0$95I6X8p^r!gHp+j_gd;9O<1~CEQQGS8) zS9Qh3#p&JM-G8rHekNmKVewU;pJRcTAog68KYo^dRo}(M>36U4Us zfgYWSiHZL3;lpWT=zNAW>Dh#mB!_@Lg%$ms8N-;aPqMn+C2HqZgz&9~Eu z4|Kp<`$q)Uw1R?y(~S>ePdonHxpV1#eSP1B;Ogo+-Pk}6#0GsZZ5!||ev2MGdh}_m z{DeR7?0-1^zVs&`AV6Vt;r3`I`OI_wgs*w=eO%_#7Kepl{B@xiyCANc(l zzIyd4y|c6PXWq9-|KM8(zIk8LPk(>a)zyFWjhT!$HJ$qX1vo@d25W<fvZQ2zUz5WRc(UnFMKHwe1| zWmlB1qdbiA(C0jmnV<}GfbKtmcu^2*P^O?MBLZKt|As~ge8&AAO~2K@zbXelK|4T<{|y4`raF{=72kC2Kn(L4YyenWgrPiv z@^mr$t{#X5VuIMeL!7Ab6_kG$&#&5p*Z{+?5U|TZ`B!7llpVmp@skYz&n^8QfPJzL z0G6K_OJM9x+Wu2gfN45phANGt{7=C>i34CV{Xqlx(fWpeAoj^N0Biu`w+MVcCUyU* zDZuzO0>4Z6fbu^T_arWW5n!E45vX8N=bxTVeFoep_G#VmNlQzAI_KTIc{6>c+04vr zx@W}zE5JNSU>!THJ{J=cqjz+4{L4A{Ob9$ZJ*S1?Ggg3klFp!+Y1@K+pK1DqI|_gq z5ZDXVpge8-cs!o|;K73#YXZ3AShj50wBvuq3NTOZ`M&qtjj#GOFfgExjg8Gn8>Vq5 z`85n+9|!iLCZF5$HJ$Iu($dm?8~-ofu}tEc+-pyke=3!im#6pk_Wo8IA|fJwD&~~F zc16osQ)EBo58U7XDuMexaPRjU@h8tXe%S{fA0NH3vGJFhuyyO!Uyl2^&EOpX{9As0 zWj+P>{@}jxH)8|r;2HdupP!vie{sJ28b&bo!8`D^x}TE$%zXNb^X1p@0PJ86`dZyj z%ce7*{^oo+6%&~I!8hQy-vQ7E)0t0ybH4l%KltWOo~8cO`T=157JqL(oq_rC%ea&4 z2NcTJe-HgFjNg-gZ$6!Y`SMHrlj}Etf7?r!zQTPPSv}{so2e>Fjs1{gzk~LGeesX%r(Lh6rbhSo_n)@@G-FTQy93;l#E)hgP@d_SGvyCp0~o(Y;Ee8{ zdVUDbHm5`2taPUOY^MAGOw*>=s7=Gst=D+p+2yON!0%Hk` zz5mAhyT4lS*T3LS^WSxUy86q&GnoHxzQ6vm8)VS}_zuqG?+3td68_x;etQAdu@sc6 zQJ&5|4(I?~3d-QOAODHpZ=hlSg(lBZ!JZWCtHHSj`0Wh93-Uk)_S%zsJ~aD>{`A0~ z9{AG(e|q3g5B%wYKRxiL2Y$8(4w6bzchKuloQW#e&S3n+P- z8!ds-%f;TJ1>)v)##>gd{PdS2Oc3VaR`fr=`O8QIO(6(N!A?pr5C#6fc~Ge@N%Vvu zaoAX2&(a6eWy_q&UwOhU)|P3J0Qc%OdhzW=F4D|pt0E4osw;%<%Dn58hAWD^XnZD= z>9~H(3bmLtxpF?a7su6J7M*x1By7YSUbxGi)Ot0P77`}P3{)&5Un{KD?`-e?r21!4vTTnN(4Y6Lin?UkSM z`MXCTC1@4A4~mvz%Rh2&EwY))LeoT=*`tMoqcEXI>TZU9WTP#l?uFv+@Dn~b(>xh2 z;>B?;Tz2SR&KVb>vGiBSB`@U7VIWFSo=LDSb9F{GF^DbmWAfpms8Sx9OX4CnBJca3 zlj9(x!dIjN?OG1X4l*imJNvRCk}F%!?SOfiOq5y^mZW)jFL@a|r-@d#f7 z2gmU8L3IZq0ynIws=}~m^#@&C%J6QFo~Mo4V`>v7MI-_!EBMMtb%_M&kvAaN)@ZVw z+`toz&WG#HkWDjnZE!6nk{e-oFdL^$YnbOCN}JC&{$#$O27@|Tn-skXr)2ml2~O!5 zX+gYoxhoc7qoU?C^3~&!U?kRFtnSEecWuH0B0OvLodgUAi}8p1 zrO6RSXHH}DMc$&|?D004DiOVMHV8kXCP@7NKB zgaZq^^O<7PoKEp72kby@W0Z!Y*Ay{&vfg#C&gG@YVR9g?FEocMUi1gSN$+V+ayF45{a zuDZDTN}mS|;BO%gEf}pjBfN2-gIrU#G5~cucA;dokXW89%>AyXJJI z9X4UlIWA|ZYHgbI z5?oFk@A=Ik7lrEQPDH!H+b`7_Y~aDb_qa=B2^Y&Ow41cU=4WDd40dp5(QS-WMN-=Y z9g;6_-JdNU;|6cPwf$ak*aJIcwL@1n$#l~zi{c{EW?T;DaW*E8DYq?Umtz{nJ&w-M zEMyTDrC&9K$d|kZe2#ws6)L=7K+{ zQw{XnV6UC$6-rW0emqm8wJoeZK)wJIcV?dST}Z;G0Arq{dVDu0&4kd%N!3F1*;*pW zR&qUiFzK=@44#QGw7k1`3t_d8&*kBV->O##t|tonFc2YWrL7_eqg+=+k;!F-`^b8> z#KWCE8%u4k@EprxqiV$VmmtiWxDLgnGu$Vs<8rppV5EajBXL4nyyZM$SWVm!wnCj-B!Wjqj5-5dNXukI2$$|Bu3Lrw}z65Lc=1G z^-#WuQOj$hwNGG?*CM_TO8Bg-1+qc>J7k5c51U8g?ZU5n?HYor;~JIjoWH-G>AoUP ztrWWLbRNqIjW#RT*WqZgPJXU7C)VaW5}MiijYbABmzoru6EmQ*N8cVK7a3|aOB#O& zBl8JY2WKfmj;h#Q!pN%9o@VNLv{OUL?rixHwOZuvX7{IJ{(EdPpuVFoQqIOa7giLVkBOKL@^smUA!tZ1CKRK}#SSM)iQHk)*R~?M!qkCruaS!#oIL1c z?J;U~&FfH#*98^G?i}pA{ z9Jg36t4=%6mhY(quYq*vSxptes9qy|7xSlH?G=S@>u>Ebe;|LVhs~@+06N<4CViBk zUiY$thvX;>Tby6z9Y1edAMQaiH zm^r3v#$Q#2T=X>bsY#D%s!bhs^M9PMAcHbCc0FMHV{u-dwlL;a1eJ63v5U*?Q_8JO zT#50!RD619#j_Uf))0ooADz~*9&lN!bBDRUgE>Vud-i5ck%vT=r^yD*^?Mp@Q^v+V zG#-?gKlr}Eeqifb{|So?HM&g91P8|av8hQoCmQXkd?7wIJwb z_^v8bbg`SAn{I*4bH$u(RZ6*xUhuA~hc=8czK8SHEKTzSxgbwi~9(OqJB&gwb^l4+m`k*Q;_?>Y-APi1{k zAHQ)P)G)f|AyjSgcCFps)Fh6Bca*Xznq36!pV6Az&m{O8$wGFD? zY&O*3*J0;_EqM#jh6^gMQKpXV?#1?>$ml1xvh8nSN>-?H=V;nJIwB07YX$e6vLxH( zqYwQ>qxwR(i4f)DLd)-$P>T-no_c!LsN@)8`e;W@)-Hj0>nJ-}Kla4-ZdPJzI&Mce zv)V_j;(3ERN3_@I$N<^|4Lf`B;8n+bX@bHbcZTopEmDI*Jfl)-pFDvo6svPRoo@(x z);_{lY<;);XzT`dBFpRmGrr}z5u1=pC^S-{ce6iXQlLGcItwJ^mZx{m$&DA_oEZ)B{_bYPq-HA zcH8WGoBG(aBU_j)vEy+_71T34@4dmSg!|M8Vf92Zj6WH7Q7t#OHQqWgFE3ARt+%!T z?oLovLVlnf?2c7pTc)~cc^($_8nyKwsN`RA-23ed3sdj(ys%pjjM+9JrctL;dy8a( z@en&CQmnV(()bu|Y%G1-4a(6x{aLytn$T-;(&{QIJB9vMox11U-1HpD@d(QkaJdEb zG{)+6Dos_L+O3NpWo^=gR?evp|CqEG?L&Ut#D*KLaRFOgOEK(Kq1@!EGcTfo+%A&I z=dLbB+d$u{sh?u)xP{PF8L%;YPPW53+@{>5W=Jt#wQpN;0_HYdw1{ksf_XhO4#2F= zyPx6Lx2<92L-;L5PD`zn6zwIH`Jk($?Qw({erA$^bC;q33hv!d!>%wRhj# zal^hk+WGNg;rJtb-EB(?czvOM=H7dl=vblBwAv>}%1@{}mnpUznfq1cE^sgsL0*4I zJ##!*B?=vI_OEVis5o+_IwMIRrpQyT_Sq~ZU%oY7c5JMIADzpD!Upz9h@iWg_>>~j zOLS;wp^i$-E?4<_cp?RiS%Rd?i;f*mOz=~(&3lo<=@(nR!_Rqiprh@weZlL!t#NCc zO!QTcInq|%#>OVgobj{~ixEUec`E25zJ~*DofsQdzIa@5^nOXj2T;8O`l--(QyU^$t?TGY^7#&FQ+2SS3B#qK*k3`ye?8jUYSajE5iBbJls75CCc(m3dk{t?- zopcER9{Z?TC)mk~gpi^kbbu>b-+a{m#8-y2^p$ka4n60w;Sc2}HMf<8JUvhCL0B&Btk)T`ctE$*qNW8L$`7!r^9T+>=<=2qaq-;ll2{`{Rg zc5a0ZUI$oG&j-qVOuKa=*v4aY#IsoM+1|c4Z)<}lEDvy;5huB@1RJPquU2U*U-;gu z=En2m+qjBzR#DEJDO`WU)hdd{Vj%^0V*KoyZ|5lzV87&g_j~NCjwv0uQVqXOb*QrQ zy|Qn`hxx(58c70$E;L(X0uZZ72M1!6oeg)(cdKO ze0gDaTz+ohR-#d)NbAH4x{I(21yjwvBQfmpLu$)|m{XolbgF!pmsqJ#D}(ylp6uC> z{bqtcI#hT#HW=wl7>p!38sKsJ`r8}lt-q%Keqy%u(xk=yiIJiUw6|5IvkS+#?JTBl z8H5(Q?l#wzazujH!8o>1xtn8#_w+397*_cy8!pQGP%K(Ga3pAjsaTbbXJlQF_+m+-UpUUent@xM zg%jqLUExj~o^vQ3Gl*>wh=_gOr2*|U64_iXb+-111aH}$TjeajM+I20xw(((>fej-@CIz4S1pi$(#}P7`4({6QS2CaQS4NPENDp>sAqD z$bH4KGzXGffkJ7R>V>)>tC)uax{UsN*dbeNC*v}#8Y#OWYwL4t$ePR?VTyIs!wea+ z5Urmc)X|^`MG~*dS6pGSbU+gPJoq*^a=_>$n4|P^w$sMBBy@f*Z^Jg6?n5?oId6f{ z$LW4M|4m502z0t7g<#Bx%X;9<=)smFolV&(V^(7Cv2-sxbxopQ!)*#ZRhTBpx1)Fc zNm1T%bONzv6@#|dz(w02AH8OXe>kQ#1FMCzO}2J_mST)+ExmBr9cva-@?;wnmWMOk z{3_~EX_xadgJGv&H@zK_8{(x84`}+c?oSBX*Ge3VdfTt&F}yCpFP?CpW+BE^cWY0^ zb&uBN!Ja3UzYHK-CTyA5=L zEMW{l3Usky#ly=7px648W31UNV@K)&Ub&zP1c7%)`{);I4b0Q<)B}3;NMG2JH=X$U zfIW4)4n9ZM`-yRj67I)YSLDK)qfUJ_ij}a#aZN~9EXrh8eZY2&=uY%2N0UFF7<~%M zsB8=erOWZ>Ct_#^tHZ|*q`H;A)5;ycw*IcmVxi8_0Xk}aJA^ath+E;xg!x+As(M#0=)3!NJR6H&9+zd#iP(m0PIW8$ z1Y^VX`>jm`W!=WpF*{ioM?C9`yOR>@0q=u7o>BP-eSHqCgMDj!2anwH?s%i2p+Q7D zzszIf5XJpE)IG4;d_(La-xenmF(tgAxK`Y4sQ}BSJEPs6N_U2vI{8=0C_F?@7<(G; zo$~G=8p+076G;`}>{MQ>t>7cm=zGtfbdDXm6||jUU|?X?CaE?(<6bKDYKeHlz}DA8 zXT={X=yp_R;HfJ9h%?eWvQ!dRgz&Su*JfNt!Wu>|XfU&68iRikRrHRW|ZxzRR^`eIGt zIeiDgVS>IeExKVRWW8-=A=yA`}`)ZkWBrZD`hpWIxBGkh&f#ijr449~m`j6{4jiJ*C!oVA8ZC?$1RM#K(_b zL9TW)kN*Y4%^-qPpMP7d4)o?Nk#>aoYHT(*g)qmRUb?**F@pnNiy6Fv9rEiUqD(^O zzyS?nBrX63BTRYduaG(0VVG2yJRe%o&rVrLjbxTaAFTd8s;<<@Qs>u(<193R8>}2_ zuwp{7;H2a*X7_jryzriZXMg?bTuegABb^87@SsKkr2)0Gyiax8KQWstw^v#ix45EVrcEhr>!NMhprl$InQMzjSFH54x5k9qHc`@9uKQzvL4ihcq{^B zPrVR=o_ic%Y>6&rMN)hTZsI7I<3&`#(nl+3y3ys9A~&^=4?PL&nd8)`OfG#n zwAMN$1&>K++c{^|7<4P=2y(B{jJsQ0a#U;HTo4ZmWZYvI{+s;Td{Yzem%0*k#)vjpB zia;J&>}ICate44SFYY3vEelqStQWFihx%^vQ@Do(sOy7yR2@WNv7Y9I^yL=nZr3mb zXKV5t@=?-Sk|b{XMhA7ZGB@2hqsx}4xwCW!in#C zI@}scZlr3-NFJ@NFaJlhyfcw{k^vvtGl`N9xSo**rDW4S}i zM9{fMPWo%4wYDG~BZ18BD+}h|GQKc-g^{++3MY>}W_uq7jGHx{mwE9fZiPCoxN$+7 zrODGGJrOkcPQUB(FD5aoS4g~7#6NR^ma7-!>mHuJfY5kTe6PpNNKC9GGRiu^L31uG z$7v`*JknQHsYB!Tm_W{a32TM099djW%5e+j0Ve_ct}IM>XLF1Ap+YvcrLV=|CKo6S zb+9Nl3_YdKP6%Cxy@6TxZ>;4&nTneadr z_ES90ydCev)LV!dN=#(*f}|ZORFdvkYBni^aLbUk>BajeWIOcmHP#8S)*2U~QKI%S zyrLmtPqb&TphJ;>yAxri#;{uyk`JJqODDw%(Z=2`1uc}br^V%>j!gS)D*q*f_-qf8&D;W1dJgQMlaH5er zN2U<%Smb7==vE}dDI8K7cKz!vs^73o9f>2sgiTzWcwY|BMYHH5%Vn7#kiw&eItCqa zIkR2~Q}>X=Ar8W|^Ms41Fm8o6IB2_j60eOeBB1Br!boW7JnoeX6Gs)?7rW0^5psc- zjS16yb>dFn>KPOF;imD}e!enuIniFzv}n$m2#gCCv4jM#ArwlzZ$7@9&XkFxZ4n!V zj3dyiwW4Ki2QG{@i>yuZXQizw_OkZI^-3otXC{!(lUpJF33gI60ak;Uqitp74|B6I zgg{b=Iz}WkhCGj1M=hu4#Aw173YxIVbISaoc z-nLZC*6Tgivd5V`K%GxhBsp@SUU60-rfc$=wb>zdJzXS&-5(NRRodFk;Kxk!S(O(a0e7oY=E( zAyS;Ow?6Q&XA+cnkCb{28_1N8H#?J!*$MmIwLq^*T_9-z^&UE@A(z9oGYtFy6EZef LrJugUA?W`A8`#=m diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/global-error.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/global-error.tsx deleted file mode 100644 index 20c175015b03..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/global-error.tsx +++ /dev/null @@ -1,23 +0,0 @@ -'use client'; - -import * as Sentry from '@sentry/nextjs'; -import NextError from 'next/error'; -import { useEffect } from 'react'; - -export default function GlobalError({ error }: { error: Error & { digest?: string } }) { - useEffect(() => { - Sentry.captureException(error); - }, [error]); - - return ( - - - {/* `NextError` is the default Next.js error page component. Its type - definition requires a `statusCode` prop. However, since the App Router - does not expose status codes for errors, we simply pass 0 to render a - generic error message. */} - - - - ); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/isr-test/[product]/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/isr-test/[product]/page.tsx deleted file mode 100644 index cd1e085e2763..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/isr-test/[product]/page.tsx +++ /dev/null @@ -1,17 +0,0 @@ -export const revalidate = 60; // ISR: revalidate every 60 seconds -export const dynamicParams = true; // Allow dynamic params beyond generateStaticParams - -export async function generateStaticParams(): Promise> { - return [{ product: 'laptop' }, { product: 'phone' }, { product: 'tablet' }]; -} - -export default async function ISRProductPage({ params }: { params: Promise<{ product: string }> }) { - const { product } = await params; - - return ( -
-

ISR Product: {product}

-
{product}
-
- ); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/isr-test/static/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/isr-test/static/page.tsx deleted file mode 100644 index f49605bd9da4..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/isr-test/static/page.tsx +++ /dev/null @@ -1,15 +0,0 @@ -export const revalidate = 60; // ISR: revalidate every 60 seconds -export const dynamicParams = true; - -export async function generateStaticParams(): Promise { - return []; -} - -export default function ISRStaticPage() { - return ( -
-

ISR Static Page

-
static-isr
-
- ); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/layout.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/layout.tsx deleted file mode 100644 index c8f9cee0b787..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/layout.tsx +++ /dev/null @@ -1,7 +0,0 @@ -export default function Layout({ children }: { children: React.ReactNode }) { - return ( - - {children} - - ); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/metrics/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/metrics/page.tsx deleted file mode 100644 index fdb7bc0a40a7..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/metrics/page.tsx +++ /dev/null @@ -1,34 +0,0 @@ -'use client'; - -import * as Sentry from '@sentry/nextjs'; - -export default function Page() { - const handleClick = async () => { - Sentry.metrics.count('test.page.count', 1, { - attributes: { - page: '/metrics', - 'random.attribute': 'Apples', - }, - }); - Sentry.metrics.distribution('test.page.distribution', 100, { - attributes: { - page: '/metrics', - 'random.attribute': 'Manzanas', - }, - }); - Sentry.metrics.gauge('test.page.gauge', 200, { - attributes: { - page: '/metrics', - 'random.attribute': 'Mele', - }, - }); - await fetch('/metrics/route-handler'); - }; - - return ( -
-

Metrics page

- -
- ); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/metrics/route-handler/route.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/metrics/route-handler/route.ts deleted file mode 100644 index 84e81960f9c9..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/metrics/route-handler/route.ts +++ /dev/null @@ -1,23 +0,0 @@ -import * as Sentry from '@sentry/nextjs'; - -export const GET = async () => { - Sentry.metrics.count('test.route.handler.count', 1, { - attributes: { - endpoint: '/metrics/route-handler', - 'random.attribute': 'Potatoes', - }, - }); - Sentry.metrics.distribution('test.route.handler.distribution', 100, { - attributes: { - endpoint: '/metrics/route-handler', - 'random.attribute': 'Patatas', - }, - }); - Sentry.metrics.gauge('test.route.handler.gauge', 200, { - attributes: { - endpoint: '/metrics/route-handler', - 'random.attribute': 'Patate', - }, - }); - return Response.json({ message: 'Bueno' }); -}; diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/nested-rsc-error/[param]/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/nested-rsc-error/[param]/page.tsx deleted file mode 100644 index 675b248026be..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/nested-rsc-error/[param]/page.tsx +++ /dev/null @@ -1,17 +0,0 @@ -import { Suspense } from 'react'; - -export const dynamic = 'force-dynamic'; - -export default async function Page() { - return ( - Loading...

}> - {/* @ts-ignore */} - ; - - ); -} - -async function Crash() { - throw new Error('I am technically uncatchable'); - return

unreachable

; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/non-isr-test/[item]/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/non-isr-test/[item]/page.tsx deleted file mode 100644 index e0bafdb24181..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/non-isr-test/[item]/page.tsx +++ /dev/null @@ -1,11 +0,0 @@ -// No generateStaticParams - this is NOT an ISR page -export default async function NonISRPage({ params }: { params: Promise<{ item: string }> }) { - const { item } = await params; - - return ( -
-

Non-ISR Dynamic Page: {item}

-
{item}
-
- ); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/page.tsx deleted file mode 100644 index 2bc0a407a355..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/page.tsx +++ /dev/null @@ -1,3 +0,0 @@ -export default function Page() { - return

Next 16 test app

; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/pageload-tracing/layout.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/pageload-tracing/layout.tsx deleted file mode 100644 index 1f0cbe478f88..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/pageload-tracing/layout.tsx +++ /dev/null @@ -1,8 +0,0 @@ -import { PropsWithChildren } from 'react'; - -export const dynamic = 'force-dynamic'; - -export default async function Layout({ children }: PropsWithChildren) { - await new Promise(resolve => setTimeout(resolve, 500)); - return <>{children}; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/pageload-tracing/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/pageload-tracing/page.tsx deleted file mode 100644 index 689735d61ddf..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/pageload-tracing/page.tsx +++ /dev/null @@ -1,14 +0,0 @@ -export const dynamic = 'force-dynamic'; - -export default async function Page() { - await new Promise(resolve => setTimeout(resolve, 1000)); - return

I am page 2

; -} - -export async function generateMetadata() { - (await fetch('https://example.com/', { cache: 'no-store' })).text(); - - return { - title: 'my title', - }; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/parameterized/[one]/beep/[two]/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/parameterized/[one]/beep/[two]/page.tsx deleted file mode 100644 index f34461c2bb07..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/parameterized/[one]/beep/[two]/page.tsx +++ /dev/null @@ -1,3 +0,0 @@ -export default function ParameterizedPage() { - return
Dynamic page two
; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/parameterized/[one]/beep/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/parameterized/[one]/beep/page.tsx deleted file mode 100644 index a7d9164c8c03..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/parameterized/[one]/beep/page.tsx +++ /dev/null @@ -1,3 +0,0 @@ -export default function BeepPage() { - return
Beep
; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/parameterized/[one]/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/parameterized/[one]/page.tsx deleted file mode 100644 index 9fa617a22381..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/parameterized/[one]/page.tsx +++ /dev/null @@ -1,3 +0,0 @@ -export default function ParameterizedPage() { - return
Dynamic page one
; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/parameterized/static/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/parameterized/static/page.tsx deleted file mode 100644 index 16ef0482d53b..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/parameterized/static/page.tsx +++ /dev/null @@ -1,3 +0,0 @@ -export default function StaticPage() { - return
Static page
; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/prefetching/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/prefetching/page.tsx deleted file mode 100644 index 4cb811ecf1b4..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/prefetching/page.tsx +++ /dev/null @@ -1,9 +0,0 @@ -import Link from 'next/link'; - -export default function Page() { - return ( - - link - - ); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/prefetching/to-be-prefetched/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/prefetching/to-be-prefetched/page.tsx deleted file mode 100644 index 83aac90d65cf..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/prefetching/to-be-prefetched/page.tsx +++ /dev/null @@ -1,5 +0,0 @@ -export const dynamic = 'force-dynamic'; - -export default function Page() { - return

Hello

; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/redirect/destination/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/redirect/destination/page.tsx deleted file mode 100644 index 5583d36b04b0..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/redirect/destination/page.tsx +++ /dev/null @@ -1,7 +0,0 @@ -export default function RedirectDestinationPage() { - return ( -
-

Redirect Destination

-
- ); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/redirect/origin/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/redirect/origin/page.tsx deleted file mode 100644 index 52615e0a054b..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/redirect/origin/page.tsx +++ /dev/null @@ -1,18 +0,0 @@ -import { redirect } from 'next/navigation'; - -async function redirectAction() { - 'use server'; - - redirect('/redirect/destination'); -} - -export default function RedirectOriginPage() { - return ( - <> - {/* @ts-ignore */} -
- -
- - ); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/route-handler/[xoxo]/edge/route.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/route-handler/[xoxo]/edge/route.ts deleted file mode 100644 index 7cd1fc7e332c..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/route-handler/[xoxo]/edge/route.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { NextResponse } from 'next/server'; - -export const runtime = 'edge'; -export const dynamic = 'force-dynamic'; - -export async function GET() { - return NextResponse.json({ message: 'Hello Edge Route Handler' }); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/route-handler/[xoxo]/node/route.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/route-handler/[xoxo]/node/route.ts deleted file mode 100644 index 5bc418f077aa..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/route-handler/[xoxo]/node/route.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { NextResponse } from 'next/server'; - -export const dynamic = 'force-dynamic'; - -export async function GET() { - return NextResponse.json({ message: 'Hello Node Route Handler' }); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/streaming-rsc-error/[param]/client-page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/streaming-rsc-error/[param]/client-page.tsx deleted file mode 100644 index 7b66c3fbdeef..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/streaming-rsc-error/[param]/client-page.tsx +++ /dev/null @@ -1,8 +0,0 @@ -'use client'; - -import { use } from 'react'; - -export function RenderPromise({ stringPromise }: { stringPromise: Promise }) { - const s = use(stringPromise); - return <>{s}; -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/streaming-rsc-error/[param]/page.tsx b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/streaming-rsc-error/[param]/page.tsx deleted file mode 100644 index 9531f9a42139..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/streaming-rsc-error/[param]/page.tsx +++ /dev/null @@ -1,18 +0,0 @@ -import { Suspense } from 'react'; -import { RenderPromise } from './client-page'; - -export const dynamic = 'force-dynamic'; - -export default async function Page() { - const crashingPromise = new Promise((_, reject) => { - setTimeout(() => { - reject(new Error('I am a data streaming error')); - }, 100); - }); - - return ( - Loading...

}> - ; - - ); -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/eslint.config.mjs b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/eslint.config.mjs deleted file mode 100644 index 60f7af38f6c2..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/eslint.config.mjs +++ /dev/null @@ -1,19 +0,0 @@ -import { dirname } from 'path'; -import { fileURLToPath } from 'url'; -import { FlatCompat } from '@eslint/eslintrc'; - -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); - -const compat = new FlatCompat({ - baseDirectory: __dirname, -}); - -const eslintConfig = [ - ...compat.extends('next/core-web-vitals', 'next/typescript'), - { - ignores: ['node_modules/**', '.next/**', 'out/**', 'build/**', 'next-env.d.ts'], - }, -]; - -export default eslintConfig; diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/instrumentation-client.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/instrumentation-client.ts deleted file mode 100644 index a170a388c465..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/instrumentation-client.ts +++ /dev/null @@ -1,15 +0,0 @@ -import * as Sentry from '@sentry/nextjs'; -import type { Log } from '@sentry/nextjs'; - -Sentry.init({ - environment: 'qa', // dynamic sampling bias to keep transactions - dsn: process.env.NEXT_PUBLIC_E2E_TEST_DSN, - tunnel: `http://localhost:3031/`, // proxy server - tracesSampleRate: 1.0, - // Verify Log type is available - beforeSendLog(log: Log) { - return log; - }, -}); - -export const onRouterTransitionStart = Sentry.captureRouterTransitionStart; diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/instrumentation.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/instrumentation.ts deleted file mode 100644 index 964f937c439a..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/instrumentation.ts +++ /dev/null @@ -1,13 +0,0 @@ -import * as Sentry from '@sentry/nextjs'; - -export async function register() { - if (process.env.NEXT_RUNTIME === 'nodejs') { - await import('./sentry.server.config'); - } - - if (process.env.NEXT_RUNTIME === 'edge') { - await import('./sentry.edge.config'); - } -} - -export const onRequestError = Sentry.captureRequestError; diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/middleware.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/middleware.ts deleted file mode 100644 index a6d31cc8c66f..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/middleware.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { getDefaultIsolationScope } from '@sentry/core'; -import * as Sentry from '@sentry/nextjs'; -import { NextResponse } from 'next/server'; -import type { NextRequest } from 'next/server'; - -export async function middleware(request: NextRequest) { - Sentry.setTag('my-isolated-tag', true); - Sentry.setTag('my-global-scope-isolated-tag', getDefaultIsolationScope().getScopeData().tags['my-isolated-tag']); // We set this tag to be able to assert that the previously set tag has not leaked into the global isolation scope - - // Streamed spans carry no scope tags, so the tests read the isolation state from this attribute instead - const activeSpan = Sentry.getActiveSpan(); - if (activeSpan) { - Sentry.getRootSpan(activeSpan).setAttribute( - 'isolation_scope.is_default', - Sentry.getIsolationScope() === getDefaultIsolationScope(), - ); - } - - if (request.headers.has('x-should-throw')) { - throw new Error('Middleware Error'); - } - - if (request.headers.has('x-should-make-request')) { - await fetch('http://localhost:3030/'); - } - - return NextResponse.next(); -} - -// See "Matching Paths" below to learn more -export const config = { - matcher: ['/api/endpoint-behind-middleware', '/api/endpoint-behind-faulty-middleware'], -}; - -export const runtime = 'experimental-edge'; diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/next.config.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/next.config.ts deleted file mode 100644 index 28162396e80e..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/next.config.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { withSentryConfig } from '@sentry/nextjs/config'; -import type { NextConfig } from 'next'; - -const nextConfig: NextConfig = {}; - -export default withSentryConfig(nextConfig, { - silent: true, -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/package.json b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/package.json deleted file mode 100644 index f0b40e15fb07..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/package.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "name": "nextjs-16-cf-workers", - "version": "0.1.0", - "private": true, - "scripts": { - "dev": "next dev", - "cf:build": "opennextjs-cloudflare build", - "cf:preview": "opennextjs-cloudflare preview --var \"E2E_TEST_DSN:$E2E_TEST_DSN\"", - "build": "next build", - "clean": "npx rimraf node_modules pnpm-lock.yaml .tmp_dev_server_logs", - "start": "pnpm cf:preview", - "lint": "eslint", - "test:prod": "TEST_ENV=production playwright test", - "test:build": "pnpm install && pnpm cf:build", - "test:build-canary": "pnpm install && pnpm add next@canary && pnpm cf:build", - "test:build-latest": "pnpm install && pnpm add next@16.3 && pnpm cf:build", - "test:assert": "pnpm test:prod" - }, - "dependencies": { - "@opentelemetry/api": "^1.9.1", - "@opennextjs/cloudflare": "^1.14.9", - "@sentry/nextjs": "file:../../packed/sentry-nextjs-packed.tgz", - "@sentry/core": "file:../../packed/sentry-core-packed.tgz", - "next": "16.2.11", - "react": "19.1.0", - "react-dom": "19.1.0" - }, - "devDependencies": { - "@playwright/test": "~1.63.0", - "@sentry-internal/test-utils": "link:../../../test-utils", - "@types/node": "^20", - "@types/react": "^19", - "@types/react-dom": "^19", - "eslint": "^9", - "eslint-config-next": "^16", - "typescript": "^5", - "wrangler": "^4.61.0" - }, - "volta": { - "node": "24.15.0", - "extends": "../../package.json" - }, - "sentryTest": { - "variants": [ - { - "build-command": "pnpm test:build-latest", - "label": "nextjs-16-cf-workers (latest)" - } - ] - } -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/playwright.config.mjs b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/playwright.config.mjs deleted file mode 100644 index 0f15639161dd..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/playwright.config.mjs +++ /dev/null @@ -1,21 +0,0 @@ -import { getPlaywrightConfig } from '@sentry-internal/test-utils'; -const testEnv = process.env.TEST_ENV; - -if (!testEnv) { - throw new Error('No test env defined'); -} - -const getStartCommand = () => { - if (testEnv === 'production') { - return 'pnpm cf:preview --port 3030'; - } - - throw new Error(`Unknown test env: ${testEnv}`); -}; - -const config = getPlaywrightConfig({ - startCommand: getStartCommand(), - port: 3030, -}); - -export default config; diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/public/file.svg b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/public/file.svg deleted file mode 100644 index 004145cddf3f..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/public/file.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/public/globe.svg b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/public/globe.svg deleted file mode 100644 index 567f17b0d7c7..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/public/globe.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/public/next.svg b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/public/next.svg deleted file mode 100644 index 5174b28c565c..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/public/next.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/public/vercel.svg b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/public/vercel.svg deleted file mode 100644 index 77053960334e..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/public/vercel.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/public/window.svg b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/public/window.svg deleted file mode 100644 index b2b2a44f6ebc..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/public/window.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/sentry.edge.config.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/sentry.edge.config.ts deleted file mode 100644 index eceebb29d45d..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/sentry.edge.config.ts +++ /dev/null @@ -1,7 +0,0 @@ -import * as Sentry from '@sentry/nextjs'; - -// Only hands over the build release: `withSentry` from `@sentry/nextjs/cloudflare` in worker.ts creates the client and -// holds the options. -Sentry.init({ - dsn: process.env.NEXT_PUBLIC_E2E_TEST_DSN, -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/sentry.server.config.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/sentry.server.config.ts deleted file mode 100644 index 1009ee9e1f39..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/sentry.server.config.ts +++ /dev/null @@ -1,7 +0,0 @@ -import * as Sentry from '@sentry/nextjs'; - -// Only hands over the build release and adds the global Next.js event processors and the turbopack tag: `withSentry` -// from `@sentry/nextjs/cloudflare` in worker.ts creates the client and holds the options. -Sentry.init({ - dsn: process.env.NEXT_PUBLIC_E2E_TEST_DSN, -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/start-event-proxy.mjs b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/start-event-proxy.mjs deleted file mode 100644 index efb664370443..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/start-event-proxy.mjs +++ /dev/null @@ -1,14 +0,0 @@ -import * as fs from 'fs'; -import * as path from 'path'; -import { startEventProxyServer } from '@sentry-internal/test-utils'; - -const packageJson = JSON.parse(fs.readFileSync(path.join(process.cwd(), 'package.json'))); - -startEventProxyServer({ - port: 3031, - proxyServerName: 'nextjs-16-cf-workers', - envelopeDumpPath: path.join( - process.cwd(), - `event-dumps/next-16-v${packageJson.dependencies.next}-${process.env.TEST_ENV}.dump`, - ), -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/async-params.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/async-params.test.ts deleted file mode 100644 index e8160d12aded..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/async-params.test.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { expect, test } from '@playwright/test'; -import fs from 'fs'; -import { isDevMode } from './isDevMode'; - -test('should not print warning for async params', async ({ page }) => { - test.skip(!isDevMode, 'should be skipped for non-dev mode'); - await page.goto('/'); - - // If the server exits with code 1, the test will fail (see instrumentation.ts) - const devStdout = fs.readFileSync('.tmp_dev_server_logs', 'utf-8'); - expect(devStdout).not.toContain('`params` should be awaited before using its properties.'); - - await expect(page.getByText('Next 16 test app')).toBeVisible(); -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/isDevMode.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/isDevMode.ts deleted file mode 100644 index d2be94232110..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/isDevMode.ts +++ /dev/null @@ -1 +0,0 @@ -export const isDevMode = !!process.env.TEST_ENV && process.env.TEST_ENV.includes('development'); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/isr-routes.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/isr-routes.test.ts deleted file mode 100644 index fe61a6263913..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/isr-routes.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { getSpanOp, waitForStreamedSpan } from '@sentry-internal/test-utils'; - -test('should remove sentry-trace and baggage meta tags on ISR dynamic route page load', async ({ page }) => { - // Navigate to ISR page - await page.goto('/isr-test/laptop'); - - // Wait for page to be fully loaded - await expect(page.locator('#isr-product-id')).toHaveText('laptop'); - - // Check that sentry-trace and baggage meta tags are removed for ISR pages - await expect(page.locator('meta[name="sentry-trace"]')).toHaveCount(0); - await expect(page.locator('meta[name="baggage"]')).toHaveCount(0); -}); - -test('should remove sentry-trace and baggage meta tags on ISR static route', async ({ page }) => { - // Navigate to ISR static page - await page.goto('/isr-test/static'); - - // Wait for page to be fully loaded - await expect(page.locator('#isr-static-marker')).toHaveText('static-isr'); - - // Check that sentry-trace and baggage meta tags are removed for ISR pages - await expect(page.locator('meta[name="sentry-trace"]')).toHaveCount(0); - await expect(page.locator('meta[name="baggage"]')).toHaveCount(0); -}); - -test('should remove meta tags for different ISR dynamic route values', async ({ page }) => { - // Test with 'phone' (one of the pre-generated static params) - await page.goto('/isr-test/phone'); - await expect(page.locator('#isr-product-id')).toHaveText('phone'); - - await expect(page.locator('meta[name="sentry-trace"]')).toHaveCount(0); - await expect(page.locator('meta[name="baggage"]')).toHaveCount(0); - - // Test with 'tablet' - await page.goto('/isr-test/tablet'); - await expect(page.locator('#isr-product-id')).toHaveText('tablet'); - - await expect(page.locator('meta[name="sentry-trace"]')).toHaveCount(0); - await expect(page.locator('meta[name="baggage"]')).toHaveCount(0); -}); - -test('should create unique traces for ISR pages on each visit', async ({ page }) => { - const traceIds: string[] = []; - - // Load the same ISR page 5 times to ensure cached HTML meta tags are consistently removed - for (let i = 0; i < 5; i++) { - const spanPromise = waitForStreamedSpan('nextjs-16-cf-workers', span => { - return span.name === '/isr-test/:product' && getSpanOp(span) === 'pageload' && span.is_segment; - }); - - if (i === 0) { - await page.goto('/isr-test/laptop'); - } else { - await page.reload(); - } - - const span = await spanPromise; - const traceId = span.trace_id; - - expect(traceId).toBeDefined(); - expect(traceId).toMatch(/[a-f0-9]{32}/); - traceIds.push(traceId!); - } - - // Verify all 5 page loads have unique trace IDs (no reuse of cached/stale meta tags) - const uniqueTraceIds = new Set(traceIds); - expect(uniqueTraceIds.size).toBe(5); -}); - -test('ISR route should be identified correctly in the route manifest', async ({ page }) => { - const spanPromise = waitForStreamedSpan('nextjs-16-cf-workers', span => { - return span.name === '/isr-test/:product' && getSpanOp(span) === 'pageload' && span.is_segment; - }); - - await page.goto('/isr-test/laptop'); - const span = await spanPromise; - - // Verify the span is properly parameterized - expect(span.name).toBe('/isr-test/:product'); - expect(span.attributes['sentry.segment.name.source']?.value).toBe('route'); -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/metrics.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/metrics.test.ts deleted file mode 100644 index 1ccbe00953de..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/metrics.test.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { waitForMetric } from '@sentry-internal/test-utils'; - -test('Should emit metrics from server and client', async ({ request, page }) => { - const clientCountPromise = waitForMetric('nextjs-16-cf-workers', async metric => { - return metric.name === 'test.page.count'; - }); - - const clientDistributionPromise = waitForMetric('nextjs-16-cf-workers', async metric => { - return metric.name === 'test.page.distribution'; - }); - - const clientGaugePromise = waitForMetric('nextjs-16-cf-workers', async metric => { - return metric.name === 'test.page.gauge'; - }); - - const serverCountPromise = waitForMetric('nextjs-16-cf-workers', async metric => { - return metric.name === 'test.route.handler.count'; - }); - - const serverDistributionPromise = waitForMetric('nextjs-16-cf-workers', async metric => { - return metric.name === 'test.route.handler.distribution'; - }); - - const serverGaugePromise = waitForMetric('nextjs-16-cf-workers', async metric => { - return metric.name === 'test.route.handler.gauge'; - }); - - await page.goto('/metrics'); - await page.getByText('Emit').click(); - const clientCount = await clientCountPromise; - const clientDistribution = await clientDistributionPromise; - const clientGauge = await clientGaugePromise; - const serverCount = await serverCountPromise; - const serverDistribution = await serverDistributionPromise; - const serverGauge = await serverGaugePromise; - - expect(clientCount).toMatchObject({ - timestamp: expect.any(Number), - trace_id: expect.any(String), - span_id: expect.any(String), - name: 'test.page.count', - type: 'counter', - value: 1, - attributes: { - page: { value: '/metrics', type: 'string' }, - 'random.attribute': { value: 'Apples', type: 'string' }, - 'sentry.environment': { value: 'qa', type: 'string' }, - 'sentry.sdk.name': { value: 'sentry.javascript.nextjs', type: 'string' }, - 'sentry.sdk.version': { value: expect.any(String), type: 'string' }, - }, - }); - - expect(clientDistribution).toMatchObject({ - timestamp: expect.any(Number), - trace_id: expect.any(String), - span_id: expect.any(String), - name: 'test.page.distribution', - type: 'distribution', - value: 100, - attributes: { - page: { value: '/metrics', type: 'string' }, - 'random.attribute': { value: 'Manzanas', type: 'string' }, - 'sentry.environment': { value: 'qa', type: 'string' }, - 'sentry.sdk.name': { value: 'sentry.javascript.nextjs', type: 'string' }, - 'sentry.sdk.version': { value: expect.any(String), type: 'string' }, - }, - }); - - expect(clientGauge).toMatchObject({ - timestamp: expect.any(Number), - trace_id: expect.any(String), - span_id: expect.any(String), - name: 'test.page.gauge', - type: 'gauge', - value: 200, - attributes: { - page: { value: '/metrics', type: 'string' }, - 'random.attribute': { value: 'Mele', type: 'string' }, - 'sentry.environment': { value: 'qa', type: 'string' }, - 'sentry.sdk.name': { value: 'sentry.javascript.nextjs', type: 'string' }, - 'sentry.sdk.version': { value: expect.any(String), type: 'string' }, - }, - }); - - expect(serverCount).toMatchObject({ - timestamp: expect.any(Number), - trace_id: expect.any(String), - name: 'test.route.handler.count', - type: 'counter', - value: 1, - attributes: { - 'random.attribute': { value: 'Potatoes', type: 'string' }, - endpoint: { value: '/metrics/route-handler', type: 'string' }, - 'sentry.environment': { value: 'qa', type: 'string' }, - 'sentry.sdk.name': { value: 'sentry.javascript.nextjs', type: 'string' }, - 'sentry.sdk.version': { value: expect.any(String), type: 'string' }, - }, - }); - - expect(serverDistribution).toMatchObject({ - timestamp: expect.any(Number), - trace_id: expect.any(String), - name: 'test.route.handler.distribution', - type: 'distribution', - value: 100, - attributes: { - 'random.attribute': { value: 'Patatas', type: 'string' }, - endpoint: { value: '/metrics/route-handler', type: 'string' }, - 'sentry.environment': { value: 'qa', type: 'string' }, - 'sentry.sdk.name': { value: 'sentry.javascript.nextjs', type: 'string' }, - 'sentry.sdk.version': { value: expect.any(String), type: 'string' }, - }, - }); - - expect(serverGauge).toMatchObject({ - timestamp: expect.any(Number), - trace_id: expect.any(String), - name: 'test.route.handler.gauge', - type: 'gauge', - value: 200, - attributes: { - 'random.attribute': { value: 'Patate', type: 'string' }, - endpoint: { value: '/metrics/route-handler', type: 'string' }, - 'sentry.environment': { value: 'qa', type: 'string' }, - 'sentry.sdk.name': { value: 'sentry.javascript.nextjs', type: 'string' }, - 'sentry.sdk.version': { value: expect.any(String), type: 'string' }, - }, - }); -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/middleware.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/middleware.test.ts deleted file mode 100644 index 32d797f6d836..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/middleware.test.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { collectStreamedSpans, collectStreamedSpansUntilSegment, getSpanOp } from '@sentry-internal/test-utils'; -import { isDevMode } from './isDevMode'; - -// `withSentry` from `@sentry/nextjs/cloudflare` wraps the Worker entry, so middleware runs inside the request's -// `http.server` segment and its span is a child of it rather than a segment of its own. -test('Should create a span for middleware inside the request segment', async ({ request }) => { - const spansPromise = collectStreamedSpansUntilSegment('nextjs-16-cf-workers', 'GET /api/endpoint-behind-middleware'); - - const response = await request.get('/api/endpoint-behind-middleware'); - expect(await response.json()).toStrictEqual({ name: 'John Doe' }); - - const spans = await spansPromise; - const segmentSpan = spans.find(span => span.is_segment)!; - const middlewareSpan = spans.find(span => span.name === 'middleware GET')!; - - expect(spans.filter(span => getSpanOp(span) === 'middleware')).toEqual([middlewareSpan]); - expect(segmentSpan.attributes['url.path']?.value).toBe('/api/endpoint-behind-middleware'); - expect(segmentSpan.attributes['sentry.segment.name.source']?.value).toBe('route'); - expect(segmentSpan.attributes['isolation_scope.is_default']).toEqual({ value: false, type: 'boolean' }); - expect(middlewareSpan.is_segment).toBe(false); - expect(middlewareSpan.parent_span_id).toBe(segmentSpan.span_id); - expect(middlewareSpan.status).toBe('ok'); - expect(getSpanOp(middlewareSpan)).toBe('middleware'); -}); - -test('Faulty middlewares', async ({ request }) => { - test.skip(isDevMode, 'Throwing crashes the dev server atm'); // https://github.com/vercel/next.js/issues/85261 - // The middleware throws, so the request span of `withSentry` keeps the middleware name, like the middleware segment - // of Node.js. - const spansPromise = collectStreamedSpans( - 'nextjs-16-cf-workers', - spans => - spans.some(span => span.is_segment && span.name === 'middleware GET') && - spans.some(span => !span.is_segment && span.name === 'middleware GET'), - ); - - request.get('/api/endpoint-behind-middleware', { headers: { 'x-should-throw': '1' } }).catch(() => { - // Noop - }); - - const spans = await spansPromise; - const segmentSpan = spans.find(span => span.is_segment)!; - const middlewareSpan = spans.find(span => !span.is_segment && span.name === 'middleware GET')!; - - expect(segmentSpan.status).toBe('error'); - expect(segmentSpan.attributes['sentry.segment.name.source']?.value).toBe('route'); - expect(middlewareSpan.parent_span_id).toBe(segmentSpan.span_id); - expect(middlewareSpan.status).toBe('error'); - expect(getSpanOp(middlewareSpan)).toBe('middleware'); -}); - -test('Should trace outgoing fetch requests inside middleware', async ({ request }) => { - test.skip(isDevMode, 'The fetch requests ends up in a separate tx in dev atm'); - - // `http.client` span names are low cardinality under span streaming, hence `GET localhost` rather - // than the full URL. - const spansPromise = collectStreamedSpans('nextjs-16-cf-workers', spans => - spans.some(span => getSpanOp(span) === 'http.client' && span.name === 'GET localhost'), - ); - - request.get('/api/endpoint-behind-middleware', { headers: { 'x-should-make-request': '1' } }).catch(() => { - // Noop - }); - - const spans = await spansPromise; - const fetchSpan = spans.find(span => getSpanOp(span) === 'http.client' && span.name === 'GET localhost')!; - - expect(fetchSpan.status).toBe('ok'); - expect(fetchSpan.attributes).toMatchObject({ - 'http.request.method': { value: 'GET', type: 'string' }, - 'http.response.status_code': { value: 200, type: 'integer' }, - 'url.full': { value: 'http://localhost:3030/', type: 'string' }, - }); -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/nested-rsc-error.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/nested-rsc-error.test.ts deleted file mode 100644 index 03f7d224e466..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/nested-rsc-error.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { waitForError, waitForStreamedSpan } from '@sentry-internal/test-utils'; - -test('Should capture errors from nested server components when `Sentry.captureRequestError` is added to the `onRequestError` hook', async ({ - page, -}) => { - const errorEventPromise = waitForError('nextjs-16-cf-workers', errorEvent => { - return !!errorEvent?.exception?.values?.some(value => value.value === 'I am technically uncatchable'); - }); - - // Matched on the error's own trace so a span from an earlier spec cannot satisfy the correlation. - const serverSpanPromise = waitForStreamedSpan('nextjs-16-cf-workers', async span => { - return ( - span.name === 'GET /nested-rsc-error/[param]' && - span.is_segment && - (await errorEventPromise).contexts?.trace?.trace_id === span.trace_id - ); - }); - - await page.goto(`/nested-rsc-error/123`); - const errorEvent = await errorEventPromise; - const serverSpan = await serverSpanPromise; - - // error event is part of the same trace as the server span - expect(errorEvent.contexts?.trace?.trace_id).toBe(serverSpan.trace_id); - - expect(errorEvent.request).toMatchObject({ - headers: expect.any(Object), - method: 'GET', - }); - - expect(errorEvent.contexts?.nextjs).toEqual({ - route_type: 'render', - router_kind: 'App Router', - router_path: '/nested-rsc-error/[param]', - request_path: '/nested-rsc-error/123', - }); - - expect(errorEvent.exception?.values?.[0]?.mechanism).toEqual({ - handled: false, - type: 'auto.function.nextjs.on_request_error', - }); -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/pageload-tracing.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/pageload-tracing.test.ts deleted file mode 100644 index d6165244752e..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/pageload-tracing.test.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { getSpanOp, waitForStreamedSpan } from '@sentry-internal/test-utils'; - -test('App router spans should be attached to the pageload request span', async ({ page }) => { - const serverSpanPromise = waitForStreamedSpan('nextjs-16-cf-workers', span => { - return span.name === 'GET /pageload-tracing' && span.is_segment; - }); - - const pageloadSpanPromise = waitForStreamedSpan('nextjs-16-cf-workers', span => { - return span.name === '/pageload-tracing' && getSpanOp(span) === 'pageload' && span.is_segment; - }); - - await page.goto(`/pageload-tracing`); - - const [serverSpan, pageloadSpan] = await Promise.all([serverSpanPromise, pageloadSpanPromise]); - - expect(pageloadSpan.trace_id).toBeTruthy(); - expect(serverSpan.trace_id).toBe(pageloadSpan.trace_id); -}); - -test('extracts HTTP request headers as span attributes', async ({ baseURL }) => { - const serverSpanPromise = waitForStreamedSpan('nextjs-16-cf-workers', span => { - const requestId = span.attributes['http.request.header.x-request-id']; - return ( - span.name === 'GET /pageload-tracing' && - span.is_segment && - requestId?.type === 'array' && - requestId.value[0] === 'nextjs-789' - ); - }); - - await fetch(`${baseURL}/pageload-tracing`, { - headers: { - 'User-Agent': 'Custom-NextJS-Agent/15.0', - 'Content-Type': 'text/html', - 'X-NextJS-Test': 'nextjs-header-value', - Accept: 'text/html, application/xhtml+xml', - 'X-Framework': 'Next.js', - 'X-Request-ID': 'nextjs-789', - }, - }); - - const serverSpan = await serverSpanPromise; - - expect(serverSpan.attributes).toMatchObject({ - 'http.request.header.user-agent': { value: ['Custom-NextJS-Agent/15.0'], type: 'array' }, - 'http.request.header.content-type': { value: ['text/html'], type: 'array' }, - 'http.request.header.x-nextjs-test': { value: ['nextjs-header-value'], type: 'array' }, - 'http.request.header.accept': { value: ['text/html, application/xhtml+xml'], type: 'array' }, - 'http.request.header.x-framework': { value: ['Next.js'], type: 'array' }, - 'http.request.header.x-request-id': { value: ['nextjs-789'], type: 'array' }, - }); -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/parameterized-routes.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/parameterized-routes.test.ts deleted file mode 100644 index 777130edc5b9..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/parameterized-routes.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { getSpanOp, waitForStreamedSpan } from '@sentry-internal/test-utils'; - -test('should create a parameterized pageload span when the `app` directory is used', async ({ page }) => { - const spanPromise = waitForStreamedSpan('nextjs-16-cf-workers', span => { - return span.name === '/parameterized/:one' && getSpanOp(span) === 'pageload' && span.is_segment; - }); - - await page.goto(`/parameterized/cappuccino`); - - const span = await spanPromise; - - expect(span.span_id).toEqual(expect.stringMatching(/[a-f0-9]{16}/)); - expect(span.trace_id).toEqual(expect.stringMatching(/[a-f0-9]{32}/)); - expect(span.attributes).toMatchObject({ - 'sentry.op': { value: 'pageload', type: 'string' }, - 'sentry.origin': { value: 'auto.pageload.nextjs.app_router_instrumentation', type: 'string' }, - 'sentry.segment.name.source': { value: 'route', type: 'string' }, - 'sentry.environment': { value: 'qa', type: 'string' }, - 'react.version': { value: expect.any(String), type: 'string' }, - }); -}); - -test('should create a span named after the static route when the `app` directory is used', async ({ page }) => { - const spanPromise = waitForStreamedSpan('nextjs-16-cf-workers', span => { - return span.name === '/parameterized/static' && getSpanOp(span) === 'pageload' && span.is_segment; - }); - - await page.goto(`/parameterized/static`); - - const span = await spanPromise; - - expect(span.span_id).toEqual(expect.stringMatching(/[a-f0-9]{16}/)); - expect(span.trace_id).toEqual(expect.stringMatching(/[a-f0-9]{32}/)); - expect(span.attributes).toMatchObject({ - 'sentry.op': { value: 'pageload', type: 'string' }, - 'sentry.origin': { value: 'auto.pageload.nextjs.app_router_instrumentation', type: 'string' }, - 'sentry.segment.name.source': { value: 'route', type: 'string' }, - 'sentry.environment': { value: 'qa', type: 'string' }, - 'react.version': { value: expect.any(String), type: 'string' }, - 'url.template': { value: '/parameterized/static', type: 'string' }, - }); -}); - -test('should create a partially parameterized pageload span when the `app` directory is used', async ({ page }) => { - const spanPromise = waitForStreamedSpan('nextjs-16-cf-workers', span => { - return span.name === '/parameterized/:one/beep' && getSpanOp(span) === 'pageload' && span.is_segment; - }); - - await page.goto(`/parameterized/cappuccino/beep`); - - const span = await spanPromise; - - expect(span.span_id).toEqual(expect.stringMatching(/[a-f0-9]{16}/)); - expect(span.trace_id).toEqual(expect.stringMatching(/[a-f0-9]{32}/)); - expect(span.attributes).toMatchObject({ - 'sentry.op': { value: 'pageload', type: 'string' }, - 'sentry.origin': { value: 'auto.pageload.nextjs.app_router_instrumentation', type: 'string' }, - 'sentry.segment.name.source': { value: 'route', type: 'string' }, - 'sentry.environment': { value: 'qa', type: 'string' }, - 'react.version': { value: expect.any(String), type: 'string' }, - }); -}); - -test('should create a nested parameterized pageload span when the `app` directory is used', async ({ page }) => { - const spanPromise = waitForStreamedSpan('nextjs-16-cf-workers', span => { - return span.name === '/parameterized/:one/beep/:two' && getSpanOp(span) === 'pageload' && span.is_segment; - }); - - await page.goto(`/parameterized/cappuccino/beep/espresso`); - - const span = await spanPromise; - - expect(span.span_id).toEqual(expect.stringMatching(/[a-f0-9]{16}/)); - expect(span.trace_id).toEqual(expect.stringMatching(/[a-f0-9]{32}/)); - expect(span.attributes).toMatchObject({ - 'sentry.op': { value: 'pageload', type: 'string' }, - 'sentry.origin': { value: 'auto.pageload.nextjs.app_router_instrumentation', type: 'string' }, - 'sentry.segment.name.source': { value: 'route', type: 'string' }, - 'sentry.environment': { value: 'qa', type: 'string' }, - 'react.version': { value: expect.any(String), type: 'string' }, - }); -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/prefetch-spans.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/prefetch-spans.test.ts deleted file mode 100644 index 53cebde1ff21..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/prefetch-spans.test.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { collectStreamedSpansUntilSegment } from '@sentry-internal/test-utils'; -import { isDevMode } from './isDevMode'; - -test('Prefetch client spans should have a http.request.prefetch attribute', async ({ page }) => { - test.skip(isDevMode, "Prefetch requests don't have the prefetch header in dev mode"); - - // The prefetch span is a child of the pageload segment span, which ends last. - const spansPromise = collectStreamedSpansUntilSegment('nextjs-16-cf-workers', '/prefetching'); - - await page.goto(`/prefetching`); - - // Make it more likely that nextjs prefetches - await page.hover('#prefetch-link'); - - const spans = await spansPromise; - - expect(spans).toContainEqual( - expect.objectContaining({ - attributes: expect.objectContaining({ - 'sentry.op': { value: 'http.client', type: 'string' }, - 'http.request.prefetch': { value: true, type: 'boolean' }, - }), - }), - ); -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/route-handler.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/route-handler.test.ts deleted file mode 100644 index f8a8420f3694..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/route-handler.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -import test, { expect } from '@playwright/test'; -import { getSpanOp, waitForStreamedSpan } from '@sentry-internal/test-utils'; - -test('Should create a span for node route handlers', async ({ request }) => { - const routehandlerSpanPromise = waitForStreamedSpan('nextjs-16-cf-workers', span => { - return span.name === 'GET /route-handler/[xoxo]/node' && span.is_segment; - }); - - const response = await request.get('/route-handler/123/node', { headers: { 'x-charly': 'gomez' } }); - expect(await response.json()).toStrictEqual({ message: 'Hello Node Route Handler' }); - - const routehandlerSpan = await routehandlerSpanPromise; - - expect(routehandlerSpan.status).toBe('ok'); - expect(getSpanOp(routehandlerSpan)).toBe('http.server'); - expect(routehandlerSpan.attributes['http.request.header.x-charly']?.value).toEqual(['gomez']); -}); - -test('Should create a span for edge route handlers', async ({ request }) => { - test.skip(true, 'OpenNext does not support the Edge runtime'); - const routehandlerSpanPromise = waitForStreamedSpan('nextjs-16-cf-workers', span => { - return span.name === 'GET /route-handler/[xoxo]/edge' && span.is_segment; - }); - - const response = await request.get('/route-handler/123/edge', { headers: { 'x-charly': 'gomez' } }); - expect(await response.json()).toStrictEqual({ message: 'Hello Edge Route Handler' }); - - const routehandlerSpan = await routehandlerSpanPromise; - - expect(routehandlerSpan.status).toBe('ok'); - expect(getSpanOp(routehandlerSpan)).toBe('http.server'); - expect(routehandlerSpan.attributes['http.request.header.x-charly']?.value).toEqual(['gomez']); -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/server-action-redirect.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/server-action-redirect.test.ts deleted file mode 100644 index a5961ae41274..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/server-action-redirect.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { waitForError, waitForStreamedSpan } from '@sentry-internal/test-utils'; - -test('Should handle server action redirect without capturing errors', async ({ page, request }) => { - // Wait for the initial pageload span - const pageLoadSpanPromise = waitForStreamedSpan('nextjs-16-cf-workers', span => { - return span.name === '/redirect/origin' && span.is_segment; - }); - - // Navigate to the origin page - await page.goto('/redirect/origin'); - - const pageLoadSpan = await pageLoadSpanPromise; - expect(pageLoadSpan).toBeDefined(); - - // Wait for the redirect span - const redirectSpanPromise = waitForStreamedSpan('nextjs-16-cf-workers', span => { - return span.name === 'GET /redirect/destination' && span.is_segment; - }); - - // The error of `/api/test-error`, requested after the redirect, is the sentinel: an error of the redirect would - // arrive before it. - const firstErrorPromise = waitForError('nextjs-16-cf-workers', () => true); - - // Click the redirect button - await page.click('button[type="submit"]'); - - await redirectSpanPromise; - - // Verify we got redirected to the destination page - await expect(page).toHaveURL('/redirect/destination'); - - await request.get('/api/test-error'); - - const firstError = await firstErrorPromise; - expect(firstError.exception?.values?.[0]?.value).toBe('This is a test error from an API route'); -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/server-components.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/server-components.test.ts deleted file mode 100644 index 344a62127543..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/server-components.test.ts +++ /dev/null @@ -1,150 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { collectStreamedSpansUntilSegment, getSpanOp } from '@sentry-internal/test-utils'; - -// Next.js emits these spans itself. The SDK attaches no op, description or function name to -// them, so asserting `undefined` pins that they stay untouched. -const nextjsSpan = { op: undefined, description: undefined, codeFunctionName: undefined }; - -// `withSentry` from `@sentry/nextjs/cloudflare` wraps the Worker entry, so the segment is its `http.server` -// span, named after the route Next.js resolved. -test('Sends a span for a request to app router with URL', async ({ page }) => { - const spansPromise = collectStreamedSpansUntilSegment( - 'nextjs-16-cf-workers', - span => - span.name === 'GET /parameterized/[one]/beep/[two]' && - span.attributes['url.path']?.value === '/parameterized/1337/beep/42', - ); - - await page.goto('/parameterized/1337/beep/42'); - - const spans = await spansPromise; - const segmentSpan = spans.find( - span => - span.name === 'GET /parameterized/[one]/beep/[two]' && - span.is_segment && - span.attributes['url.path']?.value === '/parameterized/1337/beep/42', - )!; - - expect(segmentSpan.span_id).toEqual(expect.stringMatching(/[a-f0-9]{16}/)); - expect(segmentSpan.trace_id).toEqual(expect.stringMatching(/[a-f0-9]{32}/)); - expect(segmentSpan.status).toBe('ok'); - expect(segmentSpan.attributes).toMatchObject({ - 'sentry.op': { value: 'http.server', type: 'string' }, - 'sentry.origin': { value: 'auto.http.cloudflare', type: 'string' }, - 'sentry.sample_rate': { value: 1, type: 'integer' }, - 'sentry.segment.name.source': { value: 'route', type: 'string' }, - 'http.request.method': { value: 'GET', type: 'string' }, - 'http.response.status_code': { value: 200, type: 'integer' }, - 'http.route': { value: '/parameterized/[one]/beep/[two]', type: 'string' }, - 'url.path': { value: '/parameterized/1337/beep/42', type: 'string' }, - 'next.route': { value: '/parameterized/[one]/beep/[two]', type: 'string' }, - }); - - // No child span should share the segment span's name - expect(spans.filter(span => !span.is_segment && span.name === segmentSpan.name)).toHaveLength(0); - expect(spans.filter(span => getSpanOp(span) === 'http.server')).toEqual([segmentSpan]); -}); - -test('Will create spans for every server component and metadata generation functions when visiting a page', async ({ - page, -}) => { - const spansPromise = collectStreamedSpansUntilSegment('nextjs-16-cf-workers', 'GET /nested-layout'); - - await page.goto('/nested-layout'); - - const fullSpans = await spansPromise; - const spans = fullSpans.map(span => ({ - name: span.name, - op: getSpanOp(span), - description: span.attributes['sentry.description']?.value, - codeFunctionName: span.attributes['code.function.name']?.value, - })); - - expect(spans).toContainEqual({ ...nextjsSpan, name: 'render route (app) /nested-layout' }); - expect(spans).toContainEqual({ ...nextjsSpan, name: 'build component tree' }); - // Server component spans: the name is the low-cardinality `code.function.name`, and the - // segment each one resolved for is on the description. - expect(spans).toContainEqual({ - name: 'Layout', - op: 'function', - description: 'resolve root layout server component', - codeFunctionName: 'Layout', - }); - expect(spans).toContainEqual({ - name: 'Layout', - op: 'function', - description: 'resolve layout server component "(nested-layout)"', - codeFunctionName: 'Layout', - }); - expect(spans).toContainEqual({ - name: 'Layout', - op: 'function', - description: 'resolve layout server component "nested-layout"', - codeFunctionName: 'Layout', - }); - expect(spans).toContainEqual({ - name: 'Page', - op: 'function', - description: 'resolve page server component "/nested-layout"', - codeFunctionName: 'Page', - }); - expect(spans).toContainEqual({ ...nextjsSpan, name: 'generateMetadata /(nested-layout)/nested-layout/page' }); - expect(spans).toContainEqual({ ...nextjsSpan, name: 'start response' }); -}); - -test('Will create spans for every server component and metadata generation functions when visiting a dynamic page', async ({ - page, -}) => { - const spansPromise = collectStreamedSpansUntilSegment('nextjs-16-cf-workers', 'GET /nested-layout/[dynamic]'); - - await page.goto('/nested-layout/123'); - - const fullSpans = await spansPromise; - const spans = fullSpans.map(span => ({ - name: span.name, - op: getSpanOp(span), - description: span.attributes['sentry.description']?.value, - codeFunctionName: span.attributes['code.function.name']?.value, - })); - - expect(spans).toContainEqual({ ...nextjsSpan, name: 'resolve page components' }); - expect(spans).toContainEqual({ ...nextjsSpan, name: 'render route (app) /nested-layout/[dynamic]' }); - expect(spans).toContainEqual({ ...nextjsSpan, name: 'build component tree' }); - // Server component spans: the name is the low-cardinality `code.function.name`, and the - // segment each one resolved for is on the description. - expect(spans).toContainEqual({ - name: 'Layout', - op: 'function', - description: 'resolve root layout server component', - codeFunctionName: 'Layout', - }); - expect(spans).toContainEqual({ - name: 'Layout', - op: 'function', - description: 'resolve layout server component "(nested-layout)"', - codeFunctionName: 'Layout', - }); - expect(spans).toContainEqual({ - name: 'Layout', - op: 'function', - description: 'resolve layout server component "nested-layout"', - codeFunctionName: 'Layout', - }); - expect(spans).toContainEqual({ - name: 'Layout', - op: 'function', - description: 'resolve layout server component "[dynamic]"', - codeFunctionName: 'Layout', - }); - expect(spans).toContainEqual({ - name: 'Page', - op: 'function', - description: 'resolve page server component "/nested-layout/[dynamic]"', - codeFunctionName: 'Page', - }); - expect(spans).toContainEqual({ - ...nextjsSpan, - name: 'generateMetadata /(nested-layout)/nested-layout/[dynamic]/page', - }); - expect(spans).toContainEqual({ ...nextjsSpan, name: 'start response' }); -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/streaming-rsc-error.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/streaming-rsc-error.test.ts deleted file mode 100644 index 1b991f1a9fe4..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/streaming-rsc-error.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { waitForError, waitForStreamedSpan } from '@sentry-internal/test-utils'; - -test('Should capture errors for crashing streaming promises in server components when `Sentry.captureRequestError` is added to the `onRequestError` hook', async ({ - page, -}) => { - const errorEventPromise = waitForError('nextjs-16-cf-workers', errorEvent => { - return !!errorEvent?.exception?.values?.some(value => value.value === 'I am a data streaming error'); - }); - - // Matched on the error's own trace so a span from an earlier spec cannot satisfy the correlation. - const serverSpanPromise = waitForStreamedSpan('nextjs-16-cf-workers', async span => { - return ( - span.name === 'GET /streaming-rsc-error/[param]' && - span.is_segment && - (await errorEventPromise).contexts?.trace?.trace_id === span.trace_id - ); - }); - - // The streaming RSC error can interrupt the HTTP response, causing the navigation to reject - // (e.g. net::ERR_ABORTED) even though the error and transaction are still captured. - await page.goto(`/streaming-rsc-error/123`).catch(() => {}); - const errorEvent = await errorEventPromise; - const serverSpan = await serverSpanPromise; - - // error event is part of the same trace as the server span - expect(errorEvent.contexts?.trace?.trace_id).toBe(serverSpan.trace_id); - - expect(errorEvent.request).toMatchObject({ - headers: expect.any(Object), - method: 'GET', - }); - - expect(errorEvent.contexts?.nextjs).toEqual({ - route_type: 'render', - router_kind: 'App Router', - router_path: '/streaming-rsc-error/[param]', - request_path: '/streaming-rsc-error/123', - }); - - expect(errorEvent.exception?.values?.[0]?.mechanism).toEqual({ - handled: false, - type: 'auto.function.nextjs.on_request_error', - }); -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/trace-continuation.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/trace-continuation.test.ts deleted file mode 100644 index 447aa7df0cb6..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/trace-continuation.test.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { expect, test } from '@playwright/test'; -import { collectStreamedSpans } from '@sentry-internal/test-utils'; - -test('Continues an incoming trace in the segment of the Worker request', async ({ request }) => { - const traceId = crypto.randomUUID().replace(/-/g, ''); - const parentSpanId = traceId.slice(0, 16); - - const spansPromise = collectStreamedSpans( - 'nextjs-16-cf-workers', - spans => - spans.some( - span => - span.trace_id === traceId && - span.is_segment && - span.attributes['sentry.origin']?.value === 'auto.http.cloudflare', - ) && - spans.some( - span => span.trace_id === traceId && span.name === 'render route (app) /parameterized/[one]/beep/[two]', - ), - ); - - await request.get('/parameterized/1337/beep/42', { - headers: { - 'sentry-trace': `${traceId}-${parentSpanId}-1`, - baggage: `sentry-trace_id=${traceId},sentry-sample_rate=1,sentry-sampled=true`, - }, - }); - - const spans = await spansPromise; - const segmentSpan = spans.find( - span => span.is_segment && span.attributes['sentry.origin']?.value === 'auto.http.cloudflare', - )!; - const renderSpan = spans.find(span => span.name === 'render route (app) /parameterized/[one]/beep/[two]')!; - - expect(segmentSpan.parent_span_id).toBe(parentSpanId); - expect(segmentSpan.name).toBe('GET /parameterized/[one]/beep/[two]'); - expect(renderSpan.attributes['sentry.segment.id']?.value).toBe(segmentSpan.span_id); -}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tsconfig.json b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tsconfig.json deleted file mode 100644 index cc9ed39b5aa2..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tsconfig.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2017", - "lib": ["dom", "dom.iterable", "esnext"], - "allowJs": true, - "skipLibCheck": true, - "strict": true, - "noEmit": true, - "esModuleInterop": true, - "module": "esnext", - "moduleResolution": "bundler", - "resolveJsonModule": true, - "isolatedModules": true, - "jsx": "react-jsx", - "incremental": true, - "plugins": [ - { - "name": "next" - } - ], - "paths": { - "@/*": ["./*"] - } - }, - "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts", ".next/dev/types/**/*.ts", "**/*.mts"], - "exclude": ["node_modules"] -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/wrangler.jsonc b/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/wrangler.jsonc deleted file mode 100644 index 16b0271383b1..000000000000 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/wrangler.jsonc +++ /dev/null @@ -1,74 +0,0 @@ -/** - * For more details on how to configure Wrangler, refer to: - * https://developers.cloudflare.com/workers/wrangler/configuration/ - */ -/** - * For more details on how to configure Wrangler, refer to: - * https://developers.cloudflare.com/workers/wrangler/configuration/ - */ -{ - "$schema": "node_modules/wrangler/config-schema.json", - "name": "next-cf", - "main": "./worker.ts", - // From 2026-02-19 on, wrangler uses the native `process` of workerd, whose `cwd()` is `/bundle` during a request. - // Next.js then misses its router server context and extracts the incoming trace again, see trace-continuation.test.ts. - "compatibility_date": "2026-02-19", - "compatibility_flags": ["nodejs_compat", "global_fetch_strictly_public"], - "assets": { - "binding": "ASSETS", - "directory": ".open-next/assets", - }, - "images": { - // Enable image optimization - // see https://opennext.js.org/cloudflare/howtos/image - "binding": "IMAGES", - }, - "services": [ - { - // Self-reference service binding, the service name must match the worker name - // see https://opennext.js.org/cloudflare/caching - "binding": "WORKER_SELF_REFERENCE", - "service": "next-cf", - }, - ], - "d1_databases": [ - { - "binding": "DB", - "database_name": "e2e", - "database_id": "00000000-0000-0000-0000-000000000000", - }, - ], - "observability": { - "enabled": true, - }, - /** - * Smart Placement - * Docs: https://developers.cloudflare.com/workers/configuration/smart-placement/#smart-placement - */ - // "placement": { "mode": "smart" } - /** - * Bindings - * Bindings allow your Worker to interact with resources on the Cloudflare Developer Platform, including - * databases, object storage, AI inference, real-time communication and more. - * https://developers.cloudflare.com/workers/runtime-apis/bindings/ - */ - /** - * Environment Variables - * https://developers.cloudflare.com/workers/wrangler/configuration/#environment-variables - */ - // "vars": { "MY_VARIABLE": "production_value" } - /** - * Note: Use secrets to store sensitive data. - * https://developers.cloudflare.com/workers/configuration/secrets/ - */ - /** - * Static Assets - * https://developers.cloudflare.com/workers/static-assets/binding/ - */ - // "assets": { "directory": "./public/", "binding": "ASSETS" } - /** - * Service Bindings (communicate between multiple Workers) - * https://developers.cloudflare.com/workers/wrangler/configuration/#service-bindings - */ - // "services": [{ "binding": "MY_SERVICE", "service": "my-service" }] -} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/ai-mock-server.mjs b/dev-packages/e2e-tests/test-applications/nextjs-16/ai-mock-server.mjs index b97e0cb162dc..50060d37c309 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16/ai-mock-server.mjs +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/ai-mock-server.mjs @@ -24,11 +24,12 @@ function sendJson(res, status, obj) { res.end(JSON.stringify(obj)); } -let serverPromise; +/** Port of the mock server. The Playwright global setup starts it, so the app reaches it on every runtime. */ +export const MOCK_AI_PORT = 3032; -/** Lazily starts the shared mock server and resolves to its port. */ -export function getMockAiPort() { - serverPromise ??= new Promise(resolve => { +/** Starts the mock server on `MOCK_AI_PORT` and resolves to a function that stops it. */ +export function startMockAiServer() { + return new Promise(resolve => { const server = createServer(async (req, res) => { const url = req.url || ''; @@ -84,10 +85,8 @@ export function getMockAiPort() { res.writeHead(404).end(); }); - server.listen(0, () => { - resolve(server.address().port); + server.listen(MOCK_AI_PORT, () => { + resolve(() => new Promise(done => server.close(done))); }); }); - - return serverPromise; } diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/d1/route.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/app/api/d1/route.ts similarity index 100% rename from dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/d1/route.ts rename to dev-packages/e2e-tests/test-applications/nextjs-16/app/api/d1/route.ts diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/app/api/logs/route.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/app/api/logs/route.ts new file mode 100644 index 000000000000..b289a2365f2a --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/app/api/logs/route.ts @@ -0,0 +1,9 @@ +import * as Sentry from '@sentry/nextjs'; + +// Without this Next.js prerenders the route at build time, where `init` creates no client and the log is dropped. +export const dynamic = 'force-dynamic'; + +export async function GET() { + Sentry.logger.info('e2e server log', { 'e2e.attr': 'value' }); + return Response.json({ ok: true }); +} diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/app/api/openai/route.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/app/api/openai/route.ts index 9315f9d079ce..628b7a239697 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16/app/api/openai/route.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/app/api/openai/route.ts @@ -1,13 +1,12 @@ import { NextResponse } from 'next/server'; import OpenAI from 'openai'; -import { getMockAiPort } from '../../../ai-mock-server.mjs'; +import { MOCK_AI_PORT } from '../../../ai-mock-server.mjs'; export const dynamic = 'force-dynamic'; export async function GET() { - const port = await getMockAiPort(); const client = new OpenAI({ - baseURL: `http://localhost:${port}/openai`, + baseURL: `http://localhost:${MOCK_AI_PORT}/openai`, apiKey: 'mock-api-key', }); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/otel-context/route.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/app/api/otel-context/route.ts similarity index 54% rename from dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/otel-context/route.ts rename to dev-packages/e2e-tests/test-applications/nextjs-16/app/api/otel-context/route.ts index 1ca661b60b63..26199762ea55 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/otel-context/route.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/app/api/otel-context/route.ts @@ -1,4 +1,8 @@ -import { context, createContextKey, trace } from '@opentelemetry/api'; +// The app does not install `@opentelemetry/api`: OpenNext resolves the ESM build of an installed copy when it bundles +// the Node.js middleware for Workers, but its file tracing only copies the CommonJS build. The copy in Next.js uses the +// same global OpenTelemetry API, but has no types. +// @ts-expect-error `next/dist/compiled/@opentelemetry/api` ships no type declarations +import { context, createContextKey, trace } from 'next/dist/compiled/@opentelemetry/api'; import { NextResponse } from 'next/server'; export const dynamic = 'force-dynamic'; @@ -9,7 +13,7 @@ export async function GET() { const tracer = trace.getTracer('e2e'); context.with(context.active().setValue(E2E_CONTEXT_KEY, 'e2e-value'), () => { - tracer.startActiveSpan('otel-context-outer', outer => { + tracer.startActiveSpan('otel-context-outer', (outer: { end(): void }) => { // An explicit context, as OpenTelemetry instrumentations pass it. const inner = tracer.startSpan('otel-context-inner', {}, context.active()); inner.setAttribute('e2e.context.value', String(context.active().getValue(E2E_CONTEXT_KEY))); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/test-error/route.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/app/api/test-error/route.ts similarity index 100% rename from dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/app/api/test-error/route.ts rename to dev-packages/e2e-tests/test-applications/nextjs-16/app/api/test-error/route.ts diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/propagation/test-outgoing-fetch-external-disallowed/check/route.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/app/propagation/test-outgoing-fetch-external-disallowed/check/route.ts similarity index 100% rename from dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/propagation/test-outgoing-fetch-external-disallowed/check/route.ts rename to dev-packages/e2e-tests/test-applications/nextjs-16/app/propagation/test-outgoing-fetch-external-disallowed/check/route.ts diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/propagation/test-outgoing-fetch-external-disallowed/route.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/app/propagation/test-outgoing-fetch-external-disallowed/route.ts similarity index 100% rename from dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/propagation/test-outgoing-fetch-external-disallowed/route.ts rename to dev-packages/e2e-tests/test-applications/nextjs-16/app/propagation/test-outgoing-fetch-external-disallowed/route.ts diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/propagation/test-outgoing-fetch/check/route.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/app/propagation/test-outgoing-fetch/check/route.ts similarity index 100% rename from dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/propagation/test-outgoing-fetch/check/route.ts rename to dev-packages/e2e-tests/test-applications/nextjs-16/app/propagation/test-outgoing-fetch/check/route.ts diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/propagation/test-outgoing-fetch/route.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/app/propagation/test-outgoing-fetch/route.ts similarity index 100% rename from dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/propagation/test-outgoing-fetch/route.ts rename to dev-packages/e2e-tests/test-applications/nextjs-16/app/propagation/test-outgoing-fetch/route.ts diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/propagation/utils.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/app/propagation/utils.ts similarity index 100% rename from dev-packages/e2e-tests/test-applications/nextjs-16-bun/app/propagation/utils.ts rename to dev-packages/e2e-tests/test-applications/nextjs-16/app/propagation/utils.ts diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/global-setup.mjs b/dev-packages/e2e-tests/test-applications/nextjs-16/global-setup.mjs index dc7ec8736faf..41ec97defa8f 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16/global-setup.mjs +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/global-setup.mjs @@ -1,6 +1,7 @@ import { execSync } from 'child_process'; import { dirname } from 'path'; import { fileURLToPath } from 'url'; +import { startMockAiServer } from './ai-mock-server.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); @@ -11,4 +12,8 @@ export default async function globalSetup() { cwd: __dirname, stdio: 'inherit', }); + + // The mock AI server runs here and not in the app, because a Worker can not start a `node:http` server. Playwright + // runs the returned function as teardown. + return startMockAiServer(); } diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/next.config.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/next.config.ts index 9278da016887..6893bd77fdf7 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16/next.config.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/next.config.ts @@ -5,6 +5,9 @@ import type { NextConfig } from 'next'; process.env.VERCEL = '1'; const nextConfig: NextConfig = { + // `pg` requires `pg-cloudflare` on Workers. The file tracing only copies its Node.js build, and OpenNext copies the + // whole package (with its `workerd` build) only for packages listed here. + serverExternalPackages: ['pg-cloudflare'], experimental: { authInterrupts: true, sri: { diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/open-next.config.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/open-next.config.ts similarity index 100% rename from dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/open-next.config.ts rename to dev-packages/e2e-tests/test-applications/nextjs-16/open-next.config.ts diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/package.json b/dev-packages/e2e-tests/test-applications/nextjs-16/package.json index 408cd3be5a38..80f92437f500 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16/package.json +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/package.json @@ -20,10 +20,16 @@ "test:build-latest-webpack": "pnpm install && pnpm add next@latest && pnpm build-webpack", "test:build-canary-webpack": "pnpm install && pnpm add next@canary && pnpm build-webpack", "test:assert": "pnpm test:prod && pnpm test:dev", - "test:assert-webpack": "pnpm test:prod && pnpm test:dev-webpack" + "test:assert-webpack": "pnpm test:prod && pnpm test:dev-webpack", + "test:build:cloudflare": "pnpm install && opennextjs-cloudflare build", + "test:build-latest:cloudflare": "pnpm install && pnpm add next@16.3 && opennextjs-cloudflare build", + "start:cloudflare": "opennextjs-cloudflare preview --var \"E2E_TEST_DSN:$E2E_TEST_DSN\"" }, "//": "Pin `ioredis` to 5.10.1: it's the last version before the driver publishes its own native diagnostics channels; orchestrion's config covers `ioredis <5.11.0`.", "dependencies": { + "@opennextjs/cloudflare": "^1.14.9", + "@sentry/bun": "file:../../packed/sentry-bun-packed.tgz", + "@sentry/deno": "file:../../packed/sentry-deno-packed.tgz", "@sentry/nextjs": "file:../../packed/sentry-nextjs-packed.tgz", "@sentry/core": "file:../../packed/sentry-core-packed.tgz", "@vercel/queue": "^0.1.3", @@ -47,7 +53,8 @@ "@types/react-dom": "^19", "eslint": "^9", "eslint-config-next": "^16", - "typescript": "^5" + "typescript": "^5", + "wrangler": "^4.61.0" }, "volta": { "extends": "../../package.json" @@ -67,6 +74,17 @@ { "build-command": "pnpm test:build-latest", "label": "nextjs-16 (latest, turbopack)" + }, + { + "runtime": "bun", + "assert-command": "pnpm test:prod", + "label": "nextjs-16 (bun)" + }, + { + "runtime": "cloudflare", + "build-command": "pnpm test:build:cloudflare", + "assert-command": "pnpm test:prod", + "label": "nextjs-16 (cloudflare)" } ], "optionalVariants": [ @@ -79,6 +97,18 @@ "build-command": "pnpm test:build-canary-webpack", "label": "nextjs-16 (canary, webpack)", "assert-command": "pnpm test:assert-webpack" + }, + { + "runtime": "cloudflare", + "build-command": "pnpm test:build-latest:cloudflare", + "assert-command": "pnpm test:prod", + "label": "nextjs-16 (cloudflare, latest)" + }, + { + "runtime": "deno", + "assert-command": "pnpm test:prod", + "deno-version": "v2.9.0", + "label": "nextjs-16 (deno)" } ] } diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/playwright.config.mjs b/dev-packages/e2e-tests/test-applications/nextjs-16/playwright.config.mjs index cbac80f5b7e7..7e356d81488b 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16/playwright.config.mjs +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/playwright.config.mjs @@ -1,10 +1,17 @@ -import { getPlaywrightConfig } from '@sentry-internal/test-utils'; +import { getPlaywrightConfig, getRuntime } from '@sentry-internal/test-utils'; const testEnv = process.env.TEST_ENV; if (!testEnv) { throw new Error('No test env defined'); } +const productionStartCommands = { + node: 'pnpm next start -p 3030', + bun: 'bun --bun ./node_modules/next/dist/bin/next start -p 3030', + deno: 'deno run -A ./node_modules/next/dist/bin/next start -p 3030', + cloudflare: 'pnpm start:cloudflare --port 3030', +}; + const getStartCommand = () => { if (testEnv === 'development-webpack') { return 'pnpm next dev -p 3030 --webpack 2>&1 | tee .tmp_dev_server_logs'; @@ -15,7 +22,7 @@ const getStartCommand = () => { } if (testEnv === 'production') { - return 'pnpm next start -p 3030'; + return productionStartCommands[getRuntime()]; } throw new Error(`Unknown test env: ${testEnv}`); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/sentry.server.config.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/sentry.server.config.ts index 4507c799f9f3..960f49a15533 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16/sentry.server.config.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/sentry.server.config.ts @@ -1,3 +1,5 @@ +import { bunHttpServerIntegration, fetchIntegration as bunFetchIntegration } from '@sentry/bun'; +import { fetchIntegration as denoFetchIntegration } from '@sentry/deno'; import * as Sentry from '@sentry/nextjs'; import { Log } from '@sentry/nextjs'; @@ -6,8 +8,17 @@ Sentry.init({ dsn: process.env.NEXT_PUBLIC_E2E_TEST_DSN, tunnel: `http://localhost:3031/`, // proxy server tracesSampleRate: 1.0, + tracePropagationTargets: ['http://localhost:3030/propagation/test-outgoing-fetch/check'], // debug: true, - integrations: [Sentry.vercelAIIntegration(), Sentry.nodeRuntimeMetricsIntegration({ collectionIntervalMs: 1_000 })], + integrations: [ + Sentry.vercelAIIntegration(), + Sentry.nodeRuntimeMetricsIntegration({ collectionIntervalMs: 1_000 }), + // Bun does not emit the `node:http` diagnostics channel the Node SDK uses to isolate incoming requests, so each + // request would otherwise share one trace. Next.js emits its own server spans, hence `spans: false`. + ...(process.versions.bun ? [bunHttpServerIntegration({ spans: false }), bunFetchIntegration()] : []), + // The `fetch` of Deno is not undici, so the Node SDK does not see outgoing requests. + ...(process.versions.deno ? [denoFetchIntegration()] : []), + ], // Verify Log type is available beforeSendLog(log: Log) { return log; diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/cloudflare-runtime.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/cloudflare-runtime.test.ts similarity index 79% rename from dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/cloudflare-runtime.test.ts rename to dev-packages/e2e-tests/test-applications/nextjs-16/tests/cloudflare-runtime.test.ts index d088edd32dcb..bf1c773190a7 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/cloudflare-runtime.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/cloudflare-runtime.test.ts @@ -1,9 +1,11 @@ import { expect, test } from '@playwright/test'; -import { waitForError } from '@sentry-internal/test-utils'; +import { getRuntime, waitForError } from '@sentry-internal/test-utils'; + +test.skip(getRuntime() !== 'cloudflare', 'Checks the runtime context of the Cloudflare Worker'); test.describe('Cloudflare Runtime', () => { test('Should report cloudflare as the runtime in API route error events', async ({ request }) => { - const errorEventPromise = waitForError('nextjs-16-cf-workers', errorEvent => { + const errorEventPromise = waitForError('nextjs-16', errorEvent => { return !!errorEvent?.exception?.values?.some(value => value.value?.includes('This is a test error from an API route'), ); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/d1-binding.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/d1-binding.test.ts similarity index 82% rename from dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/d1-binding.test.ts rename to dev-packages/e2e-tests/test-applications/nextjs-16/tests/d1-binding.test.ts index 43c31b94084d..e86279c27c23 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/d1-binding.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/d1-binding.test.ts @@ -1,9 +1,11 @@ import { expect, test } from '@playwright/test'; -import { collectStreamedSpansUntilSegment, getSpanOp } from '@sentry-internal/test-utils'; +import { collectStreamedSpansUntilSegment, getRuntime, getSpanOp } from '@sentry-internal/test-utils'; + +test.skip(getRuntime() !== 'cloudflare', 'D1 is a binding of Cloudflare Workers'); // `withSentry` passes an instrumented `env` to the OpenNext handler, which `getCloudflareContext()` returns. test('Should create a span for a D1 query through the `env` of `getCloudflareContext`', async ({ request }) => { - const spansPromise = collectStreamedSpansUntilSegment('nextjs-16-cf-workers', 'GET /api/d1'); + const spansPromise = collectStreamedSpansUntilSegment('nextjs-16', 'GET /api/d1'); const response = await request.get('/api/d1'); expect(await response.json()).toStrictEqual([{ one: 1 }]); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/tests/db-page.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/db-page.test.ts index 968891a085c4..e155b1bed5a1 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16/tests/db-page.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/db-page.test.ts @@ -1,18 +1,18 @@ import { expect, test } from '@playwright/test'; -import { collectStreamedSpansUntilSegment } from '@sentry-internal/test-utils'; +import { collectStreamedSpansUntilSegment, getRuntime } from '@sentry-internal/test-utils'; + +// pg is externalized, so the runtime module hook instruments it. +test('Instruments pg calls made during server-side rendering of a page', async ({ page }) => { + test.skip(getRuntime() === 'bun' || getRuntime() === 'cloudflare', 'Bun and Workers have no runtime module hook'); -test('Instruments DB calls made during server-side rendering of a page', async ({ page }) => { // The db spans are children of the segment span, which ends last. const spansPromise = collectStreamedSpansUntilSegment('nextjs-16', 'GET /db-page'); await page.goto('/db-page'); await expect(page.locator('#answer')).toHaveText('answer: 42'); - await expect(page.locator('#cached')).toHaveText('cached: 42'); const spans = await spansPromise; - // One page render produces spans from both injection paths: pg (externalized → runtime module - // hook) and ioredis (bundle-safe allowlisted → build-time loader). expect(spans).toContainEqual( expect.objectContaining({ name: 'SELECT', @@ -25,6 +25,18 @@ test('Instruments DB calls made during server-side rendering of a page', async ( }), }), ); +}); + +// ioredis is bundle-safe and allowlisted, so the build-time loader instruments it. +test('Instruments ioredis calls made during server-side rendering of a page', async ({ page }) => { + // The db spans are children of the segment span, which ends last. + const spansPromise = collectStreamedSpansUntilSegment('nextjs-16', 'GET /db-page'); + + await page.goto('/db-page'); + await expect(page.locator('#cached')).toHaveText('cached: 42'); + + const spans = await spansPromise; + expect(spans).toContainEqual( expect.objectContaining({ name: 'set localhost:6379', diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/tests/logs.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/logs.test.ts new file mode 100644 index 000000000000..25dbd2a909e6 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/logs.test.ts @@ -0,0 +1,39 @@ +import { expect, test } from '@playwright/test'; +import { getRuntime, waitForEnvelopeItem, waitForStreamedSpan } from '@sentry-internal/test-utils'; +import type { SerializedLogContainer } from '@sentry/core'; + +test('Sends a log of a route handler in the trace of the request', async ({ request }) => { + // Workers have no hostname, so the client of `@sentry/cloudflare` has no `serverName` and its logs no + // `server.address`. The other runtimes take `os.hostname()`. + const serverAddressType = getRuntime() === 'cloudflare' ? undefined : 'string'; + + const segmentSpanPromise = waitForStreamedSpan('nextjs-16', span => { + return span.name === 'GET /api/logs' && span.is_segment; + }); + + const response = await request.get('/api/logs'); + expect(await response.json()).toStrictEqual({ ok: true }); + + const segmentSpan = await segmentSpanPromise; + // The log can arrive before the segment, so the search starts at the first envelope of the proxy. + const logEnvelopeItem = await waitForEnvelopeItem( + 'nextjs-16', + envelopeItem => + envelopeItem[0].type === 'log' && + (envelopeItem[1] as SerializedLogContainer).items.some(log => log.trace_id === segmentSpan.trace_id), + 0, + ); + const logs = (logEnvelopeItem[1] as SerializedLogContainer).items.filter( + log => log.trace_id === segmentSpan.trace_id, + ); + + expect(logs).toHaveLength(1); + expect(logs[0]!.body).toBe('e2e server log'); + expect(logs[0]!.level).toBe('info'); + expect(logs[0]!.severity_number).toBe(9); + expect(logs[0]!.attributes?.['e2e.attr']).toEqual({ value: 'value', type: 'string' }); + expect(logs[0]!.attributes?.['sentry.environment']).toEqual({ value: 'qa', type: 'string' }); + expect(logs[0]!.attributes?.['sentry.sdk.name']).toEqual({ value: 'sentry.javascript.nextjs', type: 'string' }); + expect(logs[0]!.attributes?.['sentry.trace.parent_span_id']?.value).toMatch(/^[a-f0-9]{16}$/); + expect(logs[0]!.attributes?.['server.address']?.type).toBe(serverAddressType); +}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/tests/metrics.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/metrics.test.ts index 43edb917d526..85e7c2635f12 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16/tests/metrics.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/metrics.test.ts @@ -1,7 +1,11 @@ import { expect, test } from '@playwright/test'; -import { waitForMetric } from '@sentry-internal/test-utils'; +import { getRuntime, waitForMetric } from '@sentry-internal/test-utils'; test('Should emit metrics from server and client', async ({ request, page }) => { + // Workers have no hostname, so the client of `@sentry/cloudflare` has no `serverName` and its metrics no + // `server.address`. The other runtimes take `os.hostname()`. + const serverAddressType = getRuntime() === 'cloudflare' ? undefined : 'string'; + const clientCountPromise = waitForMetric('nextjs-16', async metric => { return metric.name === 'test.page.count'; }); @@ -90,7 +94,6 @@ test('Should emit metrics from server and client', async ({ request, page }) => type: 'counter', value: 1, attributes: { - 'server.address': { value: expect.any(String), type: 'string' }, 'random.attribute': { value: 'Potatoes', type: 'string' }, endpoint: { value: '/metrics/route-handler', type: 'string' }, 'sentry.environment': { value: 'qa', type: 'string' }, @@ -106,7 +109,6 @@ test('Should emit metrics from server and client', async ({ request, page }) => type: 'distribution', value: 100, attributes: { - 'server.address': { value: expect.any(String), type: 'string' }, 'random.attribute': { value: 'Patatas', type: 'string' }, endpoint: { value: '/metrics/route-handler', type: 'string' }, 'sentry.environment': { value: 'qa', type: 'string' }, @@ -122,7 +124,6 @@ test('Should emit metrics from server and client', async ({ request, page }) => type: 'gauge', value: 200, attributes: { - 'server.address': { value: expect.any(String), type: 'string' }, 'random.attribute': { value: 'Patate', type: 'string' }, endpoint: { value: '/metrics/route-handler', type: 'string' }, 'sentry.environment': { value: 'qa', type: 'string' }, @@ -130,4 +131,7 @@ test('Should emit metrics from server and client', async ({ request, page }) => 'sentry.sdk.version': { value: expect.any(String), type: 'string' }, }, }); + expect(serverCount.attributes?.['server.address']?.type).toBe(serverAddressType); + expect(serverDistribution.attributes?.['server.address']?.type).toBe(serverAddressType); + expect(serverGauge.attributes?.['server.address']?.type).toBe(serverAddressType); }); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/tests/middleware.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/middleware.test.ts index f91168e54038..d3b3b3dcd05c 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16/tests/middleware.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/middleware.test.ts @@ -2,12 +2,15 @@ import { expect, test } from '@playwright/test'; import { collectStreamedSpans, collectStreamedSpansUntilSegment, + getRuntime, getSpanOp, waitForStreamedSpan, } from '@sentry-internal/test-utils'; import { isDevMode } from './isDevMode'; test('Should create a span for middleware', async ({ request }) => { + test.skip(getRuntime() === 'cloudflare', 'On Workers the middleware span is a child of the request span'); + const spansPromise = collectStreamedSpansUntilSegment('nextjs-16', 'middleware GET'); const routeSpanPromise = waitForStreamedSpan('nextjs-16', span => { @@ -44,8 +47,33 @@ test('Should create a span for middleware', async ({ request }) => { expect(routeSpan.attributes['isolation_scope.has_proxy_marker']).toEqual({ value: false, type: 'boolean' }); }); +// `withSentry` from `@sentry/nextjs/cloudflare` wraps the Worker entry, so middleware runs inside the request's +// `http.server` segment and its span is a child of it rather than a segment of its own. +test('Should create a span for middleware inside the request segment of the Worker', async ({ request }) => { + test.skip(getRuntime() !== 'cloudflare', 'Only on Workers the request span of `withSentry` is the segment'); + + const spansPromise = collectStreamedSpansUntilSegment('nextjs-16', 'GET /api/endpoint-behind-middleware'); + + const response = await request.get('/api/endpoint-behind-middleware'); + expect(await response.json()).toStrictEqual({ name: 'John Doe' }); + + const spans = await spansPromise; + const segmentSpan = spans.find(span => span.is_segment)!; + const middlewareSpan = spans.find(span => span.name === 'middleware GET')!; + + expect(spans.filter(span => getSpanOp(span) === 'middleware')).toEqual([middlewareSpan]); + expect(segmentSpan.attributes['url.path']?.value).toBe('/api/endpoint-behind-middleware'); + expect(segmentSpan.attributes['sentry.segment.name.source']?.value).toBe('route'); + expect(segmentSpan.attributes['isolation_scope.is_default']).toEqual({ value: false, type: 'boolean' }); + expect(middlewareSpan.is_segment).toBe(false); + expect(middlewareSpan.parent_span_id).toBe(segmentSpan.span_id); + expect(middlewareSpan.status).toBe('ok'); + expect(getSpanOp(middlewareSpan)).toBe('middleware'); +}); + test('Faulty middlewares', async ({ request }) => { test.skip(isDevMode, 'Throwing crashes the dev server atm'); // https://github.com/vercel/next.js/issues/85261 + test.skip(getRuntime() === 'cloudflare', 'On Workers the middleware span is a child of the request span'); const middlewareSpanPromise = waitForStreamedSpan('nextjs-16', span => { return span.name === 'middleware GET' && span.is_segment; }); @@ -65,9 +93,39 @@ test('Faulty middlewares', async ({ request }) => { // await test.step('should record exceptions', async () => { ... }); }); +test('Faulty middlewares inside the request segment of the Worker', async ({ request }) => { + test.skip(getRuntime() !== 'cloudflare', 'Only on Workers the request span of `withSentry` is the segment'); + // The middleware throws, so the request span of `withSentry` keeps the middleware name, like the middleware segment + // of Node.js. + const spansPromise = collectStreamedSpans( + 'nextjs-16', + spans => + spans.some(span => span.is_segment && span.name === 'middleware GET') && + spans.some(span => !span.is_segment && span.name === 'middleware GET'), + ); + + request.get('/api/endpoint-behind-middleware', { headers: { 'x-should-throw': '1' } }).catch(() => { + // Noop + }); + + const spans = await spansPromise; + const segmentSpan = spans.find(span => span.is_segment)!; + const middlewareSpan = spans.find(span => !span.is_segment && span.name === 'middleware GET')!; + + expect(segmentSpan.status).toBe('error'); + expect(segmentSpan.attributes['sentry.segment.name.source']?.value).toBe('route'); + expect(middlewareSpan.parent_span_id).toBe(segmentSpan.span_id); + expect(middlewareSpan.status).toBe('error'); + expect(getSpanOp(middlewareSpan)).toBe('middleware'); +}); + test('Should trace outgoing fetch requests inside middleware', async ({ request }) => { test.skip(isDevMode, 'The fetch requests ends up in a separate tx in dev atm'); + // Only Node.js has the undici diagnostics channel. On the other runtimes the fetch integration of the runtime + // SDK creates this span, which has no `sentry.kind`, `url.path` and `url.scheme`. + const isNode = getRuntime() === 'node'; + // In some builds (especially webpack) the fetch span is not a child of the middleware segment but a // segment of its own, so this waits for either. `http.client` span names are low cardinality under // span streaming, hence `GET localhost` rather than the full URL. @@ -83,17 +141,15 @@ test('Should trace outgoing fetch requests inside middleware', async ({ request const fetchSpan = spans.find(span => getSpanOp(span) === 'http.client' && span.name === 'GET localhost')!; expect(fetchSpan.status).toBe('ok'); - expect(fetchSpan.attributes).toMatchObject({ - 'http.request.method': { value: 'GET', type: 'string' }, - 'http.response.status_code': { value: 200, type: 'integer' }, - 'sentry.kind': { value: 'client', type: 'string' }, - 'sentry.op': { value: 'http.client', type: 'string' }, - 'sentry.origin': { value: 'auto.http.node_fetch', type: 'string' }, - 'server.address': { value: 'localhost', type: 'string' }, - 'server.port': { value: 3030, type: 'integer' }, - 'url.domain': { value: 'localhost', type: 'string' }, - 'url.full': { value: 'http://localhost:3030/', type: 'string' }, - 'url.path': { value: '/', type: 'string' }, - 'url.scheme': { value: 'http', type: 'string' }, - }); + expect(fetchSpan.attributes['http.request.method']?.value).toBe('GET'); + expect(fetchSpan.attributes['http.response.status_code']?.value).toBe(200); + expect(fetchSpan.attributes['sentry.op']?.value).toBe('http.client'); + expect(fetchSpan.attributes['sentry.origin']?.value).toBe(isNode ? 'auto.http.node_fetch' : 'auto.http.fetch'); + expect(fetchSpan.attributes['server.address']?.value).toBe('localhost'); + expect(fetchSpan.attributes['server.port']?.value).toBe(3030); + expect(fetchSpan.attributes['url.domain']?.value).toBe('localhost'); + expect(fetchSpan.attributes['url.full']?.value).toBe('http://localhost:3030/'); + expect(fetchSpan.attributes['sentry.kind']?.value).toBe(isNode ? 'client' : undefined); + expect(fetchSpan.attributes['url.path']?.value).toBe(isNode ? '/' : undefined); + expect(fetchSpan.attributes['url.scheme']?.value).toBe(isNode ? 'http' : undefined); }); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/tests/node-runtime-metrics.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/node-runtime-metrics.test.ts index 5295101f1e20..7f611cf3daa9 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16/tests/node-runtime-metrics.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/node-runtime-metrics.test.ts @@ -1,5 +1,7 @@ import { expect, test } from '@playwright/test'; -import { waitForMetric } from '@sentry-internal/test-utils'; +import { getRuntime, waitForMetric } from '@sentry-internal/test-utils'; + +test.skip(getRuntime() === 'cloudflare', 'Node.js runtime metrics do not exist on Workers'); const EXPECTED_ATTRIBUTES = { 'sentry.environment': { value: 'qa', type: 'string' }, diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/otel-context.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/otel-context.test.ts similarity index 95% rename from dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/otel-context.test.ts rename to dev-packages/e2e-tests/test-applications/nextjs-16/tests/otel-context.test.ts index 5303c62a984f..1bc31009a6b1 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/otel-context.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/otel-context.test.ts @@ -2,7 +2,7 @@ import { expect, test } from '@playwright/test'; import { collectStreamedSpansUntilSegment } from '@sentry-internal/test-utils'; test('Keeps OpenTelemetry context values and parents', async ({ request }) => { - const spansPromise = collectStreamedSpansUntilSegment('nextjs-16-cf-workers', 'GET /api/otel-context'); + const spansPromise = collectStreamedSpansUntilSegment('nextjs-16', 'GET /api/otel-context'); const response = await request.get('/api/otel-context'); expect(await response.json()).toStrictEqual({ ok: true }); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/propagation.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/propagation.test.ts similarity index 73% rename from dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/propagation.test.ts rename to dev-packages/e2e-tests/test-applications/nextjs-16/tests/propagation.test.ts index e82fb6453474..e758cd769551 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-bun/tests/propagation.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/propagation.test.ts @@ -5,12 +5,14 @@ test('Propagates trace for outgoing fetch requests', async ({ baseURL, request } // Inbound span, outbound span and the http.client span in between all share one trace, and // `collectStreamedSpans` evaluates a single trace at a time, so requiring all three together // keeps them paired. - const spansPromise = collectStreamedSpans('nextjs-16-bun', spans => { + const spansPromise = collectStreamedSpans('nextjs-16', spans => { return ( spans.some(span => span.name === 'GET /propagation/test-outgoing-fetch' && span.is_segment) && spans.some(span => span.name === 'GET /propagation/test-outgoing-fetch/check' && span.is_segment) && spans.some( - span => getSpanOp(span) === 'http.client' && span.attributes['sentry.origin']?.value === 'auto.http.fetch', + span => + getSpanOp(span) === 'http.client' && + String(span.attributes['url.full']?.value).endsWith('/propagation/test-outgoing-fetch/check'), ) ); }); @@ -23,7 +25,9 @@ test('Propagates trace for outgoing fetch requests', async ({ baseURL, request } span => span.name === 'GET /propagation/test-outgoing-fetch/check' && span.is_segment, )!; const httpClientSpan = spans.find( - span => getSpanOp(span) === 'http.client' && span.attributes['sentry.origin']?.value === 'auto.http.fetch', + span => + getSpanOp(span) === 'http.client' && + String(span.attributes['url.full']?.value).endsWith('/propagation/test-outgoing-fetch/check'), ); expect(inboundSpan.trace_id).toStrictEqual(expect.any(String)); @@ -33,10 +37,10 @@ test('Propagates trace for outgoing fetch requests', async ({ baseURL, request } expect(httpClientSpan?.span_id).toStrictEqual(expect.any(String)); expect(inboundSpan.parent_span_id).toBe(httpClientSpan?.span_id); - expect(headers).toMatchObject({ - baggage: expect.any(String), - 'sentry-trace': `${outboundSpan.trace_id}-${httpClientSpan?.span_id}-1`, - }); + expect(headers['sentry-trace']).toBe(`${outboundSpan.trace_id}-${httpClientSpan?.span_id}-1`); + expect(headers.baggage).toContain(`sentry-trace_id=${outboundSpan.trace_id}`); + expect(headers.baggage).toContain('sentry-sampled=true'); + expect(headers.baggage).toContain('sentry-environment=qa'); }); test('Does not propagate outgoing fetch requests not covered by tracePropagationTargets', async ({ @@ -44,11 +48,11 @@ test('Does not propagate outgoing fetch requests not covered by tracePropagation request, }) => { // These two spans are deliberately in different traces, so they are matched by their unique names. - const inboundSpanPromise = waitForStreamedSpan('nextjs-16-bun', span => { + const inboundSpanPromise = waitForStreamedSpan('nextjs-16', span => { return span.name === 'GET /propagation/test-outgoing-fetch-external-disallowed/check' && span.is_segment; }); - const outboundSpanPromise = waitForStreamedSpan('nextjs-16-bun', span => { + const outboundSpanPromise = waitForStreamedSpan('nextjs-16', span => { return span.name === 'GET /propagation/test-outgoing-fetch-external-disallowed' && span.is_segment; }); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/tests/server-action-redirect.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/server-action-redirect.test.ts index 27b82ae35c61..3ebe985701f4 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16/tests/server-action-redirect.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/server-action-redirect.test.ts @@ -1,7 +1,7 @@ import { expect, test } from '@playwright/test'; import { waitForError, waitForStreamedSpan } from '@sentry-internal/test-utils'; -test('Should handle server action redirect without capturing errors', async ({ page }) => { +test('Should handle server action redirect without capturing errors', async ({ page, request }) => { // Wait for the initial pageload span const pageLoadSpanPromise = waitForStreamedSpan('nextjs-16', span => { return span.name === '/redirect/origin' && span.is_segment; @@ -18,10 +18,9 @@ test('Should handle server action redirect without capturing errors', async ({ p return span.name === 'GET /redirect/destination' && span.is_segment; }); - // No error should be captured - const redirectErrorPromise = waitForError('nextjs-16', async errorEvent => { - return !!errorEvent; - }); + // The error of `/api/test-error`, requested after the redirect, is the sentinel: an error of the redirect would + // arrive before it. + const firstErrorPromise = waitForError('nextjs-16', () => true); // Click the redirect button await page.click('button[type="submit"]'); @@ -31,17 +30,8 @@ test('Should handle server action redirect without capturing errors', async ({ p // Verify we got redirected to the destination page await expect(page).toHaveURL('/redirect/destination'); - // Wait for potential errors with a 2 second timeout - const errorTimeout = new Promise((_, reject) => - setTimeout(() => reject(new Error('No error captured (timeout)')), 2000), - ); - - // We expect this to timeout since no error should be captured during the redirect - try { - await Promise.race([redirectErrorPromise, errorTimeout]); - throw new Error('Expected no error to be captured, but an error was found'); - } catch (e) { - // If we get a timeout error (as expected), no error was captured - expect((e as Error).message).toBe('No error captured (timeout)'); - } + await request.get('/api/test-error'); + + const firstError = await firstErrorPromise; + expect(firstError.exception?.values?.[0]?.value).toBe('This is a test error from an API route'); }); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/tests/server-components.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/server-components.test.ts index b271cfd1ac52..e7e0de8b1202 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16/tests/server-components.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/server-components.test.ts @@ -1,5 +1,5 @@ import { expect, test } from '@playwright/test'; -import { collectStreamedSpansUntilSegment, getSpanOp } from '@sentry-internal/test-utils'; +import { collectStreamedSpansUntilSegment, getRuntime, getSpanOp } from '@sentry-internal/test-utils'; import { isTurbopackDevMode } from './isDevMode'; // Next.js emits these spans itself. The SDK attaches no op, description or function name to @@ -8,6 +8,7 @@ const nextjsSpan = { op: undefined, description: undefined, codeFunctionName: un test('Sends a span for a request to app router with URL', async ({ page }) => { test.skip(isTurbopackDevMode, 'Turbopack intermittently returns 404 for nested dynamic routes in dev mode'); + test.skip(getRuntime() === 'cloudflare', 'On Workers the segment is the request span of `withSentry`'); const spansPromise = collectStreamedSpansUntilSegment( 'nextjs-16', @@ -45,6 +46,49 @@ test('Sends a span for a request to app router with URL', async ({ page }) => { // No child span should share the segment span's name expect(spans.filter(span => !span.is_segment && span.name === segmentSpan.name)).toHaveLength(0); + expect(spans.filter(span => getSpanOp(span) === 'http.server')).toEqual([segmentSpan]); +}); + +// `withSentry` from `@sentry/nextjs/cloudflare` wraps the Worker entry, so the segment is its `http.server` span with +// the attributes of `@sentry/cloudflare`, named after the route Next.js resolved. +test('Sends the request span of the Worker for a request to app router with URL', async ({ page }) => { + test.skip(getRuntime() !== 'cloudflare', 'Only on Workers the request span of `withSentry` is the segment'); + + const spansPromise = collectStreamedSpansUntilSegment( + 'nextjs-16', + span => + span.name === 'GET /parameterized/[one]/beep/[two]' && + span.attributes['url.path']?.value === '/parameterized/1337/beep/42', + ); + + await page.goto('/parameterized/1337/beep/42'); + + const spans = await spansPromise; + const segmentSpan = spans.find( + span => + span.name === 'GET /parameterized/[one]/beep/[two]' && + span.is_segment && + span.attributes['url.path']?.value === '/parameterized/1337/beep/42', + )!; + + expect(segmentSpan.span_id).toEqual(expect.stringMatching(/[a-f0-9]{16}/)); + expect(segmentSpan.trace_id).toEqual(expect.stringMatching(/[a-f0-9]{32}/)); + expect(segmentSpan.status).toBe('ok'); + expect(segmentSpan.attributes).toMatchObject({ + 'sentry.op': { value: 'http.server', type: 'string' }, + 'sentry.origin': { value: 'auto.http.cloudflare', type: 'string' }, + 'sentry.sample_rate': { value: 1, type: 'integer' }, + 'sentry.segment.name.source': { value: 'route', type: 'string' }, + 'http.request.method': { value: 'GET', type: 'string' }, + 'http.response.status_code': { value: 200, type: 'integer' }, + 'http.route': { value: '/parameterized/[one]/beep/[two]', type: 'string' }, + 'url.path': { value: '/parameterized/1337/beep/42', type: 'string' }, + 'next.route': { value: '/parameterized/[one]/beep/[two]', type: 'string' }, + }); + + // No child span should share the segment span's name + expect(spans.filter(span => !span.is_segment && span.name === segmentSpan.name)).toHaveLength(0); + expect(spans.filter(span => getSpanOp(span) === 'http.server')).toEqual([segmentSpan]); }); test('Will create spans for every server component and metadata generation functions when visiting a page', async ({ diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/tests/trace-continuation.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/trace-continuation.test.ts new file mode 100644 index 000000000000..88ebdc16d554 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/trace-continuation.test.ts @@ -0,0 +1,37 @@ +import { expect, test } from '@playwright/test'; +import { collectStreamedSpans, getRuntime } from '@sentry-internal/test-utils'; + +test('Continues an incoming trace in the segment of the request', async ({ request }) => { + // On Workers the segment is the request span of `withSentry`, on the other runtimes the request span of Next.js. + const segmentOrigin = getRuntime() === 'cloudflare' ? 'auto.http.cloudflare' : 'auto'; + const traceId = crypto.randomUUID().replace(/-/g, ''); + const parentSpanId = traceId.slice(0, 16); + + const spansPromise = collectStreamedSpans( + 'nextjs-16', + spans => + spans.some(span => span.trace_id === traceId && span.is_segment) && + spans.some( + span => span.trace_id === traceId && span.name === 'render route (app) /parameterized/[one]/beep/[two]', + ), + ); + + await request.get('/parameterized/1337/beep/42', { + headers: { + 'sentry-trace': `${traceId}-${parentSpanId}-1`, + baggage: `sentry-trace_id=${traceId},sentry-sample_rate=1,sentry-sampled=true`, + }, + }); + + const spans = await spansPromise; + const segmentSpan = spans.find(span => span.trace_id === traceId && span.is_segment)!; + const renderSpan = spans.find( + span => span.trace_id === traceId && span.name === 'render route (app) /parameterized/[one]/beep/[two]', + )!; + + expect(spans.filter(span => span.trace_id === traceId && span.is_segment)).toEqual([segmentSpan]); + expect(segmentSpan.parent_span_id).toBe(parentSpanId); + expect(segmentSpan.attributes['sentry.origin']?.value).toBe(segmentOrigin); + expect(segmentSpan.name).toBe('GET /parameterized/[one]/beep/[two]'); + expect(renderSpan.attributes['sentry.segment.id']?.value).toBe(segmentSpan.span_id); +}); diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/worker-bundle.test.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/worker-bundle.test.ts similarity index 95% rename from dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/worker-bundle.test.ts rename to dev-packages/e2e-tests/test-applications/nextjs-16/tests/worker-bundle.test.ts index dbd927e2053b..af16d1add37d 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/tests/worker-bundle.test.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/tests/worker-bundle.test.ts @@ -1,9 +1,12 @@ import { expect, test } from '@playwright/test'; +import { getRuntime } from '@sentry-internal/test-utils'; import * as fs from 'fs'; import { createRequire } from 'module'; import * as path from 'path'; import { isDevMode } from './isDevMode'; +test.skip(getRuntime() !== 'cloudflare', 'Checks the worker bundle of OpenNext'); + /** * The orchestrion bundler plugins are build-time-only, and their module-scope side effects break * on Workers (an unawaited `WebAssembly.compile()` crashed every cold start, issue #22794). The diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/worker.ts b/dev-packages/e2e-tests/test-applications/nextjs-16/worker.ts similarity index 68% rename from dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/worker.ts rename to dev-packages/e2e-tests/test-applications/nextjs-16/worker.ts index 6718e015a32c..fb5708783e58 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-16-cf-workers/worker.ts +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/worker.ts @@ -1,8 +1,8 @@ import * as Sentry from '@sentry/nextjs/cloudflare'; -// @ts-expect-error generated by `opennextjs-cloudflare build` +// @ts-ignore the file exists only after `opennextjs-cloudflare build` import handler from './.open-next/worker.js'; -// @ts-expect-error generated by `opennextjs-cloudflare build` +// @ts-ignore the file exists only after `opennextjs-cloudflare build` export { DOQueueHandler, DOShardedTagCache, BucketCachePurge } from './.open-next/worker.js'; export default Sentry.withSentry( @@ -11,6 +11,7 @@ export default Sentry.withSentry( dsn: env.E2E_TEST_DSN, tunnel: `http://localhost:3031/`, // proxy server tracesSampleRate: 1.0, + tracePropagationTargets: ['http://localhost:3030/propagation/test-outgoing-fetch/check'], integrations: [Sentry.vercelAIIntegration()], }), handler, diff --git a/dev-packages/e2e-tests/test-applications/nextjs-16/wrangler.jsonc b/dev-packages/e2e-tests/test-applications/nextjs-16/wrangler.jsonc new file mode 100644 index 000000000000..f6bfe6a4ec38 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/nextjs-16/wrangler.jsonc @@ -0,0 +1,36 @@ +{ + "$schema": "node_modules/wrangler/config-schema.json", + "name": "nextjs-16", + "main": "./worker.ts", + // From 2026-02-19 on, wrangler uses the native `process` of workerd, whose `cwd()` is `/bundle` during a request. + // Next.js then misses its router server context and extracts the incoming trace again, see trace-continuation.test.ts. + "compatibility_date": "2026-02-19", + "compatibility_flags": ["nodejs_compat", "global_fetch_strictly_public"], + "assets": { + "binding": "ASSETS", + "directory": ".open-next/assets", + }, + "images": { + // Enable image optimization + // see https://opennext.js.org/cloudflare/howtos/image + "binding": "IMAGES", + }, + "services": [ + { + // Self-reference service binding, the service name must match the worker name + // see https://opennext.js.org/cloudflare/caching + "binding": "WORKER_SELF_REFERENCE", + "service": "nextjs-16", + }, + ], + "d1_databases": [ + { + "binding": "DB", + "database_name": "e2e", + "database_id": "00000000-0000-0000-0000-000000000000", + }, + ], + "observability": { + "enabled": true, + }, +} From 4f9a027988acf007a0ad8d360403221a8069a482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Peer=20St=C3=B6cklmair?= Date: Thu, 8 Oct 2026 14:28:41 +0300 Subject: [PATCH 27/47] feat(cloudflare): Wrap vinext Worker entries with `@sentry/nextjs/cloudflare` (#25000) part of #19514 part of JS-1803 When a Vite build uses vinext and `@sentry/nextjs/cloudflare` resolves from the Worker entry, the Vite plugin of `@sentry/cloudflare` now wraps the entry with `withSentry` from `@sentry/nextjs/cloudflare`. Other builds, and versions of `@sentry/nextjs` without that entry, keep `@sentry/cloudflare`. ```diff -import * as __SENTRY__ from '@sentry/cloudflare'; +import * as __SENTRY__ from '@sentry/nextjs/cloudflare'; const __SENTRY_DEFAULT_EXPORT__ = { fetch() { ... } }; export default __SENTRY__.withSentry(() => undefined, __SENTRY_DEFAULT_EXPORT__); ``` ## Why With `withSentry` of `@sentry/cloudflare`, a vinext app on Workers sends Cloudflare data but no Next.js spans. Now it gets the Next.js spans without a config change. ## Traces The same app and requests: vinext on Workers before and after this PR, and `next start` on Node.js. | Request | vinext before | vinext after | Next.js on Node.js | | --- | --- | --- | --- | | `GET /parameterized/42` | [`GET`, 1 span](https://sentry-sdks.sentry.io/explore/traces/trace/0f98d8ca6ef645e3ae5bb60c039eeb6d/) | [`GET /parameterized/[id]`, 7 spans](https://sentry-sdks.sentry.io/explore/traces/trace/a26de83affd24b0c8fe7990e567ae8b6/) | [`GET /parameterized/[id]`, 7 spans](https://sentry-sdks.sentry.io/explore/traces/trace/c24bdbb12b134e80a5417fe9d2a59c56/) | | `GET /api/hello` | [`GET`, 1 span](https://sentry-sdks.sentry.io/explore/traces/trace/df5b9f7976c5487daa7004cc9361eeeb/) | [`GET /api/hello`, 4 spans](https://sentry-sdks.sentry.io/explore/traces/trace/75b4a831b48f4bb1adb13648f5c6419a/) | [`GET /api/hello`, 4 spans](https://sentry-sdks.sentry.io/explore/traces/trace/ea8990c798144e2db75ee88e8bd6e634/) | | `GET /api/throw` (error) | [`GET`, 1 span](https://sentry-sdks.sentry.io/explore/traces/trace/b6389e0368e44b269e5e07ba9823c29a/) | [`GET /api/throw`, 3 spans](https://sentry-sdks.sentry.io/explore/traces/trace/07caa661217e4f14bb38b182167d08c7/) | [`GET /api/throw`, 3 spans](https://sentry-sdks.sentry.io/explore/traces/trace/c9a3a73e3c7a44d4b7ff6519c8760a7b/) | Co-authored-by: Claude Opus 5.5 --- .../cloudflare/src/vite/autoInstrument.ts | 29 +++++- packages/cloudflare/src/vite/transform.ts | 4 +- .../test/vite/autoInstrument.test.ts | 91 ++++++++++++++++++- 3 files changed, 121 insertions(+), 3 deletions(-) diff --git a/packages/cloudflare/src/vite/autoInstrument.ts b/packages/cloudflare/src/vite/autoInstrument.ts index 6c7e8f9734fc..c4550f2fa12f 100644 --- a/packages/cloudflare/src/vite/autoInstrument.ts +++ b/packages/cloudflare/src/vite/autoInstrument.ts @@ -15,17 +15,39 @@ function normalizePath(path: string): string { // `.html`, … — sharing the entry's basename must never be treated as the entry. const JS_EXTENSION_REGEX = /\.[cm]?[jt]sx?$/; +// Re-exports everything the generated code uses, and adds the Next.js span handling and OpenTelemetry context. +const NEXTJS_CLOUDFLARE_MODULE = '@sentry/nextjs/cloudflare'; + +/** + * Whether `@sentry/nextjs/cloudflare` resolves from `importer`. Vite throws instead of returning `null` when + * `@sentry/nextjs` is installed in a version without that entry. + */ +async function resolvesNextjsCloudflare(resolver: ModuleResolver, importer: string): Promise { + try { + return !!(await resolver.resolve?.(NEXTJS_CLOUDFLARE_MODULE, importer)); + } catch { + return false; + } +} + export function sentryCloudflareAutoInstrumentPlugin(options: { wranglerConfigPath?: string } = {}) { let wranglerConfig: WranglerConfig | undefined; let entryFilePath: string | undefined; let optionsFn = ENV_FALLBACK_OPTIONS_FN; let optionsImport: string | undefined; + let isVinext = false; return { name: 'sentry-cloudflare-auto-instrument', - configResolved(config: { root: string; logger?: { warn(msg: string): void } }): void { + configResolved(config: { + root: string; + plugins?: readonly { name?: string }[]; + logger?: { warn(msg: string): void }; + }): void { + isVinext = config.plugins?.some(plugin => plugin.name?.startsWith('vinext:')) ?? false; + const result = resolveWranglerConfig(config.root, options.wranglerConfigPath); if (!result) { // An explicit path that fails is a misconfiguration worth naming; @@ -119,6 +141,10 @@ export function sentryCloudflareAutoInstrumentPlugin(options: { wranglerConfigPa }) : undefined; + // A vinext app with `@sentry/nextjs` installed gets `withSentry` of `@sentry/nextjs/cloudflare`. + const sentryModule = + isVinext && (await resolvesNextjsCloudflare(this, normalizedId)) ? NEXTJS_CLOUDFLARE_MODULE : undefined; + // No registration import is injected here: the orchestrion plugin's // subscribe-injection makes each bundled package self-register its channel // subscriber on the global marker, so wrapping the entry with `withSentry` @@ -129,6 +155,7 @@ export function sentryCloudflareAutoInstrumentPlugin(options: { wranglerConfigPa optionsFn, optionsImport, sameWorkerBindings: wranglerConfig.sameWorkerBindings, + sentryModule, }); const wrappedClasses = result?.wrappedClasses ?? new Set(); diff --git a/packages/cloudflare/src/vite/transform.ts b/packages/cloudflare/src/vite/transform.ts index 1fdf647831c1..3674ed5dd032 100644 --- a/packages/cloudflare/src/vite/transform.ts +++ b/packages/cloudflare/src/vite/transform.ts @@ -114,6 +114,8 @@ export interface TransformContext { optionsFn: string; /** Import statement prepended when `optionsFn` references a separate module. */ optionsImport?: string; + /** Module the generated code imports `withSentry` and the class wrappers from. Defaults to `@sentry/cloudflare`. */ + sentryModule?: string; /** @see {@link import('./wranglerConfig').WranglerConfig.sameWorkerBindings} */ sameWorkerBindings?: readonly SameWorkerBinding[]; } @@ -198,7 +200,7 @@ export function applyAutoInstrumentTransforms( ms.prepend(buildMergedOptionsDeclaration(sameWorkerBindings, ctx.optionsFn, state)); } if (ctx.optionsImport) ms.prepend(ctx.optionsImport); - ms.prepend("import * as __SENTRY__ from '@sentry/cloudflare';\n"); + ms.prepend(`import * as __SENTRY__ from '${ctx.sentryModule ?? '@sentry/cloudflare'}';\n`); return { code: ms.toString(), diff --git a/packages/cloudflare/test/vite/autoInstrument.test.ts b/packages/cloudflare/test/vite/autoInstrument.test.ts index 7303e667ee6c..f9654b0d3b5d 100644 --- a/packages/cloudflare/test/vite/autoInstrument.test.ts +++ b/packages/cloudflare/test/vite/autoInstrument.test.ts @@ -2,7 +2,7 @@ import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { parse } from 'acorn'; -import { afterEach, describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { sentryCloudflareAutoInstrumentPlugin } from '../../src/vite/autoInstrument'; function parseJS(code: string) { @@ -55,6 +55,95 @@ describe('sentryCloudflareAutoInstrumentPlugin', () => { expect(result.code).toContain('__SENTRY__.withSentry('); }); + it('wraps the entry with `@sentry/nextjs/cloudflare` in a vinext build', async () => { + const dir = writeTempDir({ 'wrangler.toml': 'main = "src/index.ts"' }); + const plugin = sentryCloudflareAutoInstrumentPlugin(); + plugin.configResolved({ root: dir, plugins: [{ name: 'vinext:config' }] }); + const resolve = vi + .fn() + .mockResolvedValue({ id: join(dir, 'node_modules/@sentry/nextjs/build/esm/cloudflare/index.js') }); + + const code = 'export default { fetch() { return new Response("ok"); } };'; + const result = await plugin.transform.call( + { parse: (c: string) => parseJS(c), resolve }, + code, + join(dir, 'src/index.ts'), + ); + + expect(resolve).toHaveBeenCalledWith('@sentry/nextjs/cloudflare', join(dir, 'src/index.ts')); + expect(result.code).toBe( + "import * as __SENTRY__ from '@sentry/nextjs/cloudflare';\n" + + 'const __SENTRY_DEFAULT_EXPORT__ = { fetch() { return new Response("ok"); } };\n' + + 'export default __SENTRY__.withSentry(() => undefined, __SENTRY_DEFAULT_EXPORT__);\n', + ); + }); + + it('wraps the entry with `@sentry/cloudflare` when `@sentry/nextjs/cloudflare` resolves outside a vinext build', async () => { + const dir = writeTempDir({ 'wrangler.toml': 'main = "src/index.ts"' }); + const plugin = sentryCloudflareAutoInstrumentPlugin(); + plugin.configResolved({ root: dir, plugins: [{ name: 'vite:react' }] }); + const resolve = vi + .fn() + .mockResolvedValue({ id: join(dir, 'node_modules/@sentry/nextjs/build/esm/cloudflare/index.js') }); + + const code = 'export default { fetch() { return new Response("ok"); } };'; + const result = await plugin.transform.call( + { parse: (c: string) => parseJS(c), resolve }, + code, + join(dir, 'src/index.ts'), + ); + + expect(result.code).toBe( + "import * as __SENTRY__ from '@sentry/cloudflare';\n" + + 'const __SENTRY_DEFAULT_EXPORT__ = { fetch() { return new Response("ok"); } };\n' + + 'export default __SENTRY__.withSentry(() => undefined, __SENTRY_DEFAULT_EXPORT__);\n', + ); + }); + + it('wraps the entry with `@sentry/cloudflare` in a vinext build without `@sentry/nextjs`', async () => { + const dir = writeTempDir({ 'wrangler.toml': 'main = "src/index.ts"' }); + const plugin = sentryCloudflareAutoInstrumentPlugin(); + plugin.configResolved({ root: dir, plugins: [{ name: 'vinext:config' }] }); + const resolve = vi.fn().mockResolvedValue(null); + + const code = 'export default { fetch() { return new Response("ok"); } };'; + const result = await plugin.transform.call( + { parse: (c: string) => parseJS(c), resolve }, + code, + join(dir, 'src/index.ts'), + ); + + expect(result.code).toBe( + "import * as __SENTRY__ from '@sentry/cloudflare';\n" + + 'const __SENTRY_DEFAULT_EXPORT__ = { fetch() { return new Response("ok"); } };\n' + + 'export default __SENTRY__.withSentry(() => undefined, __SENTRY_DEFAULT_EXPORT__);\n', + ); + }); + + it('wraps the entry with `@sentry/cloudflare` in a vinext build with an `@sentry/nextjs` without the `./cloudflare` entry', async () => { + const dir = writeTempDir({ 'wrangler.toml': 'main = "src/index.ts"' }); + const plugin = sentryCloudflareAutoInstrumentPlugin(); + plugin.configResolved({ root: dir, plugins: [{ name: 'vinext:config' }] }); + const resolve = vi + .fn() + .mockRejectedValue( + new Error('"./cloudflare" is not exported under the conditions ["workerd"] from package @sentry/nextjs'), + ); + + const code = 'export default { fetch() { return new Response("ok"); } };'; + const result = await plugin.transform.call( + { parse: (c: string) => parseJS(c), resolve }, + code, + join(dir, 'src/index.ts'), + ); + + expect(result.code).toBe( + "import * as __SENTRY__ from '@sentry/cloudflare';\n" + + 'const __SENTRY_DEFAULT_EXPORT__ = { fetch() { return new Response("ok"); } };\n' + + 'export default __SENTRY__.withSentry(() => undefined, __SENTRY_DEFAULT_EXPORT__);\n', + ); + }); + it('leaves an already-manually-wrapped entry untouched', async () => { const { transform: tx, entryPath } = createPlugin('main = "src/index.ts"'); From b1638a0c92c7020e580739c1367e4fd7d581fa1d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 13:44:42 +0200 Subject: [PATCH 28/47] feat(deps): Bump next from 16.2.11 to 16.3.8 in /dev-packages/e2e-tests/test-applications/nextjs-sourcemaps (#25170) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [next](https://github.com/vercel/next.js) from 16.2.11 to 16.3.8.
Release notes

Sourced from next's releases.

v16.3.8

This release contains security fixes for the following advisories:

High:

Medium:

Low:

v16.3.7

[!NOTE] This release is backporting bug fixes. It does not include all pending features/changes on canary.

Core Changes

  • turbo-tasks-backend: fix strongly consistent read hanging on a canceled task (#98931)

Credits

Huge thanks to @​lukesandberg for helping!

v16.3.6

This release contains a security fix for GHSA-vcvr-r3jv-pc5j: Remote Code Execution in next/og ImageResponse

v16.3.5

The following bug fixes have been backported. It does not include all pending features/changes on canary.

  • next/image: Skip 0-byte entries when initializing disk LRU cache (#98185)
  • next/image: Reject empty images when reading/writing to the disk cache (#98186)
  • Emit whole-app server NFTs when output: 'standalone' is used with an adapter (#98167)
  • Add CSP nonce to script tags of loading and template files (#98403)
  • Fix use cache prerender signal retention (#98448)

v16.3.4

Follow-up release to v16.3.3 re-enabling AVIF Image Optimization (#97949).

The following bug fixes have been backported. It does not include all pending features/changes on canary.

  • testmode: Fix infinite recursion in testmode passthrough fetch (#97691)
  • Fix build error when aliasing typescript to @​typescript/typescript6 (#97997)
  • Fix unset crossOrigin in Turbopack manifests (#97930)

Credits

... (truncated)

Commits
  • b0fad0d v16.3.8
  • 719e4c6 [lts-active] Scope response cache keys to their source route (#218)
  • e92db45 [lts-active] Fix metadata propagation for deduplicated nested caches (#223)
  • 40c2ba9 [lts-active] Match Next data paths case-sensitively (#196)
  • 2d9f50a [lts-active] Fix MCP middleware DNS rebinding (#213)
  • bd9214f [lts-active] Fix draft mode leaks through cross-request 'use cache' dedupli...
  • 8db4a62 [lts-active][webpack] Ensure dynamicParams is respected in `opengraph-image...
  • e002ad6 [lts-active] fix(next/image): Pin DNS resolution when fetching external image...
  • 4c20699 v16.3.7
  • 2521aec [backport] turbo-tasks-backend: fix strongly consistent read hanging on a can...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=next&package-manager=npm_and_yarn&previous-version=16.2.11&new-version=16.3.8)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/getsentry/sentry-javascript/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../e2e-tests/test-applications/nextjs-sourcemaps/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-packages/e2e-tests/test-applications/nextjs-sourcemaps/package.json b/dev-packages/e2e-tests/test-applications/nextjs-sourcemaps/package.json index ee79cb9ff340..ddfd68285ec3 100644 --- a/dev-packages/e2e-tests/test-applications/nextjs-sourcemaps/package.json +++ b/dev-packages/e2e-tests/test-applications/nextjs-sourcemaps/package.json @@ -10,7 +10,7 @@ }, "dependencies": { "@sentry/nextjs": "file:../../packed/sentry-nextjs-packed.tgz", - "next": "16.2.11", + "next": "16.3.8", "react": "19.1.0", "react-dom": "19.1.0", "typescript": "~5.0.0" From b6616ce3808270295027e6eee6328e250d7e63ae Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Thu, 8 Oct 2026 13:44:58 +0200 Subject: [PATCH 29/47] test(node): Port AMQP integration tests to span streaming (#25158) Port AMQP tests to span streaming. Part of #24137 --------- Co-authored-by: GPT-6 --- .../amqplib/instrument-span-streaming.mjs | 10 - .../suites/tracing/amqplib/instrument.mjs | 1 - .../suites/tracing/amqplib/test.ts | 173 +++++++----------- 3 files changed, 65 insertions(+), 119 deletions(-) delete mode 100644 dev-packages/node-integration-tests/suites/tracing/amqplib/instrument-span-streaming.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/amqplib/instrument-span-streaming.mjs b/dev-packages/node-integration-tests/suites/tracing/amqplib/instrument-span-streaming.mjs deleted file mode 100644 index 54c513068360..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/amqplib/instrument-span-streaming.mjs +++ /dev/null @@ -1,10 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - traceLifecycle: 'stream', - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - transport: loggingTransport, -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/amqplib/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/amqplib/instrument.mjs index 170ad6f6a702..46a27dd03b74 100644 --- a/dev-packages/node-integration-tests/suites/tracing/amqplib/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/amqplib/instrument.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/amqplib/test.ts b/dev-packages/node-integration-tests/suites/tracing/amqplib/test.ts index fade84b952a4..9e162e07490c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/amqplib/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/amqplib/test.ts @@ -1,4 +1,19 @@ -import type { TransactionEvent } from '@sentry/core'; +import { + MESSAGING_DESTINATION_NAME, + MESSAGING_OPERATION_NAME, + MESSAGING_OPERATION_TYPE, + MESSAGING_RABBITMQ_DESTINATION_ROUTING_KEY, + MESSAGING_SYSTEM, + NETWORK_PROTOCOL_NAME, + NETWORK_PROTOCOL_VERSION, + SENTRY_KIND, + SENTRY_OP, + SENTRY_ORIGIN, + SERVER_ADDRESS, + SERVER_PORT, + URL_FULL, +} from '@sentry/conventions/attributes'; +import { QUEUE_PROCESS, QUEUE_PUBLISH } from '@sentry/conventions/op'; import { afterAll, describe, expect } from 'vitest'; import { cleanupChildProcesses, createEsmAndCjsTests, describeWithDockerCompose } from '../../../utils/runner'; @@ -8,37 +23,35 @@ import { cleanupChildProcesses, createEsmAndCjsTests, describeWithDockerCompose // default (empty) exchange with the queue name as the routing key. const expectedProducerSpan = (routingKey: string) => expect.objectContaining({ - op: 'queue.publish', - data: expect.objectContaining({ - 'messaging.system': 'rabbitmq', - 'messaging.operation.name': 'send', - 'messaging.operation.type': 'send', - 'messaging.destination.name': routingKey, - 'messaging.rabbitmq.destination.routing_key': routingKey, - 'network.protocol.name': 'AMQP', - 'network.protocol.version': '0.9.1', - 'server.address': 'localhost', - 'server.port': 5672, - 'url.full': 'amqp://sentry:***@localhost:5672/', - 'sentry.kind': 'producer', - 'sentry.op': 'queue.publish', - 'sentry.origin': 'auto.amqplib.publisher', + attributes: expect.objectContaining({ + [MESSAGING_SYSTEM]: { type: 'string', value: 'rabbitmq' }, + [MESSAGING_OPERATION_NAME]: { type: 'string', value: 'send' }, + [MESSAGING_OPERATION_TYPE]: { type: 'string', value: 'send' }, + [MESSAGING_DESTINATION_NAME]: { type: 'string', value: routingKey }, + [MESSAGING_RABBITMQ_DESTINATION_ROUTING_KEY]: { type: 'string', value: routingKey }, + [NETWORK_PROTOCOL_NAME]: { type: 'string', value: 'AMQP' }, + [NETWORK_PROTOCOL_VERSION]: { type: 'string', value: '0.9.1' }, + [SERVER_ADDRESS]: { type: 'string', value: 'localhost' }, + [SERVER_PORT]: { type: 'integer', value: 5672 }, + [URL_FULL]: { type: 'string', value: 'amqp://sentry:***@localhost:5672/' }, + [SENTRY_KIND]: { type: 'string', value: 'producer' }, + [SENTRY_OP]: { type: 'string', value: QUEUE_PUBLISH }, + [SENTRY_ORIGIN]: { type: 'string', value: 'auto.amqplib.publisher' }, }), status: 'ok', }); const EXPECTED_MESSAGE_SPAN_CONSUMER = expect.objectContaining({ - op: 'queue.process', - data: expect.objectContaining({ - 'messaging.system': 'rabbitmq', + attributes: expect.objectContaining({ + [MESSAGING_SYSTEM]: { type: 'string', value: 'rabbitmq' }, // The delivery carries the default exchange (''), so the routing key is the destination. - 'messaging.destination.name': 'queue1', - 'messaging.rabbitmq.destination.routing_key': 'queue1', - 'messaging.operation.name': 'process', - 'messaging.operation.type': 'process', - 'sentry.kind': 'consumer', - 'sentry.op': 'queue.process', - 'sentry.origin': 'auto.amqplib.consumer', + [MESSAGING_DESTINATION_NAME]: { type: 'string', value: 'queue1' }, + [MESSAGING_RABBITMQ_DESTINATION_ROUTING_KEY]: { type: 'string', value: 'queue1' }, + [MESSAGING_OPERATION_NAME]: { type: 'string', value: 'process' }, + [MESSAGING_OPERATION_TYPE]: { type: 'string', value: 'process' }, + [SENTRY_KIND]: { type: 'string', value: 'consumer' }, + [SENTRY_OP]: { type: 'string', value: QUEUE_PROCESS }, + [SENTRY_ORIGIN]: { type: 'string', value: 'auto.amqplib.consumer' }, }), status: 'ok', }); @@ -59,40 +72,27 @@ describeWithDockerCompose('amqplib auto-instrumentation', { workingDirectory: [_ 'instrument.mjs', (createTestRunner, test) => { test('should be able to send and receive messages', { timeout: 60_000 }, async () => { - // The producer ('root span') and consumer ('queue1 process') transactions can - // arrive in any order, so we collect them and assert after both are received. - const receivedTransactions: TransactionEvent[] = []; - await createTestRunner() .expect({ - transaction: (transaction: TransactionEvent) => { - receivedTransactions.push(transaction); - }, - }) - .expect({ - transaction: (transaction: TransactionEvent) => { - receivedTransactions.push(transaction); - - // The producer span is a child of the manually-started 'root span' transaction, so we - // identify it by its origin rather than by transaction name. The consumer span is its - // own transaction, identified by the origin on its trace context. - const producer = receivedTransactions.find(t => - t.spans?.some(s => s.data?.['sentry.origin'] === 'auto.amqplib.publisher'), - ); - const consumer = receivedTransactions.find( - t => t.contexts?.trace?.data?.['sentry.origin'] === 'auto.amqplib.consumer', + span: container => { + const producer = container.items.find(span => span.is_segment && span.name === 'root span'); + const consumer = container.items.find( + t => t.attributes[SENTRY_ORIGIN]?.value === 'auto.amqplib.consumer', ); expect(producer).toBeDefined(); expect(consumer).toBeDefined(); - expect(producer!.transaction).toBe('root span'); - expect(consumer!.transaction).toBe('queue1 process'); + expect(producer!.name).toBe('root span'); + expect(consumer!.name).toBe('process queue1'); - const producerSpan = producer!.spans?.find(s => s.data?.['sentry.origin'] === 'auto.amqplib.publisher'); + const producerSpan = container.items.find( + s => s.attributes[SENTRY_ORIGIN]?.value === 'auto.amqplib.publisher', + ); + expect(producerSpan?.name).toBe('send queue1'); expect(producerSpan).toMatchObject(expectedProducerSpan('queue1')); - expect(consumer!.contexts?.trace).toMatchObject(EXPECTED_MESSAGE_SPAN_CONSUMER); + expect(consumer!).toMatchObject(EXPECTED_MESSAGE_SPAN_CONSUMER); }, }) .start() @@ -108,35 +108,23 @@ describeWithDockerCompose('amqplib auto-instrumentation', { workingDirectory: [_ 'instrument.mjs', (createTestRunner, test) => { test('marks the consumer span as errored when the message is rejected', { timeout: 60_000 }, async () => { - // The error scenario emits the producer ('root span') and the rejected consumer - // ('queue1 process') transactions in any order, so we collect both and assert on the consumer. - const receivedTransactions: TransactionEvent[] = []; - await createTestRunner() .expect({ - transaction: (transaction: TransactionEvent) => { - receivedTransactions.push(transaction); - }, - }) - .expect({ - transaction: (transaction: TransactionEvent) => { - receivedTransactions.push(transaction); - - const consumer = receivedTransactions.find( - t => t.contexts?.trace?.data?.['sentry.origin'] === 'auto.amqplib.consumer', + span: container => { + const consumer = container.items.find( + t => t.attributes[SENTRY_ORIGIN]?.value === 'auto.amqplib.consumer', ); expect(consumer).toBeDefined(); - expect(consumer!.transaction).toBe('queue-error process'); - expect(consumer!.contexts?.trace).toMatchObject( + expect(consumer!.name).toBe('process queue-error'); + expect(consumer!).toMatchObject( expect.objectContaining({ - op: 'queue.process', - status: 'internal_error', - data: expect.objectContaining({ - 'messaging.system': 'rabbitmq', - 'sentry.kind': 'consumer', - 'sentry.op': 'queue.process', - 'sentry.origin': 'auto.amqplib.consumer', + status: 'error', + attributes: expect.objectContaining({ + [MESSAGING_SYSTEM]: { type: 'string', value: 'rabbitmq' }, + [SENTRY_KIND]: { type: 'string', value: 'consumer' }, + [SENTRY_OP]: { type: 'string', value: QUEUE_PROCESS }, + [SENTRY_ORIGIN]: { type: 'string', value: 'auto.amqplib.consumer' }, }), }), ); @@ -160,11 +148,11 @@ describeWithDockerCompose('amqplib auto-instrumentation', { workingDirectory: [_ async () => { await createTestRunner() .expect({ - transaction: (transaction: TransactionEvent) => { - expect(transaction.transaction).toBe('root span'); + span: container => { + expect(container.items.find(span => span.is_segment)?.name).toBe('root span'); - const producerSpans = transaction.spans?.filter( - s => s.data?.['sentry.origin'] === 'auto.amqplib.publisher', + const producerSpans = container.items.filter( + s => s.attributes[SENTRY_ORIGIN]?.value === 'auto.amqplib.publisher', ); // The confirm channel internally calls the base publish; the instrumentation must not @@ -181,35 +169,4 @@ describeWithDockerCompose('amqplib auto-instrumentation', { workingDirectory: [_ { additionalDependencies }, ); }); - - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-span-streaming.mjs', (createTestRunner, test) => { - test('names streamed spans after the messaging conventions', { timeout: 60_000 }, async () => { - await createTestRunner() - .ignore('event') - .expect({ - span: container => { - // `sendToQueue` publishes to the default exchange, which has no name. Its routing key is the - // queue name, so it is the destination rather than per-message data. - for (const origin of ['auto.amqplib.publisher', 'auto.amqplib.consumer']) { - const span = container.items.find(item => item.attributes['sentry.origin']?.value === origin); - expect(span).toBeDefined(); - expect(span!.attributes['messaging.destination.name']?.value).toBe('queue1'); - expect(span!.attributes['messaging.rabbitmq.destination.routing_key']?.value).toBe('queue1'); - } - - const producerSpan = container.items.find( - span => span.attributes['sentry.origin']?.value === 'auto.amqplib.publisher', - ); - expect(producerSpan!.name).toBe('send queue1'); - - const consumerSpan = container.items.find( - span => span.attributes['sentry.origin']?.value === 'auto.amqplib.consumer', - ); - expect(consumerSpan!.name).toBe('process queue1'); - }, - }) - .start() - .completed(); - }); - }); }); From 8acc0c8786c93dc94ce831a3076053cc2838a949 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Thu, 8 Oct 2026 13:46:09 +0200 Subject: [PATCH 30/47] test(node): Port pool and DataLoader integration tests to span streaming (#25156) Port generic-pool and dataloader tests to span streaming. Part of #24137 --------- Co-authored-by: GPT-6 --- .../suites/tracing/dataloader/instrument.mjs | 1 - .../suites/tracing/dataloader/test.ts | 135 ++++++++---------- .../tracing/genericPool-v2/instrument.mjs | 1 - .../suites/tracing/genericPool-v2/test.ts | 48 ++++--- .../suites/tracing/genericPool/instrument.mjs | 1 - .../suites/tracing/genericPool/test.ts | 49 ++++--- 6 files changed, 106 insertions(+), 129 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/dataloader/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/dataloader/instrument.mjs index 22bf57f14364..46a27dd03b74 100644 --- a/dev-packages/node-integration-tests/suites/tracing/dataloader/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/dataloader/instrument.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: process.env.STREAMED === 'true' ? 'stream' : 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/dataloader/test.ts b/dev-packages/node-integration-tests/suites/tracing/dataloader/test.ts index b49004cf02a9..620a4311d083 100644 --- a/dev-packages/node-integration-tests/suites/tracing/dataloader/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/dataloader/test.ts @@ -1,4 +1,13 @@ -import type { SerializedStreamedSpanContainer } from '@sentry/core'; +import { + CACHE_KEY, + CACHE_OPERATION, + DB_COLLECTION_NAME, + DB_OPERATION_NAME, + SENTRY_KIND, + SENTRY_OP, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import { CACHE_GET, CACHE_PUT, CACHE_REMOVE } from '@sentry/conventions/op'; import { afterAll, describe, expect } from 'vitest'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; @@ -6,8 +15,7 @@ import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runn // enabled (via INJECT_ORCHESTRION) the OTel `Dataloader` integration is swapped for the // diagnostics-channel one, which stamps a different origin. const ORIGIN = 'auto.db.dataloader'; -const CACHE_GET_OP = 'cache.get'; -const CACHE_MUTATION_OPS = { prime: 'cache.put', clear: 'cache.remove', clearAll: 'cache.remove' } as const; +const CACHE_MUTATION_OPS = { prime: CACHE_PUT, clear: CACHE_REMOVE, clearAll: CACHE_REMOVE } as const; describe('dataloader auto-instrumentation', () => { afterAll(() => { @@ -18,30 +26,28 @@ describe('dataloader auto-instrumentation', () => { test('instruments load, loadMany, batch, prime, clear and clearAll', async () => { const runner = createRunner() .expect({ - transaction: event => { - expect(event.transaction).toBe('GET /load'); + span: container => { + expect(container.items.find(span => span.is_segment)?.name).toBe('GET /load'); - const spans = event.spans || []; + const spans = container.items; - const loadSpan = spans.find(span => span.description === 'dataloader.load'); + const loadSpan = spans.find(span => span.attributes[DB_OPERATION_NAME]?.value === 'load'); expect(loadSpan).toBeDefined(); - expect(loadSpan?.op).toBe(CACHE_GET_OP); - expect(loadSpan?.origin).toBe(ORIGIN); + expect(loadSpan?.attributes[SENTRY_OP]?.value).toBe(CACHE_GET); + expect(loadSpan?.attributes[SENTRY_ORIGIN]?.value).toBe(ORIGIN); expect(loadSpan?.status).toBe('ok'); - expect(loadSpan?.data?.['sentry.origin']).toBe(ORIGIN); - expect(loadSpan?.data?.['sentry.op']).toBe(CACHE_GET_OP); - expect(loadSpan?.data?.['cache.key']).toEqual(['user-1']); - expect(loadSpan?.data?.['db.operation.name']).toBe('load'); + expect(loadSpan?.attributes[CACHE_KEY]?.value).toEqual(['user-1']); + expect(loadSpan?.attributes[DB_OPERATION_NAME]?.value).toBe('load'); // A direct operation is a client call; the deferred `batch` below gets no kind - expect(loadSpan?.data?.['sentry.kind']).toBe('client'); + expect(loadSpan?.attributes[SENTRY_KIND]?.value).toBe('client'); - const batchSpan = spans.find(span => span.description === 'dataloader.batch'); + const batchSpan = spans.find(span => span.attributes[DB_OPERATION_NAME]?.value === 'batch'); expect(batchSpan).toBeDefined(); - expect(batchSpan?.op).toBe(CACHE_GET_OP); - expect(batchSpan?.origin).toBe(ORIGIN); + expect(batchSpan?.attributes[SENTRY_OP]?.value).toBe(CACHE_GET); + expect(batchSpan?.attributes[SENTRY_ORIGIN]?.value).toBe(ORIGIN); expect(batchSpan?.status).toBe('ok'); - expect(batchSpan?.data?.['cache.key']).toEqual(['user-1']); - expect(batchSpan?.data?.['sentry.kind']).toBeUndefined(); + expect(batchSpan?.attributes[CACHE_KEY]?.value).toEqual(['user-1']); + expect(batchSpan?.attributes[SENTRY_KIND]?.value).toBeUndefined(); // The batch span links back to the load span that triggered it expect(batchSpan?.links).toEqual([ @@ -54,57 +60,62 @@ describe('dataloader auto-instrumentation', () => { // Locks down the async behavior: `load` encloses the deferred `batch` span expect(batchSpan?.parent_span_id).toBe(loadSpan?.span_id); expect(loadSpan?.start_timestamp).toBeLessThanOrEqual(batchSpan?.start_timestamp ?? 0); - expect(loadSpan?.timestamp).toBeGreaterThanOrEqual(batchSpan?.timestamp ?? 0); + expect(loadSpan?.end_timestamp).toBeGreaterThanOrEqual(batchSpan?.end_timestamp ?? 0); }, }) .expect({ - transaction: event => { - expect(event.transaction).toBe('GET /load-many'); + span: container => { + expect(container.items.find(span => span.is_segment)?.name).toBe('GET /load-many'); - const loadManySpan = (event.spans || []).find(span => span.description === 'dataloader.loadMany'); + const loadManySpan = container.items.find(span => span.attributes[DB_OPERATION_NAME]?.value === 'loadMany'); expect(loadManySpan).toBeDefined(); - expect(loadManySpan?.op).toBe(CACHE_GET_OP); - expect(loadManySpan?.origin).toBe(ORIGIN); + expect(loadManySpan?.attributes[SENTRY_OP]?.value).toBe(CACHE_GET); + expect(loadManySpan?.attributes[SENTRY_ORIGIN]?.value).toBe(ORIGIN); expect(loadManySpan?.status).toBe('ok'); - expect(loadManySpan?.data?.['sentry.origin']).toBe(ORIGIN); - expect(loadManySpan?.data?.['sentry.op']).toBe(CACHE_GET_OP); - expect(loadManySpan?.data?.['cache.key']).toEqual(['user-1', 'user-2']); + expect(loadManySpan?.attributes[CACHE_KEY]?.value).toEqual(['user-1', 'user-2']); }, }) .expect({ - transaction: event => { - expect(event.transaction).toBe('GET /cache-ops'); + span: container => { + expect(container.items.find(span => span.is_segment)?.name).toBe('GET /cache-ops'); - const spans = event.spans || []; + const spans = container.items; // prime writes to the cache, clear/clearAll remove from it for (const [operation, op] of Object.entries(CACHE_MUTATION_OPS)) { - const span = spans.find(s => s.description === `dataloader.${operation}`); + const span = spans.find(s => s.attributes[DB_OPERATION_NAME]?.value === operation); expect(span, `expected a dataloader.${operation} span`).toBeDefined(); - expect(span?.origin).toBe(ORIGIN); + expect(span?.attributes[SENTRY_ORIGIN]?.value).toBe(ORIGIN); expect(span?.status).toBe('ok'); - expect(span?.op).toBe(op); - expect(span?.data?.['sentry.origin']).toBe(ORIGIN); - expect(span?.data?.['db.operation.name']).toBe(operation); + expect(span?.name).toBe(op); + expect(span?.attributes[SENTRY_OP]?.value).toBe(op); + expect(span?.attributes[DB_OPERATION_NAME]?.value).toBe(operation); } // `clearAll` takes no key, the other two act on a single key - expect(spans.find(s => s.description === 'dataloader.prime')?.data?.['cache.key']).toEqual(['user-1']); - expect(spans.find(s => s.description === 'dataloader.clear')?.data?.['cache.key']).toEqual(['user-1']); - expect(spans.find(s => s.description === 'dataloader.clearAll')?.data?.['cache.key']).toBeUndefined(); + expect( + spans.find(s => s.attributes[DB_OPERATION_NAME]?.value === 'prime')?.attributes[CACHE_KEY]?.value, + ).toEqual(['user-1']); + expect( + spans.find(s => s.attributes[DB_OPERATION_NAME]?.value === 'clear')?.attributes[CACHE_KEY]?.value, + ).toEqual(['user-1']); + expect( + spans.find(s => s.attributes[DB_OPERATION_NAME]?.value === 'clearAll')?.attributes[CACHE_KEY]?.value, + ).toBeUndefined(); }, }) .expect({ - transaction: event => { - expect(event.transaction).toBe('GET /named'); + span: container => { + expect(container.items.find(span => span.is_segment)?.name).toBe('GET /named'); - // A named dataloader includes its name in the span description - const namedLoadSpan = (event.spans || []).find(span => span.description === 'dataloader.load usersLoader'); + const namedLoadSpan = container.items.find(span => span.attributes[DB_OPERATION_NAME]?.value === 'load'); expect(namedLoadSpan).toBeDefined(); - expect(namedLoadSpan?.op).toBe(CACHE_GET_OP); - expect(namedLoadSpan?.origin).toBe(ORIGIN); + expect(namedLoadSpan?.attributes[SENTRY_OP]?.value).toBe(CACHE_GET); + expect(namedLoadSpan?.attributes[SENTRY_ORIGIN]?.value).toBe(ORIGIN); expect(namedLoadSpan?.status).toBe('ok'); - expect(namedLoadSpan?.data?.['db.collection.name']).toBe('usersLoader'); + expect(namedLoadSpan?.name).toBe(CACHE_GET); + expect(namedLoadSpan?.attributes[CACHE_OPERATION]?.value).toBe('get'); + expect(namedLoadSpan?.attributes[DB_COLLECTION_NAME]?.value).toBe('usersLoader'); }, }) .start(); @@ -115,37 +126,5 @@ describe('dataloader auto-instrumentation', () => { await runner.makeRequest('get', '/named'); await runner.completed(); }, 30_000); - - test('names spans after the cache operation when streamed', async () => { - const runner = createRunner() - .withEnv({ STREAMED: 'true' }) - .expect({ - span: (container: SerializedStreamedSpanContainer) => { - const namedLoadSpan = container.items.find( - span => span.attributes?.['db.operation.name']?.value === 'load', - ); - expect(namedLoadSpan?.name).toBe('cache.get'); - expect(namedLoadSpan?.attributes?.['sentry.op']?.value).toBe(CACHE_GET_OP); - expect(namedLoadSpan?.attributes?.['cache.operation']?.value).toBe('get'); - // The loader name is no longer part of the span name, it moved to `db.collection.name`. - expect(namedLoadSpan?.attributes?.['db.collection.name']?.value).toBe('usersLoader'); - }, - }) - .expect({ - span: (container: SerializedStreamedSpanContainer) => { - for (const [operation, op] of Object.entries(CACHE_MUTATION_OPS)) { - const span = container.items.find(item => item.attributes?.['db.operation.name']?.value === operation); - expect(span, `expected a ${operation} span`).toBeDefined(); - expect(span?.name).toBe(op); - expect(span?.attributes?.['sentry.op']?.value).toBe(op); - } - }, - }) - .start(); - - await runner.makeRequest('get', '/named'); - await runner.makeRequest('get', '/cache-ops'); - await runner.completed(); - }, 30_000); }); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/genericPool-v2/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/genericPool-v2/instrument.mjs index 170ad6f6a702..46a27dd03b74 100644 --- a/dev-packages/node-integration-tests/suites/tracing/genericPool-v2/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/genericPool-v2/instrument.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/genericPool-v2/test.ts b/dev-packages/node-integration-tests/suites/tracing/genericPool-v2/test.ts index 5f744edbbf3f..08f30f1545ef 100644 --- a/dev-packages/node-integration-tests/suites/tracing/genericPool-v2/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/genericPool-v2/test.ts @@ -1,3 +1,5 @@ +import { ERROR_TYPE, SENTRY_OP, SENTRY_ORIGIN } from '@sentry/conventions/attributes'; +import { DB } from '@sentry/conventions/op'; import { afterAll, describe, expect } from 'vitest'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; @@ -12,32 +14,30 @@ describe('genericPool v2 auto instrumentation', () => { 'instrument.mjs', (createRunner, test) => { test('should auto-instrument `generic-pool` v2 when calling pool.acquire()', async () => { - const EXPECTED_TRANSACTION = { - transaction: 'Test Transaction', - spans: expect.arrayContaining([ + const EXPECTED_SPANS = { + items: expect.arrayContaining([ + expect.objectContaining({ name: 'Test Transaction', is_segment: true }), expect.objectContaining({ - description: 'generic-pool.acquire', - op: 'db', - origin: 'auto.db.generic_pool', - data: expect.objectContaining({ - 'sentry.origin': 'auto.db.generic_pool', + name: 'generic-pool.acquire', + attributes: expect.objectContaining({ + [SENTRY_OP]: { type: 'string', value: DB }, + [SENTRY_ORIGIN]: { type: 'string', value: 'auto.db.generic_pool' }, }), status: 'ok', }), expect.objectContaining({ - description: 'generic-pool.acquire', - op: 'db', - origin: 'auto.db.generic_pool', - data: expect.objectContaining({ - 'sentry.origin': 'auto.db.generic_pool', + name: 'generic-pool.acquire', + attributes: expect.objectContaining({ + [SENTRY_OP]: { type: 'string', value: DB }, + [SENTRY_ORIGIN]: { type: 'string', value: 'auto.db.generic_pool' }, }), status: 'ok', }), ]), }; - await createRunner().expect({ transaction: EXPECTED_TRANSACTION }).start().completed(); + await createRunner().expect({ span: EXPECTED_SPANS }).start().completed(); }); }, { additionalDependencies: { 'generic-pool': '^2.5.0' } }, @@ -49,20 +49,22 @@ describe('genericPool v2 auto instrumentation', () => { 'instrument.mjs', (createRunner, test) => { test('marks the `generic-pool.acquire` span as errored when acquiring fails', async () => { - const EXPECTED_TRANSACTION = { - transaction: 'Test Transaction', - spans: expect.arrayContaining([ + const EXPECTED_SPANS = { + items: expect.arrayContaining([ + expect.objectContaining({ name: 'Test Transaction', is_segment: true }), expect.objectContaining({ - description: 'generic-pool.acquire', - op: 'db', - origin: 'auto.db.generic_pool', - data: expect.objectContaining({ 'sentry.origin': 'auto.db.generic_pool', 'error.type': 'Error' }), - status: 'internal_error', + name: 'generic-pool.acquire', + attributes: expect.objectContaining({ + [SENTRY_OP]: { type: 'string', value: DB }, + [SENTRY_ORIGIN]: { type: 'string', value: 'auto.db.generic_pool' }, + [ERROR_TYPE]: { type: 'string', value: 'Error' }, + }), + status: 'error', }), ]), }; - await createRunner().expect({ transaction: EXPECTED_TRANSACTION }).start().completed(); + await createRunner().expect({ span: EXPECTED_SPANS }).start().completed(); }); }, { additionalDependencies: { 'generic-pool': '^2.5.0' } }, diff --git a/dev-packages/node-integration-tests/suites/tracing/genericPool/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/genericPool/instrument.mjs index 170ad6f6a702..46a27dd03b74 100644 --- a/dev-packages/node-integration-tests/suites/tracing/genericPool/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/genericPool/instrument.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/genericPool/test.ts b/dev-packages/node-integration-tests/suites/tracing/genericPool/test.ts index 2c607666691a..bce47d5f779f 100644 --- a/dev-packages/node-integration-tests/suites/tracing/genericPool/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/genericPool/test.ts @@ -1,3 +1,5 @@ +import { ERROR_TYPE, SENTRY_OP, SENTRY_ORIGIN } from '@sentry/conventions/attributes'; +import { DB } from '@sentry/conventions/op'; import { afterAll, describe, expect } from 'vitest'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; @@ -8,54 +10,51 @@ describe('genericPool auto instrumentation', () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { test('should auto-instrument `genericPool` package when calling pool.require()', async () => { - const EXPECTED_TRANSACTION = { - transaction: 'Test Transaction', - spans: expect.arrayContaining([ + const EXPECTED_SPANS = { + items: expect.arrayContaining([ + expect.objectContaining({ name: 'Test Transaction', is_segment: true }), expect.objectContaining({ - description: 'generic-pool.acquire', - op: 'db', - origin: 'auto.db.generic_pool', - data: expect.objectContaining({ - 'sentry.origin': 'auto.db.generic_pool', + name: 'generic-pool.acquire', + attributes: expect.objectContaining({ + [SENTRY_OP]: { type: 'string', value: DB }, + [SENTRY_ORIGIN]: { type: 'string', value: 'auto.db.generic_pool' }, }), status: 'ok', }), expect.objectContaining({ - description: 'generic-pool.acquire', - op: 'db', - origin: 'auto.db.generic_pool', - data: expect.objectContaining({ - 'sentry.origin': 'auto.db.generic_pool', + name: 'generic-pool.acquire', + attributes: expect.objectContaining({ + [SENTRY_OP]: { type: 'string', value: DB }, + [SENTRY_ORIGIN]: { type: 'string', value: 'auto.db.generic_pool' }, }), status: 'ok', }), ]), }; - await createRunner().expect({ transaction: EXPECTED_TRANSACTION }).start().completed(); + await createRunner().expect({ span: EXPECTED_SPANS }).start().completed(); }); }); createEsmAndCjsTests(__dirname, 'scenario-error.mjs', 'instrument.mjs', (createRunner, test) => { test('marks the `generic-pool.acquire` span as errored when acquiring fails', async () => { - const EXPECTED_TRANSACTION = { - transaction: 'Test Transaction', - spans: expect.arrayContaining([ + const EXPECTED_SPANS = { + items: expect.arrayContaining([ + expect.objectContaining({ name: 'Test Transaction', is_segment: true }), expect.objectContaining({ - description: 'generic-pool.acquire', - op: 'db', - origin: 'auto.db.generic_pool', - data: expect.objectContaining({ - 'sentry.origin': 'auto.db.generic_pool', - 'error.type': 'TimeoutError', + name: 'generic-pool.acquire', + attributes: expect.objectContaining({ + [SENTRY_OP]: { type: 'string', value: DB }, + [SENTRY_ORIGIN]: { type: 'string', value: 'auto.db.generic_pool' }, + [ERROR_TYPE]: { type: 'string', value: 'TimeoutError' }, }), - status: 'internal_error', + status: 'error', }), ]), }; - await createRunner().expect({ transaction: EXPECTED_TRANSACTION }).start().completed(); + await createRunner().expect({ span: EXPECTED_SPANS }).start().completed(); }); }); }); From cf24f0cb7428f5978105f560dfe22f886a16196d Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Thu, 8 Oct 2026 13:46:29 +0200 Subject: [PATCH 31/47] test(node): Port Knex integration tests to span streaming (#25154) Port the Knex MySQL2 and Postgres tests to default span streaming, preserving their assertions and folding in the existing query-name checks. Part of #24137 --------- Co-authored-by: GPT-6 --- .../knex/mysql2/instrument-span-streaming.mjs | 10 - .../suites/tracing/knex/mysql2/instrument.mjs | 1 - .../suites/tracing/knex/mysql2/test.ts | 153 +++++++-------- .../knex/pg/instrument-span-streaming.mjs | 10 - .../suites/tracing/knex/pg/instrument.mjs | 1 - .../suites/tracing/knex/pg/test.ts | 181 ++++++++---------- 6 files changed, 159 insertions(+), 197 deletions(-) delete mode 100644 dev-packages/node-integration-tests/suites/tracing/knex/mysql2/instrument-span-streaming.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/knex/pg/instrument-span-streaming.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/knex/mysql2/instrument-span-streaming.mjs b/dev-packages/node-integration-tests/suites/tracing/knex/mysql2/instrument-span-streaming.mjs deleted file mode 100644 index 53b9511a21f0..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/knex/mysql2/instrument-span-streaming.mjs +++ /dev/null @@ -1,10 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/knex/mysql2/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/knex/mysql2/instrument.mjs index 170ad6f6a702..46a27dd03b74 100644 --- a/dev-packages/node-integration-tests/suites/tracing/knex/mysql2/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/knex/mysql2/instrument.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/knex/mysql2/test.ts b/dev-packages/node-integration-tests/suites/tracing/knex/mysql2/test.ts index 98cb1127906b..ec3947f54f4a 100644 --- a/dev-packages/node-integration-tests/suites/tracing/knex/mysql2/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/knex/mysql2/test.ts @@ -1,3 +1,16 @@ +import { + DB_NAMESPACE, + DB_OPERATION_NAME, + DB_QUERY_SUMMARY, + DB_QUERY_TEXT, + DB_SYSTEM_NAME, + DB_USER, + SENTRY_OP, + SENTRY_ORIGIN, + SERVER_ADDRESS, + SERVER_PORT, +} from '@sentry/conventions/attributes'; +import { DB } from '@sentry/conventions/op'; import { describe, expect } from 'vitest'; import { createEsmAndCjsTests, describeWithDockerCompose } from '../../../../utils/runner'; @@ -9,93 +22,75 @@ describeWithDockerCompose('knex auto instrumentation', { workingDirectory: [__di describe('with `mysql2` client', () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { test('should auto-instrument `knex` package', { timeout: 60_000 }, async () => { - const EXPECTED_TRANSACTION = { - transaction: 'Test Transaction', - spans: expect.arrayContaining([ - expect.objectContaining({ - data: expect.objectContaining({ - 'knex.version': KNEX_VERSION, - 'db.system.name': 'mysql2', - 'db.namespace': 'tests', - 'db.user': 'root', - 'sentry.origin': ORIGIN, - 'sentry.op': 'db', - 'server.address': 'localhost', - 'server.port': 3307, - }), - status: 'ok', - description: - 'create table `User` (`id` int unsigned not null auto_increment primary key, `createdAt` timestamp(?) not null default CURRENT_TIMESTAMP(?), `email` text not null, `name` text not null)', - origin: ORIGIN, + const EXPECTED_SPANS = [ + expect.objectContaining({ + attributes: expect.objectContaining({ + 'knex.version': { type: 'string', value: KNEX_VERSION }, + [DB_SYSTEM_NAME]: { type: 'string', value: 'mysql2' }, + [DB_NAMESPACE]: { type: 'string', value: 'tests' }, + [DB_USER]: { type: 'string', value: 'root' }, + [SENTRY_ORIGIN]: { type: 'string', value: ORIGIN }, + [SENTRY_OP]: { type: 'string', value: DB }, + [SERVER_ADDRESS]: { type: 'string', value: 'localhost' }, + [SERVER_PORT]: { type: 'integer', value: 3307 }, + [DB_QUERY_TEXT]: { + type: 'string', + value: + 'create table `User` (`id` int unsigned not null auto_increment primary key, `createdAt` timestamp(?) not null default CURRENT_TIMESTAMP(?), `email` text not null, `name` text not null)', + }, + [DB_QUERY_SUMMARY]: { type: 'string', value: 'create table `User`' }, }), - expect.objectContaining({ - data: expect.objectContaining({ - 'knex.version': KNEX_VERSION, - 'db.system.name': 'mysql2', - 'db.namespace': 'tests', - 'db.user': 'root', - 'sentry.origin': ORIGIN, - 'sentry.op': 'db', - 'server.address': 'localhost', - 'server.port': 3307, - }), - status: 'ok', - description: 'insert into `User` (`email`, `name`) values (?, ?)', - origin: ORIGIN, + status: 'ok', + name: 'create table `User`', + }), + expect.objectContaining({ + attributes: expect.objectContaining({ + 'knex.version': { type: 'string', value: KNEX_VERSION }, + [DB_SYSTEM_NAME]: { type: 'string', value: 'mysql2' }, + [DB_NAMESPACE]: { type: 'string', value: 'tests' }, + [DB_USER]: { type: 'string', value: 'root' }, + [SENTRY_ORIGIN]: { type: 'string', value: ORIGIN }, + [SENTRY_OP]: { type: 'string', value: DB }, + [SERVER_ADDRESS]: { type: 'string', value: 'localhost' }, + [SERVER_PORT]: { type: 'integer', value: 3307 }, + [DB_QUERY_TEXT]: { type: 'string', value: 'insert into `User` (`email`, `name`) values (?, ?)' }, + [DB_QUERY_SUMMARY]: { type: 'string', value: 'insert `User`' }, }), + status: 'ok', + name: 'insert `User`', + }), - expect.objectContaining({ - data: expect.objectContaining({ - 'knex.version': KNEX_VERSION, - 'db.operation.name': 'select', - 'db.sql.table': 'User', - 'db.system.name': 'mysql2', - 'db.namespace': 'tests', - 'db.query.text': 'select * from `User`', - 'db.user': 'root', - 'sentry.origin': ORIGIN, - 'sentry.op': 'db', - }), - status: 'ok', - description: 'select * from `User`', - origin: ORIGIN, + expect.objectContaining({ + attributes: expect.objectContaining({ + 'knex.version': { type: 'string', value: KNEX_VERSION }, + [DB_OPERATION_NAME]: { type: 'string', value: 'select' }, + 'db.sql.table': { type: 'string', value: 'User' }, + [DB_SYSTEM_NAME]: { type: 'string', value: 'mysql2' }, + [DB_NAMESPACE]: { type: 'string', value: 'tests' }, + [DB_QUERY_TEXT]: { type: 'string', value: 'select * from `User`' }, + [DB_USER]: { type: 'string', value: 'root' }, + [SENTRY_ORIGIN]: { type: 'string', value: ORIGIN }, + [SENTRY_OP]: { type: 'string', value: DB }, + [DB_QUERY_SUMMARY]: { type: 'string', value: 'select `User`' }, }), - ]), - }; - - await createRunner().expect({ transaction: EXPECTED_TRANSACTION }).start().completed(); - }); - }); + status: 'ok', + name: 'select `User`', + }), + expect.objectContaining({ + attributes: expect.objectContaining({ + [DB_QUERY_TEXT]: { type: 'string', value: 'drop table `User`' }, + [DB_QUERY_SUMMARY]: { type: 'string', value: 'drop table `User`' }, + }), + name: 'drop table `User`', + }), + ]; - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-span-streaming.mjs', (createRunner, test) => { - test('should name spans after the query summary with span streaming', { timeout: 60_000 }, async () => { await createRunner() .expect({ span: container => { - // The `mysql2` driver spans underneath the knex spans come from a different integration and - // are named after the full statement, so they are filtered out here. - const knexSpans = container.items.filter(item => item.attributes['sentry.origin']?.value === ORIGIN); - - expect( - knexSpans.map(span => ({ - name: span.name, - summary: span.attributes['db.query.summary']?.value, - text: span.attributes['db.query.text']?.value, - })), - ).toEqual([ - { - name: 'create table `User`', - summary: 'create table `User`', - text: 'create table `User` (`id` int unsigned not null auto_increment primary key, `createdAt` timestamp(?) not null default CURRENT_TIMESTAMP(?), `email` text not null, `name` text not null)', - }, - { - name: 'insert `User`', - summary: 'insert `User`', - text: 'insert into `User` (`email`, `name`) values (?, ?)', - }, - { name: 'select `User`', summary: 'select `User`', text: 'select * from `User`' }, - { name: 'drop table `User`', summary: 'drop table `User`', text: 'drop table `User`' }, - ]); + expect(container.items.find(span => span.is_segment)?.name).toBe('Test Transaction'); + const knexSpans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === ORIGIN); + expect(knexSpans).toEqual(EXPECTED_SPANS); }, }) .start() diff --git a/dev-packages/node-integration-tests/suites/tracing/knex/pg/instrument-span-streaming.mjs b/dev-packages/node-integration-tests/suites/tracing/knex/pg/instrument-span-streaming.mjs deleted file mode 100644 index 53b9511a21f0..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/knex/pg/instrument-span-streaming.mjs +++ /dev/null @@ -1,10 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/knex/pg/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/knex/pg/instrument.mjs index 170ad6f6a702..46a27dd03b74 100644 --- a/dev-packages/node-integration-tests/suites/tracing/knex/pg/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/knex/pg/instrument.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/knex/pg/test.ts b/dev-packages/node-integration-tests/suites/tracing/knex/pg/test.ts index 5d1da5ad89ff..ae9ea260eab4 100644 --- a/dev-packages/node-integration-tests/suites/tracing/knex/pg/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/knex/pg/test.ts @@ -1,3 +1,15 @@ +import { + DB_NAMESPACE, + DB_OPERATION_NAME, + DB_QUERY_SUMMARY, + DB_QUERY_TEXT, + DB_SYSTEM_NAME, + SENTRY_OP, + SENTRY_ORIGIN, + SERVER_ADDRESS, + SERVER_PORT, +} from '@sentry/conventions/attributes'; +import { DB } from '@sentry/conventions/op'; import { describe, expect } from 'vitest'; import { createEsmAndCjsTests, describeWithDockerCompose } from '../../../../utils/runner'; @@ -9,112 +21,89 @@ describe('knex auto instrumentation', () => { describeWithDockerCompose('with `pg` client', { workingDirectory: [__dirname] }, () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { test('should auto-instrument `knex` package', { timeout: 60_000 }, async () => { - const EXPECTED_TRANSACTION = { - transaction: 'Test Transaction', - spans: expect.arrayContaining([ - expect.objectContaining({ - data: expect.objectContaining({ - 'knex.version': KNEX_VERSION, - 'db.system.name': 'postgresql', - 'db.namespace': 'tests', - 'sentry.origin': ORIGIN, - 'sentry.op': 'db', - 'server.address': 'localhost', - 'server.port': 5445, - }), - status: 'ok', - description: - 'create table "User" ("id" serial primary key, "createdAt" timestamptz(?) not null default CURRENT_TIMESTAMP(?), "email" text not null, "name" text not null)', - origin: ORIGIN, + const EXPECTED_SPANS = [ + expect.objectContaining({ + attributes: expect.objectContaining({ + 'knex.version': { type: 'string', value: KNEX_VERSION }, + [DB_SYSTEM_NAME]: { type: 'string', value: 'postgresql' }, + [DB_NAMESPACE]: { type: 'string', value: 'tests' }, + [SENTRY_ORIGIN]: { type: 'string', value: ORIGIN }, + [SENTRY_OP]: { type: 'string', value: DB }, + [SERVER_ADDRESS]: { type: 'string', value: 'localhost' }, + [SERVER_PORT]: { type: 'integer', value: 5445 }, + [DB_QUERY_TEXT]: { + type: 'string', + value: + 'create table "User" ("id" serial primary key, "createdAt" timestamptz(?) not null default CURRENT_TIMESTAMP(?), "email" text not null, "name" text not null)', + }, + [DB_QUERY_SUMMARY]: { type: 'string', value: 'create table "User"' }, }), - expect.objectContaining({ - data: expect.objectContaining({ - 'knex.version': KNEX_VERSION, - 'db.system.name': 'postgresql', - 'db.namespace': 'tests', - 'sentry.origin': ORIGIN, - 'sentry.op': 'db', - 'server.address': 'localhost', - 'server.port': 5445, - }), - status: 'ok', - // In the knex-otel spans, the placeholders (e.g., `$1`) are replaced by a `?`. - description: 'insert into "User" ("email", "name") values (?, ?)', - origin: ORIGIN, + status: 'ok', + name: 'create table "User"', + }), + expect.objectContaining({ + attributes: expect.objectContaining({ + 'knex.version': { type: 'string', value: KNEX_VERSION }, + [DB_SYSTEM_NAME]: { type: 'string', value: 'postgresql' }, + [DB_NAMESPACE]: { type: 'string', value: 'tests' }, + [SENTRY_ORIGIN]: { type: 'string', value: ORIGIN }, + [SENTRY_OP]: { type: 'string', value: DB }, + [SERVER_ADDRESS]: { type: 'string', value: 'localhost' }, + [SERVER_PORT]: { type: 'integer', value: 5445 }, + [DB_QUERY_TEXT]: { type: 'string', value: 'insert into "User" ("email", "name") values (?, ?)' }, + [DB_QUERY_SUMMARY]: { type: 'string', value: 'insert "User"' }, }), + status: 'ok', + // In the knex-otel spans, the placeholders (e.g., `$1`) are replaced by a `?`. + name: 'insert "User"', + }), - expect.objectContaining({ - data: expect.objectContaining({ - 'knex.version': KNEX_VERSION, - 'db.operation.name': 'select', - 'db.sql.table': 'User', - 'db.system.name': 'postgresql', - 'db.namespace': 'tests', - 'db.query.text': 'select * from "User"', - 'sentry.origin': ORIGIN, - 'sentry.op': 'db', - }), - status: 'ok', - description: 'select * from "User"', - origin: ORIGIN, + expect.objectContaining({ + attributes: expect.objectContaining({ + 'knex.version': { type: 'string', value: KNEX_VERSION }, + [DB_OPERATION_NAME]: { type: 'string', value: 'select' }, + 'db.sql.table': { type: 'string', value: 'User' }, + [DB_SYSTEM_NAME]: { type: 'string', value: 'postgresql' }, + [DB_NAMESPACE]: { type: 'string', value: 'tests' }, + [DB_QUERY_TEXT]: { type: 'string', value: 'select * from "User"' }, + [SENTRY_ORIGIN]: { type: 'string', value: ORIGIN }, + [SENTRY_OP]: { type: 'string', value: DB }, + [DB_QUERY_SUMMARY]: { type: 'string', value: 'select "User"' }, }), + status: 'ok', + name: 'select "User"', + }), - expect.objectContaining({ - data: expect.objectContaining({ - 'knex.version': KNEX_VERSION, - 'db.operation.name': 'select', - 'db.sql.table': 'DoesNotExist', - 'db.system.name': 'postgresql', - 'db.namespace': 'tests', - 'db.query.text': 'select * from "DoesNotExist"', - 'sentry.origin': ORIGIN, - 'sentry.op': 'db', - }), - status: 'internal_error', - description: 'select * from "DoesNotExist"', - origin: ORIGIN, + expect.objectContaining({ + attributes: expect.objectContaining({ + 'knex.version': { type: 'string', value: KNEX_VERSION }, + [DB_OPERATION_NAME]: { type: 'string', value: 'select' }, + 'db.sql.table': { type: 'string', value: 'DoesNotExist' }, + [DB_SYSTEM_NAME]: { type: 'string', value: 'postgresql' }, + [DB_NAMESPACE]: { type: 'string', value: 'tests' }, + [DB_QUERY_TEXT]: { type: 'string', value: 'select * from "DoesNotExist"' }, + [SENTRY_ORIGIN]: { type: 'string', value: ORIGIN }, + [SENTRY_OP]: { type: 'string', value: DB }, + [DB_QUERY_SUMMARY]: { type: 'string', value: 'select "DoesNotExist"' }, }), - ]), - }; - - await createRunner().expect({ transaction: EXPECTED_TRANSACTION }).start().completed(); - }); - }); + status: 'error', + name: 'select "DoesNotExist"', + }), + expect.objectContaining({ + attributes: expect.objectContaining({ + [DB_QUERY_TEXT]: { type: 'string', value: 'drop table "User"' }, + [DB_QUERY_SUMMARY]: { type: 'string', value: 'drop table "User"' }, + }), + name: 'drop table "User"', + }), + ]; - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-span-streaming.mjs', (createRunner, test) => { - test('should name spans after the query summary with span streaming', { timeout: 60_000 }, async () => { await createRunner() .expect({ span: container => { - // The `pg` driver spans underneath the knex spans come from a different integration and - // are named after the full statement, so they are filtered out here. - const knexSpans = container.items.filter(item => item.attributes['sentry.origin']?.value === ORIGIN); - - expect( - knexSpans.map(span => ({ - name: span.name, - summary: span.attributes['db.query.summary']?.value, - text: span.attributes['db.query.text']?.value, - })), - ).toEqual([ - { - name: 'create table "User"', - summary: 'create table "User"', - text: 'create table "User" ("id" serial primary key, "createdAt" timestamptz(?) not null default CURRENT_TIMESTAMP(?), "email" text not null, "name" text not null)', - }, - { - name: 'insert "User"', - summary: 'insert "User"', - text: 'insert into "User" ("email", "name") values (?, ?)', - }, - { name: 'select "User"', summary: 'select "User"', text: 'select * from "User"' }, - { - name: 'select "DoesNotExist"', - summary: 'select "DoesNotExist"', - text: 'select * from "DoesNotExist"', - }, - { name: 'drop table "User"', summary: 'drop table "User"', text: 'drop table "User"' }, - ]); + expect(container.items.find(span => span.is_segment)?.name).toBe('Test Transaction'); + const knexSpans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === ORIGIN); + expect(knexSpans).toEqual(EXPECTED_SPANS); }, }) .start() From b4bd17e1c1f4e2f09fefb12fe07d1f5b2129fab3 Mon Sep 17 00:00:00 2001 From: Lukas Stracke Date: Thu, 8 Oct 2026 13:52:00 +0200 Subject: [PATCH 32/47] chore(nitro): Remove stale bundler-plugin-core rollup external (#25149) Niitro depends and imports from `@sentry/bundler-plugins` since 11.0.0. Nitro's rollup `external` entry for `@sentry/bundler-plugin-core` was outdated and doesn't match any dependency anymore. found while working on #25147 for v10 and checking v11 Co-authored-by: Claude Opus 5.5 (1M context) --- packages/nitro/rollup.npm.config.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nitro/rollup.npm.config.mjs b/packages/nitro/rollup.npm.config.mjs index 08f1f53421dd..aebbc9aff288 100644 --- a/packages/nitro/rollup.npm.config.mjs +++ b/packages/nitro/rollup.npm.config.mjs @@ -5,7 +5,7 @@ export default [ makeBaseNPMConfig({ entrypoints: ['src/index.ts', 'src/runtime/plugins/server.ts'], packageSpecificConfig: { - external: [/^nitro/, /^srvx/, /^@sentry\/opentelemetry/, '@sentry/bundler-plugin-core'], + external: [/^nitro/, /^srvx/, /^@sentry\/opentelemetry/], }, }), { emitCjs: false }, From de43b569afe3dd19733675823e59b6eb3086e872 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 11:52:48 +0000 Subject: [PATCH 33/47] chore(deps-dev): Bump next from 15.5.24 to 15.5.27 (#25171) Bumps [next](https://github.com/vercel/next.js) from 15.5.24 to 15.5.27.
Release notes

Sourced from next's releases.

v15.5.27

This release contains security fixes for the following advisories:

Medium:

v15.5.26

This release contains additional security hardening for next/og. For more information, check out https://nextjs.org/blog/nextjs-security-update-september-22-2026

v15.5.25

Follow-up release to v15.5.24 re-enabling AVIF Image Optimization when newer versions of sharp are installed (#97954).

Commits
  • def7eff v15.5.27
  • 52c94ab [lts-maintenance] Scope response cache keys to their source route (#219)
  • 785f4c2 [lts-maintenance] Match Next data paths case-sensitively (#197)
  • 7217d63 [lts-maintenance][webpack] Ensure dynamicParams is respected in `opengraph-...
  • d5979ba [lts-maintenance][ci] Remove PR creation from deploy test workflow (#99093)
  • 812ae7d [lts-maintenance][test] Adjust deploy assertions for platform drift and the e...
  • 6936da5 v15.5.26
  • b49b6ca [maintenance-lts] Harden next/og SVG serialization
  • f391939 [15.5.x][test] Disable deploy tests triggering bot challenge (#98132)
  • 013ee1d v15.5.25
  • Additional commits viewable in compare view

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- packages/nextjs/package.json | 2 +- yarn.lock | 406 +++++++++++++++++------------------ 2 files changed, 204 insertions(+), 204 deletions(-) diff --git a/packages/nextjs/package.json b/packages/nextjs/package.json index 97488a84ec01..68c9b2baaf57 100644 --- a/packages/nextjs/package.json +++ b/packages/nextjs/package.json @@ -94,7 +94,7 @@ "stacktrace-parser": "^0.1.11" }, "devDependencies": { - "next": "15.5.24", + "next": "15.5.27", "react": "^18.3.1", "react-dom": "^18.3.1" }, diff --git a/yarn.lock b/yarn.lock index 43f25e6bf314..abf7f58320c1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4927,12 +4927,12 @@ optionalDependencies: "@img/sharp-libvips-darwin-arm64" "1.2.4" -"@img/sharp-darwin-arm64@0.35.4": - version "0.35.4" - resolved "https://registry.yarnpkg.com/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.4.tgz#bc10b262de2fc80088013f5f998298d1c8909fbf" - integrity sha512-Uhfl4V4lhP2nbUVF9+hyH1+luj86f1gUFeo8ALYxFoULoU+G87D43BfeMP8XHsk9boxAnCY/bf2EHwhA7MuGsA== +"@img/sharp-darwin-arm64@0.35.5": + version "0.35.5" + resolved "https://registry.yarnpkg.com/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.5.tgz#b1e0276d4e9bf26a7fde7b2adcbe076c71015fc2" + integrity sha512-QRUlFQ0WxvdWyqqG/WtI3iupfD5rBzmCHXSdPsY91sAtVtTo7Q4cb6zOccZ3gqEqkr0f1As1ehLqmEpDsRf+lg== optionalDependencies: - "@img/sharp-libvips-darwin-arm64" "1.3.3" + "@img/sharp-libvips-darwin-arm64" "1.3.4" "@img/sharp-darwin-x64@0.33.5": version "0.33.5" @@ -4948,19 +4948,19 @@ optionalDependencies: "@img/sharp-libvips-darwin-x64" "1.2.4" -"@img/sharp-darwin-x64@0.35.4": - version "0.35.4" - resolved "https://registry.yarnpkg.com/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.4.tgz#76c49ff04fb3f9d846b0d0575841b7ee59beec68" - integrity sha512-hWniXY3bG5qKpkKrAwPe4y+VTPmf086YQAnkxWh7uA1YrlRouWGa0M0Mxj3ZjnXFkv7/TD1bTy9lGUK26vRvWw== +"@img/sharp-darwin-x64@0.35.5": + version "0.35.5" + resolved "https://registry.yarnpkg.com/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.5.tgz#bff811f9610ef322415e87f7551fd5d773122928" + integrity sha512-+BR255RhDlpygUpOc/Jdt1nT6DQ3XG/ERo5wbcdOf5Q320dKtPCKPLR1LJs9VGXRaMa8l1uUa0tkCNOXiAxZUw== optionalDependencies: - "@img/sharp-libvips-darwin-x64" "1.3.3" + "@img/sharp-libvips-darwin-x64" "1.3.4" -"@img/sharp-freebsd-wasm32@0.35.4": - version "0.35.4" - resolved "https://registry.yarnpkg.com/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.4.tgz#ba55fd603c5d1d01a1cb14ffb4d970bc6ced80e1" - integrity sha512-lIsKw/BU+kjB4eZjxrYrZmwOJYi3Ajrv66iAlBmUPyKc3HpnloevB1g3wxGD9P/5BbQ1brBGl65VRRrCvQDEqA== +"@img/sharp-freebsd-wasm32@0.35.5": + version "0.35.5" + resolved "https://registry.yarnpkg.com/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.5.tgz#53b6041958608afa2f5f08323091101407e99086" + integrity sha512-Y/z91nEZ4uIBX5X3nfTovjU9lHNKFYbL2lpHCLVNmXQK03VIZvXBBt0KxbPGp2SdGSF+2mQU4e+hQaWOt86iAw== dependencies: - "@img/sharp-wasm32" "0.35.4" + "@img/sharp-wasm32" "0.35.5" "@img/sharp-libvips-darwin-arm64@1.0.4": version "1.0.4" @@ -4972,10 +4972,10 @@ resolved "https://registry.yarnpkg.com/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz#2894c0cb87d42276c3889942e8e2db517a492c43" integrity sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g== -"@img/sharp-libvips-darwin-arm64@1.3.3": - version "1.3.3" - resolved "https://registry.yarnpkg.com/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.3.tgz#08a6cf4fb4ee8d45f99404a569dcaa6b29391d97" - integrity sha512-suTBPTDGrI9WodccaDdwZItTSaBYASlBk1NSfElSHrUfzu3szG6lvIF58+WiFvnfzuK8ZBFS5zE00PxqxnRiPg== +"@img/sharp-libvips-darwin-arm64@1.3.4": + version "1.3.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.4.tgz#f29cbba302384f9a08bdb04d14c57b5cf71da4a4" + integrity sha512-5R89nBYiRdUlSWJxPhO+GVtaXzXSxKnRu/xqMn3KTA3L9EB9Oy/P+Nn2f2vlhPuUdy/Zusb2DarbyTpGCfEDuw== "@img/sharp-libvips-darwin-x64@1.0.4": version "1.0.4" @@ -4987,10 +4987,10 @@ resolved "https://registry.yarnpkg.com/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz#e63681f4539a94af9cd17246ed8881734386f8cc" integrity sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg== -"@img/sharp-libvips-darwin-x64@1.3.3": - version "1.3.3" - resolved "https://registry.yarnpkg.com/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.3.tgz#1461e6fb310a869b3c13504589b8dd5c06624da8" - integrity sha512-FVJZ5mITMobmXIz/hPDTw0EintTW5H3WfrxwLqEqjiIihlu+hVRyGrFQ60xl0Lxn7Bt3zdpevPaQi0HEzqz9fw== +"@img/sharp-libvips-darwin-x64@1.3.4": + version "1.3.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.4.tgz#527f52721a89dd75614663aef87162a781739bab" + integrity sha512-iR2OKH80yi0U+dUplyh3/xdpFvps6YkCwsXenIJxqxR1v9o+xtKTGbS9H7cps+2Vxjc8B1j96p75NmTGjIhtpQ== "@img/sharp-libvips-linux-arm64@1.0.4": version "1.0.4" @@ -5002,10 +5002,10 @@ resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz#b1b288b36864b3bce545ad91fa6dadcf1a4ad318" integrity sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw== -"@img/sharp-libvips-linux-arm64@1.3.3": - version "1.3.3" - resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.3.tgz#cb938a3971b9a36329bc99427f1e5b72e266ba2e" - integrity sha512-0DaL0A6Xu6sQSQFwe4iVCrKWU2cCTItnRsYsCdxAMm9NF6twAA9BKnoqy4hqz4+azQ0JHuA26qiUKsf1XJ/v5A== +"@img/sharp-libvips-linux-arm64@1.3.4": + version "1.3.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.4.tgz#78c11c6b0a4b4045c72dacacdec3b10f3e521349" + integrity sha512-Y3dgX/6lE2QhQb+Gxy0WZxfg9MEm/JBjamZpS2IklP7xIQoKN4hzAm7KcMVGtaVDt3neE9OKBC7vAfonA/Lr1A== "@img/sharp-libvips-linux-arm@1.0.5": version "1.0.5" @@ -5017,30 +5017,30 @@ resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz#b9260dd1ebe6f9e3bdbcbdcac9d2ac125f35852d" integrity sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A== -"@img/sharp-libvips-linux-arm@1.3.3": - version "1.3.3" - resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.3.tgz#71032941b9fcbf8ebc3c1b62949927b844cbad62" - integrity sha512-3rbU4vqXXc3hY/OiXdl52xZvT0F1yEngWfvqudtPJg/KkyiaQw2DRsFrNzpmLvfavbwOq3qXn36GP8obHRULQA== +"@img/sharp-libvips-linux-arm@1.3.4": + version "1.3.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.4.tgz#f5ecd8ebb63690e580438e308ce73015b49516a8" + integrity sha512-LmRtTsOHuvM2+wlO2Db37dx5MiZhB0FvSunciw48YjdOkZz9KAiRbm8ujeMOA1INqmei5NapFxYEK1D1ZSidmw== "@img/sharp-libvips-linux-ppc64@1.2.4": version "1.2.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz#4b83ecf2a829057222b38848c7b022e7b4d07aa7" integrity sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA== -"@img/sharp-libvips-linux-ppc64@1.3.3": - version "1.3.3" - resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.3.tgz#8246d2dec15234fd1cc54cc55dcba1aebc26c414" - integrity sha512-cdn1OvUBwsXhbC0zSzJnNzf5MZ/mTrobawDvNXBTxe8VtqKAm0sRuEY2Evzovb/w9JMk4TvRxqt1mekSuJz64w== +"@img/sharp-libvips-linux-ppc64@1.3.4": + version "1.3.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.4.tgz#b66a4d25bd042eff66c8076b1e4bd99c6d19246e" + integrity sha512-Le6boB8Tai0Nis+gIxIpKx68UDVVIqdR8Tin5Yf1z2LJJQLDJvCDRqRu+jC2qCoD+eIomonmOwB4smBRxfVpYQ== "@img/sharp-libvips-linux-riscv64@1.2.4": version "1.2.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz#880b4678009e5a2080af192332b00b0aaf8a48de" integrity sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA== -"@img/sharp-libvips-linux-riscv64@1.3.3": - version "1.3.3" - resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.3.tgz#59ff663c2aa6b418857357cd77ca4ccaece95876" - integrity sha512-HjPVx7yKz+0lqdhDlTw1tt90wamBoxhiXpvl1XZpJLiHH4RCJ5yDTqH+VlYPv2fwFs89JFw4c1IexYOcQUi4IQ== +"@img/sharp-libvips-linux-riscv64@1.3.4": + version "1.3.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.4.tgz#9dee8b3fd801c0ff5d6083d6558d3c89b8dedc1b" + integrity sha512-aHkkIEHPRdQEegJN20MLmGtxYD9R2wQr3Cwpddnu5+YKMt6Uzax7S9h5gpZTo8wyrGuZSlfQ63OevL5mTyOC7Q== "@img/sharp-libvips-linux-s390x@1.0.4": version "1.0.4" @@ -5052,10 +5052,10 @@ resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz#74f343c8e10fad821b38f75ced30488939dc59ec" integrity sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ== -"@img/sharp-libvips-linux-s390x@1.3.3": - version "1.3.3" - resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.3.tgz#bc17451f80d9f31663b3e906149f159a0ecb1ca8" - integrity sha512-neWLh+3yCNThxnfy3c4BbVBeGgt9aftno+XbT56iK28RgeDs3UOFWviLWlUu0bArYVYJaFDK+RRohbicUNCm8Q== +"@img/sharp-libvips-linux-s390x@1.3.4": + version "1.3.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.4.tgz#55a80f118f9ffd7653d20cefde41fca6ba0241e2" + integrity sha512-ra/mB6MikESDUO7Yg+Mi95bFBb9GsObURuhnOv3OqknjGe9sZrG8tCe9q0xSIGrtLgvgw0gKnFWcK4blSgQOuQ== "@img/sharp-libvips-linux-x64@1.0.4": version "1.0.4" @@ -5067,10 +5067,10 @@ resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz#df4183e8bd8410f7d61b66859a35edeab0a531ce" integrity sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw== -"@img/sharp-libvips-linux-x64@1.3.3": - version "1.3.3" - resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.3.tgz#c7d1f83bc1cfe222cd9dc927f3c5137aa9791e76" - integrity sha512-4vKmvAst9nrowcqquKFAyZJUDolUaIp8uRiN0mWFguJ1IplC9/pitXtlnnlU4aa/eJw3J7i67V+pwUL+wZGdsA== +"@img/sharp-libvips-linux-x64@1.3.4": + version "1.3.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.4.tgz#a0de3840ae386476eb4045d6d2b056a8062c548b" + integrity sha512-GJ//SSXbnwSDes02umB3nDJLFcQzw8a18V8fyhqr6tV515tOEMdImjjxj1AoafMRz56F3PHgftnj1QEKSU1zkw== "@img/sharp-libvips-linuxmusl-arm64@1.0.4": version "1.0.4" @@ -5082,10 +5082,10 @@ resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz#c8d6b48211df67137541007ee8d1b7b1f8ca8e06" integrity sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw== -"@img/sharp-libvips-linuxmusl-arm64@1.3.3": - version "1.3.3" - resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.3.tgz#303dfe2a8e6d1fd279ead559f4d2ccff2545e2a2" - integrity sha512-Y9kQaLMuNoB0bPYOOdcZMaseNrFpPodIWWMrx+CZyydf2xn68j9WYc6sWWRrDwNkzCQjKYfc68L7jKjGlHMibw== +"@img/sharp-libvips-linuxmusl-arm64@1.3.4": + version "1.3.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.4.tgz#420f2418eaec33cef4346e43d96a7864bb3fcbb4" + integrity sha512-hvulFwtjUcagsis6BBxHwGFwWoNZjgYmULGVrZcyfNbjA8hKILbRxGg15/7w5HDyXHXUos/j6baAWqnCyQ2DWA== "@img/sharp-libvips-linuxmusl-x64@1.0.4": version "1.0.4" @@ -5097,10 +5097,10 @@ resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz#be11c75bee5b080cbee31a153a8779448f919f75" integrity sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg== -"@img/sharp-libvips-linuxmusl-x64@1.3.3": - version "1.3.3" - resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.3.tgz#3750f70ad46ad87e4a08ee8d372ab9a7e17e2b6c" - integrity sha512-fj8Mv0HHfD1Rr+4I68+3agJynxDWtBFgicTbSOb9Bke6pIwzGcJ+RX/yHjmiEGFMCavY/dxvem7MyNaJF+wDiw== +"@img/sharp-libvips-linuxmusl-x64@1.3.4": + version "1.3.4" + resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.4.tgz#eb5696f7fa521f5f7f3d962fd9ade41deeba6a51" + integrity sha512-6zXKeE/p39I1AmA3cJG35eyBGNqNddLnUXjhwBnsGjFPWqf5VKkDBEqaEkPDoTEtkxwi2vv8Tcr2mDyP4So7Fg== "@img/sharp-linux-arm64@0.33.5": version "0.33.5" @@ -5116,12 +5116,12 @@ optionalDependencies: "@img/sharp-libvips-linux-arm64" "1.2.4" -"@img/sharp-linux-arm64@0.35.4": - version "0.35.4" - resolved "https://registry.yarnpkg.com/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.4.tgz#c2f53bf913fa1b46762b192c527a2b3647a211a8" - integrity sha512-De4jpEnAU8Hd5oT0j1G3uL4ZvTuipVMn7YC6vPaJhy6/7EwEae0SVAoBrUMYQbkLGDm85taVWwuPc1a44LTzCQ== +"@img/sharp-linux-arm64@0.35.5": + version "0.35.5" + resolved "https://registry.yarnpkg.com/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.5.tgz#ffaea1b88310b5bc40de48859a7f8f01d07e5e99" + integrity sha512-LYVx5JTsOM2CBzmxreh+nl64/3H6Xb09iSLknqH47z2T2DFFxDeFLP5y4dJwe6H7uGQlHPyEEtIqyo3DYsRwdQ== optionalDependencies: - "@img/sharp-libvips-linux-arm64" "1.3.3" + "@img/sharp-libvips-linux-arm64" "1.3.4" "@img/sharp-linux-arm@0.33.5": version "0.33.5" @@ -5137,12 +5137,12 @@ optionalDependencies: "@img/sharp-libvips-linux-arm" "1.2.4" -"@img/sharp-linux-arm@0.35.4": - version "0.35.4" - resolved "https://registry.yarnpkg.com/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.4.tgz#de8da0da2ee552267cae771597332a863378e43b" - integrity sha512-7OAS8gI0EReKGVN2HssHlM6umJgxF5VI3xN0p9FA91p/YO+ou5hiNghLdZ5BEHztwaaK5+bLKRf8x/o2L2nk9A== +"@img/sharp-linux-arm@0.35.5": + version "0.35.5" + resolved "https://registry.yarnpkg.com/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.5.tgz#950645b8695c200bfdb3c6eccf17c32c26968c23" + integrity sha512-LEaXK2WdXVK5ykcw0buWyPMsmLLL2vpHLD6yrNSW+JGEL3BZPA4tpKN6iaMc4AxTTAoaX/sU1rOL51lcIz48ZQ== optionalDependencies: - "@img/sharp-libvips-linux-arm" "1.3.3" + "@img/sharp-libvips-linux-arm" "1.3.4" "@img/sharp-linux-ppc64@0.34.5": version "0.34.5" @@ -5151,12 +5151,12 @@ optionalDependencies: "@img/sharp-libvips-linux-ppc64" "1.2.4" -"@img/sharp-linux-ppc64@0.35.4": - version "0.35.4" - resolved "https://registry.yarnpkg.com/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.4.tgz#c40d7c5f8ce216c35224c58edcef8e6445935256" - integrity sha512-2oYZJeIl4kCcMGk4ouZVjnkCtFrpQFlNEtJ6GbxzhHQchwH0NH/qEb9ykmOl29dqwMq+JhFdZn+1ak2FKhI9fQ== +"@img/sharp-linux-ppc64@0.35.5": + version "0.35.5" + resolved "https://registry.yarnpkg.com/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.5.tgz#af60c7db090411f7af2df2bc1aa764b93b6b8add" + integrity sha512-QVxAAq8evVRI9ia2vqgwrmWucn5Dfv+JdWzj75pD8omHLPSP7f8p20O8jxzjCcuCEQEOtYOZUmX1hkiZ0kdevA== optionalDependencies: - "@img/sharp-libvips-linux-ppc64" "1.3.3" + "@img/sharp-libvips-linux-ppc64" "1.3.4" "@img/sharp-linux-riscv64@0.34.5": version "0.34.5" @@ -5165,12 +5165,12 @@ optionalDependencies: "@img/sharp-libvips-linux-riscv64" "1.2.4" -"@img/sharp-linux-riscv64@0.35.4": - version "0.35.4" - resolved "https://registry.yarnpkg.com/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.4.tgz#032ca206e713b1179efe0e18d525188c67277730" - integrity sha512-cPbNChoRURAWdebDIHSenxRpgEdy7JkPydSnUxRm9VvKD7m0/xVaR/8Fzlu81pk5nHEvHH87UZUA7cTtwnbJSA== +"@img/sharp-linux-riscv64@0.35.5": + version "0.35.5" + resolved "https://registry.yarnpkg.com/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.5.tgz#afcd42ac0592330fbf4fff5904574e0e049ba0a4" + integrity sha512-LtdreXguaavKODPIfzJ4kffx7UNt1omwtK0rch4EBbbSTXPnxWmYSayXdLJw0fJzQ97kHt1gL/yh4tvU+nCyRQ== optionalDependencies: - "@img/sharp-libvips-linux-riscv64" "1.3.3" + "@img/sharp-libvips-linux-riscv64" "1.3.4" "@img/sharp-linux-s390x@0.33.5": version "0.33.5" @@ -5186,12 +5186,12 @@ optionalDependencies: "@img/sharp-libvips-linux-s390x" "1.2.4" -"@img/sharp-linux-s390x@0.35.4": - version "0.35.4" - resolved "https://registry.yarnpkg.com/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.4.tgz#19dac7501a15c8748bc7a11219fca391e424b6d7" - integrity sha512-RY0JFY8Fd6RonCBtHz+DvadaPkXDSI1AUn6yWL9TipqkZ1vY8w8evqdgyDFnkm4/K1ve1TvZiaePP5oSd4+WVQ== +"@img/sharp-linux-s390x@0.35.5": + version "0.35.5" + resolved "https://registry.yarnpkg.com/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.5.tgz#092e584a8f36f2f90ba98652d6f45432edea87cb" + integrity sha512-UZasTOFiYzotTsGOCu42BfUzP6Tu6Do/947iRm1RsLKvlllxwGcn4RN27LibGWceix4Y+Pmw3jsnTcCQIgWjqA== optionalDependencies: - "@img/sharp-libvips-linux-s390x" "1.3.3" + "@img/sharp-libvips-linux-s390x" "1.3.4" "@img/sharp-linux-x64@0.33.5": version "0.33.5" @@ -5207,12 +5207,12 @@ optionalDependencies: "@img/sharp-libvips-linux-x64" "1.2.4" -"@img/sharp-linux-x64@0.35.4": - version "0.35.4" - resolved "https://registry.yarnpkg.com/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.4.tgz#7d1d414d4a538e0704ba6b76ce6547e60f6211ce" - integrity sha512-9qvvEAuk8k89TfWUoX2htWjbAMX8p+NxCppjpcg5k6xMsjhBQPTsoIh36h9Qde4WRuGpJeYnOjdosDn/cnv+OA== +"@img/sharp-linux-x64@0.35.5": + version "0.35.5" + resolved "https://registry.yarnpkg.com/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.5.tgz#f624f6b98c6579084b9fb363d7cb83e2315de79c" + integrity sha512-SxFtLTeJInhAA9Q836kux2vZNeOBQEx658qvbboZScr0wIARym3IcGmW7KpVD5sbVg0Ojy+udFQdayYIZyoNog== optionalDependencies: - "@img/sharp-libvips-linux-x64" "1.3.3" + "@img/sharp-libvips-linux-x64" "1.3.4" "@img/sharp-linuxmusl-arm64@0.33.5": version "0.33.5" @@ -5228,12 +5228,12 @@ optionalDependencies: "@img/sharp-libvips-linuxmusl-arm64" "1.2.4" -"@img/sharp-linuxmusl-arm64@0.35.4": - version "0.35.4" - resolved "https://registry.yarnpkg.com/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.4.tgz#9534c93807822f4282d079a322facc6db53d50f6" - integrity sha512-KB5jxpfWQTr0nc3xdHtWChdbifHrBGsd2SM62Eyxrl8afikm+f5qGBU75SJIZBT/S1MC8XyacdlXBMSWq6OURA== +"@img/sharp-linuxmusl-arm64@0.35.5": + version "0.35.5" + resolved "https://registry.yarnpkg.com/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.5.tgz#92d0904211086d10bd3c929de715dd8a81fa7268" + integrity sha512-9HbMclmI1zlNkFRs3z9/eBtDjfD0sGlrX1z6b1qwmiFY5ElDLh4BC0LPBdVp7z1DXFiKlIcznf+ZlsuZzLxQqg== optionalDependencies: - "@img/sharp-libvips-linuxmusl-arm64" "1.3.3" + "@img/sharp-libvips-linuxmusl-arm64" "1.3.4" "@img/sharp-linuxmusl-x64@0.33.5": version "0.33.5" @@ -5249,12 +5249,12 @@ optionalDependencies: "@img/sharp-libvips-linuxmusl-x64" "1.2.4" -"@img/sharp-linuxmusl-x64@0.35.4": - version "0.35.4" - resolved "https://registry.yarnpkg.com/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.4.tgz#f488e7a0291991bff643cef82b5afd3cfd70fcf2" - integrity sha512-f+eZJZIQNEEd26RPSW+76chwOf1XtA2Y/O+5ocVyLliHkeih3e+jhLVBdNTd2rS3IbNXK8+ug93Vf5ZXtF5Lxg== +"@img/sharp-linuxmusl-x64@0.35.5": + version "0.35.5" + resolved "https://registry.yarnpkg.com/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.5.tgz#1d146eab6dfcdc6429f29b032b8bc3f0180b6d80" + integrity sha512-4KOphqB035HrVdqLZfCgMzzERrQkkzOwRhl4OAkRO1YCldbaFjySXMaK534Mo0V+LndnlJk+sbUyLeU0ULyD1A== optionalDependencies: - "@img/sharp-libvips-linuxmusl-x64" "1.3.3" + "@img/sharp-libvips-linuxmusl-x64" "1.3.4" "@img/sharp-wasm32@0.33.5": version "0.33.5" @@ -5270,29 +5270,29 @@ dependencies: "@emnapi/runtime" "^1.7.0" -"@img/sharp-wasm32@0.35.4": - version "0.35.4" - resolved "https://registry.yarnpkg.com/@img/sharp-wasm32/-/sharp-wasm32-0.35.4.tgz#81f71f3fc0d8447dc3af6cb378e24e758ff54225" - integrity sha512-zQnl4Kwp7Q6NHsENtU2T/00Zi+w3AQNwz3+UaTyVBy2FpXrzXzGjndpK61onhZjRtRpQXxCTeqw19bVyXOh7jA== +"@img/sharp-wasm32@0.35.5": + version "0.35.5" + resolved "https://registry.yarnpkg.com/@img/sharp-wasm32/-/sharp-wasm32-0.35.5.tgz#5369559c3a4c32e08ce92b6ffba2434175a45a62" + integrity sha512-Ptsga1su4tQx+LLF1ECS9U6nz5kmrXKo6XVbtR48Ke3ZRxxgaWBu7IDtEe1quo8hiupwm6WFqxVlXaSf7IINGQ== dependencies: "@emnapi/runtime" "^1.11.3" -"@img/sharp-webcontainers-wasm32@0.35.4": - version "0.35.4" - resolved "https://registry.yarnpkg.com/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.4.tgz#77d2e8892d3a3825f0dc95ab02ceb613cb213286" - integrity sha512-ESfNkywmCfPNyaZjxooddJQiQ+l/nTpGEOGthxiLnIHXC/CmcBixnfwUleX9mCz9ovrUUvKMap/pm8RYbzfwaA== +"@img/sharp-webcontainers-wasm32@0.35.5": + version "0.35.5" + resolved "https://registry.yarnpkg.com/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.5.tgz#3674eddf41eb85d0e92896d7da01ca00310256b9" + integrity sha512-hfhF/FmoQyTUkA0bIKFOtw536BQSeBMe6BF6QyWlrPxT754+TFLaZ7sKKTfvvM0yJgKgaYTwnFCIZ/GuDw5SUA== dependencies: - "@img/sharp-wasm32" "0.35.4" + "@img/sharp-wasm32" "0.35.5" "@img/sharp-win32-arm64@0.34.5": version "0.34.5" resolved "https://registry.yarnpkg.com/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz#3706e9e3ac35fddfc1c87f94e849f1b75307ce0a" integrity sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g== -"@img/sharp-win32-arm64@0.35.4": - version "0.35.4" - resolved "https://registry.yarnpkg.com/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.4.tgz#4101b44e3fb97d6f1a73c2a9c6142d491d1aa831" - integrity sha512-iNdlBX9gLVvqe2I3uIJSIKTq6wckP/DYxZtcqxm09x5Gi24DnFBmPAWZmr60ZyYMG0xlzo6goG3670ar+RXvRw== +"@img/sharp-win32-arm64@0.35.5": + version "0.35.5" + resolved "https://registry.yarnpkg.com/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.5.tgz#e3c94326eb0a46ade843b84cf50a3917b79d323f" + integrity sha512-X4t7g+7ZA5DKblCBEXGjUqqemj4vczING/5viFwAL8h4N3qYeyjwdCvRLHi4EdOUI+2Z7UFlp1VM+p/AuEtm6Q== "@img/sharp-win32-ia32@0.33.5": version "0.33.5" @@ -5304,10 +5304,10 @@ resolved "https://registry.yarnpkg.com/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz#0b71166599b049e032f085fb9263e02f4e4788de" integrity sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg== -"@img/sharp-win32-ia32@0.35.4": - version "0.35.4" - resolved "https://registry.yarnpkg.com/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.4.tgz#8d4755bd26d5ff34fb89c7907bd89c080a5dc9f5" - integrity sha512-kqRsbaa5CS6KHlpxnN7WhE6vAAugXyZButpRdvDWetlv6Qv4N9WTcrWzF7tXfB9T7MsoadqdI8hmwLq6UlLvtw== +"@img/sharp-win32-ia32@0.35.5": + version "0.35.5" + resolved "https://registry.yarnpkg.com/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.5.tgz#cc5dac453dc3500aac2e63f182b952af0d437bc1" + integrity sha512-5Zm82LoBc43nhwNybZlG7Y1KO//Zhsn306fQl29ZOuStHLGTo3BWL83q3cznX0poxSAMuYL1On/BHBxkBeKr6A== "@img/sharp-win32-x64@0.33.5": version "0.33.5" @@ -5319,10 +5319,10 @@ resolved "https://registry.yarnpkg.com/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz#a81ffb00e69267cd0a1d626eaedb8a8430b2b2f8" integrity sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw== -"@img/sharp-win32-x64@0.35.4": - version "0.35.4" - resolved "https://registry.yarnpkg.com/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.4.tgz#0b67bd0b1f01123a982afcc6ba0dc00b877b8a0a" - integrity sha512-XtmnYhBcrORsJ4XJngyzr/EWP0hRZLAZRFaApdKuviyqF78+ylxh2y06ZmtULAMOnObJ3ucpN0AcwSWnMowTRg== +"@img/sharp-win32-x64@0.35.5": + version "0.35.5" + resolved "https://registry.yarnpkg.com/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.5.tgz#1785356b8cb396e58430e03865281280acb818b1" + integrity sha512-x76eH0vEiHlcMQu8Y8IenntaACtddpT6W0wmXtWrnKcnKI7ME5DdgqhAD6SEWOEl1v2zDvkZDhFA9KnURwpfqg== "@ioredis/commands@1.10.0": version "1.10.0" @@ -5719,50 +5719,50 @@ path-to-regexp "8.4.2" tslib "2.8.1" -"@next/env@15.5.24": - version "15.5.24" - resolved "https://registry.yarnpkg.com/@next/env/-/env-15.5.24.tgz#baee5641d0d93acac87618e1b8004acce594bbad" - integrity sha512-mBDF7T0XKZjs9SpUAl0buizVO+O02ULjOvWX8o/AZo/5AGw/UAS1Zzcylmd4pqbftzmKQi+L/nB4jgBYKEAl5Q== - -"@next/swc-darwin-arm64@15.5.24": - version "15.5.24" - resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-15.5.24.tgz#7a9134e91872bcdb80f070f409903eb81780d04b" - integrity sha512-AGdNLvxZNY6eR2iSnV+6wUa8CiHTMr4F7g3uHH7fT4ICIJBE00R9u4tzN/Vuwsw0cOi8MTD2HJcTCb6siMH88Q== - -"@next/swc-darwin-x64@15.5.24": - version "15.5.24" - resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-15.5.24.tgz#67a91d19b1618b5d1237d874cb52042173ab3269" - integrity sha512-9HrQajBMmGcrrrvDfRimiCrbAPh3E6uHJmwBovYr6Yrmi9p9PZqI876BrXX280wICh3o2XwUlp4blkB0NNBqFg== - -"@next/swc-linux-arm64-gnu@15.5.24": - version "15.5.24" - resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-15.5.24.tgz#b235d3e13c65aea3ed9dc1dde5b557b306969223" - integrity sha512-rl9LSfE75si0WT3cDgdUC1XYCKS+TgxC+/IjitmeycrAG18X/plIP1/vy8dd/HPycYcIvE688PD7FuvEAiEAew== - -"@next/swc-linux-arm64-musl@15.5.24": - version "15.5.24" - resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-15.5.24.tgz#c3a3dd495602cf2b79e72d008226b4e438cfe321" - integrity sha512-TlNAnpsjxSF3aAUtqnfmtXXf8m9sIDBlmF3c7bTAlnshUYu2U0OxN2uf5d0gcFwqHVEdivJNBcCaqNOwPGNimw== - -"@next/swc-linux-x64-gnu@15.5.24": - version "15.5.24" - resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-15.5.24.tgz#b1c74074f90d2cb2a2d1fe6dda5a73b8aa9f2151" - integrity sha512-7dwtlhr0SLndqTG1z9ncRkbJswDZiKWlxzFyXDvJ2RDZRDRHp8zyMJ4D9UH/FgnQeXxxB6gZy2pMcIUoNKQ4pA== - -"@next/swc-linux-x64-musl@15.5.24": - version "15.5.24" - resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-15.5.24.tgz#528bba9b6818c92ae6da6b0f3db1aef690d7ed86" - integrity sha512-kGZxM+WhkYs0276lFrMkj7PRtXT3Btp6cwvfSO/cCVLxJJttB5Ccnl2niaCgUja8HgSbEVnMHpg3FJWoOJ9e/g== - -"@next/swc-win32-arm64-msvc@15.5.24": - version "15.5.24" - resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-15.5.24.tgz#1257886635fe6a3dc873cd2b0c6134b601a60cf8" - integrity sha512-jBDDkZ/qKAqkWivWDMkJSXUzbzV0QKRBKJjEHUAvSB97Hzw7NLzJ6yV56Lts/wjir7s4P31GYgpbS6ZL+hasAA== - -"@next/swc-win32-x64-msvc@15.5.24": - version "15.5.24" - resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-15.5.24.tgz#4b86cda0ebfba20b2dac1f20abc56630be5c05c9" - integrity sha512-JqtwjvvorjacQ0spgjmUJoxySoYgPwdT1sFdQ0/zmW4iMlP2hjYlCoJIyS7o6Epb4Fug8eco3HXFoBDUCDeH7Q== +"@next/env@15.5.27": + version "15.5.27" + resolved "https://registry.yarnpkg.com/@next/env/-/env-15.5.27.tgz#4c4ec6e4a38c5d654bae847426959e84bfca7229" + integrity sha512-WC5hvVqiuFOcRyz6gKPeJkiVvSMf3scNeAV49CdWtVqxmUlzh8UKR6YHt9yiNkh8LngtzqSlVz07/Ma3ZAzchA== + +"@next/swc-darwin-arm64@15.5.27": + version "15.5.27" + resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-15.5.27.tgz#b119e1e3e6c3b8ceca7a8613021442f94ac9c4cb" + integrity sha512-fBA9QurmK4+uj4wNhmRTuO5zMVRaBHCvp6H93zTG/5i6Yi6TewbvbesXB8YQNj8T99Cwrow87hHiFV6wL/hyMA== + +"@next/swc-darwin-x64@15.5.27": + version "15.5.27" + resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-15.5.27.tgz#c40d0c9866a3395e9abee09ad352cdec28724e6c" + integrity sha512-FGZwJwry8qZNHxD/nfwrCwCHHrqx6epqItkkxbYcBQEQ2uw5S43bY83fQB5zbAsZE7TELMjoYkbhsQ4oKM7wwg== + +"@next/swc-linux-arm64-gnu@15.5.27": + version "15.5.27" + resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-15.5.27.tgz#44ace267964a522f7df666bbde2a9e241374b722" + integrity sha512-GQulAaJYNF1RYuatlsw9/uzbKz4if2V7Ic9nVO+P0ljYPm4yBRTFIl6g5YkKTw40lMLbpy9edLoH44g3jSHMYA== + +"@next/swc-linux-arm64-musl@15.5.27": + version "15.5.27" + resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-15.5.27.tgz#a2867ac0fe08f14d0f35780d0b05ade53c36cb43" + integrity sha512-URFFsUmu8Pi0/1qdnUI7tx7keZw1Mt5zHfrIX1RfDG9Nih8DYxK54WZsDJUX2ZTlii1jgotL7qcbwKnSAES6xQ== + +"@next/swc-linux-x64-gnu@15.5.27": + version "15.5.27" + resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-15.5.27.tgz#a9a98bc6e9093f1983325202718981cc8b98b089" + integrity sha512-aJp74z7uvU5jQhyH7lyk+4qSRcTdKbKT9NQE3jiO9kWYkoHZjKi0zzHDbCoZ+v2sJDEiJ/z58R0UlWMGWNF3HA== + +"@next/swc-linux-x64-musl@15.5.27": + version "15.5.27" + resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-15.5.27.tgz#8f5e14ed78c4e12845383ddabad3c1473ce001d3" + integrity sha512-ILl2bQ8MbpvyMpY8AXljb3BXs0MJSC8HXZ6QV5AWpVS7ZELqNPtrHvhUk+SNarvjvICXX8bCS287TCi+gUFVYg== + +"@next/swc-win32-arm64-msvc@15.5.27": + version "15.5.27" + resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-15.5.27.tgz#1f0542804a2b03e0748c59543cdcc911de3cb08d" + integrity sha512-1MB9Rh5y8TOPN2x7naNTJNkCZQav9c70AI7EBYXxsEewwOffC4DjjwBwKf20NBvcA+u+5jDv7nVmHxunlP2d7Q== + +"@next/swc-win32-x64-msvc@15.5.27": + version "15.5.27" + resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-15.5.27.tgz#d3f0e0fe2f0edd0611f1d68f78bb1e366a6546be" + integrity sha512-XtYgpGOswgnyBWXzOEG6q9RhvnSw0CEZ0MbWqCsl5PISgt9EzvJvNW5mU5n1PsphX/lnamzxjgQXH2GUJL8nqg== "@ngtools/webpack@14.2.13": version "14.2.13" @@ -20461,26 +20461,26 @@ new-find-package-json@^2.0.0: dependencies: debug "^4.3.4" -next@15.5.24: - version "15.5.24" - resolved "https://registry.yarnpkg.com/next/-/next-15.5.24.tgz#27852f3cfcaeb2631fbd4795cd9d529081e8af70" - integrity sha512-Y+xn8EQCoC3ZbsFPyzE+tE8XOdrWeUdUF7NeXbmg9DsgAxl5UYxlsrvgVESHTyTGigoTa1bCUrxn70F5bqt0Gw== +next@15.5.27: + version "15.5.27" + resolved "https://registry.yarnpkg.com/next/-/next-15.5.27.tgz#bd149e4dcc483bfff31fa390842cfb8b7ff9a522" + integrity sha512-F82CrlPZ8GRxBH9RIIoR3qU1t5RT3dv1K8moe8SjlDEquiFipvEI13R1Om90TaUOEwShkWCMlg8+gvTNzfMhvg== dependencies: - "@next/env" "15.5.24" + "@next/env" "15.5.27" "@swc/helpers" "0.5.15" caniuse-lite "^1.0.30001579" postcss "8.4.31" styled-jsx "5.1.6" optionalDependencies: - "@next/swc-darwin-arm64" "15.5.24" - "@next/swc-darwin-x64" "15.5.24" - "@next/swc-linux-arm64-gnu" "15.5.24" - "@next/swc-linux-arm64-musl" "15.5.24" - "@next/swc-linux-x64-gnu" "15.5.24" - "@next/swc-linux-x64-musl" "15.5.24" - "@next/swc-win32-arm64-msvc" "15.5.24" - "@next/swc-win32-x64-msvc" "15.5.24" - sharp "^0.34.3 || ^0.35.3" + "@next/swc-darwin-arm64" "15.5.27" + "@next/swc-darwin-x64" "15.5.27" + "@next/swc-linux-arm64-gnu" "15.5.27" + "@next/swc-linux-arm64-musl" "15.5.27" + "@next/swc-linux-x64-gnu" "15.5.27" + "@next/swc-linux-x64-musl" "15.5.27" + "@next/swc-win32-arm64-msvc" "15.5.27" + "@next/swc-win32-x64-msvc" "15.5.27" + sharp "^0.34.3 || ^0.35.4" nf3@^0.3.11, nf3@^0.3.16: version "0.3.16" @@ -24932,40 +24932,40 @@ sharp@^0.33.3: "@img/sharp-win32-ia32" "0.33.5" "@img/sharp-win32-x64" "0.33.5" -"sharp@^0.34.3 || ^0.35.3": - version "0.35.4" - resolved "https://registry.yarnpkg.com/sharp/-/sharp-0.35.4.tgz#361df3b2959daeb380541289960456c6be0f92de" - integrity sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA== +"sharp@^0.34.3 || ^0.35.4": + version "0.35.5" + resolved "https://registry.yarnpkg.com/sharp/-/sharp-0.35.5.tgz#43fb539872ec5f53cc7162eb850dbf6514692fbf" + integrity sha512-Ywn4OnzGukp7CDMrp08RQ50YKmuwG47brZgIVPTvBaaAfQlRlygrRqSrxdCiL9M+LlzLBiJ68IR1QqvzHyjC7g== dependencies: "@img/colour" "^1.1.0" detect-libc "^2.1.2" semver "^7.8.5" optionalDependencies: - "@img/sharp-darwin-arm64" "0.35.4" - "@img/sharp-darwin-x64" "0.35.4" - "@img/sharp-freebsd-wasm32" "0.35.4" - "@img/sharp-libvips-darwin-arm64" "1.3.3" - "@img/sharp-libvips-darwin-x64" "1.3.3" - "@img/sharp-libvips-linux-arm" "1.3.3" - "@img/sharp-libvips-linux-arm64" "1.3.3" - "@img/sharp-libvips-linux-ppc64" "1.3.3" - "@img/sharp-libvips-linux-riscv64" "1.3.3" - "@img/sharp-libvips-linux-s390x" "1.3.3" - "@img/sharp-libvips-linux-x64" "1.3.3" - "@img/sharp-libvips-linuxmusl-arm64" "1.3.3" - "@img/sharp-libvips-linuxmusl-x64" "1.3.3" - "@img/sharp-linux-arm" "0.35.4" - "@img/sharp-linux-arm64" "0.35.4" - "@img/sharp-linux-ppc64" "0.35.4" - "@img/sharp-linux-riscv64" "0.35.4" - "@img/sharp-linux-s390x" "0.35.4" - "@img/sharp-linux-x64" "0.35.4" - "@img/sharp-linuxmusl-arm64" "0.35.4" - "@img/sharp-linuxmusl-x64" "0.35.4" - "@img/sharp-webcontainers-wasm32" "0.35.4" - "@img/sharp-win32-arm64" "0.35.4" - "@img/sharp-win32-ia32" "0.35.4" - "@img/sharp-win32-x64" "0.35.4" + "@img/sharp-darwin-arm64" "0.35.5" + "@img/sharp-darwin-x64" "0.35.5" + "@img/sharp-freebsd-wasm32" "0.35.5" + "@img/sharp-libvips-darwin-arm64" "1.3.4" + "@img/sharp-libvips-darwin-x64" "1.3.4" + "@img/sharp-libvips-linux-arm" "1.3.4" + "@img/sharp-libvips-linux-arm64" "1.3.4" + "@img/sharp-libvips-linux-ppc64" "1.3.4" + "@img/sharp-libvips-linux-riscv64" "1.3.4" + "@img/sharp-libvips-linux-s390x" "1.3.4" + "@img/sharp-libvips-linux-x64" "1.3.4" + "@img/sharp-libvips-linuxmusl-arm64" "1.3.4" + "@img/sharp-libvips-linuxmusl-x64" "1.3.4" + "@img/sharp-linux-arm" "0.35.5" + "@img/sharp-linux-arm64" "0.35.5" + "@img/sharp-linux-ppc64" "0.35.5" + "@img/sharp-linux-riscv64" "0.35.5" + "@img/sharp-linux-s390x" "0.35.5" + "@img/sharp-linux-x64" "0.35.5" + "@img/sharp-linuxmusl-arm64" "0.35.5" + "@img/sharp-linuxmusl-x64" "0.35.5" + "@img/sharp-webcontainers-wasm32" "0.35.5" + "@img/sharp-win32-arm64" "0.35.5" + "@img/sharp-win32-ia32" "0.35.5" + "@img/sharp-win32-x64" "0.35.5" sharp@^0.34.5: version "0.34.5" From 540c79858013c1230500b641a81f31eae05d885c Mon Sep 17 00:00:00 2001 From: Andrei <168741329+andreiborza@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:09:04 +0200 Subject: [PATCH 34/47] feat(server-utils): Record LangChain TypeSafeClassifier runs as gen_ai.evaluate spans (#25120) ## What `TypeSafeClassifier` runs from `@langchain/typesafe` now show up as `gen_ai.evaluate` spans with model, provider, token usage, and the state, questions and answers when gen_ai recording is on. - Standalone `classifier.invoke()` calls are traced automatically. - Calls from inside agents (for example the TypeSafe model router and auto-mode middleware) are traced too. ## Why The classifier calls Jev with `fetch` directly, so the TypeSafe integration did not see it, and LangChain reported it as a generic `invoke_agent` span without any evaluation data. Closes: #24859 --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: Charly Gomez --- ...ypesafe-classifier-inherited-callbacks.mjs | 61 +++++++++++++ .../v1/scenario-typesafe-classifier.mjs | 43 ++++++++++ .../suites/tracing/langchain/v1/test.ts | 84 ++++++++++++++++++ .../server-utils/src/ai/langchain/index.ts | 26 ++++-- .../src/ai/langchain/typesafe-classifier.ts | 86 +++++++++++++++++++ .../server-utils/src/ai/langchain/utils.ts | 16 ++++ .../server-utils/src/ai/typesafe/index.ts | 10 ++- .../src/integrations/langchain.ts | 30 +++++-- .../config/channel-integration-definitions.ts | 1 + .../src/orchestrion/config/langchain.ts | 15 +++- .../tracing/langchain-invoke-agent.test.ts | 36 +++++++- 11 files changed, 393 insertions(+), 15 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier-inherited-callbacks.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier.mjs create mode 100644 packages/server-utils/src/ai/langchain/typesafe-classifier.ts diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier-inherited-callbacks.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier-inherited-callbacks.mjs new file mode 100644 index 000000000000..642945063c88 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier-inherited-callbacks.mjs @@ -0,0 +1,61 @@ +import { BaseCallbackHandler } from '@langchain/core/callbacks/base'; +import { RunnableLambda } from '@langchain/core/runnables'; +import { TypeSafeClassifier } from '@langchain/typesafe'; +// Sets up the AsyncLocalStorage that passes a parent run's config to its children, as it is inside `createAgent`. +import '@langchain/langgraph'; +import * as Sentry from '@sentry/node'; +import express from 'express'; + +function startMockTypeSafeServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/systemone', (req, res) => { + res.json({ + model: 'jev-1.13', + answers: { urgent: { type: 'noul', noul: 0.9 } }, + usage: { input_tokens: 30, output_tokens: 2 }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +// Stands in for a tracer such as LangSmith, which the user passes to the parent run. +class RecordingHandler extends BaseCallbackHandler { + name = 'RecordingHandler'; + runs = []; + + handleChainStart(chain, _inputs, _runId, parentRunId) { + this.runs.push(`${chain.id.at(-1)}:${parentRunId ? 'child' : 'root'}`); + } +} + +async function run() { + const server = await startMockTypeSafeServer(); + const baseUrl = `http://localhost:${server.address().port}`; + + await Sentry.startSpan({ op: 'function', name: 'main' }, async span => { + const classifier = new TypeSafeClassifier({ + apiKey: 'mock-api-key', + baseUrl, + questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } }, + }); + + // Like the `@langchain/typesafe` middlewares, call the classifier without a config. + const parent = RunnableLambda.from(input => classifier.invoke(input)).withConfig({ runName: 'parent' }); + const recorder = new RecordingHandler(); + await parent.invoke('My payouts have been failing.', { callbacks: [recorder] }); + + span.setAttribute('test.recorded_runs', recorder.runs.join(',')); + }); + + await Sentry.flush(2000); + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier.mjs new file mode 100644 index 000000000000..eaaf7a9c7e52 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier.mjs @@ -0,0 +1,43 @@ +import { HumanMessage } from '@langchain/core/messages'; +import { TypeSafeClassifier } from '@langchain/typesafe'; +import * as Sentry from '@sentry/node'; +import express from 'express'; + +function startMockTypeSafeServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/systemone', (req, res) => { + res.json({ + model: 'jev-1.13', + answers: { urgent: { type: 'noul', noul: 0.9 } }, + usage: { input_tokens: 30, output_tokens: 2 }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockTypeSafeServer(); + const baseUrl = `http://localhost:${server.address().port}`; + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const classifier = new TypeSafeClassifier({ + apiKey: 'mock-api-key', + baseUrl, + questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } }, + }); + + await classifier.invoke('My payouts have been failing.'); + await classifier.invoke(new HumanMessage('My card was charged twice.')); + }); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts index 41158cb85db1..ae4f7a1554ef 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts @@ -3,6 +3,7 @@ import { GEN_AI_CONVERSATION_ID, GEN_AI_INPUT_MESSAGES, GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, GEN_AI_PROVIDER_NAME, GEN_AI_REQUEST_MAX_TOKENS, GEN_AI_REQUEST_MODEL, @@ -18,6 +19,7 @@ import { SENTRY_OP, SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; +import { GEN_AI_EVALUATE } from '@sentry/conventions/op'; import { GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; import { createEsmTests } from '../../../../utils/runner/createEsmAndCjsTests'; @@ -233,6 +235,88 @@ describe('LangChain integration (v1)', () => { }, ); + createEsmAndCjsTests( + __dirname, + 'scenario-typesafe-classifier.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('records a TypeSafeClassifier run as a gen_ai.evaluate span', async () => { + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = (await spansPromise).filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain', + ); + expect(spans.map(span => span.name)).toEqual(['evaluate jev-latest', 'evaluate jev-latest']); + + const [evaluateSpan, messageEvaluateSpan] = spans.sort((a, b) => a.start_timestamp - b.start_timestamp); + expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE); + expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(evaluateSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('evaluate'); + expect(evaluateSpan.attributes[GEN_AI_PROVIDER_NAME].value).toBe('typesafe'); + expect(evaluateSpan.attributes[GEN_AI_REQUEST_MODEL].value).toBe('jev-latest'); + expect(evaluateSpan.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('jev-1.13'); + expect(evaluateSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(30); + expect(evaluateSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(2); + expect(evaluateSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(32); + expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ + { + type: 'evaluation', + state: 'My payouts have been failing.', + questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } }, + }, + ]); + // A message is recorded as the transcript line the classifier sends, not as LangChain's serialized form. + expect(JSON.parse(messageEvaluateSpan!.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ + { + type: 'evaluation', + state: 'user: My card was charged twice.', + questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } }, + }, + ]); + expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([ + { type: 'evaluation', answers: { urgent: { type: 'noul', noul: 0.9 } } }, + ]); + }); + }, + { + additionalDependencies: { + langchain: '^1.0.0', + '@langchain/core': '^1.0.0', + '@langchain/typesafe': '^0.0.2', + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-typesafe-classifier-inherited-callbacks.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('keeps the parent run callbacks for a TypeSafeClassifier called without config', async () => { + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = await spansPromise; + const segment = spans.find(span => span.is_segment && span.name === 'main'); + expect(segment!.attributes['test.recorded_runs'].value).toBe('RunnableLambda:root,TypeSafeClassifier:child'); + expect(spans.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_EVALUATE)).toHaveLength(1); + }); + }, + { + additionalDependencies: { + langchain: '^1.0.0', + '@langchain/core': '^1.0.0', + '@langchain/typesafe': '^0.0.2', + '@langchain/langgraph': '^1.0.0', + }, + }, + ); + createEsmTests( __dirname, 'scenario-openai-before-langchain.mjs', diff --git a/packages/server-utils/src/ai/langchain/index.ts b/packages/server-utils/src/ai/langchain/index.ts index 9088e24cc523..f7b5408d1d59 100644 --- a/packages/server-utils/src/ai/langchain/index.ts +++ b/packages/server-utils/src/ai/langchain/index.ts @@ -15,6 +15,11 @@ import { import { GEN_AI_CHAT, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op'; import { resolveAIRecordingOptions } from '../core/utils'; import { LANGCHAIN_ORIGIN } from './constants'; +import { + addTypeSafeClassifierResponseAttributes, + isTypeSafeClassifier, + startTypeSafeClassifierSpan, +} from './typesafe-classifier'; import type { LangChainCallbackHandler, LangChainLLMResult, @@ -43,6 +48,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): // Internal state - single instance tracks all spans const spanMap = new Map(); + const evaluateRunIds = new Set(); /** * Exit a span and clean up @@ -51,8 +57,9 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): const span = spanMap.get(runId); if (span?.isRecording()) { span.end(); - spanMap.delete(runId); } + spanMap.delete(runId); + evaluateRunIds.delete(runId); }; /** @@ -215,6 +222,13 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): _runType?: string, runName?: string, ) { + // A Jev call is a real model call, so it is recorded inside an agent too. + if (isTypeSafeClassifier(chain)) { + spanMap.set(runId, startTypeSafeClassifierSpan(chain, inputs, metadata, recordInputs)); + evaluateRunIds.add(runId); + return; + } + // Skip chain spans when inside an agent context (createReactAgent). // The agent already creates an invoke_agent span; internal chain steps // (ChannelWrite, Branch, prompt, etc.) are noise. @@ -264,14 +278,16 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): handleChainEnd(outputs: unknown, runId: string) { const span = spanMap.get(runId); if (span?.isRecording()) { - // Add outputs if recordOutputs is enabled - if (recordOutputs) { + if (evaluateRunIds.has(runId)) { + addTypeSafeClassifierResponseAttributes(span, outputs, recordOutputs); + } else if (recordOutputs) { span.setAttributes({ 'langchain.chain.outputs': JSON.stringify(outputs), }); } - exitSpan(runId); } + // Also for a sampled-out span, which would otherwise stay tracked. + exitSpan(runId); }, // Chain Error Handler @@ -281,8 +297,8 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): const span = spanMap.get(runId); if (span?.isRecording()) { span.setStatus({ code: SPAN_STATUS_ERROR, message: 'internal_error' }); - exitSpan(runId); } + exitSpan(runId); }, // Tool Start Handler diff --git a/packages/server-utils/src/ai/langchain/typesafe-classifier.ts b/packages/server-utils/src/ai/langchain/typesafe-classifier.ts new file mode 100644 index 000000000000..e254685a4561 --- /dev/null +++ b/packages/server-utils/src/ai/langchain/typesafe-classifier.ts @@ -0,0 +1,86 @@ +import { SENTRY_ORIGIN } from '@sentry/conventions/attributes'; +import type { Span } from '@sentry/core'; +import { isObjectLike } from '@sentry/core'; +import { addResponseAttributes, startEvaluateSpan } from '../typesafe'; +import { LANGCHAIN_ORIGIN } from './constants'; +import { getAgentNameFromMetadata, getConversationIdFromMetadata } from './utils'; + +// `TypeSafeClassifier` from `@langchain/typesafe` calls Jev with `fetch`, not through `@typesafe-ai/sdk`, +// so the TypeSafe integration does not see it. Its serialized id is `[...lc_namespace, lc_name()]`; only +// the class name is matched, as the namespace is the part that changes between versions. +const TYPESAFE_CLASSIFIER_NAME = 'TypeSafeClassifier'; + +/** The package's default, used when the classifier is constructed without a `model`. */ +const DEFAULT_TYPESAFE_CLASSIFIER_MODEL = 'jev-latest'; + +// The `state` the classifier sends to Jev (messages rendered as transcript lines), keyed by the input +// passed to `invoke()`. LangChain hands that same object to `handleChainStart`, so the span records what +// Jev received rather than LangChain's serialized messages. +const wireStates = new WeakMap(); + +/** Record the `state` a `TypeSafeClassifier` sends for `input`, using the classifier's own serializer. */ +export function recordTypeSafeClassifierState(classifier: unknown, input: unknown): void { + // A string state is sent as is, so there is nothing to record. + if (!isObjectLike(input) || !isObjectLike(classifier) || typeof classifier.payload !== 'function') { + return; + } + + try { + const body: unknown = JSON.parse(classifier.payload(input)); + if (isObjectLike(body)) { + wireStates.set(input, body.state); + } + } catch { + // The classifier rejects the same input itself; the span keeps LangChain's form of it. + } +} + +// Typed loosely: LangChain's `Serialized` union does not match our handler's chain type. +export function isTypeSafeClassifier(chain: unknown): boolean { + return isObjectLike(chain) && Array.isArray(chain.id) && chain.id[chain.id.length - 1] === TYPESAFE_CLASSIFIER_NAME; +} + +/** Start an `evaluate` span for a `TypeSafeClassifier` run, from its serialized constructor arguments. */ +export function startTypeSafeClassifierSpan( + chain: unknown, + inputs: Record, + metadata: Record | undefined, + recordInputs: boolean, +): Span { + const kwargs = isObjectLike(chain) && isObjectLike(chain.kwargs) ? chain.kwargs : {}; + const model = typeof kwargs.model === 'string' ? kwargs.model : DEFAULT_TYPESAFE_CLASSIFIER_MODEL; + + return startEvaluateSpan({ model, state: getState(inputs), questions: kwargs.questions }, undefined, recordInputs, { + [SENTRY_ORIGIN]: LANGCHAIN_ORIGIN, + ...getAgentNameFromMetadata(metadata), + ...getConversationIdFromMetadata(metadata), + }); +} + +/** The classifier result reports usage in camelCase, the TypeSafe helpers read the API's snake_case. */ +export function addTypeSafeClassifierResponseAttributes(span: Span, outputs: unknown, recordOutputs: boolean): void { + if (!isObjectLike(outputs)) { + return; + } + + const usage = isObjectLike(outputs.usage) ? outputs.usage : {}; + addResponseAttributes( + span, + { + model: outputs.model, + answers: outputs.answers, + usage: { input_tokens: usage.inputTokens, output_tokens: usage.outputTokens }, + }, + recordOutputs, + ); +} + +function getState(inputs: Record): unknown { + // LangChain hands a string or array input to callbacks wrapped as `{ input }`. + const keys = Object.keys(inputs); + const input = inputs.input; + const state = + keys.length === 1 && keys[0] === 'input' && (typeof input === 'string' || Array.isArray(input)) ? input : inputs; + + return isObjectLike(state) && wireStates.has(state) ? wireStates.get(state) : state; +} diff --git a/packages/server-utils/src/ai/langchain/utils.ts b/packages/server-utils/src/ai/langchain/utils.ts index 067a82fc2d75..8ae219e42437 100644 --- a/packages/server-utils/src/ai/langchain/utils.ts +++ b/packages/server-utils/src/ai/langchain/utils.ts @@ -564,6 +564,22 @@ export function extractToolDefinitions(extraParams?: Record): s return JSON.stringify(toolDefs); } +// A run started without config inherits the parent run's config, which LangChain keeps in a global +// AsyncLocalStorage. `ensureConfig` lets an explicit `callbacks` key replace the inherited one. +const LANGCHAIN_ALS_KEY = Symbol.for('ls:tracing_async_local_storage'); +const LANGCHAIN_CHILD_CONFIG_KEY = Symbol.for('lc:child_config'); + +/** The callbacks LangChain gives a run started without config: the ones of the parent run, if any. */ +export function getInheritedLangChainCallbacks(): unknown { + const storage = (globalThis as Record)[LANGCHAIN_ALS_KEY] as + | { getStore?: () => unknown } + | undefined; + const store = typeof storage?.getStore === 'function' ? storage.getStore() : undefined; + const extra = isObjectLike(store) ? (store.extra as Record | undefined) : undefined; + const config = isObjectLike(extra) ? extra[LANGCHAIN_CHILD_CONFIG_KEY] : undefined; + return isObjectLike(config) ? config.callbacks : undefined; +} + /** Duck-types a LangChain `CallbackManager` (avoids coupling to a specific `@langchain/core` resolution). */ function isCallbackManager(value: unknown): value is { addHandler: (handler: unknown, inherit?: boolean) => void; diff --git a/packages/server-utils/src/ai/typesafe/index.ts b/packages/server-utils/src/ai/typesafe/index.ts index af4115b0137d..9552e5e96a4d 100644 --- a/packages/server-utils/src/ai/typesafe/index.ts +++ b/packages/server-utils/src/ai/typesafe/index.ts @@ -21,7 +21,12 @@ import { TYPESAFE_ORIGIN, TYPESAFE_PROVIDER_NAME } from './constants'; * Start the span for a `systemOne(request)` call. The request model falls back to the client's * `defaultModel`, the same way the SDK resolves it. */ -export function startEvaluateSpan(request: unknown, client: unknown, recordInputs: boolean): Span { +export function startEvaluateSpan( + request: unknown, + client: unknown, + recordInputs: boolean, + attributes?: SpanAttributes, +): Span { const params = isObjectLike(request) ? request : {}; const defaultModel = isObjectLike(client) ? client.defaultModel : undefined; const model = @@ -30,7 +35,8 @@ export function startEvaluateSpan(request: unknown, client: unknown, recordInput return startInactiveSpan({ name: model ? `evaluate ${model}` : 'evaluate', op: GEN_AI_EVALUATE, - attributes: getRequestAttributes(params, model, recordInputs), + // Extra attributes (e.g. another integration's origin) are set at start, so samplers see them. + attributes: { ...getRequestAttributes(params, model, recordInputs), ...attributes }, }); } diff --git a/packages/server-utils/src/integrations/langchain.ts b/packages/server-utils/src/integrations/langchain.ts index e623bbbed9f0..c10b9c79b6ae 100644 --- a/packages/server-utils/src/integrations/langchain.ts +++ b/packages/server-utils/src/integrations/langchain.ts @@ -6,8 +6,13 @@ import { GOOGLE_GENAI_INTEGRATION_NAME } from '../ai/google-genai/constants'; import { createLangChainCallbackHandler } from '../ai/langchain'; import { LANGCHAIN_INTEGRATION_NAME } from '../ai/langchain/constants'; import { _INTERNAL_getLangChainEmbeddingsSpanOptions } from '../ai/langchain/embeddings'; +import { recordTypeSafeClassifierState } from '../ai/langchain/typesafe-classifier'; import type { LangChainOptions } from '../ai/langchain/types'; -import { getConversationIdMetadataFromConfig, _INTERNAL_mergeLangChainCallbackHandler } from '../ai/langchain/utils'; +import { + getConversationIdMetadataFromConfig, + getInheritedLangChainCallbacks, + _INTERNAL_mergeLangChainCallbackHandler, +} from '../ai/langchain/utils'; import { MISTRAL_INTEGRATION_NAME } from '../ai/mistral/constants'; import { OPENAI_INTEGRATION_NAME } from '../ai/openai/constants'; import { GROQ_INTEGRATION_NAME } from './groq'; @@ -35,6 +40,7 @@ const SKIPPED_PROVIDERS = [ // The chat-model channels carry the live args array of `invoke(input, options)` / `_streamIterator(input, options)`. interface RunnableChannelContext { arguments: unknown[]; + self?: unknown; } // The embeddings channels carry the instance (`self`) and the `embedQuery(text)` / `embedDocuments(texts)` args. @@ -77,8 +83,6 @@ function instrumentChatModels(options: LangChainOptions): void { // callback dispatch then creates the spans, exactly as in the OTel path, so no span is opened // here — a `start` subscriber (which also makes orchestrion wrap the function) is enough. const injectHandler = (message: unknown): void => { - markProvidersSkipped(); - const args = (message as RunnableChannelContext).arguments; if (!Array.isArray(args)) { return; @@ -90,7 +94,11 @@ function instrumentChatModels(options: LangChainOptions): void { args[1] = callOptions; } - callOptions.callbacks = _INTERNAL_mergeLangChainCallbackHandler(callOptions.callbacks, sentryHandler); + // Without callbacks of its own, the call would otherwise lose the parent run and its tracers (e.g. LangSmith). + callOptions.callbacks = _INTERNAL_mergeLangChainCallbackHandler( + callOptions.callbacks ?? getInheritedLangChainCallbacks(), + sentryHandler, + ); const conversationIdMetadata = getConversationIdMetadataFromConfig(callOptions); if (Object.keys(conversationIdMetadata).length) { @@ -99,8 +107,20 @@ function instrumentChatModels(options: LangChainOptions): void { }; for (const channelName of [CHANNELS.LANGCHAIN_CHAT_MODEL_INVOKE, CHANNELS.LANGCHAIN_CHAT_MODEL_STREAM]) { - diagnosticsChannel.tracingChannel(channelName).start.subscribe(injectHandler); + diagnosticsChannel.tracingChannel(channelName).start.subscribe(message => { + markProvidersSkipped(); + injectHandler(message); + }); } + + // `TypeSafeClassifier` calls Jev with `fetch`, not through a provider SDK, so nothing needs skipping. + diagnosticsChannel + .tracingChannel(CHANNELS.LANGCHAIN_TYPESAFE_CLASSIFIER_INVOKE) + .start.subscribe(message => { + injectHandler(message); + const { self, arguments: args } = message as RunnableChannelContext; + recordTypeSafeClassifierState(self, args?.[0]); + }); } // Embeddings don't use the callback system. Wrap the method in its own span. diff --git a/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts b/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts index 51d25ab28328..4a133a2d89a7 100644 --- a/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts +++ b/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts @@ -43,6 +43,7 @@ export const CHANNEL_INTEGRATION_DEFINITIONS = [ '@langchain/google-common', '@langchain/google-genai', '@langchain/mistralai', + '@langchain/typesafe', ], }, { exportName: 'langGraphIntegration', modules: ['@langchain/langgraph'] }, diff --git a/packages/server-utils/src/orchestrion/config/langchain.ts b/packages/server-utils/src/orchestrion/config/langchain.ts index 4697613a9448..f6d2684b86a3 100644 --- a/packages/server-utils/src/orchestrion/config/langchain.ts +++ b/packages/server-utils/src/orchestrion/config/langchain.ts @@ -61,7 +61,19 @@ const embeddingsConfig = EMBEDDINGS_PROVIDERS.flatMap(({ name, versionRange, met ), ); -export const langchainConfig = [...chatModelConfig, ...embeddingsConfig] satisfies InstrumentationConfig[]; +// `TypeSafeClassifier` (`@langchain/typesafe`) is a plain `Runnable`, not a chat model, so the chat-model +// hooks never add the Sentry callback handler to its runs. +const typesafeClassifierConfig = ['dist/classifier.cjs', 'dist/classifier.js'].map(filePath => ({ + channelName: 'typesafeClassifierInvoke', + module: { name: '@langchain/typesafe', versionRange: '>=0.0.1 <1.0.0', filePath }, + functionQuery: { className: 'TypeSafeClassifier', methodName: 'invoke', kind: 'Async' as const }, +})); + +export const langchainConfig = [ + ...chatModelConfig, + ...embeddingsConfig, + ...typesafeClassifierConfig, +] satisfies InstrumentationConfig[]; // The embeddings channel strings the subscriber binds to, derived from the provider list above so that // adding a provider is a single edit that both instruments it and subscribes the listener to it. @@ -74,4 +86,5 @@ export const langchainModuleNames = getModuleNames(langchainConfig); export const langchainChannels = { LANGCHAIN_CHAT_MODEL_INVOKE: 'orchestrion:@langchain/core:chatModelInvoke', LANGCHAIN_CHAT_MODEL_STREAM: 'orchestrion:@langchain/core:chatModelStream', + LANGCHAIN_TYPESAFE_CLASSIFIER_INVOKE: 'orchestrion:@langchain/typesafe:typesafeClassifierInvoke', } as const; diff --git a/packages/server-utils/test/ai/lib/tracing/langchain-invoke-agent.test.ts b/packages/server-utils/test/ai/lib/tracing/langchain-invoke-agent.test.ts index 8e316a69a763..3a36432167ea 100644 --- a/packages/server-utils/test/ai/lib/tracing/langchain-invoke-agent.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/langchain-invoke-agent.test.ts @@ -1,6 +1,13 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { GEN_AI_OPERATION_NAME, GEN_AI_PIPELINE_NAME } from '@sentry/conventions/attributes'; -import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; +import { + GEN_AI_AGENT_NAME, + GEN_AI_OPERATION_NAME, + GEN_AI_PIPELINE_NAME, + SENTRY_OP, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import { GEN_AI_EVALUATE } from '@sentry/conventions/op'; +import { getClient, getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; import type { Span } from '@sentry/core'; import { createLangChainCallbackHandler } from '../../../../src/ai/langchain'; import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; @@ -95,4 +102,29 @@ describe('LangChain invoke_agent span names', () => { expect(span.data?.[GEN_AI_PIPELINE_NAME]).toBeUndefined(); expect(span.data?.['langchain.chain.name']).toBeUndefined(); }); + + it('records a TypeSafeClassifier run inside an agent, with its attributes set at span start', () => { + const endedSpans = setupClient('stream'); + const startAttributes: Record[] = []; + getClient()!.on('spanStart', span => startAttributes.push(spanToStaticSpanJSON(span).data)); + + const handler = createLangChainCallbackHandler(); + // LangChain passes the classifier serialized as `[...lc_namespace, lc_name()]`. + const classifier = { name: undefined, id: ['langchain', 'classifiers', 'typesafe', 'TypeSafeClassifier'] }; + + handler.handleChainStart?.(classifier, { input: 'My payouts have been failing.' }, 'run-1', undefined, undefined, { + __sentry_langgraph__: true, + lc_agent_name: 'support_agent', + }); + handler.handleChainEnd?.({ model: 'jev-1.13', answers: {}, usage: {} }, 'run-1'); + + expect(startAttributes).toEqual([ + expect.objectContaining({ + [SENTRY_OP]: GEN_AI_EVALUATE, + [SENTRY_ORIGIN]: 'auto.ai.langchain', + [GEN_AI_AGENT_NAME]: 'support_agent', + }), + ]); + expect(endedSpans.map(span => spanToStaticSpanJSON(span).description)).toEqual(['evaluate jev-latest']); + }); }); From 1fee6d12201c4423421570d9faceb01908dd2e8b Mon Sep 17 00:00:00 2001 From: Andrei <168741329+andreiborza@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:09:05 +0200 Subject: [PATCH 35/47] test(node): Cover Jev calls inside createAgent and LangGraph (#25132) ## What Adds tests that check Jev calls made inside agents show up as `gen_ai.evaluate` spans. - A LangChain `createAgent` with the TypeSafe model router middleware. - A LangGraph `StateGraph` node that calls `TypeSafeClassifier` directly. ## Why These Jev calls were invisible inside agents. #25120 fixes that, and these tests keep it from regressing. Closes: #25032 --------- Co-authored-by: Claude Opus 5.5 --- .../node-suites/excludes.ts | 3 +- .../v1/scenario-typesafe-model-router.mjs | 69 +++++++++++++++ .../suites/tracing/langchain/v1/test.ts | 60 +++++++++++++ .../langgraph/v1/instrument-with-pii.mjs | 10 +++ .../tracing/langgraph/v1/instrument.mjs | 10 +++ .../v1/scenario-typesafe-classifier-node.mjs | 37 ++++++++ .../suites/tracing/langgraph/v1/test.ts | 88 +++++++++++++++++++ 7 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-model-router.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/langgraph/v1/instrument-with-pii.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/langgraph/v1/instrument.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/langgraph/v1/scenario-typesafe-classifier-node.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/langgraph/v1/test.ts diff --git a/dev-packages/bun-integration-tests/node-suites/excludes.ts b/dev-packages/bun-integration-tests/node-suites/excludes.ts index d27790a79a0f..a8bba26f4713 100644 --- a/dev-packages/bun-integration-tests/node-suites/excludes.ts +++ b/dev-packages/bun-integration-tests/node-suites/excludes.ts @@ -100,7 +100,7 @@ export const NO_AUTO_INSTRUMENTATION = [ 'suites/tracing/knex/**', 'suites/tracing/koa/test.ts', 'suites/tracing/langchain/**', - 'suites/tracing/langgraph/test.ts', + 'suites/tracing/langgraph/**', 'suites/tracing/lru-memoizer/test.ts', 'suites/tracing/mastra/test.ts', 'suites/tracing/mcp-handler-exact-once/test.ts', @@ -244,6 +244,7 @@ const BUN_BUILD_NOT_TRIAGED = [ 'suites/tracing/google-genai-v2/test.ts', 'suites/tracing/google-genai/test.ts', 'suites/tracing/langchain/v1/test.ts', + 'suites/tracing/langgraph/v1/test.ts', 'suites/tracing/mastra/test.ts', 'suites/tracing/mcp-handler-exact-once/test.ts', 'suites/tracing/mcp-server-streamed/test.ts', diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-model-router.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-model-router.mjs new file mode 100644 index 000000000000..d9d56910381b --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-model-router.mjs @@ -0,0 +1,69 @@ +import { ChatAnthropic } from '@langchain/anthropic'; +import { modelRouterMiddleware } from '@langchain/typesafe/middleware'; +import * as Sentry from '@sentry/node'; +import express from 'express'; +import { createAgent } from 'langchain'; + +function startMockServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/messages', (req, res) => { + res.json({ + id: 'msg_router_test', + type: 'message', + role: 'assistant', + content: [{ type: 'text', text: 'A refund is on its way.' }], + model: req.body.model, + stop_reason: 'end_turn', + stop_sequence: null, + usage: { input_tokens: 10, output_tokens: 5 }, + }); + }); + + app.post('/v1/systemone', (req, res) => { + res.json({ + model: 'jev-1.13', + answers: { + model_route: { type: 'choice', choice: 'fast', probabilities: { fast: 0.9, smart: 0.1 }, confidence: 0.8 }, + }, + usage: { input_tokens: 40, output_tokens: 3 }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockServer(); + const baseUrl = `http://localhost:${server.address().port}`; + + const model = name => new ChatAnthropic({ model: name, apiKey: 'mock-api-key', clientOptions: { baseURL: baseUrl } }); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const agent = createAgent({ + name: 'support_agent', + model: model('claude-3-5-sonnet-20241022'), + middleware: [ + modelRouterMiddleware({ + instructions: 'Pick the model for this request.', + choices: { + fast: { model: model('claude-3-5-haiku-20241022'), criteria: 'Simple requests' }, + smart: { model: model('claude-3-5-sonnet-20241022'), criteria: 'Complex requests' }, + }, + classifierOptions: { apiKey: 'mock-api-key', baseUrl }, + }), + ], + }); + + await agent.invoke({ messages: [{ role: 'user', content: 'Where is my refund?' }] }); + }); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts index ae4f7a1554ef..f14ccb93169d 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts @@ -1,5 +1,6 @@ import { afterAll, describe, expect } from 'vitest'; import { + GEN_AI_AGENT_NAME, GEN_AI_CONVERSATION_ID, GEN_AI_INPUT_MESSAGES, GEN_AI_OPERATION_NAME, @@ -317,6 +318,65 @@ describe('LangChain integration (v1)', () => { }, ); + createEsmAndCjsTests( + __dirname, + 'scenario-typesafe-model-router.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('records the Jev model router call inside a createAgent run', async () => { + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = await spansPromise; + const evaluateSpan = spans.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_EVALUATE)!; + const chatSpan = spans.find(span => span.name === 'chat claude-3-5-haiku-20241022')!; + + // The router classifies in `beforeAgent`, inside the agent run, where other chain steps are skipped. + expect(evaluateSpan.name).toBe('evaluate jev-latest'); + expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(evaluateSpan.attributes[GEN_AI_AGENT_NAME].value).toBe('support_agent'); + expect(evaluateSpan.parent_span_id).toBe(chatSpan.parent_span_id); + expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ + { + type: 'evaluation', + // The router passes the latest human message, which the classifier sends as a transcript line. + state: 'user: Where is my refund?', + questions: { + model_route: { + type: 'choice', + instructions: 'Pick the model for this request.', + criteria: { fast: 'Simple requests', smart: 'Complex requests' }, + }, + }, + }, + ]); + expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([ + { + type: 'evaluation', + answers: { + model_route: { + type: 'choice', + choice: 'fast', + probabilities: { fast: 0.9, smart: 0.1 }, + confidence: 0.8, + }, + }, + }, + ]); + }); + }, + { + additionalDependencies: { + langchain: '^1.0.0', + '@langchain/core': '^1.0.0', + '@langchain/anthropic': '^1.0.0', + '@langchain/typesafe': '^0.0.2', + }, + }, + ); + createEsmTests( __dirname, 'scenario-openai-before-langchain.mjs', diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/instrument-with-pii.mjs new file mode 100644 index 000000000000..1c507ba84b9c --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/instrument-with-pii.mjs @@ -0,0 +1,10 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + dataCollection: { genAI: { inputs: true, outputs: true } }, + transport: loggingTransport, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/instrument.mjs new file mode 100644 index 000000000000..cc192fb89834 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/instrument.mjs @@ -0,0 +1,10 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + dataCollection: { genAI: { inputs: false, outputs: false } }, + transport: loggingTransport, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/scenario-typesafe-classifier-node.mjs b/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/scenario-typesafe-classifier-node.mjs new file mode 100644 index 000000000000..33256a644def --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/scenario-typesafe-classifier-node.mjs @@ -0,0 +1,37 @@ +import { Annotation, END, START, StateGraph } from '@langchain/langgraph'; +import { TypeSafeClassifier } from '@langchain/typesafe'; +import * as Sentry from '@sentry/node'; + +// Answer in-process through the classifier's `fetch` option, so no mock server is needed. +async function mockTypeSafeFetch() { + return new Response( + JSON.stringify({ + model: 'jev-1.13', + answers: { urgent: { type: 'noul', noul: 0.9 } }, + usage: { input_tokens: 30, output_tokens: 2 }, + }), + { headers: { 'content-type': 'application/json' } }, + ); +} + +async function run() { + const classifier = new TypeSafeClassifier({ + apiKey: 'mock-api-key', + fetch: mockTypeSafeFetch, + questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } }, + }); + + const State = Annotation.Root({ ticket: Annotation(), urgent: Annotation() }); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const graph = new StateGraph(State) + .addNode('triage', async state => ({ urgent: (await classifier.invoke(state.ticket)).nouls.urgent.noul })) + .addEdge(START, 'triage') + .addEdge('triage', END) + .compile({ name: 'triage_graph' }); + + await graph.invoke({ ticket: 'My payouts have been failing.' }); + }); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/test.ts b/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/test.ts new file mode 100644 index 000000000000..c6a71539de1a --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/test.ts @@ -0,0 +1,88 @@ +import { afterAll, describe, expect } from 'vitest'; +import { + GEN_AI_AGENT_NAME, + GEN_AI_INPUT_MESSAGES, + GEN_AI_OUTPUT_MESSAGES, + SENTRY_OP, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import { GEN_AI_EVALUATE } from '@sentry/conventions/op'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; + +describe('LangGraph integration (v1)', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + createEsmAndCjsTests( + __dirname, + 'scenario-typesafe-classifier-node.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('records a TypeSafeClassifier call made inside a StateGraph node', async () => { + const runner = createRunner(); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = await spansPromise; + const rootSpan = spans.find(span => span.is_segment && span.name === 'main')!; + const agentSpan = spans.find(span => span.name === 'invoke_agent triage_graph')!; + const evaluateSpan = spans.find(span => span.name === 'evaluate jev-latest')!; + + expect(agentSpan.parent_span_id).toBe(rootSpan.span_id); + expect(evaluateSpan.parent_span_id).toBe(agentSpan.span_id); + expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE); + expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(evaluateSpan.attributes[GEN_AI_AGENT_NAME].value).toBe('triage_graph'); + expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ + { + type: 'evaluation', + state: 'My payouts have been failing.', + questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } }, + }, + ]); + expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([ + { type: 'evaluation', answers: { urgent: { type: 'noul', noul: 0.9 } } }, + ]); + }); + }, + { + additionalDependencies: { + langchain: '^1.0.0', + '@langchain/core': '^1.0.0', + '@langchain/langgraph': '^1.0.0', + '@langchain/typesafe': '^0.0.2', + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-typesafe-classifier-node.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('omits the classifier input and output when genAI recording is off', async () => { + const runner = createRunner(); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = await spansPromise; + const evaluateSpan = spans.find(span => span.name === 'evaluate jev-latest')!; + + expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE); + expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); + expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); + }); + }, + { + additionalDependencies: { + langchain: '^1.0.0', + '@langchain/core': '^1.0.0', + '@langchain/langgraph': '^1.0.0', + '@langchain/typesafe': '^0.0.2', + }, + }, + ); +}); From dffd6b1bf3945d98d23e0f362ec366abf7153f85 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Thu, 8 Oct 2026 14:28:50 +0200 Subject: [PATCH 36/47] test(node): Port Vercel AI v5-v7 tests to span streaming (#25090) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port Vercel AI v5–v7 integration tests to default span streaming while retaining v4 as transaction coverage in `v4-static`, including the original transaction-tag and error-correlation checks. Remove the separate streaming-only suites and fold their unique GenAI assertions into the corresponding retained suites. V4 keeps the default data-collection opt-in check, and v6 retains telemetry-metadata coverage. Closes #24138 --------- Co-authored-by: GPT-6 --- .../span-streaming-v4/instrument-with-pii.mjs | 12 - .../vercelai/span-streaming-v4/instrument.mjs | 11 - .../scenario-error-in-tool.mjs | 46 - .../vercelai/span-streaming-v4/scenario.mjs | 82 -- .../vercelai/span-streaming-v4/test.ts | 297 ------- .../span-streaming-v6/instrument-with-pii.mjs | 11 - .../vercelai/span-streaming-v6/instrument.mjs | 11 - .../scenario-error-in-tool.mjs | 47 - .../vercelai/span-streaming-v6/scenario.mjs | 95 -- .../vercelai/span-streaming-v6/test.ts | 366 -------- .../{ => v4-static}/instrument-with-pii.mjs | 2 +- .../vercelai/{ => v4-static}/instrument.mjs | 0 .../scenario-conversation-id.mjs | 0 .../{ => v4-static}/scenario-embeddings.mjs | 0 .../scenario-error-in-tool-express.mjs | 0 .../scenario-error-in-tool.mjs | 0 .../scenario-generate-object.mjs | 0 .../{ => v4-static}/scenario-stream-text.mjs | 0 .../scenario-system-instructions.mjs | 0 .../vercelai/{ => v4-static}/scenario.mjs | 0 .../tracing/vercelai/{ => v4-static}/test.ts | 440 ++++++---- .../vercelai/v5/instrument-with-pii.mjs | 1 - .../suites/tracing/vercelai/v5/instrument.mjs | 1 - .../suites/tracing/vercelai/v5/test.ts | 182 ++-- .../vercelai/v6_v7/instrument-with-pii.mjs | 1 - .../tracing/vercelai/v6_v7/instrument.mjs | 1 - .../scenario-telemetry-metadata.mjs | 0 .../suites/tracing/vercelai/v6_v7/test.ts | 808 ++++++++++++------ 28 files changed, 891 insertions(+), 1523 deletions(-) delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-pii.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument-with-pii.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-error-in-tool.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts rename dev-packages/node-integration-tests/suites/tracing/vercelai/{ => v4-static}/instrument-with-pii.mjs (82%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{ => v4-static}/instrument.mjs (100%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{ => v4-static}/scenario-conversation-id.mjs (100%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{ => v4-static}/scenario-embeddings.mjs (100%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{ => v4-static}/scenario-error-in-tool-express.mjs (100%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{ => v4-static}/scenario-error-in-tool.mjs (100%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{ => v4-static}/scenario-generate-object.mjs (100%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{ => v4-static}/scenario-stream-text.mjs (100%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{ => v4-static}/scenario-system-instructions.mjs (100%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{ => v4-static}/scenario.mjs (100%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{ => v4-static}/test.ts (57%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{span-streaming-v6 => v6_v7}/scenario-telemetry-metadata.mjs (100%) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-pii.mjs deleted file mode 100644 index c0abcf114ae1..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-pii.mjs +++ /dev/null @@ -1,12 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - // inputs and outputs are enabeld by default when opting into dataCollection - dataCollection: {}, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs deleted file mode 100644 index 5e0b6fb5592f..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: false, outputs: false } }, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs deleted file mode 100644 index a305d0bd2dd8..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs +++ /dev/null @@ -1,46 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { generateText } from 'ai'; -import { MockLanguageModelV1 } from 'ai/test'; -import { z } from 'zod'; - -async function run() { - Sentry.setTag('test-tag', 'test-value'); - - await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - try { - await generateText({ - model: new MockLanguageModelV1({ - doGenerate: async () => ({ - rawCall: { rawPrompt: null, rawSettings: {} }, - finishReason: 'tool-calls', - usage: { promptTokens: 15, completionTokens: 25 }, - text: 'Tool call completed!', - toolCalls: [ - { - toolCallType: 'function', - toolCallId: 'call-1', - toolName: 'getWeather', - args: '{ "location": "San Francisco" }', - }, - ], - }), - }), - tools: { - getWeather: { - parameters: z.object({ location: z.string() }), - execute: async () => { - throw new Error('Error in tool'); - }, - }, - }, - prompt: 'What is the weather in San Francisco?', - }); - } catch { - // Expected error - we want the spans to still be flushed - } - }); - - await Sentry.flush(2000); -} - -run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario.mjs deleted file mode 100644 index ef8cb19c3646..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario.mjs +++ /dev/null @@ -1,82 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { generateText } from 'ai'; -import { MockLanguageModelV1 } from 'ai/test'; -import { z } from 'zod'; - -async function run() { - await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - await generateText({ - model: new MockLanguageModelV1({ - doGenerate: async () => ({ - rawCall: { rawPrompt: null, rawSettings: {} }, - finishReason: 'stop', - usage: { promptTokens: 10, completionTokens: 20 }, - text: 'First span here!', - }), - }), - prompt: 'Where is the first span?', - }); - - // This span should have input and output prompts attached because telemetry is explicitly enabled. - await generateText({ - experimental_telemetry: { isEnabled: true }, - model: new MockLanguageModelV1({ - doGenerate: async () => ({ - rawCall: { rawPrompt: null, rawSettings: {} }, - finishReason: 'stop', - usage: { promptTokens: 10, completionTokens: 20 }, - text: 'Second span here!', - }), - }), - prompt: 'Where is the second span?', - }); - - // This span should include tool calls and tool results - await generateText({ - model: new MockLanguageModelV1({ - doGenerate: async () => ({ - rawCall: { rawPrompt: null, rawSettings: {} }, - finishReason: 'tool-calls', - usage: { promptTokens: 15, completionTokens: 25 }, - text: 'Tool call completed!', - toolCalls: [ - { - toolCallType: 'function', - toolCallId: 'call-1', - toolName: 'getWeather', - args: '{ "location": "San Francisco" }', - }, - ], - }), - }), - tools: { - getWeather: { - description: 'Get the current weather for a location', - parameters: z.object({ location: z.string() }), - execute: async args => { - return `Weather in ${args.location}: Sunny, 72°F`; - }, - }, - }, - prompt: 'What is the weather in San Francisco?', - }); - - // This span should not be captured because we've disabled telemetry - await generateText({ - experimental_telemetry: { isEnabled: false }, - model: new MockLanguageModelV1({ - doGenerate: async () => ({ - rawCall: { rawPrompt: null, rawSettings: {} }, - finishReason: 'stop', - usage: { promptTokens: 10, completionTokens: 20 }, - text: 'Third span here!', - }), - }), - prompt: 'Where is the third span?', - }); - }); - - await Sentry.flush(2000); -} - -run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts deleted file mode 100644 index 50385f0e0c40..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts +++ /dev/null @@ -1,297 +0,0 @@ -import { afterAll, describe, expect } from 'vitest'; -import { - GEN_AI_INPUT_MESSAGES, - GEN_AI_OPERATION_NAME, - GEN_AI_OUTPUT_MESSAGES, - GEN_AI_REQUEST_MODEL, - GEN_AI_RESPONSE_MODEL, - GEN_AI_TOOL_CALL_ARGUMENTS, - GEN_AI_TOOL_CALL_RESULT, - GEN_AI_TOOL_DEFINITIONS, - GEN_AI_TOOL_DESCRIPTION, - GEN_AI_TOOL_NAME, - GEN_AI_USAGE_INPUT_TOKENS, - GEN_AI_USAGE_OUTPUT_TOKENS, - GEN_AI_USAGE_TOTAL_TOKENS, - SENTRY_OP, - SENTRY_ORIGIN, -} from '@sentry/conventions/attributes'; -import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; -import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; - -/** - * Helper to match a typed attribute value in a SerializedStreamedSpan. - * Streamed span attributes are `{ value: X, type: Y }` objects, unlike transaction - * span `data` which stores values directly. - */ -function attr(value: unknown) { - return expect.objectContaining({ value }); -} - -const expectedOrigin = 'auto.vercelai.channel'; - -// The channel path serializes the SDK-parsed args object (compact). -const toolCallArgs = '{\\"location\\":\\"San Francisco\\"}'; - -describe('Vercel AI integration (streaming v4)', () => { - afterAll(() => { - cleanupChildProcesses(); - }); - - const EXPECTED_SPANS_DEFAULT_PII_FALSE = { - items: expect.arrayContaining([ - // First span - invoke_agent for simple generateText - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Second span - generate_content for simple generateText - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Third span - invoke_agent for explicit telemetry generateText - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Fourth span - tool call invoke_agent - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Fifth span - tool call generate_content - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Sixth span - execute_tool - // Note: gen_ai.tool.description is NOT present when genAI recording disabled because ai.prompt.tools is not recorded - expect.objectContaining({ - name: 'execute_tool getWeather', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), - [GEN_AI_TOOL_NAME]: attr('getWeather'), - [GEN_AI_OPERATION_NAME]: attr('execute_tool'), - [SENTRY_OP]: attr('gen_ai.execute_tool'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - ]), - }; - - const EXPECTED_SPANS_DEFAULT_PII_TRUE = { - items: expect.arrayContaining([ - // First span - invoke_agent with input/output messages (PII enabled) - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: attr('[{"role":"user","content":"Where is the first span?"}]'), - [GEN_AI_OUTPUT_MESSAGES]: attr( - '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Second span - generate_content with input/output messages - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: expect.objectContaining({ value: expect.any(String) }), - [GEN_AI_OUTPUT_MESSAGES]: attr( - '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Third span - explicit telemetry invoke_agent with messages - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: attr('[{"role":"user","content":"Where is the second span?"}]'), - [GEN_AI_OUTPUT_MESSAGES]: attr( - '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Fourth span - tool call invoke_agent with messages - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: attr('[{"role":"user","content":"What is the weather in San Francisco?"}]'), - [GEN_AI_OUTPUT_MESSAGES]: attr( - `[{"role":"assistant","parts":[{"type":"text","content":"Tool call completed!"},{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"${toolCallArgs}"}],"finish_reason":"tool_call"}]`, - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Fifth span - tool call generate_content with tool definitions - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_DEFINITIONS]: expect.objectContaining({ - value: expect.stringContaining('getWeather'), - }), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Sixth span - execute_tool with description and input/output - expect.objectContaining({ - name: 'execute_tool getWeather', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), - [GEN_AI_TOOL_DESCRIPTION]: attr('Get the current weather for a location'), - [GEN_AI_TOOL_CALL_ARGUMENTS]: expect.objectContaining({ value: expect.any(String) }), - [GEN_AI_TOOL_NAME]: attr('getWeather'), - [GEN_AI_TOOL_CALL_RESULT]: expect.objectContaining({ value: expect.any(String) }), - [GEN_AI_OPERATION_NAME]: attr('execute_tool'), - [SENTRY_OP]: attr('gen_ai.execute_tool'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - ]), - }; - - const EXPECTED_SPANS_ERROR_IN_TOOL = { - items: expect.arrayContaining([ - expect.objectContaining({ - name: 'invoke_agent', - status: 'error', - attributes: expect.objectContaining({ - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - expect.objectContaining({ - name: 'execute_tool getWeather', - status: 'error', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), - [GEN_AI_TOOL_NAME]: attr('getWeather'), - [GEN_AI_OPERATION_NAME]: attr('execute_tool'), - [SENTRY_OP]: attr('gen_ai.execute_tool'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - ]), - }; - - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { - test('creates ai related spans in streaming mode with genAI recording disabled', async () => { - await createRunner().expect({ span: EXPECTED_SPANS_DEFAULT_PII_FALSE }).start().completed(); - }); - }); - - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { - test('creates ai related spans in streaming mode with genAI recording enabled', async () => { - await createRunner().expect({ span: EXPECTED_SPANS_DEFAULT_PII_TRUE }).start().completed(); - }); - }); - - createEsmAndCjsTests(__dirname, 'scenario-error-in-tool.mjs', 'instrument.mjs', (createRunner, test) => { - test('normalizes error status in streaming mode', async () => { - await createRunner().ignore('event').expect({ span: EXPECTED_SPANS_ERROR_IN_TOOL }).start().completed(); - }); - }); -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument-with-pii.mjs deleted file mode 100644 index 657bed0a3a8c..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument-with-pii.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: true, outputs: true } }, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument.mjs deleted file mode 100644 index 5e0b6fb5592f..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: false, outputs: false } }, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-error-in-tool.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-error-in-tool.mjs deleted file mode 100644 index 51bdae176158..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-error-in-tool.mjs +++ /dev/null @@ -1,47 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { generateText, tool } from 'ai'; -import { MockLanguageModelV3 } from 'ai/test'; -import { z } from 'zod'; - -async function run() { - await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - try { - await generateText({ - model: new MockLanguageModelV3({ - doGenerate: async () => ({ - finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, - usage: { - inputTokens: { total: 15, noCache: 15, cached: 0 }, - outputTokens: { total: 25, noCache: 25, cached: 0 }, - totalTokens: { total: 40, noCache: 40, cached: 0 }, - }, - content: [ - { - type: 'tool-call', - toolCallId: 'call-1', - toolName: 'getWeather', - input: JSON.stringify({ location: 'San Francisco' }), - }, - ], - warnings: [], - }), - }), - tools: { - getWeather: tool({ - inputSchema: z.object({ location: z.string() }), - execute: async () => { - throw new Error('Error in tool'); - }, - }), - }, - prompt: 'What is the weather in San Francisco?', - }); - } catch { - // Expected error - we want the spans to still be flushed - } - }); - - await Sentry.flush(2000); -} - -run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario.mjs deleted file mode 100644 index bf5e43a32e65..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario.mjs +++ /dev/null @@ -1,95 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { generateText, tool } from 'ai'; -import { MockLanguageModelV3 } from 'ai/test'; -import { z } from 'zod'; - -async function run() { - await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - await generateText({ - model: new MockLanguageModelV3({ - doGenerate: async () => ({ - finishReason: { unified: 'stop', raw: 'stop' }, - usage: { - inputTokens: { total: 10, noCache: 10, cached: 0 }, - outputTokens: { total: 20, noCache: 20, cached: 0 }, - totalTokens: { total: 30, noCache: 30, cached: 0 }, - }, - content: [{ type: 'text', text: 'First span here!' }], - warnings: [], - }), - }), - prompt: 'Where is the first span?', - }); - - // This span should have input and output prompts attached because telemetry is explicitly enabled. - await generateText({ - experimental_telemetry: { isEnabled: true }, - model: new MockLanguageModelV3({ - doGenerate: async () => ({ - finishReason: { unified: 'stop', raw: 'stop' }, - usage: { - inputTokens: { total: 10, noCache: 10, cached: 0 }, - outputTokens: { total: 20, noCache: 20, cached: 0 }, - totalTokens: { total: 30, noCache: 30, cached: 0 }, - }, - content: [{ type: 'text', text: 'Second span here!' }], - warnings: [], - }), - }), - prompt: 'Where is the second span?', - }); - - // This span should include tool calls and tool results - await generateText({ - model: new MockLanguageModelV3({ - doGenerate: async () => ({ - finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, - usage: { - inputTokens: { total: 15, noCache: 15, cached: 0 }, - outputTokens: { total: 25, noCache: 25, cached: 0 }, - totalTokens: { total: 40, noCache: 40, cached: 0 }, - }, - content: [ - { - type: 'tool-call', - toolCallId: 'call-1', - toolName: 'getWeather', - input: JSON.stringify({ location: 'San Francisco' }), - }, - ], - warnings: [], - }), - }), - tools: { - getWeather: tool({ - description: 'Get the current weather for a location', - inputSchema: z.object({ location: z.string() }), - execute: async ({ location }) => `Weather in ${location}: Sunny, 72°F`, - }), - }, - prompt: 'What is the weather in San Francisco?', - }); - - // This span should not be captured because we've disabled telemetry - await generateText({ - experimental_telemetry: { isEnabled: false }, - model: new MockLanguageModelV3({ - doGenerate: async () => ({ - finishReason: { unified: 'stop', raw: 'stop' }, - usage: { - inputTokens: { total: 10, noCache: 10, cached: 0 }, - outputTokens: { total: 20, noCache: 20, cached: 0 }, - totalTokens: { total: 30, noCache: 30, cached: 0 }, - }, - content: [{ type: 'text', text: 'Third span here!' }], - warnings: [], - }), - }), - prompt: 'Where is the third span?', - }); - }); - - await Sentry.flush(2000); -} - -run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts deleted file mode 100644 index 68c88b131783..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts +++ /dev/null @@ -1,366 +0,0 @@ -import { afterAll, describe, expect } from 'vitest'; -import { - GEN_AI_INPUT_MESSAGES, - GEN_AI_OPERATION_NAME, - GEN_AI_OUTPUT_MESSAGES, - GEN_AI_REQUEST_MODEL, - GEN_AI_RESPONSE_MODEL, - GEN_AI_TOOL_CALL_ARGUMENTS, - GEN_AI_TOOL_CALL_RESULT, - GEN_AI_TOOL_DEFINITIONS, - GEN_AI_TOOL_DESCRIPTION, - GEN_AI_TOOL_NAME, - GEN_AI_USAGE_INPUT_TOKENS, - GEN_AI_USAGE_OUTPUT_TOKENS, - GEN_AI_USAGE_TOTAL_TOKENS, - SENTRY_OP, - SENTRY_ORIGIN, -} from '@sentry/conventions/attributes'; -import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; -import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; - -/** - * Helper to match a typed attribute value in a SerializedStreamedSpan. - * Streamed span attributes are `{ value: X, type: Y }` objects, unlike transaction - * span `data` which stores values directly. - */ -function attr(value: unknown) { - return expect.objectContaining({ value }); -} - -describe('Vercel AI integration (streaming, v6)', () => { - afterAll(() => { - cleanupChildProcesses(); - }); - - const origin = 'auto.vercelai.channel'; - - const EXPECTED_SPANS_DEFAULT_PII_FALSE = { - items: expect.arrayContaining([ - // First span - invoke_agent for simple generateText - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Second span - generate_content for simple generateText - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Third span - invoke_agent for explicit telemetry generateText - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Fourth span - tool call invoke_agent - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Fifth span - tool call generate_content - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Sixth span - execute_tool - // Note: gen_ai.tool.description is NOT present when genAI recording disabled because ai.prompt.tools is not recorded - expect.objectContaining({ - name: 'execute_tool getWeather', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), - [GEN_AI_TOOL_NAME]: attr('getWeather'), - [GEN_AI_OPERATION_NAME]: attr('execute_tool'), - [SENTRY_OP]: attr('gen_ai.execute_tool'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - ]), - }; - - const EXPECTED_SPANS_DEFAULT_PII_TRUE = { - items: expect.arrayContaining([ - // First span - invoke_agent with input/output messages (PII enabled) - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: attr('[{"role":"user","content":"Where is the first span?"}]'), - [GEN_AI_OUTPUT_MESSAGES]: attr( - '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Second span - generate_content with input/output messages - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: expect.objectContaining({ value: expect.any(String) }), - [GEN_AI_OUTPUT_MESSAGES]: attr( - '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Third span - explicit telemetry invoke_agent with messages - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: attr('[{"role":"user","content":"Where is the second span?"}]'), - [GEN_AI_OUTPUT_MESSAGES]: attr( - '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Fourth span - tool call invoke_agent with messages (V6: no text part, only tool_call) - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: attr('[{"role":"user","content":"What is the weather in San Francisco?"}]'), - [GEN_AI_OUTPUT_MESSAGES]: attr( - '[{"role":"assistant","parts":[{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"{\\"location\\":\\"San Francisco\\"}"}],"finish_reason":"tool_call"}]', - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Fifth span - tool call generate_content with tool definitions - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_DEFINITIONS]: expect.objectContaining({ - value: expect.stringContaining('getWeather'), - }), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Sixth span - execute_tool with description and input/output - expect.objectContaining({ - name: 'execute_tool getWeather', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), - [GEN_AI_TOOL_DESCRIPTION]: attr('Get the current weather for a location'), - [GEN_AI_TOOL_CALL_ARGUMENTS]: expect.objectContaining({ value: expect.any(String) }), - [GEN_AI_TOOL_NAME]: attr('getWeather'), - [GEN_AI_TOOL_CALL_RESULT]: expect.objectContaining({ value: expect.any(String) }), - [GEN_AI_OPERATION_NAME]: attr('execute_tool'), - [SENTRY_OP]: attr('gen_ai.execute_tool'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - ]), - }; - - const EXPECTED_SPANS_ERROR_IN_TOOL = { - items: expect.arrayContaining([ - expect.objectContaining({ - name: 'invoke_agent', - attributes: expect.objectContaining({ - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - expect.objectContaining({ - name: 'execute_tool getWeather', - status: 'error', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), - [GEN_AI_TOOL_NAME]: attr('getWeather'), - [GEN_AI_OPERATION_NAME]: attr('execute_tool'), - [SENTRY_OP]: attr('gen_ai.execute_tool'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - ]), - }; - - createEsmAndCjsTests( - __dirname, - 'scenario.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('creates ai related spans in streaming mode with genAI recording disabled', async () => { - await createRunner().expect({ span: EXPECTED_SPANS_DEFAULT_PII_FALSE }).start().completed(); - }); - }, - { - additionalDependencies: { - ai: '^6.0.0', - }, - }, - ); - - createEsmAndCjsTests( - __dirname, - 'scenario.mjs', - 'instrument-with-pii.mjs', - (createRunner, test) => { - test('creates ai related spans in streaming mode with genAI recording enabled', async () => { - await createRunner().expect({ span: EXPECTED_SPANS_DEFAULT_PII_TRUE }).start().completed(); - }); - }, - { - additionalDependencies: { - ai: '^6.0.0', - }, - }, - ); - - function spanWithMetadata(name: string, op: string, requestId: string, extra: Record = {}) { - return expect.objectContaining({ - name, - attributes: expect.objectContaining({ - 'vercel.ai.telemetry.metadata.requestId': attr(requestId), - 'vercel.ai.telemetry.metadata.tenantId': attr('acme'), - ...extra, - [SENTRY_OP]: attr(op), - }), - }); - } - - const EXPECTED_SPANS_TELEMETRY_METADATA = { - items: expect.arrayContaining([ - spanWithMetadata('invoke_agent support-chat', 'gen_ai.invoke_agent', 'req_generate', { - 'vercel.ai.telemetry.metadata.attempt': { type: 'string', value: '2' }, - }), - spanWithMetadata('generate_content mock-model-id', 'gen_ai.generate_content', 'req_generate', { - 'vercel.ai.telemetry.metadata.attempt': { type: 'string', value: '2' }, - }), - spanWithMetadata('invoke_agent support-stream', 'gen_ai.invoke_agent', 'req_stream'), - spanWithMetadata('generate_content mock-model-id', 'gen_ai.generate_content', 'req_stream'), - spanWithMetadata('embeddings mock-model-id', 'gen_ai.embeddings', 'req_embed'), - spanWithMetadata('embeddings mock-model-id', 'gen_ai.embeddings', 'req_embed_many'), - ]), - }; - - createEsmAndCjsTests( - __dirname, - 'scenario-telemetry-metadata.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('records experimental_telemetry.metadata as span attributes', async () => { - await createRunner().expect({ span: EXPECTED_SPANS_TELEMETRY_METADATA }).start().completed(); - }); - }, - { - additionalDependencies: { - ai: '^6.0.0', - }, - }, - ); - - createEsmAndCjsTests( - __dirname, - 'scenario-error-in-tool.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('normalizes error status in streaming mode', async () => { - await createRunner().ignore('event').expect({ span: EXPECTED_SPANS_ERROR_IN_TOOL }).start().completed(); - }); - }, - { - additionalDependencies: { - ai: '^6.0.0', - }, - }, - ); -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/instrument-with-pii.mjs similarity index 82% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-with-pii.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/instrument-with-pii.mjs index bdf24d6bbd0e..43ea2c5b8d86 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/instrument-with-pii.mjs @@ -6,6 +6,6 @@ Sentry.init({ dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: true, outputs: true } }, + dataCollection: {}, transport: loggingTransport, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/instrument.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/instrument.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/instrument.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-conversation-id.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-conversation-id.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-conversation-id.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-conversation-id.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-embeddings.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-embeddings.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-embeddings.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-embeddings.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error-in-tool-express.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-error-in-tool-express.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error-in-tool-express.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-error-in-tool-express.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error-in-tool.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-error-in-tool.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error-in-tool.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-error-in-tool.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-generate-object.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-generate-object.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-generate-object.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-generate-object.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-stream-text.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-stream-text.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-stream-text.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-stream-text.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-system-instructions.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-system-instructions.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-system-instructions.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-system-instructions.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/test.ts similarity index 57% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/test.ts index e6ec70fa35ff..99604cb8a599 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/test.ts @@ -1,8 +1,10 @@ import type { Event } from '@sentry/node'; import { afterAll, describe, expect } from 'vitest'; import { + GEN_AI_CONVERSATION_ID, GEN_AI_EMBEDDINGS_INPUT, GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, GEN_AI_OUTPUT_MESSAGES, GEN_AI_PROVIDER_NAME, GEN_AI_REQUEST_MODEL, @@ -16,12 +18,15 @@ import { GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, + SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; -import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; -import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; -import { getStringAttributeValue } from '../../../utils'; +import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; +import { getStringAttributeValue } from '../../../../utils'; const expectedOrigin = 'auto.vercelai.channel'; +const toolCallArgs = '{\\"location\\":\\"San Francisco\\"}'; describe('Vercel AI integration (v4)', () => { afterAll(() => { @@ -42,17 +47,17 @@ describe('Vercel AI integration (v4)', () => { span.attributes[GEN_AI_INPUT_MESSAGES] === undefined && span.attributes[GEN_AI_USAGE_INPUT_TOKENS].value === 10, ); - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan!.name).toBe('invoke_agent'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); expect(firstInvokeAgentSpan!.status).toBe('ok'); - expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(firstInvokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('mock-model-id'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('mock-model-id'); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(20); expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(30); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); const firstGenerateContentSpan = container.items.find( span => @@ -61,16 +66,33 @@ describe('Vercel AI integration (v4)', () => { span.attributes[GEN_AI_INPUT_MESSAGES] === undefined && span.attributes[GEN_AI_USAGE_INPUT_TOKENS].value === 10, ); - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan!.name).toBe('generate_content mock-model-id'); + expect(firstGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(firstGenerateContentSpan!.status).toBe('ok'); - expect(firstGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(firstGenerateContentSpan!.attributes['vercel.ai.operationId'].value).toBe( - 'ai.generateText.doGenerate', - ); + expect(firstGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(firstGenerateContentSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('mock-provider'); - expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(firstGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); const secondInvokeAgentSpan = container.items.find( span => @@ -78,13 +100,29 @@ describe('Vercel AI integration (v4)', () => { span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the second span?"}]', ); - expect(secondInvokeAgentSpan).toBeDefined(); - expect(secondInvokeAgentSpan!.name).toBe('invoke_agent'); + expect(secondInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(secondInvokeAgentSpan!.status).toBe('ok'); - expect(secondInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(secondInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"Where is the second span?"}]', - ); + expect(secondInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(secondInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toBe( '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', ); @@ -94,21 +132,25 @@ describe('Vercel AI integration (v4)', () => { span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes('Second span here!'), ); - expect(secondGenerateContentSpan).toBeDefined(); - expect(secondGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(secondGenerateContentSpan!.status).toBe('ok'); - expect(secondGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(secondGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); expect(secondGenerateContentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toContain('Second span here!'); const toolInvokeAgentSpan = container.items.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 15, ); - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan!.name).toBe('invoke_agent'); + expect(toolInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(toolInvokeAgentSpan!.status).toBe('ok'); - expect(toolInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(15); + expect(toolInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(25); expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(40); @@ -117,17 +159,37 @@ describe('Vercel AI integration (v4)', () => { span.name === 'generate_content mock-model-id' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 15, ); - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan!.name).toBe('generate_content mock-model-id'); + expect(toolGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 40, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(toolGenerateContentSpan!.status).toBe('ok'); - expect(toolGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(15); + expect(toolGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan!.name).toBe('execute_tool getWeather'); + expect(toolExecutionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolExecutionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'execute_tool', + }); expect(toolExecutionSpan!.status).toBe('ok'); - expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE].value).toBe('call-1'); }, @@ -150,14 +212,28 @@ describe('Vercel AI integration (v4)', () => { span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the first span?"}]', ); - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan!.name).toBe('invoke_agent'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 30 }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ type: 'integer', value: 10 }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(firstInvokeAgentSpan!.status).toBe('ok'); - expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(firstInvokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"Where is the first span?"}]', - ); expect(firstInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toBe( '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', ); @@ -167,15 +243,45 @@ describe('Vercel AI integration (v4)', () => { span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes('First span here!'), ); - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan!.name).toBe('generate_content mock-model-id'); + expect(firstGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(firstGenerateContentSpan!.status).toBe('ok'); - expect(firstGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(firstGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(firstGenerateContentSpan!.attributes['vercel.ai.operationId'].value).toBe( 'ai.generateText.doGenerate', ); - expect(firstGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); - expect(firstGenerateContentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toContain('First span here!'); + expect(firstGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]?.value).toEqual(expect.any(String)); + expect(firstGenerateContentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toEqual({ + type: 'string', + value: + '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', + }); const secondInvokeAgentSpan = container.items.find( span => @@ -183,23 +289,42 @@ describe('Vercel AI integration (v4)', () => { span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the second span?"}]', ); - expect(secondInvokeAgentSpan).toBeDefined(); - expect(secondInvokeAgentSpan!.name).toBe('invoke_agent'); + expect(secondInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toEqual({ + type: 'string', + value: + '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', + }); expect(secondInvokeAgentSpan!.status).toBe('ok'); - expect(secondInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(secondInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"Where is the second span?"}]', - ); + expect(secondInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); const secondGenerateContentSpan = container.items.find( span => span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes('Second span here!'), ); - expect(secondGenerateContentSpan).toBeDefined(); - expect(secondGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(secondGenerateContentSpan!.status).toBe('ok'); - expect(secondGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); const toolInvokeAgentSpan = container.items.find( span => @@ -207,38 +332,76 @@ describe('Vercel AI integration (v4)', () => { span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"What is the weather in San Francisco?"}]', ); - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan!.name).toBe('invoke_agent'); + expect(toolInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 40 }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ type: 'integer', value: 25 }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ type: 'integer', value: 15 }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(toolInvokeAgentSpan!.status).toBe('ok'); - expect(toolInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(toolInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"What is the weather in San Francisco?"}]', - ); - expect(toolInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeDefined(); + expect(toolInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toEqual({ + type: 'string', + value: `[{"role":"assistant","parts":[{"type":"text","content":"Tool call completed!"},{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"${toolCallArgs}"}],"finish_reason":"tool_call"}]`, + }); const toolGenerateContentSpan = container.items.find( span => span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_TOOL_DEFINITIONS]?.value)?.includes('getWeather'), ); - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan!.name).toBe('generate_content mock-model-id'); + expect(toolGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 40, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(toolGenerateContentSpan!.status).toBe('ok'); - expect(toolGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(toolGenerateContentSpan!.attributes[GEN_AI_TOOL_DEFINITIONS].value).toContain('getWeather'); + expect(toolGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); + expect(toolGenerateContentSpan!.attributes[GEN_AI_TOOL_DEFINITIONS]).toEqual({ + type: 'string', + value: expect.stringContaining('getWeather'), + }); expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(15); const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan!.name).toBe('execute_tool getWeather'); + expect(toolExecutionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolExecutionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'execute_tool', + }); + expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'call-1', + }); expect(toolExecutionSpan!.status).toBe('ok'); - expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_DESCRIPTION].value).toBe( 'Get the current weather for a location', ); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ARGUMENTS]).toBeDefined(); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_RESULT]).toBeDefined(); + expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ARGUMENTS]?.value).toEqual(expect.any(String)); + expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_RESULT]?.value).toEqual(expect.any(String)); }, }) .start() @@ -266,25 +429,37 @@ describe('Vercel AI integration (v4)', () => { const invokeAgentSpan = container.items.find( span => span.name === 'invoke_agent' && span.status === 'error', ); - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan!.name).toBe('invoke_agent'); - expect(invokeAgentSpan!.status).toBe('error'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(invokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan!.name).toBe('generate_content mock-model-id'); + expect(generateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 40 }); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ type: 'integer', value: 25 }); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ type: 'integer', value: 15 }); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(generateContentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText.doGenerate'); const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolSpan).toBeDefined(); - expect(toolSpan!.name).toBe('execute_tool getWeather'); + expect(toolSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'execute_tool' }); + expect(toolSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]).toEqual({ type: 'string', value: 'call-1' }); expect(toolSpan!.status).toBe('error'); - expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); - expect(toolSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); + expect(toolSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); + expect(toolSpan!.attributes[SENTRY_ORIGIN].value).toBe(expectedOrigin); expect(toolSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); }, }) @@ -296,10 +471,8 @@ describe('Vercel AI integration (v4)', () => { .start() .completed(); - expect(transactionEvent).toBeDefined(); expect(transactionEvent!.transaction).toBe('main'); - expect(errorEvent).toBeDefined(); expect(errorEvent!.tags).toMatchObject({ 'test-tag': 'test-value' }); // The tool error bubbles out of the `ai` call as the SDK's wrapped `AI_ToolExecutionError`. The @@ -339,23 +512,16 @@ describe('Vercel AI integration (v4)', () => { const invokeAgentSpan = container.items.find( span => span.name === 'invoke_agent' && span.status === 'error', ); - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan!.name).toBe('invoke_agent'); - expect(invokeAgentSpan!.status).toBe('error'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan!.name).toBe('generate_content mock-model-id'); expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolSpan).toBeDefined(); - expect(toolSpan!.name).toBe('execute_tool getWeather'); expect(toolSpan!.status).toBe('error'); - expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); expect(toolSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); }, }) @@ -369,11 +535,9 @@ describe('Vercel AI integration (v4)', () => { await runner.makeRequest('get', '/test/error-in-tool', { expectError: true }); await runner.completed(); - expect(transactionEvent).toBeDefined(); expect(transactionEvent!.transaction).toBe('GET /test/error-in-tool'); expect(transactionEvent!.tags).toMatchObject({ 'test-tag': 'test-value' }); - expect(errorEvent).toBeDefined(); expect(errorEvent!.tags).toMatchObject({ 'test-tag': 'test-value' }); expect(errorEvent!.contexts!.trace!.trace_id).toBe(transactionEvent!.contexts!.trace!.trace_id); @@ -411,17 +575,13 @@ describe('Vercel AI integration (v4)', () => { span: container => { expect(container.items).toHaveLength(2); const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan!.name).toBe('invoke_agent'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS].value).toBe( JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]), ); const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); }, }) .start() @@ -438,21 +598,17 @@ describe('Vercel AI integration (v4)', () => { span: container => { expect(container.items).toHaveLength(2); const embedSpan = container.items.find(span => span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 10); - expect(embedSpan).toBeDefined(); expect(embedSpan!.name).toBe('embeddings mock-model-id'); expect(embedSpan!.status).toBe('ok'); - expect(embedSpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); + expect(embedSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.embeddings'); expect(embedSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('mock-model-id'); - expect(embedSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); const embedManySpan = container.items.find( span => span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 20, ); - expect(embedManySpan).toBeDefined(); expect(embedManySpan!.name).toBe('embeddings mock-model-id'); expect(embedManySpan!.status).toBe('ok'); - expect(embedManySpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); - expect(embedManySpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(20); + expect(embedManySpan!.attributes[SENTRY_OP].value).toBe('gen_ai.embeddings'); }, }) .start() @@ -470,20 +626,16 @@ describe('Vercel AI integration (v4)', () => { const embedSpan = container.items.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', ); - expect(embedSpan).toBeDefined(); expect(embedSpan!.name).toBe('embeddings mock-model-id'); expect(embedSpan!.status).toBe('ok'); - expect(embedSpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); - expect(embedSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT].value).toBe('Embedding test!'); + expect(embedSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.embeddings'); const embedManySpan = container.items.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === '["First input","Second input"]', ); - expect(embedManySpan).toBeDefined(); expect(embedManySpan!.name).toBe('embeddings mock-model-id'); expect(embedManySpan!.status).toBe('ok'); - expect(embedManySpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); - expect(embedManySpan!.attributes[GEN_AI_EMBEDDINGS_INPUT].value).toBe('["First input","Second input"]'); + expect(embedManySpan!.attributes[SENTRY_OP].value).toBe('gen_ai.embeddings'); }, }) .start() @@ -499,16 +651,12 @@ describe('Vercel AI integration (v4)', () => { span: container => { expect(container.items).toHaveLength(2); const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan!.name).toBe('invoke_agent'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(invokeAgentSpan!.attributes['gen_ai.conversation.id'].value).toBe('conv-a'); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[GEN_AI_CONVERSATION_ID].value).toBe('conv-a'); const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(generateContentSpan!.attributes['gen_ai.conversation.id'].value).toBe('conv-a'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[GEN_AI_CONVERSATION_ID].value).toBe('conv-a'); }, }) .start() @@ -525,10 +673,9 @@ describe('Vercel AI integration (v4)', () => { expect(container.items).toHaveLength(2); const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan).toBeDefined(); expect(invokeAgentSpan!.status).toBe('ok'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(invokeAgentSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_ORIGIN].value).toBe(expectedOrigin); expect(invokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.streamText'); expect(invokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('mock-model-id'); // Aggregated over the drained stream: v4 reports `promptTokens`/`completionTokens`, which the @@ -539,10 +686,9 @@ describe('Vercel AI integration (v4)', () => { expect(invokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toContain('Stream response!'); const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(generateContentSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[SENTRY_ORIGIN].value).toBe(expectedOrigin); expect(generateContentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.streamText.doStream'); }, }) @@ -561,31 +707,29 @@ describe('Vercel AI integration (v4)', () => { // generateObject (invoke_agent) const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan).toBeDefined(); expect(invokeAgentSpan!.status).toBe('ok'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateObject'); - expect(invokeAgentSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); - expect(invokeAgentSpan!.attributes['gen_ai.operation.name'].value).toBe('invoke_agent'); - expect(invokeAgentSpan!.attributes['gen_ai.response.model'].value).toBe('mock-model-id'); - expect(invokeAgentSpan!.attributes['gen_ai.usage.input_tokens'].value).toBe(15); - expect(invokeAgentSpan!.attributes['gen_ai.usage.output_tokens'].value).toBe(25); - expect(invokeAgentSpan!.attributes['gen_ai.usage.total_tokens'].value).toBe(40); + expect(invokeAgentSpan!.attributes[SENTRY_ORIGIN].value).toBe(expectedOrigin); + expect(invokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('invoke_agent'); + expect(invokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('mock-model-id'); + expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(15); + expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(25); + expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(40); // generateObject.doGenerate (generate_content) const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(generateContentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateObject.doGenerate'); - expect(generateContentSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); - expect(generateContentSpan!.attributes['gen_ai.operation.name'].value).toBe('generate_content'); - expect(generateContentSpan!.attributes['gen_ai.provider.name'].value).toBe('mock-provider'); - expect(generateContentSpan!.attributes['gen_ai.request.model'].value).toBe('mock-model-id'); - expect(generateContentSpan!.attributes['gen_ai.response.model'].value).toBe('mock-model-id'); - expect(generateContentSpan!.attributes['gen_ai.usage.input_tokens'].value).toBe(15); - expect(generateContentSpan!.attributes['gen_ai.usage.output_tokens'].value).toBe(25); - expect(generateContentSpan!.attributes['gen_ai.usage.total_tokens'].value).toBe(40); + expect(generateContentSpan!.attributes[SENTRY_ORIGIN].value).toBe(expectedOrigin); + expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); + expect(generateContentSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('mock-provider'); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('mock-model-id'); + expect(generateContentSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('mock-model-id'); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(15); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(25); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(40); }, }) .start() diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument-with-pii.mjs index 7ca11a34673e..7a021162097c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument-with-pii.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument.mjs index 42052d281304..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts index 00430e451229..b608a1f717fc 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts @@ -1,3 +1,4 @@ +import type { SerializedStreamedSpanContainer } from '@sentry/core'; import type { Event } from '@sentry/node'; import { afterAll, describe, expect } from 'vitest'; import { @@ -14,13 +15,13 @@ import { GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, + SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; import { getStringAttributeValue } from '../../../../utils'; -const expectedOrigin = 'auto.vercelai.channel'; - describe('Vercel AI integration (v5)', () => { afterAll(() => { cleanupChildProcesses(); @@ -33,99 +34,76 @@ describe('Vercel AI integration (v5)', () => { (createRunner, test) => { test('creates ai spans when dataCollection.genAi has inputs and outputs disabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(7); - const firstInvokeAgentSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(7); + const firstInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes['vercel.ai.operationId'].value === 'ai.generateText' && span.attributes[GEN_AI_INPUT_MESSAGES] === undefined && span.attributes[GEN_AI_USAGE_INPUT_TOKENS].value === 10, ); - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan!.name).toBe('invoke_agent'); expect(firstInvokeAgentSpan!.status).toBe('ok'); - expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(firstInvokeAgentSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); - expect(firstInvokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('mock-model-id'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('mock-model-id'); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(20); expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(30); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); - const firstGenerateContentSpan = container.items.find( + const firstGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes['vercel.ai.operationId'].value === 'ai.generateText.doGenerate' && span.attributes[GEN_AI_INPUT_MESSAGES] === undefined && span.attributes[GEN_AI_USAGE_INPUT_TOKENS].value === 10, ); - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(firstGenerateContentSpan!.status).toBe('ok'); - expect(firstGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(firstGenerateContentSpan!.attributes['vercel.ai.operationId'].value).toBe( - 'ai.generateText.doGenerate', - ); + expect(firstGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(firstGenerateContentSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('mock-provider'); - expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(firstGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); - const secondInvokeAgentSpan = container.items.find( + const secondInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the second span?"}]', ); - expect(secondInvokeAgentSpan).toBeDefined(); - expect(secondInvokeAgentSpan!.name).toBe('invoke_agent'); expect(secondInvokeAgentSpan!.status).toBe('ok'); - expect(secondInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(secondInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"Where is the second span?"}]', - ); + expect(secondInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(secondInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toBe( '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', ); - const secondGenerateContentSpan = container.items.find( + const secondGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes( 'Second span here!', ), ); - expect(secondGenerateContentSpan).toBeDefined(); - expect(secondGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(secondGenerateContentSpan!.status).toBe('ok'); - expect(secondGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); - const toolInvokeAgentSpan = container.items.find( + const toolInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 15, ); - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan!.name).toBe('invoke_agent'); expect(toolInvokeAgentSpan!.status).toBe('ok'); - const toolGenerateContentSpan = container.items.find( + const toolGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 15, ); - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(toolGenerateContentSpan!.status).toBe('ok'); - const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan!.name).toBe('execute_tool getWeather'); + const toolExecutionSpan = spans.find(span => span.name === 'execute_tool getWeather'); expect(toolExecutionSpan!.status).toBe('ok'); - expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); - expect(toolExecutionSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); + expect(toolExecutionSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE].value).toBe('call-1'); }, @@ -148,99 +126,80 @@ describe('Vercel AI integration (v5)', () => { (createRunner, test) => { test('creates ai spans for dataCollection defaults', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(7); - const firstInvokeAgentSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(7); + const firstInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the first span?"}]', ); - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan!.name).toBe('invoke_agent'); expect(firstInvokeAgentSpan!.status).toBe('ok'); - expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(firstInvokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"Where is the first span?"}]', - ); expect(firstInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toBe( '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', ); - const firstGenerateContentSpan = container.items.find( + const firstGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes('First span here!'), ); - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(firstGenerateContentSpan!.status).toBe('ok'); - expect(firstGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(firstGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(firstGenerateContentSpan!.attributes['vercel.ai.operationId'].value).toBe( 'ai.generateText.doGenerate', ); expect(firstGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); expect(firstGenerateContentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toContain('First span here!'); - const secondInvokeAgentSpan = container.items.find( + const secondInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the second span?"}]', ); - expect(secondInvokeAgentSpan).toBeDefined(); - expect(secondInvokeAgentSpan!.name).toBe('invoke_agent'); expect(secondInvokeAgentSpan!.status).toBe('ok'); - expect(secondInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(secondInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"Where is the second span?"}]', - ); + expect(secondInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); - const secondGenerateContentSpan = container.items.find( + const secondGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes( 'Second span here!', ), ); - expect(secondGenerateContentSpan).toBeDefined(); - expect(secondGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(secondGenerateContentSpan!.status).toBe('ok'); - expect(secondGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); - const toolInvokeAgentSpan = container.items.find( + const toolInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"What is the weather in San Francisco?"}]', ); - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan!.name).toBe('invoke_agent'); expect(toolInvokeAgentSpan!.status).toBe('ok'); - expect(toolInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"What is the weather in San Francisco?"}]', - ); - const toolGenerateContentSpan = container.items.find( + const toolGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes[GEN_AI_TOOL_DEFINITIONS] !== undefined, ); - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(toolGenerateContentSpan!.status).toBe('ok'); - expect(toolGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(toolGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(toolGenerateContentSpan!.attributes[GEN_AI_TOOL_DEFINITIONS]).toBeDefined(); expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(15); - const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan!.name).toBe('execute_tool getWeather'); + const toolExecutionSpan = spans.find(span => span.name === 'execute_tool getWeather'); expect(toolExecutionSpan!.status).toBe('ok'); - expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_DESCRIPTION].value).toBe( 'Get the current weather for a location', @@ -266,39 +225,17 @@ describe('Vercel AI integration (v5)', () => { 'instrument.mjs', (createRunner, test) => { test('captures error in tool', async () => { - let transactionEvent: Event | undefined; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; let errorEvent: Event | undefined; await createRunner() // The tool error is captured while the tool is running (mid-transaction), so the error event - // and the transaction/span envelopes can arrive in either order — assert content, not wire order. + // and the span containers can arrive in either order — assert content, not wire order. .unordered() - .expect({ - transaction: transaction => { - transactionEvent = transaction; - }, - }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan!.name).toBe('invoke_agent'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - - const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - - const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolSpan).toBeDefined(); - expect(toolSpan!.name).toBe('execute_tool getWeather'); - expect(toolSpan!.status).toBe('error'); - expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); - expect(toolSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); - expect(toolSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .expect({ @@ -309,20 +246,29 @@ describe('Vercel AI integration (v5)', () => { .start() .completed(); - expect(transactionEvent).toBeDefined(); - expect(transactionEvent!.transaction).toBe('main'); + const segment = receivedSpans.find(span => span.is_segment && span.name === 'main'); + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel'); + expect(spans).toHaveLength(3); + const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + + const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); + expect(generateContentSpan!.status).toBe('ok'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); + + const toolSpan = spans.find(span => span.name === 'execute_tool getWeather'); + expect(toolSpan!.status).toBe('error'); + expect(toolSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); + expect(toolSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); - expect(errorEvent).toBeDefined(); expect(errorEvent!.level).toBe('error'); - expect(errorEvent!.tags).toEqual( - expect.objectContaining({ - 'vercel.ai.tool.name': 'getWeather', - 'vercel.ai.tool.callId': 'call-1', - }), - ); + expect(errorEvent!.tags).toEqual({ + 'vercel.ai.tool.name': 'getWeather', + 'vercel.ai.tool.callId': 'call-1', + }); - // Trace id should be the same for the transaction and error event - expect(transactionEvent!.contexts!.trace!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id); + // Trace id should be the same for the segment and error event + expect(segment!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id); }); }, { diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-pii.mjs index 7ca11a34673e..7a021162097c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-pii.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs index 42052d281304..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-telemetry-metadata.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-telemetry-metadata.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-telemetry-metadata.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-telemetry-metadata.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts index 805a65313a66..893cea265ebb 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -1,3 +1,4 @@ +import type { SerializedStreamedSpanContainer } from '@sentry/core'; import type { Event } from '@sentry/node'; import { afterAll, describe, expect } from 'vitest'; import { @@ -22,6 +23,8 @@ import { GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_REASONING_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, + SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests, createEsmTests } from '../../../../utils/runner'; @@ -47,104 +50,239 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('creates ai spans for dataCollection defaults', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - expect(container.items).toHaveLength(7); - const firstInvokeAgentSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(7); + const firstInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the first span?"}]', )!; - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan.name).toBe('invoke_agent'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(firstInvokeAgentSpan.status).toBe('ok'); - expect(firstInvokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(firstInvokeAgentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); expect(firstInvokeAgentSpan.attributes['vercel.ai.operationId']?.value).toBe('ai.generateText'); - expect(firstInvokeAgentSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value).toBe( - '[{"role":"user","content":"Where is the first span?"}]', - ); expect(firstInvokeAgentSpan.attributes[GEN_AI_OUTPUT_MESSAGES]?.value).toBe( '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', ); - const firstGenerateContentSpan = container.items.find( + const firstGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && (span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value as string | undefined)?.includes('First span here!'), )!; - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(firstGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(firstGenerateContentSpan.status).toBe('ok'); - expect(firstGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(firstGenerateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); expect(firstGenerateContentSpan.attributes['vercel.ai.operationId']?.value).toBe( 'ai.generateText.doGenerate', ); - expect(firstGenerateContentSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); - expect(firstGenerateContentSpan.attributes[GEN_AI_OUTPUT_MESSAGES]?.value).toContain('First span here!'); - - const secondInvokeAgentSpan = container.items.find( + expect(firstGenerateContentSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value).toEqual(expect.any(String)); + expect(firstGenerateContentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toEqual({ + type: 'string', + value: + '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', + }); + + const secondInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the second span?"}]', )!; - expect(secondInvokeAgentSpan).toBeDefined(); - expect(secondInvokeAgentSpan.name).toBe('invoke_agent'); + expect(secondInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 183, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 92, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 91, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toEqual({ + type: 'string', + value: + '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"Where is the second span?"}]', + }); expect(secondInvokeAgentSpan.status).toBe('ok'); - expect(secondInvokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(secondInvokeAgentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); - const secondGenerateContentSpan = container.items.find( + const secondGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && (span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value as string | undefined)?.includes('Second span here!'), )!; - expect(secondGenerateContentSpan).toBeDefined(); - expect(secondGenerateContentSpan.name).toBe('generate_content mock-model-id'); expect(secondGenerateContentSpan.status).toBe('ok'); - expect(secondGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(secondGenerateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); - const toolInvokeAgentSpan = container.items.find( + const toolInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"What is the weather in San Francisco?"}]', )!; - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan.name).toBe('invoke_agent'); + expect(toolInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolInvokeAgentSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.invoke_agent', + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 40, + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toEqual({ + type: 'string', + value: + '[{"role":"assistant","parts":[{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"{\\"location\\":\\"San Francisco\\"}"}],"finish_reason":"tool_call"}]', + }); expect(toolInvokeAgentSpan.status).toBe('ok'); - expect(toolInvokeAgentSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value).toBe( - '[{"role":"user","content":"What is the weather in San Francisco?"}]', - ); - const toolGenerateContentSpan = container.items.find( + const toolGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes[GEN_AI_TOOL_DEFINITIONS] !== undefined, )!; - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(toolGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 40, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(toolGenerateContentSpan.status).toBe('ok'); - expect(toolGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); - expect(toolGenerateContentSpan.attributes[GEN_AI_TOOL_DEFINITIONS]).toBeDefined(); + expect(toolGenerateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); + expect(toolGenerateContentSpan!.attributes[GEN_AI_TOOL_DEFINITIONS]).toEqual({ + type: 'string', + value: expect.stringContaining('getWeather'), + }); expect(toolGenerateContentSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); - const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather')!; - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan.name).toBe('execute_tool getWeather'); + const toolExecutionSpan = spans.find(span => span.name === 'execute_tool getWeather')!; + expect(toolExecutionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolExecutionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'execute_tool', + }); + expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'call-1', + }); expect(toolExecutionSpan.status).toBe('ok'); - expect(toolExecutionSpan.attributes['sentry.op']?.value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); expect(toolExecutionSpan.attributes[GEN_AI_TOOL_DESCRIPTION]?.value).toBe( 'Get the current weather for a location', ); - expect(toolExecutionSpan.attributes[GEN_AI_TOOL_CALL_ARGUMENTS]).toBeDefined(); - expect(toolExecutionSpan.attributes[GEN_AI_TOOL_CALL_RESULT]).toBeDefined(); + expect(toolExecutionSpan.attributes[GEN_AI_TOOL_CALL_ARGUMENTS]?.value).toEqual(expect.any(String)); + expect(toolExecutionSpan.attributes[GEN_AI_TOOL_CALL_RESULT]?.value).toEqual(expect.any(String)); }, }) .start() @@ -165,62 +303,104 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('creates ai spans when dataCollection.genAi has inputs and outputs disabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - expect(container.items).toHaveLength(7); - const firstInvokeAgentSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(7); + const firstInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes['vercel.ai.operationId']?.value === 'ai.generateText' && span.attributes[GEN_AI_INPUT_MESSAGES] === undefined && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 10, )!; - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan.name).toBe('invoke_agent'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); expect(firstInvokeAgentSpan.status).toBe('ok'); - expect(firstInvokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); - expect(firstInvokeAgentSpan.attributes['vercel.ai.operationId']?.value).toBe('ai.generateText'); + expect(firstInvokeAgentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); expect(firstInvokeAgentSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('mock-model-id'); expect(firstInvokeAgentSpan.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('mock-model-id'); - expect(firstInvokeAgentSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); expect(firstInvokeAgentSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20); expect(firstInvokeAgentSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(30); - expect(firstInvokeAgentSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); - const firstGenerateContentSpan = container.items.find( + const firstGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes['vercel.ai.operationId']?.value === 'ai.generateText.doGenerate' && span.attributes[GEN_AI_INPUT_MESSAGES] === undefined && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 10, )!; - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(firstGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(firstGenerateContentSpan.status).toBe('ok'); - expect(firstGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); - expect(firstGenerateContentSpan.attributes['vercel.ai.operationId']?.value).toBe( - 'ai.generateText.doGenerate', - ); + expect(firstGenerateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); expect(firstGenerateContentSpan.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('mock-provider'); - expect(firstGenerateContentSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); - expect(firstGenerateContentSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); - const secondInvokeAgentSpan = container.items.find( + const secondInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes['vercel.ai.operationId']?.value === 'ai.generateText' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 91, )!; - expect(secondInvokeAgentSpan).toBeDefined(); - expect(secondInvokeAgentSpan.name).toBe('invoke_agent'); + expect(secondInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 183, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 92, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 91, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(secondInvokeAgentSpan.status).toBe('ok'); - expect(secondInvokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(secondInvokeAgentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); expect(secondInvokeAgentSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value).toBe( '[{"role":"user","content":"Where is the second span?"}]', ); @@ -228,16 +408,14 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', ); - const secondGenerateContentSpan = container.items.find( + const secondGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes['vercel.ai.operationId']?.value === 'ai.generateText.doGenerate' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 91, )!; - expect(secondGenerateContentSpan).toBeDefined(); - expect(secondGenerateContentSpan.name).toBe('generate_content mock-model-id'); expect(secondGenerateContentSpan.status).toBe('ok'); - expect(secondGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(secondGenerateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); expect(secondGenerateContentSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); expect(secondGenerateContentSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value as string).toContain( 'Where is the second span?', @@ -246,27 +424,79 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe 'Second span here!', ); - const toolInvokeAgentSpan = container.items.find( + const toolInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 15, )!; - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan.name).toBe('invoke_agent'); + expect(toolInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolInvokeAgentSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.invoke_agent', + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 40, + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(toolInvokeAgentSpan.status).toBe('ok'); - const toolGenerateContentSpan = container.items.find( + const toolGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 15, )!; - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(toolGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(toolGenerateContentSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.generate_content', + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 40, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(toolGenerateContentSpan.status).toBe('ok'); - const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather')!; - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan.name).toBe('execute_tool getWeather'); + const toolExecutionSpan = spans.find(span => span.name === 'execute_tool getWeather')!; + expect(toolExecutionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolExecutionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'execute_tool', + }); expect(toolExecutionSpan.status).toBe('ok'); - expect(toolExecutionSpan.attributes['sentry.op']?.value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); expect(toolExecutionSpan.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]?.value).toBe('call-1'); }, @@ -288,39 +518,15 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe 'instrument.mjs', (createRunner, test) => { test('captures error in tool', async () => { - let transactionEvent: Event | undefined; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; let errorEvent: Event | undefined; await createRunner() - .expect({ - transaction: transaction => { - transactionEvent = transaction; - }, - }) + .unordered() .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - expect(container.items).toHaveLength(3); - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent')!; - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan.name).toBe('invoke_agent'); - expect(invokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); - - const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id')!; - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan.name).toBe('generate_content mock-model-id'); - expect(generateContentSpan.status).toBe('ok'); - expect(generateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); - - const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather')!; - expect(toolSpan).toBeDefined(); - expect(toolSpan.name).toBe('execute_tool getWeather'); - expect(toolSpan.status).toBe('error'); - expect(toolSpan.attributes['sentry.op']?.value).toBe('gen_ai.execute_tool'); - expect(toolSpan.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .expect({ @@ -331,20 +537,58 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe .start() .completed(); - expect(transactionEvent).toBeDefined(); - expect(transactionEvent!.transaction).toBe('main'); + const segment = receivedSpans.find(span => span.is_segment && span.name === 'main'); + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel'); + expect(spans).toHaveLength(3); + const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent')!; + expect(invokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(invokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(invokeAgentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); + + const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id')!; + expect(generateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 40, + }); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(generateContentSpan.status).toBe('ok'); + expect(generateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); + + const toolSpan = spans.find(span => span.name === 'execute_tool getWeather')!; + expect(toolSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'execute_tool' }); + expect(toolSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]).toEqual({ type: 'string', value: 'call-1' }); + expect(toolSpan.status).toBe('error'); + expect(toolSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); + expect(toolSpan.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); - expect(errorEvent).toBeDefined(); expect(errorEvent!.level).toBe('error'); - expect(errorEvent!.tags).toEqual( - expect.objectContaining({ - 'vercel.ai.tool.name': 'getWeather', - 'vercel.ai.tool.callId': 'call-1', - }), - ); - - // Trace id should be the same for the transaction and error event - expect(transactionEvent!.contexts!.trace!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id); + expect(errorEvent!.tags).toEqual({ + 'vercel.ai.tool.name': 'getWeather', + 'vercel.ai.tool.callId': 'call-1', + }); + + // Trace id should be the same for the segment and error event + expect(segment!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id); }); }, { @@ -361,27 +605,25 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('creates ai related spans', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - expect(container.items).toHaveLength(7); - const invokeAgentSpans = container.items.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(7); + const invokeAgentSpans = spans.filter( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent', ); expect(invokeAgentSpans).toHaveLength(3); - const generateContentSpans = container.items.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + const generateContentSpans = spans.filter( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', ); expect(generateContentSpans).toHaveLength(3); - const toolSpan = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.execute_tool', - ); + const toolSpan = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.execute_tool'); expect(toolSpan).toBeDefined(); }, }) @@ -403,46 +645,40 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('creates spans for ToolLoopAgent with tool calls', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - expect(container.items).toHaveLength(4); - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent weather_agent')!; - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan.name).toBe('invoke_agent weather_agent'); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(4); + const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent weather_agent')!; expect(invokeAgentSpan.status).toBe('ok'); - expect(invokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('mock-model-id'); - const toolCallsGenerateContentSpan = container.items.find( + const toolCallsGenerateContentSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["tool-calls"]', )!; - expect(toolCallsGenerateContentSpan).toBeDefined(); expect(toolCallsGenerateContentSpan.name).toBe('generate_content mock-model-id'); expect(toolCallsGenerateContentSpan.status).toBe('ok'); - expect(toolCallsGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(toolCallsGenerateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); expect(toolCallsGenerateContentSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); expect(toolCallsGenerateContentSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20); - const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather')!; - expect(toolSpan).toBeDefined(); - expect(toolSpan.name).toBe('execute_tool getWeather'); + const toolSpan = spans.find(span => span.name === 'execute_tool getWeather')!; expect(toolSpan.status).toBe('ok'); - expect(toolSpan.attributes['sentry.op']?.value).toBe('gen_ai.execute_tool'); + expect(toolSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); expect(toolSpan.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); expect(toolSpan.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]?.value).toBe('call-1'); - const finalGenerateContentSpan = container.items.find( + const finalGenerateContentSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["stop"]', )!; - expect(finalGenerateContentSpan).toBeDefined(); expect(finalGenerateContentSpan.name).toBe('generate_content mock-model-id'); expect(finalGenerateContentSpan.status).toBe('ok'); - expect(finalGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(finalGenerateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); expect(finalGenerateContentSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); expect(finalGenerateContentSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(25); }, @@ -465,18 +701,16 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('parents concurrent calls that share one model instance correctly', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - const invokeAgents = container.items.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', ); - const generateContents = container.items.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + const invokeAgents = spans.filter(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent'); + const generateContents = spans.filter( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', ); // Two concurrent operations -> two invoke_agent + two generate_content spans. @@ -516,14 +750,16 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe // collapse onto whichever operation resolved the shared model last. test('parents concurrent streamText calls that share one model instance correctly', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const invokeAgents = container.items.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', ); - const generateContents = container.items.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + const invokeAgents = spans.filter(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent'); + const generateContents = spans.filter( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', ); // Two concurrent operations -> two invoke_agent + two generate_content spans. @@ -560,23 +796,19 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('creates streamText spans with the model call parented to invoke_agent', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - const invokeAgent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - )!; - expect(invokeAgent).toBeDefined(); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText'); - const generateContent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + const generateContent = spans.find( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', )!; - expect(generateContent).toBeDefined(); expect(generateContent.parent_span_id).toBe(invokeAgent.span_id); expect(generateContent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText.doStream'); @@ -619,13 +851,14 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('captures usage, tool calls and output across a multi-step streamText', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const invokeAgent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - )!; - expect(invokeAgent).toBeDefined(); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent.status).toBe('ok'); expect(invokeAgent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText'); // Usage is summed across the two streamed model calls (10+15, 20+25, 30+40). @@ -633,8 +866,8 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe expect(invokeAgent.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(45); expect(invokeAgent.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(70); - const generateContents = container.items.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + const generateContents = spans.filter( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', ); expect(generateContents).toHaveLength(2); generateContents.forEach(span => expect(span.parent_span_id).toBe(invokeAgent.span_id)); @@ -643,7 +876,6 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe const toolStep = generateContents.find( span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["tool-calls"]', )!; - expect(toolStep).toBeDefined(); expect(toolStep.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); expect(toolStep.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20); const toolStepOutput = toolStep.attributes[GEN_AI_OUTPUT_MESSAGES]?.value as string; @@ -654,15 +886,13 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe const textStep = generateContents.find( span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["stop"]', )!; - expect(textStep).toBeDefined(); expect(textStep.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); expect(textStep.attributes[GEN_AI_OUTPUT_MESSAGES]?.value).toContain('Sunny, 72°F.'); // A tool span is emitted for the streamed tool call. Its parent and recorded input/output // vary by path during stream consumption (tool i/o is covered by the non-stream scenario // and the v7 path), so here we just assert the span exists with the right name/status. - const executeTool = container.items.find(span => span.name === 'execute_tool getWeather')!; - expect(executeTool).toBeDefined(); + const executeTool = spans.find(span => span.name === 'execute_tool getWeather')!; expect(executeTool.status).toBe('ok'); expect(executeTool.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); }, @@ -685,21 +915,21 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('captures streamed structured output (streamText with experimental_output)', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const invokeAgent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - )!; - expect(invokeAgent).toBeDefined(); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent.status).toBe('ok'); expect(invokeAgent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText'); expect(invokeAgent.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(30); - const generateContent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + const generateContent = spans.find( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', )!; - expect(generateContent).toBeDefined(); expect(generateContent.parent_span_id).toBe(invokeAgent.span_id); expect(generateContent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(12); expect(generateContent.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(18); @@ -729,25 +959,21 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('finishes spans with an error status when the operation rejects', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); // The model throws, so the operation rejects. The spans must still be *finished* (and // therefore present in the transaction) with an error status — not left open. - const invokeAgent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - )!; - expect(invokeAgent).toBeDefined(); + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent.status).toBe('error'); - const generateContent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + const generateContent = spans.find( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', )!; - expect(generateContent).toBeDefined(); expect(generateContent.status).toBe('error'); }, }) @@ -769,17 +995,16 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('derives provider-metadata token breakdown and system instructions', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - const generateContent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + const generateContent = spans.find( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', )!; - expect(generateContent).toBeDefined(); // Cache/reasoning token breakdown is derived from the model's `providerMetadata` by the // channel subscriber, which v6 reaches through the orchestrion adapter, so the shape is the @@ -789,9 +1014,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe // The per-response `responseId` is not a conversation id and must not be recorded as one. expect(generateContent.attributes[GEN_AI_CONVERSATION_ID]).toBeUndefined(); - const invokeAgent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - )!; + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent).toBeDefined(); // The system prompt is supplied via the v7-only `instructions` option. Only the channel @@ -821,17 +1044,15 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('derives gen_ai.conversation.id from the OpenAI `conversation` provider option', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); const genAiSpans = container.items.filter(s => - String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.'), + String(s.attributes[SENTRY_OP]?.value ?? '').startsWith('gen_ai.'), ); const conversationIdOf = (span: (typeof genAiSpans)[number]) => span.attributes[GEN_AI_CONVERSATION_ID]?.value; - const invokeAgentSpans = genAiSpans.filter( - s => s.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - ); + const invokeAgentSpans = genAiSpans.filter(s => s.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent'); expect(invokeAgentSpans).toHaveLength(4); const [firstTurn, secondTurn, chainedTurn, apiTurn] = invokeAgentSpans.sort( (a, b) => a.start_timestamp - b.start_timestamp, @@ -850,7 +1071,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe // Model-call and tool spans carry their operation's id, even though their start events // do not carry `providerOptions`. const modelCallSpans = genAiSpans.filter( - s => s.attributes['sentry.op']?.value === 'gen_ai.generate_content', + s => s.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', ); expect(modelCallSpans.map(conversationIdOf).sort()).toEqual([ 'conv-from-api', @@ -858,7 +1079,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe 'conv_azure', undefined, ]); - const toolSpan = genAiSpans.find(s => s.attributes['sentry.op']?.value === 'gen_ai.execute_tool')!; + const toolSpan = genAiSpans.find(s => s.attributes[SENTRY_OP]?.value === 'gen_ai.execute_tool')!; expect(toolSpan).toBeDefined(); expect(conversationIdOf(toolSpan)).toBe('conv_azure'); }, @@ -881,21 +1102,21 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('reads cache token counts from the SDK usage object', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const generateContent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + const generateContent = spans.find( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', )!; - expect(generateContent).toBeDefined(); expect(generateContent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(120); expect(generateContent.attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]?.value).toBe(80); expect(generateContent.attributes[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]?.value).toBe(20); - const invokeAgent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - )!; - expect(invokeAgent).toBeDefined(); + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent.attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]?.value).toBe(80); expect(invokeAgent.attributes[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]?.value).toBe(20); }, @@ -924,32 +1145,30 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe test('creates embeddings spans for embed and embedMany', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); - const embedSpan = container.items.find( + const embedSpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', )!; - expect(embedSpan).toBeDefined(); expect(embedSpan.name).toBe('embeddings mock-model-id'); expect(embedSpan.status).toBe('ok'); - expect(embedSpan.attributes['sentry.op']?.value).toBe('gen_ai.embeddings'); + expect(embedSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.embeddings'); expect(embedSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('mock-model-id'); expect(embedSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); - const embedManySpan = container.items.find( + const embedManySpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === '["First input","Second input"]', ); if (embedManyInstrumented) { - expect(embedManySpan).toBeDefined(); expect(embedManySpan!.name).toBe('embeddings mock-model-id'); expect(embedManySpan!.status).toBe('ok'); - expect(embedManySpan!.attributes['sentry.op']?.value).toBe('gen_ai.embeddings'); + expect(embedManySpan!.attributes[SENTRY_OP]?.value).toBe('gen_ai.embeddings'); expect(embedManySpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(20); } else { expect(embedManySpan).toBeUndefined(); @@ -978,28 +1197,26 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe // channel directly — v6 is exercised here, v4 in the base suite. test.skipIf(version === '7')('creates spans for generateObject', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan).toBeDefined(); + const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); expect(invokeAgentSpan!.status).toBe('ok'); - expect(invokeAgentSpan!.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes['vercel.ai.operationId']?.value).toBe('ai.generateObject'); expect(invokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('mock-model-id'); expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(25); expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(40); - const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); + const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); expect(generateContentSpan!.attributes['vercel.ai.operationId']?.value).toBe( 'ai.generateObject.doGenerate', ); @@ -1032,9 +1249,8 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe .expect({ span: container => { const invokeAgent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent', )!; - expect(invokeAgent).toBeDefined(); expect(invokeAgent.status).toBe('error'); expect(invokeAgent.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('mock-model-id'); expect(invokeAgent.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined(); @@ -1042,9 +1258,8 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe expect(invokeAgent.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); const generateContent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', )!; - expect(generateContent).toBeDefined(); expect(generateContent.status).toBe('error'); }, }) @@ -1072,16 +1287,14 @@ describe('Vercel AI integration experimental_evaluate', () => { (createRunner, test) => { test('creates an evaluate span', async () => { await createRunner() - .unordered() .expect({ span: container => { const evaluateSpan = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.evaluate', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate', )!; - expect(evaluateSpan).toBeDefined(); expect(evaluateSpan.name).toBe('evaluate typesafe-ai/jev'); expect(evaluateSpan.status).toBe('ok'); - expect(evaluateSpan.attributes['sentry.origin']?.value).toBe('auto.vercelai.channel'); + expect(evaluateSpan.attributes[SENTRY_ORIGIN]?.value).toBe('auto.vercelai.channel'); expect(evaluateSpan.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('evaluate'); expect(evaluateSpan.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('gateway'); expect(evaluateSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('typesafe-ai/jev'); @@ -1145,13 +1358,11 @@ describe('Vercel AI integration experimental_evaluate', () => { (createRunner, test) => { test('does not record inputs or outputs when recording is off', async () => { await createRunner() - .unordered() .expect({ span: container => { const evaluateSpan = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.evaluate', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate', )!; - expect(evaluateSpan).toBeDefined(); expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); // State, questions and answers must not come back through another attribute. Only the attributes @@ -1172,3 +1383,52 @@ describe('Vercel AI integration experimental_evaluate', () => { }, ); }); + +describe('Vercel AI telemetry metadata (v6)', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + createEsmAndCjsTests( + __dirname, + 'scenario-telemetry-metadata.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('records experimental_telemetry.metadata as span attributes', async () => { + await createRunner() + .expect({ + span: container => { + for (const [name, op, requestId] of [ + ['invoke_agent support-chat', 'gen_ai.invoke_agent', 'req_generate'], + ['generate_content mock-model-id', 'gen_ai.generate_content', 'req_generate'], + ['invoke_agent support-stream', 'gen_ai.invoke_agent', 'req_stream'], + ['generate_content mock-model-id', 'gen_ai.generate_content', 'req_stream'], + ['embeddings mock-model-id', 'gen_ai.embeddings', 'req_embed'], + ['embeddings mock-model-id', 'gen_ai.embeddings', 'req_embed_many'], + ]) { + const span = container.items.find( + span => + span.name === name && + span.attributes['vercel.ai.telemetry.metadata.requestId']?.value === requestId, + ); + expect(span?.attributes[SENTRY_OP]).toEqual({ type: 'string', value: op }); + expect(span?.attributes['vercel.ai.telemetry.metadata.tenantId']).toEqual({ + type: 'string', + value: 'acme', + }); + } + const generateSpans = container.items.filter( + span => span.attributes['vercel.ai.telemetry.metadata.requestId']?.value === 'req_generate', + ); + for (const span of generateSpans) { + expect(span.attributes['vercel.ai.telemetry.metadata.attempt']).toEqual({ type: 'string', value: '2' }); + } + }, + }) + .start() + .completed(); + }); + }, + { additionalDependencies: { ai: '^6.0.0' } }, + ); +}); From 0db19493efa80b9a1840caedf19e70eb090e8a3b Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Thu, 8 Oct 2026 14:31:36 +0200 Subject: [PATCH 37/47] test(node): Port Tedious integration tests to span streaming (#25155) Port Tedious tests to span streaming. Part of #24137 --------- Co-authored-by: GPT-6 --- .../tedious/instrument-span-streaming.mjs | 10 - .../suites/tracing/tedious/instrument.mjs | 1 - .../suites/tracing/tedious/test.ts | 221 +++++++----------- 3 files changed, 83 insertions(+), 149 deletions(-) delete mode 100644 dev-packages/node-integration-tests/suites/tracing/tedious/instrument-span-streaming.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/tedious/instrument-span-streaming.mjs b/dev-packages/node-integration-tests/suites/tracing/tedious/instrument-span-streaming.mjs deleted file mode 100644 index 53b9511a21f0..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/tedious/instrument-span-streaming.mjs +++ /dev/null @@ -1,10 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/tedious/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/tedious/instrument.mjs index 170ad6f6a702..46a27dd03b74 100644 --- a/dev-packages/node-integration-tests/suites/tracing/tedious/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/tedious/instrument.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/tedious/test.ts b/dev-packages/node-integration-tests/suites/tracing/tedious/test.ts index eb5c81a0a49d..1d24ab0e7041 100644 --- a/dev-packages/node-integration-tests/suites/tracing/tedious/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/tedious/test.ts @@ -1,3 +1,15 @@ +import { + DB_NAMESPACE, + DB_QUERY_SUMMARY, + DB_QUERY_TEXT, + DB_SYSTEM_NAME, + DB_USER, + SENTRY_OP, + SENTRY_ORIGIN, + SERVER_ADDRESS, + SERVER_PORT, +} from '@sentry/conventions/attributes'; +import { DB } from '@sentry/conventions/op'; import { afterAll, expect } from 'vitest'; import { conditionalTest } from '../../../utils'; import { cleanupChildProcesses, createEsmAndCjsTests, describeWithDockerCompose } from '../../../utils/runner'; @@ -9,36 +21,36 @@ describeWithDockerCompose('tedious auto instrumentation', { workingDirectory: [_ cleanupChildProcesses(); }); - const dbSpan = (overrides: Record) => + const dbSpan = (text: string, status = 'ok') => expect.objectContaining({ - op: 'db', - origin: ORIGIN, - data: expect.objectContaining({ - 'sentry.origin': ORIGIN, - 'sentry.op': 'db', - 'db.system.name': 'mssql', - 'db.namespace': 'master', - 'db.user': 'sa', - 'server.address': '127.0.0.1', - 'server.port': 1433, + status, + attributes: expect.objectContaining({ + [DB_QUERY_TEXT]: { type: 'string', value: text }, + [SENTRY_ORIGIN]: { type: 'string', value: ORIGIN }, + [SENTRY_OP]: { type: 'string', value: DB }, + [DB_SYSTEM_NAME]: { type: 'string', value: 'mssql' }, + [DB_NAMESPACE]: { type: 'string', value: 'master' }, + [DB_USER]: { type: 'string', value: 'sa' }, + [SERVER_ADDRESS]: { type: 'string', value: '127.0.0.1' }, + [SERVER_PORT]: { type: 'integer', value: 1433 }, }), - ...overrides, }); - const EXPECTED_TRANSACTION = { - transaction: 'Test Transaction', - spans: expect.arrayContaining([ - dbSpan({ description: 'SELECT ? + ? AS solution', status: 'ok' }), - dbSpan({ description: 'SELECT ?; SELECT ?', status: 'ok' }), - dbSpan({ description: 'select !', status: 'internal_error' }), - dbSpan({ description: '[dbo].[test_proced]', status: 'ok' }), - dbSpan({ description: 'INSERT INTO [dbo].[test_prepared] VALUES (@val1, @val2)', status: 'ok' }), + const EXPECTED_SPANS = { + items: expect.arrayContaining([ + dbSpan('SELECT ? + ? AS solution'), + dbSpan('SELECT ?; SELECT ?'), + dbSpan('select !', 'error'), + dbSpan('[dbo].[test_proced]'), + dbSpan('INSERT INTO [dbo].[test_prepared] VALUES (@val1, @val2)'), expect.objectContaining({ - description: 'execBulkLoad test_bulk master', - op: 'db', - origin: ORIGIN, + name: 'execBulkLoad test_bulk', status: 'ok', - data: expect.objectContaining({ 'db.sql.table': 'test_bulk' }), + attributes: expect.objectContaining({ + 'db.sql.table': { type: 'string', value: 'test_bulk' }, + [SENTRY_OP]: { type: 'string', value: DB }, + [SENTRY_ORIGIN]: { type: 'string', value: ORIGIN }, + }), }), ]), }; @@ -47,9 +59,9 @@ describeWithDockerCompose('tedious auto instrumentation', { workingDirectory: [_ test('should auto-instrument `tedious` package', async () => { await createTestRunner() .expect({ - transaction: transaction => { - expect(transaction.transaction).toBe(EXPECTED_TRANSACTION.transaction); - expect(transaction.spans).toEqual(EXPECTED_TRANSACTION.spans); + span: container => { + expect(container.items.find(span => span.is_segment)?.name).toBe('Test Transaction'); + expect(container.items).toEqual(EXPECTED_SPANS.items); const CREATE_PROCEDURE = 'CREATE OR ALTER PROCEDURE [dbo].[test_proced] @inputVal varchar(?), @outputCount int OUTPUT AS set @outputCount = LEN(@inputVal)'; @@ -65,26 +77,43 @@ describeWithDockerCompose('tedious auto instrumentation', { workingDirectory: [_ const SELECT_STRING_LITERAL = 'SELECT c1, c2 FROM [dbo].[test_bulk] WHERE c2 = ?'; expect( - (transaction.spans ?? []) - .filter(span => span.origin === ORIGIN) - .map(span => ({ name: span.description, text: span.data?.['db.query.text'] })), + container.items + .filter(span => span.attributes[SENTRY_ORIGIN]?.value === ORIGIN) + .map(span => ({ + name: span.name, + summary: span.attributes[DB_QUERY_SUMMARY]?.value, + text: span.attributes[DB_QUERY_TEXT]?.value, + })), ).toEqual([ - { name: 'SELECT ? + ? AS solution', text: 'SELECT ? + ? AS solution' }, - { name: 'SELECT ?; SELECT ?', text: 'SELECT ?; SELECT ?' }, - { name: 'select !', text: 'select !' }, - { name: CREATE_PROCEDURE, text: CREATE_PROCEDURE }, - { name: '[dbo].[test_proced]', text: '[dbo].[test_proced]' }, - { name: CREATE_PREPARED_TABLE, text: CREATE_PREPARED_TABLE }, - { name: INSERT_PREPARED, text: INSERT_PREPARED }, - { name: INSERT_PREPARED, text: INSERT_PREPARED }, - { name: CREATE_BULK_TABLE, text: CREATE_BULK_TABLE }, - { name: 'execBulkLoad test_bulk master', text: undefined }, - { name: INSERT_BULK, text: INSERT_BULK }, - { name: SELECT_PREPARED, text: SELECT_PREPARED }, - { name: SELECT_JOIN, text: SELECT_JOIN }, - { name: SELECT_INLINE_LITERAL, text: SELECT_INLINE_LITERAL }, - { name: SELECT_PARAMETERIZED, text: SELECT_PARAMETERIZED }, - { name: SELECT_STRING_LITERAL, text: SELECT_STRING_LITERAL }, + { name: 'SELECT', summary: 'SELECT', text: 'SELECT ? + ? AS solution' }, + { name: 'SELECT', summary: 'SELECT', text: 'SELECT ?; SELECT ?' }, + { name: 'select', summary: 'select', text: 'select !' }, + { name: 'CREATE', summary: 'CREATE', text: CREATE_PROCEDURE }, + { name: 'callProcedure [dbo].[test_proced]', summary: undefined, text: '[dbo].[test_proced]' }, + { name: 'if', summary: 'if', text: CREATE_PREPARED_TABLE }, + { name: 'INSERT [dbo].[test_prepared]', summary: 'INSERT [dbo].[test_prepared]', text: INSERT_PREPARED }, + { name: 'INSERT [dbo].[test_prepared]', summary: 'INSERT [dbo].[test_prepared]', text: INSERT_PREPARED }, + { name: 'if', summary: 'if', text: CREATE_BULK_TABLE }, + { name: 'insert', summary: 'insert', text: INSERT_BULK }, + { name: 'execBulkLoad test_bulk', summary: undefined, text: undefined }, + { name: 'SELECT [dbo].[test_prepared]', summary: 'SELECT [dbo].[test_prepared]', text: SELECT_PREPARED }, + { + // TODO: (check if correct) Both sides of the join survive into the summary. + name: 'SELECT [dbo].[test_prepared] [dbo].[test_bulk]', + summary: 'SELECT [dbo].[test_prepared] [dbo].[test_bulk]', + text: SELECT_JOIN, + }, + { + name: 'SELECT [dbo].[test_prepared]', + summary: 'SELECT [dbo].[test_prepared]', + text: SELECT_INLINE_LITERAL, + }, + { + name: 'SELECT [dbo].[test_prepared]', + summary: 'SELECT [dbo].[test_prepared]', + text: SELECT_PARAMETERIZED, + }, + { name: 'SELECT [dbo].[test_bulk]', summary: 'SELECT [dbo].[test_bulk]', text: SELECT_STRING_LITERAL }, ]); }, }) @@ -98,13 +127,13 @@ describeWithDockerCompose('tedious auto instrumentation', { workingDirectory: [_ createEsmAndCjsTests( __dirname, 'scenario.mjs', - 'instrument-span-streaming.mjs', + 'instrument.mjs', (createTestRunner, test) => { test('should auto-instrument `tedious` package', async () => { await createTestRunner() .expect({ span: container => { - const dbSpans = container.items.filter(item => item.attributes['sentry.origin']?.value === ORIGIN); + const dbSpans = container.items.filter(item => item.attributes[SENTRY_ORIGIN]?.value === ORIGIN); expect(dbSpans.map(span => span.name)).toEqual( expect.arrayContaining([ @@ -116,13 +145,12 @@ describeWithDockerCompose('tedious auto instrumentation', { workingDirectory: [_ ]), ); expect(dbSpans.find(span => span.name === 'select')?.status).toBe('error'); - expect(dbSpans[0]?.attributes).toMatchObject({ - 'db.system.name': { value: 'mssql' }, - 'db.namespace': { value: 'master' }, - 'db.user': { value: 'sa' }, - 'server.address': { value: '127.0.0.1' }, - 'server.port': { value: 1433 }, - }); + const attributes = dbSpans[0]?.attributes; + expect(attributes?.[DB_SYSTEM_NAME]).toEqual({ type: 'string', value: 'mssql' }); + expect(attributes?.[DB_NAMESPACE]).toEqual({ type: 'string', value: 'master' }); + expect(attributes?.[DB_USER]).toEqual({ type: 'string', value: 'sa' }); + expect(attributes?.[SERVER_ADDRESS]).toEqual({ type: 'string', value: '127.0.0.1' }); + expect(attributes?.[SERVER_PORT]).toEqual({ type: 'integer', value: 1433 }); }, }) .start() @@ -132,87 +160,4 @@ describeWithDockerCompose('tedious auto instrumentation', { workingDirectory: [_ { additionalDependencies: { tedious: '^20' } }, ); }); - - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-span-streaming.mjs', (createTestRunner, test) => { - test('should name spans after the query summary with span streaming', async () => { - await createTestRunner() - .expect({ - span: container => { - const dbSpans = container.items.filter(item => item.attributes['sentry.origin']?.value === ORIGIN); - - expect( - dbSpans.map(span => ({ - name: span.name, - summary: span.attributes['db.query.summary']?.value, - text: span.attributes['db.query.text']?.value, - })), - ).toEqual([ - { name: 'SELECT', summary: 'SELECT', text: 'SELECT ? + ? AS solution' }, - { name: 'SELECT', summary: 'SELECT', text: 'SELECT ?; SELECT ?' }, - { name: 'select', summary: 'select', text: 'select !' }, - { - name: 'CREATE', - summary: 'CREATE', - text: 'CREATE OR ALTER PROCEDURE [dbo].[test_proced] @inputVal varchar(?), @outputCount int OUTPUT AS set @outputCount = LEN(@inputVal)', - }, - { name: 'callProcedure [dbo].[test_proced]', summary: undefined, text: '[dbo].[test_proced]' }, - { - name: 'if', - summary: 'if', - text: 'if object_id(?) is null CREATE TABLE [dbo].[test_prepared] (c1 int, c2 int)', - }, - { - name: 'INSERT [dbo].[test_prepared]', - summary: 'INSERT [dbo].[test_prepared]', - text: 'INSERT INTO [dbo].[test_prepared] VALUES (@val1, @val2)', - }, - { - name: 'INSERT [dbo].[test_prepared]', - summary: 'INSERT [dbo].[test_prepared]', - text: 'INSERT INTO [dbo].[test_prepared] VALUES (@val1, @val2)', - }, - { - name: 'if', - summary: 'if', - text: 'if object_id(?) is null CREATE TABLE [dbo].[test_bulk] (c1 int, c2 varchar(?))', - }, - { - name: 'insert', - summary: 'insert', - text: 'insert bulk test_bulk([c1] int, [c2] nvarchar(?)) WITH (KEEP_NULLS)', - }, - { name: 'execBulkLoad test_bulk', summary: undefined, text: undefined }, - { - name: 'SELECT [dbo].[test_prepared]', - summary: 'SELECT [dbo].[test_prepared]', - text: 'SELECT c1, c2 FROM [dbo].[test_prepared]', - }, - { - // TODO: (check if correct) Both sides of the join survive into the summary. - name: 'SELECT [dbo].[test_prepared] [dbo].[test_bulk]', - summary: 'SELECT [dbo].[test_prepared] [dbo].[test_bulk]', - text: 'SELECT p.c1 FROM [dbo].[test_prepared] p INNER JOIN [dbo].[test_bulk] b ON p.c1 = b.c1', - }, - { - name: 'SELECT [dbo].[test_prepared]', - summary: 'SELECT [dbo].[test_prepared]', - text: 'SELECT c1, c2 FROM [dbo].[test_prepared] WHERE c1 = ?', - }, - { - name: 'SELECT [dbo].[test_prepared]', - summary: 'SELECT [dbo].[test_prepared]', - text: 'SELECT c1, c2 FROM [dbo].[test_prepared] WHERE c1 = @c1', - }, - { - name: 'SELECT [dbo].[test_bulk]', - summary: 'SELECT [dbo].[test_bulk]', - text: 'SELECT c1, c2 FROM [dbo].[test_bulk] WHERE c2 = ?', - }, - ]); - }, - }) - .start() - .completed(); - }); - }); }); From 4971a72e0e0d14a34e03e79ab4cabf8397f6f0ba Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Thu, 8 Oct 2026 14:44:13 +0200 Subject: [PATCH 38/47] test(node): Port MySQL integration tests to span streaming (#25148) Port MySQL and MySQL2 diagnostics-channel tests to default span streaming. Keep the original MySQL2 suite static for transaction coverage. Part of #24137 --------- Co-authored-by: GPT-6 --- .../suites/tracing/mysql/instrument.mjs | 1 - .../suites/tracing/mysql/test.ts | 342 +++++++----------- .../mysql2-tracing-channel/instrument.mjs | 1 - .../tracing/mysql2-tracing-channel/test.ts | 68 ++-- .../suites/tracing/mysql2/test.ts | 11 +- 5 files changed, 182 insertions(+), 241 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/mysql/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/mysql/instrument.mjs index 450e0dd3ebfe..46a27dd03b74 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mysql/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/mysql/instrument.mjs @@ -6,5 +6,4 @@ Sentry.init({ release: '1.0', tracesSampleRate: 1.0, transport: loggingTransport, - traceLifecycle: process.env.STREAMED === 'true' ? 'stream' : 'static', }); diff --git a/dev-packages/node-integration-tests/suites/tracing/mysql/test.ts b/dev-packages/node-integration-tests/suites/tracing/mysql/test.ts index e0db6de0be75..2a9cb3c3010a 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mysql/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/mysql/test.ts @@ -1,18 +1,35 @@ +import { + DB_CONNECTION_STRING, + DB_QUERY_SUMMARY, + DB_QUERY_TEXT, + DB_SYSTEM_NAME, + DB_USER, + SENTRY_ENVIRONMENT, + SENTRY_IS_LOCALHOST, + SENTRY_KIND, + SENTRY_OP, + SENTRY_ORIGIN, + SENTRY_RELEASE, + SENTRY_SDK_NAME, + SENTRY_SDK_VERSION, + SENTRY_SEGMENT_ID, + SENTRY_SEGMENT_NAME, + SENTRY_TRACE_LIFECYCLE, + SERVER_ADDRESS, + SERVER_PORT, +} from '@sentry/conventions/attributes'; +import { DB } from '@sentry/conventions/op'; import type { AddressInfo, Server } from 'node:net'; import { afterAll, beforeAll, describe, expect } from 'vitest'; -import { cleanupChildProcesses, createCjsTests, createEsmAndCjsTests } from '../../../utils/runner'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; import { startMysqlTestServer } from './mysql-test-server'; -import type { SerializedStreamedSpanContainer } from '@sentry/core'; -import { SENTRY_TRACE_LIFECYCLE, SENTRY_OP } from '@sentry/conventions/attributes'; describe('mysql auto instrumentation', () => { // A minimal in-process MySQL server (on a random free port) so the client's // connection handshake succeeds. Without it, `createPool()` queries fail at // connection acquisition — before `connection.query` runs — so the // diagnostics-channel instrumentation (which hooks `connection.query`) never - // sees them. Queries still error (the server rejects them), so spans keep - // `status: internal_error` as the assertions expect. The port is passed to - // each scenario via the `MYSQL_PORT` env var. + // sees them. The port is passed to each scenario via the `MYSQL_PORT` env var. let mysqlServer: Server; let mysqlPort: number; beforeAll(async () => { @@ -26,36 +43,97 @@ describe('mysql auto instrumentation', () => { cleanupChildProcesses(); }); - // Builds the expected transaction. When `origin` is given, the spans must also - // carry that `sentry.origin`, which is how we assert that the - // diagnostics-channel instrumentation (not the OTel one) produced them. A - // scenario can pass `override` to replace the default transaction expectation - // (e.g. the streamed-error scenario, which runs a different, failing query). - function expectedTransaction( + function expectedSpans( port: number, origin: string | undefined, override: Record | undefined, - ): Record { - const span = (description: string): ReturnType => - expect.objectContaining({ - description, - op: 'db', - ...(origin ? { origin } : {}), - data: expect.objectContaining({ - ...(origin ? { 'sentry.origin': origin } : {}), - 'db.system.name': 'mysql', - 'server.address': 'localhost', - 'server.port': port, - 'db.user': 'root', - }), - status: 'ok', - }); + ): unknown { + const COMMON_ATTRIBUTES = { + // These spans belong to a script with no incoming request, so there is nothing to judge. + [SENTRY_IS_LOCALHOST]: { type: 'boolean', value: false }, + [DB_CONNECTION_STRING]: { + type: 'string', + value: expect.stringMatching(/^jdbc:mysql:\/\/localhost:.*/), + }, + [DB_SYSTEM_NAME]: { + type: 'string', + value: 'mysql', + }, + [DB_USER]: { + type: 'string', + value: 'root', + }, + [SERVER_ADDRESS]: { + type: 'string', + value: 'localhost', + }, + [SERVER_PORT]: { + type: 'integer', + value: port, + }, + [SENTRY_KIND]: { + type: 'string', + value: 'client', + }, + [SENTRY_ENVIRONMENT]: { + type: 'string', + value: 'production', + }, + [SENTRY_OP]: { + type: 'string', + value: DB, + }, + [SENTRY_ORIGIN]: { + type: 'string', + value: origin, + }, + [SENTRY_RELEASE]: { + type: 'string', + value: '1.0', + }, + [SENTRY_SDK_NAME]: { + type: 'string', + value: 'sentry.javascript.node', + }, + [SENTRY_SDK_VERSION]: { + type: 'string', + value: expect.any(String), + }, + [SENTRY_SEGMENT_ID]: { + type: 'string', + value: expect.stringMatching(/^[\da-f]{16}$/), + }, + [SENTRY_SEGMENT_NAME]: { + type: 'string', + value: 'Test Transaction', + }, + [SENTRY_TRACE_LIFECYCLE]: { + type: 'string', + value: 'stream', + }, + }; - return { - transaction: 'Test Transaction', - spans: expect.arrayContaining([span('SELECT ? + ? AS solution'), span('SELECT NOW()')]), - ...(override ?? {}), + const COMMON_SPAN_PROPS = { + end_timestamp: expect.any(Number), + is_segment: false, + parent_span_id: expect.stringMatching(/^[\da-f]{16}$/), + span_id: expect.stringMatching(/^[\da-f]{16}$/), + start_timestamp: expect.any(Number), + status: 'ok', + trace_id: expect.stringMatching(/^[\da-f]{32}$/), }; + + const span = (queryText: string) => ({ + name: 'SELECT', + attributes: { + ...COMMON_ATTRIBUTES, + [DB_QUERY_TEXT]: { type: 'string', value: queryText }, + [DB_QUERY_SUMMARY]: { type: 'string', value: 'SELECT' }, + }, + ...COMMON_SPAN_PROPS, + }); + + return override?.spans ?? [span('SELECT ? + ? AS solution'), span('SELECT NOW()')]; } const CHANNEL_ORIGIN = 'auto.db.mysql'; @@ -90,26 +168,27 @@ describe('mysql auto instrumentation', () => { 'scenario-streamError.mjs', 'streamed query error', { - // The transaction itself succeeds (status `ok`); only the failing query's child span is errored. - spans: expect.arrayContaining([ + // The segment span succeeds (status `ok`); only the failing query span is errored. + spans: [ expect.objectContaining({ - description: 'SELECT * FROM does_not_exist', - op: 'db', + name: 'SELECT does_not_exist', // A failing streamed query emits `error`, which marks the span as errored - status: 'internal_error', - data: expect.objectContaining({ - 'db.system.name': 'mysql', - 'db.user': 'root', + status: 'error', + attributes: expect.objectContaining({ + [SENTRY_OP]: { type: 'string', value: DB }, + [DB_QUERY_TEXT]: { type: 'string', value: 'SELECT * FROM does_not_exist' }, + [DB_SYSTEM_NAME]: { type: 'string', value: 'mysql' }, + [DB_USER]: { type: 'string', value: 'root' }, }), }), - ]), + ], }, ], ] as const; for (const { label, env, flags, origin, failsOnEsm } of CASES) { describe(label, () => { - for (const [scenario, description, transactionOverride] of SCENARIOS) { + for (const [scenario, description, spanOverride] of SCENARIOS) { createEsmAndCjsTests( __dirname, scenario, @@ -119,7 +198,13 @@ describe('mysql auto instrumentation', () => { await createRunner() .withEnv({ ...env, MYSQL_PORT: String(mysqlPort) }) .withFlags(...flags) - .expect({ transaction: expectedTransaction(mysqlPort, origin, transactionOverride) }) + .expect({ + span: container => { + expect(container.items.find(span => span.is_segment)?.name).toBe('Test Transaction'); + const spans = container.items.filter(span => span.attributes[SENTRY_OP]?.value === DB); + expect(spans).toEqual(expectedSpans(mysqlPort, origin, spanOverride)); + }, + }) .start() .completed(); }); @@ -140,12 +225,14 @@ describe('mysql auto instrumentation', () => { .withFlags(...flags) .withEnv({ ...env, MYSQL_PORT: String(mysqlPort) }) .expect({ - transaction: (transaction): void => { - const transactionSpanId = transaction.contexts?.trace?.span_id; - const spans = transaction.spans ?? []; - const mysqlSpan = spans.find(span => span.description === 'SELECT ? + ? AS solution'); - const listenerSpan = spans.find(span => span.description === 'listener-child'); - const innerSpan = spans.find(span => span.description === 'inner-span'); + span: (container): void => { + const transactionSpanId = container.items.find(span => span.is_segment)?.span_id; + const spans = container.items; + const mysqlSpan = spans.find( + span => span.attributes[DB_QUERY_TEXT]?.value === 'SELECT ? + ? AS solution', + ); + const listenerSpan = spans.find(span => span.name === 'listener-child'); + const innerSpan = spans.find(span => span.name === 'inner-span'); expect(transactionSpanId).toBeDefined(); expect(mysqlSpan).toBeDefined(); @@ -169,163 +256,4 @@ describe('mysql auto instrumentation', () => { ); }); } - - describe('streamed', () => { - const assertMysqlSpans = (container: SerializedStreamedSpanContainer): void => { - const segmentSpan = container.items.find(item => item.is_segment); - expect(segmentSpan?.name).toBe('Test Transaction'); - - const dbSpans = container.items.filter(spanItem => spanItem.attributes[SENTRY_OP]?.value === 'db'); - - expect(dbSpans.length).toBe(2); - - const COMMON_ATTRIBUTES = { - // These spans belong to a script with no incoming request, so there is nothing to judge. - 'sentry.is_localhost': { type: 'boolean', value: false }, - 'db.connection_string': { - type: 'string', - value: expect.stringMatching(/^jdbc:mysql:\/\/localhost:.*/), - }, - 'db.system.name': { - type: 'string', - value: 'mysql', - }, - 'db.user': { - type: 'string', - value: 'root', - }, - 'server.address': { - type: 'string', - value: 'localhost', - }, - 'server.port': { - type: 'integer', - value: expect.any(Number), - }, - 'sentry.kind': { - type: 'string', - value: 'client', - }, - 'sentry.environment': { - type: 'string', - value: 'production', - }, - 'sentry.op': { - type: 'string', - value: 'db', - }, - 'sentry.origin': { - type: 'string', - value: 'auto.db.mysql', - }, - 'sentry.release': { - type: 'string', - value: '1.0', - }, - 'sentry.sdk.name': { - type: 'string', - value: 'sentry.javascript.node', - }, - 'sentry.sdk.version': { - type: 'string', - value: expect.any(String), - }, - 'sentry.segment.id': { - type: 'string', - value: expect.stringMatching(/^[\da-f]{16}$/), - }, - 'sentry.segment.name': { - type: 'string', - value: 'Test Transaction', - }, - [SENTRY_TRACE_LIFECYCLE]: { - type: 'string', - value: 'stream', - }, - }; - - const COMMON_SPAN_PROPS = { - end_timestamp: expect.any(Number), - is_segment: false, - parent_span_id: expect.stringMatching(/^[\da-f]{16}$/), - span_id: expect.stringMatching(/^[\da-f]{16}$/), - start_timestamp: expect.any(Number), - status: 'ok', - trace_id: expect.stringMatching(/^[\da-f]{32}$/), - }; - - expect(dbSpans).toEqual([ - { - attributes: { - ...COMMON_ATTRIBUTES, - 'db.query.text': { - type: 'string', - value: 'SELECT ? + ? AS solution', - }, - 'db.query.summary': { - type: 'string', - value: 'SELECT', - }, - }, - name: 'SELECT', - ...COMMON_SPAN_PROPS, - }, - { - attributes: { - ...COMMON_ATTRIBUTES, - 'db.query.text': { - type: 'string', - value: 'SELECT NOW()', - }, - 'db.query.summary': { - type: 'string', - value: 'SELECT', - }, - }, - name: 'SELECT', - ...COMMON_SPAN_PROPS, - }, - ]); - }; - - describe('with connection.connect()', () => { - createCjsTests(__dirname, 'scenario-withConnect.mjs', 'instrument.mjs', (createTestRunner, test) => { - test('should auto-instrument `mysql` package when using connection.connect()', async () => { - await createTestRunner() - .withEnv({ STREAMED: 'true', MYSQL_PORT: String(mysqlPort) }) - .expect({ - span: assertMysqlSpans, - }) - .start() - .completed(); - }); - }); - }); - - describe('query without callback', () => { - createCjsTests(__dirname, 'scenario-withoutCallback.mjs', 'instrument.mjs', (createTestRunner, test) => { - test('should auto-instrument `mysql` package when using query without callback', async () => { - await createTestRunner() - .withEnv({ STREAMED: 'true', MYSQL_PORT: String(mysqlPort) }) - .expect({ span: assertMysqlSpans }) - .start() - .completed(); - }); - }); - }); - - describe('without connection.connect()', () => { - createCjsTests(__dirname, 'scenario-withoutConnect.mjs', 'instrument.mjs', (createTestRunner, test) => { - test('should auto-instrument `mysql` package without connection.connect()', async () => { - await createTestRunner() - .withEnv({ STREAMED: 'true', MYSQL_PORT: String(mysqlPort) }) - .expect({ - span: assertMysqlSpans, - }) - .start() - .completed(); - }); - }); - }); - }); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/mysql2-tracing-channel/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/mysql2-tracing-channel/instrument.mjs index 170ad6f6a702..46a27dd03b74 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mysql2-tracing-channel/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/mysql2-tracing-channel/instrument.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/mysql2-tracing-channel/test.ts b/dev-packages/node-integration-tests/suites/tracing/mysql2-tracing-channel/test.ts index b243da576bb3..6e67cc51482d 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mysql2-tracing-channel/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/mysql2-tracing-channel/test.ts @@ -1,3 +1,13 @@ +import { + DB_OPERATION_NAME, + DB_QUERY_TEXT, + DB_SYSTEM_NAME, + SENTRY_OP, + SENTRY_ORIGIN, + SERVER_ADDRESS, + SERVER_PORT, +} from '@sentry/conventions/attributes'; +import { DB } from '@sentry/conventions/op'; import { afterAll, describe, expect } from 'vitest'; import { cleanupChildProcesses, createEsmAndCjsTests, describeWithDockerCompose } from '../../../utils/runner'; @@ -13,22 +23,21 @@ describe('mysql2 tracing channel Test', () => { const expectedQuerySpan = (queryText: string) => expect.objectContaining({ - description: queryText, - op: 'db', - origin: 'auto.db.mysql2.diagnostic_channel', - data: expect.objectContaining({ - 'sentry.origin': 'auto.db.mysql2.diagnostic_channel', - 'db.system.name': 'mysql', - 'db.operation.name': 'SELECT', - 'db.query.text': queryText, - 'server.address': 'localhost', - 'server.port': 3308, + attributes: expect.objectContaining({ + [SENTRY_ORIGIN]: { type: 'string', value: 'auto.db.mysql2.diagnostic_channel' }, + [DB_SYSTEM_NAME]: { type: 'string', value: 'mysql' }, + [DB_OPERATION_NAME]: { type: 'string', value: 'SELECT' }, + [DB_QUERY_TEXT]: { type: 'string', value: queryText }, + [SERVER_ADDRESS]: { type: 'string', value: 'localhost' }, + [SERVER_PORT]: { type: 'integer', value: 3308 }, + [SENTRY_OP]: { type: 'string', value: DB }, }), + name: 'SELECT', }); - const EXPECTED_TRANSACTION = { - transaction: 'Test Transaction', - spans: expect.arrayContaining([ + const EXPECTED_SPANS = { + items: expect.arrayContaining([ + expect.objectContaining({ name: 'Test Transaction', is_segment: true }), expectedQuerySpan('SELECT ? + ? AS solution'), // the inlined literal is redacted out of `db.query.text` expectedQuerySpan('SELECT ? AS leaked'), @@ -36,10 +45,13 @@ describe('mysql2 tracing channel Test', () => { expectedQuerySpan('SELECT ? AS answer'), // a failing query produces a span with an error status expect.objectContaining({ - description: 'SELECT * FROM does_not_exist', - op: 'db', - status: 'internal_error', - origin: 'auto.db.mysql2.diagnostic_channel', + attributes: expect.objectContaining({ + [SENTRY_OP]: { type: 'string', value: DB }, + [SENTRY_ORIGIN]: { type: 'string', value: 'auto.db.mysql2.diagnostic_channel' }, + [DB_QUERY_TEXT]: { type: 'string', value: 'SELECT * FROM does_not_exist' }, + }), + name: 'SELECT does_not_exist', + status: 'error', }), ]), }; @@ -47,19 +59,21 @@ describe('mysql2 tracing channel Test', () => { describeWithDockerCompose('with pg docker compose', { workingDirectory: [__dirname] }, () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createTestRunner, test) => { test('subscribes to mysql2 >= 3.20.0 diagnostics channels with stable semconv attributes', async () => { - await createTestRunner().expect({ transaction: EXPECTED_TRANSACTION }).start().completed(); + await createTestRunner().expect({ span: EXPECTED_SPANS }).start().completed(); }, 30_000); test('does not double-instrument: the legacy IITM mysql2 patcher does not fire on 3.20.0+', async () => { await createTestRunner() .expect({ - transaction: event => { - expect(event.transaction).toBe('Test Transaction'); - const spans = event.spans || []; + span: container => { + expect(container.items.find(span => span.is_segment)?.name).toBe('Test Transaction'); + const spans = container.items.filter(span => !span.is_segment); // The monkey-patch path (origin `auto.db.mysql2`) must be inactive on 3.20.0+. - expect(spans.find(span => span.origin === 'auto.db.mysql2')).toBeUndefined(); + expect(spans.find(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.db.mysql2')).toBeUndefined(); // ...while the diagnostics-channel path is active. - expect(spans.find(span => span.origin === 'auto.db.mysql2.diagnostic_channel')).toBeDefined(); + expect( + spans.find(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.db.mysql2.diagnostic_channel'), + ).toBeDefined(); }, }) .start() @@ -69,11 +83,11 @@ describe('mysql2 tracing channel Test', () => { test('never leaks raw values into db.query.text', async () => { await createTestRunner() .expect({ - transaction: event => { - expect(event.transaction).toBe('Test Transaction'); - const spans = event.spans || []; + span: container => { + expect(container.items.find(span => span.is_segment)?.name).toBe('Test Transaction'); + const spans = container.items.filter(span => !span.is_segment); for (const span of spans) { - const queryText = span.data?.['db.query.text']; + const queryText = span.attributes[DB_QUERY_TEXT]?.value; if (typeof queryText === 'string') { expect(queryText).not.toContain('super-secret'); } diff --git a/dev-packages/node-integration-tests/suites/tracing/mysql2/test.ts b/dev-packages/node-integration-tests/suites/tracing/mysql2/test.ts index 9733d4c7b614..619fd607a06e 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mysql2/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/mysql2/test.ts @@ -1,3 +1,4 @@ +import { DB } from '@sentry/conventions/op'; import { afterAll, expect } from 'vitest'; import { cleanupChildProcesses, createEsmAndCjsTests, describeWithDockerCompose } from '../../../utils/runner'; @@ -15,7 +16,7 @@ describeWithDockerCompose('mysql2 auto instrumentation', { workingDirectory: [__ spans: expect.arrayContaining([ expect.objectContaining({ description: 'SELECT ? + ? AS solution', - op: 'db', + op: DB, origin: ORIGIN, data: expect.objectContaining({ 'db.system.name': 'mysql', @@ -28,7 +29,7 @@ describeWithDockerCompose('mysql2 auto instrumentation', { workingDirectory: [__ // bind values are left as `?` placeholders in `db.statement` (not inlined) expect.objectContaining({ description: 'SELECT ? as a, ? as b, NOW() as c', - op: 'db', + op: DB, origin: ORIGIN, data: expect.objectContaining({ 'db.system.name': 'mysql', @@ -41,7 +42,7 @@ describeWithDockerCompose('mysql2 auto instrumentation', { workingDirectory: [__ // a single non-array bind value is also left as a `?` placeholder in `db.statement` expect.objectContaining({ description: 'SELECT ? AS scalar_value', - op: 'db', + op: DB, origin: ORIGIN, data: expect.objectContaining({ 'db.system.name': 'mysql', @@ -51,7 +52,7 @@ describeWithDockerCompose('mysql2 auto instrumentation', { workingDirectory: [__ // `execute` is instrumented the same way as `query` expect.objectContaining({ description: 'SELECT ? AS answer', - op: 'db', + op: DB, origin: ORIGIN, data: expect.objectContaining({ 'db.system.name': 'mysql', @@ -61,7 +62,7 @@ describeWithDockerCompose('mysql2 auto instrumentation', { workingDirectory: [__ // a failing query produces a span with an error status expect.objectContaining({ description: 'SELECT * FROM does_not_exist', - op: 'db', + op: DB, status: 'internal_error', origin: ORIGIN, data: expect.objectContaining({ From 04f7cb984a497a42ec856d1556b725cb9717e3a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Peer=20St=C3=B6cklmair?= Date: Thu, 8 Oct 2026 15:49:02 +0300 Subject: [PATCH 39/47] feat(server-utils): Add instrumentation for pi-durable (#24993) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `piDurableIntegration` for [`@earendil-works/pi-durable`](https://github.com/earendil-works/pi/tree/main/packages/durable), Earendil's new durable agent harness. It is on by default in Node, Bun and Deno, and in a Worker built with `@sentry/cloudflare/vite`, which covers pi-durable in a Durable Object through the Agents SDK `PiHarness`. Each run becomes its own `invoke_agent` trace with `chat` and `execute_tool` children, and the first run of a subagent conversation nests under the tool call that created it. pi-durable has no telemetry hooks (`pi-telemetry` exists, but no pi package emits spans through it yet), so the integration hooks `Harness.open()` and wraps the `models` and `registry` it receives. The wrapped options keep the caller's options as prototype, because pi-durable reads `settings`, `env` and `conversationCreated` at every use. Decisions worth a look: - **Runs are tracked per conversation.** pi-durable hands run control to a new `pi.generation` task after every tool round, so no single task spans a run. The run span ends when the commit that removes `pi.live.run` goes through, or when the Harness closes with the run in flight. - **Each run starts a new trace in clean scopes.** The scheduler runs task phases in whatever async context last woke it, often an unrelated request. Forking that context attached runs to the wrong request and the wrong conversation. Only the client of that context is kept, because `@sentry/cloudflare` binds the client to the scope of each request, not to the default scope. - **`gen_ai.conversation.id` is `:`.** pi-durable numbers conversations per storage from 1, so the bare id would merge every root conversation into one. The cost is a new id after a restart (on Cloudflare, after every restart of the Durable Object), until pi-durable has a stable storage id. - **Tool spans record the result the model receives**, read from the committed result entry. That covers output streamed through `api.output()` (all of `bash`) and results an `afterTool` hook redacted. - **Errors:** failures pi-durable only passes to `onReport` (throwing hooks or sections) and task phases that throw are captured. Throws of the built-in coding tools are not, because `bash` throws for every non-zero exit, a normal result for a coding agent. The built-in tools are recognized by their factories (`createBashTool()` and friends), so an app that registers them in an extension of its own gets the same treatment. The span keeps its error status. - **The provider SDK integrations are skipped process-wide** from the first run on, as the Flue and LangChain integrations do: pi-ai sends its requests through the `openai`, `@anthropic-ai/sdk` and `@google/genai` clients, or through the Workers AI binding with `createAI()` of `agents/models/pi-ai`, and those integrations cannot tell pi-ai's calls from the app's own. Bedrock requests go through the AWS SDK, which `awsIntegration` still reports twice; that skip is a follow-up. Messages, token usage and finish reasons go through the pi-ai mapper from the Flue fix, and the system prompt and tools are replayed from pi-durable's positional system messages. Not covered yet: spans for tool calls pi-durable answers without running `execute()`, spans for custom tasks, agent names (pi-durable has none), a link between a run and the request that submitted it, a marker for compaction requests, and an error event for a failed model request (the `chat` span carries the status). 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: Andrei <168741329+andreiborza@users.noreply.github.com> --- CHANGELOG.md | 4 + .../node-suites/excludes.ts | 1 + .../suites/tracing/pi-durable/instrument.mjs | 12 + .../pi-durable/scenario-compaction.mjs | 38 + .../pi-durable/scenario-concurrent.mjs | 61 + .../pi-durable/scenario-interrupted.mjs | 114 ++ .../tracing/pi-durable/scenario-reports.mjs | 70 ++ .../pi-durable/scenario-restart-abort.mjs | 57 + .../tracing/pi-durable/scenario-retention.mjs | 76 ++ .../tracing/pi-durable/scenario-subagent.mjs | 121 ++ .../tracing/pi-durable/scenario-tools.mjs | 110 ++ .../suites/tracing/pi-durable/scenario.mjs | 63 + .../suites/tracing/pi-durable/test.ts | 747 ++++++++++++ packages/astro/src/index.server.ts | 1 + packages/aws-serverless/src/index.ts | 1 + packages/bun/src/index.ts | 1 + packages/deno/src/index.ts | 1 + packages/elysia/src/index.ts | 1 + packages/google-cloud-serverless/src/index.ts | 1 + packages/node/src/index.ts | 1 + .../server-utils/src/ai/pi-ai/messages.ts | 74 ++ .../server-utils/src/ai/pi-ai/providers.ts | 11 +- .../src/ai/pi-durable/constants.ts | 23 + .../server-utils/src/ai/pi-durable/index.ts | 243 ++++ .../server-utils/src/ai/pi-durable/models.ts | 213 ++++ .../server-utils/src/ai/pi-durable/runs.ts | 245 ++++ .../server-utils/src/ai/pi-durable/tools.ts | 258 ++++ .../server-utils/src/ai/pi-durable/types.ts | 141 +++ .../server-utils/src/ai/pi-durable/utils.ts | 5 + packages/server-utils/src/index.ts | 1 + .../server-utils/src/integrations/index.ts | 2 + .../src/integrations/pi-durable.ts | 76 ++ .../server-utils/src/orchestrion/channels.ts | 2 + .../config/channel-integration-definitions.ts | 1 + .../src/orchestrion/config/index.ts | 2 + .../src/orchestrion/config/pi-durable.ts | 35 + .../server-utils/src/utils/withCleanScopes.ts | 20 + .../lib/tracing/pi-durable-retention.test.ts | 80 ++ .../test/ai/lib/tracing/pi-durable.test.ts | 1048 +++++++++++++++++ .../test/ai/lib/utils/pi-ai-messages.test.ts | 24 + 40 files changed, 3981 insertions(+), 4 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/pi-durable/instrument.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-compaction.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-concurrent.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-interrupted.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-reports.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-restart-abort.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-retention.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-subagent.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-tools.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/pi-durable/test.ts create mode 100644 packages/server-utils/src/ai/pi-durable/constants.ts create mode 100644 packages/server-utils/src/ai/pi-durable/index.ts create mode 100644 packages/server-utils/src/ai/pi-durable/models.ts create mode 100644 packages/server-utils/src/ai/pi-durable/runs.ts create mode 100644 packages/server-utils/src/ai/pi-durable/tools.ts create mode 100644 packages/server-utils/src/ai/pi-durable/types.ts create mode 100644 packages/server-utils/src/ai/pi-durable/utils.ts create mode 100644 packages/server-utils/src/integrations/pi-durable.ts create mode 100644 packages/server-utils/src/orchestrion/config/pi-durable.ts create mode 100644 packages/server-utils/src/utils/withCleanScopes.ts create mode 100644 packages/server-utils/test/ai/lib/tracing/pi-durable-retention.test.ts create mode 100644 packages/server-utils/test/ai/lib/tracing/pi-durable.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index a693a101773e..4b6d793fae6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ - "You miss 100 percent of the chances you don't take. — Wayne Gretzky" — Michael Scott +- **feat(server-utils)** Add instrumentation for pi-durable ([#24993](https://github.com/getsentry/sentry-javascript/pull/24993/)) + +The new `piDurableIntegration` traces agents built with [pi-durable](https://github.com/earendil-works/pi/tree/main/packages/durable): each run, with its model requests and tool calls. + Work in this release was contributed by @zkasuran. Thank you for your contribution! ## 11.5.0 diff --git a/dev-packages/bun-integration-tests/node-suites/excludes.ts b/dev-packages/bun-integration-tests/node-suites/excludes.ts index a8bba26f4713..5b99ffc866ab 100644 --- a/dev-packages/bun-integration-tests/node-suites/excludes.ts +++ b/dev-packages/bun-integration-tests/node-suites/excludes.ts @@ -124,6 +124,7 @@ export const NO_AUTO_INSTRUMENTATION = [ 'suites/tracing/openai/v6/test.ts', 'suites/tracing/openai/v7/test.ts', 'suites/tracing/orchestrion-lazy-registration/test.ts', + 'suites/tracing/pi-durable/test.ts', 'suites/tracing/postgres-streamed/test.ts', 'suites/tracing/postgres/test.ts', 'suites/tracing/postgresjs-streamed/test.ts', diff --git a/dev-packages/node-integration-tests/suites/tracing/pi-durable/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/pi-durable/instrument.mjs new file mode 100644 index 000000000000..f1f018ef471a --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/pi-durable/instrument.mjs @@ -0,0 +1,12 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + ...(process.env.PI_DURABLE_RECORDING === 'off' + ? { dataCollection: { genAI: { inputs: false, outputs: false } } } + : {}), +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-compaction.mjs b/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-compaction.mjs new file mode 100644 index 000000000000..18c98f77f2ba --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-compaction.mjs @@ -0,0 +1,38 @@ +import { BACKGROUND_CONTEXT } from '@earendil-works/chord/context'; +import { createModels } from '@earendil-works/pi-ai/models'; +import { fauxAssistantMessage, fauxProvider } from '@earendil-works/pi-ai/providers/faux'; +import { createRegistry, Harness, MemoryStorage } from '@earendil-works/pi-durable'; + +const context = BACKGROUND_CONTEXT; + +// A context window so small that pi-durable compacts the conversation before it can answer the +// third question. The compaction request belongs to that run. +const faux = fauxProvider({ provider: 'faux', models: [{ id: 'tiny', contextWindow: 60 }] }); +const models = createModels(); +models.setProvider(faux.provider); +faux.setResponses([ + fauxAssistantMessage('First answer with quite a lot of detail so the transcript grows beyond the window.'), + fauxAssistantMessage('Second answer with even more detail so that the next request needs a compaction first.'), + fauxAssistantMessage('Summary of the conversation.'), + fauxAssistantMessage('Third answer.'), +]); + +const harness = await Harness.open( + new MemoryStorage(), + { + models, + registry: createRegistry(), + settings: { compaction: { enabled: true, reserveTokens: 0, backgroundTokens: 0, keepRecentTokens: 1 } }, + }, + context, +); +const root = await harness.root(context, { agent: { model: { provider: 'faux', modelId: 'tiny' } } }); +for (const content of [ + 'First question with some words in it.', + 'Second question with some more words.', + 'Third question.', +]) { + await (await root.submit({ type: 'input', content }, context)).wait(context); +} +await harness.waitForIdle(context); +await harness.close(context); diff --git a/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-concurrent.mjs b/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-concurrent.mjs new file mode 100644 index 000000000000..1fdf6b110bd2 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-concurrent.mjs @@ -0,0 +1,61 @@ +import * as Sentry from '@sentry/node'; +import { BACKGROUND_CONTEXT } from '@earendil-works/chord/context'; +import { Type } from '@earendil-works/pi-ai'; +import { createModels } from '@earendil-works/pi-ai/models'; +import { fauxAssistantMessage, fauxProvider, fauxToolCall } from '@earendil-works/pi-ai/providers/faux'; +import { createRegistry, defineExtension, defineTool, Harness, MemoryStorage } from '@earendil-works/pi-durable'; + +const context = BACKGROUND_CONTEXT; + +// Two conversations run at the same time, each submitted from its own request with its own scope +// data. Each response is derived from the transcript, and the delays make the two runs interleave. +const delay = ms => new Promise(resolve => setTimeout(resolve, ms)); +const respond = async transcript => { + const lastUser = [...transcript.messages].reverse().find(message => message.role === 'user'); + const prompt = + typeof lastUser.content === 'string' ? lastUser.content : lastUser.content.map(part => part.text).join(''); + const afterTool = transcript.messages.some(message => message.role === 'toolResult'); + await delay(prompt.includes('A') ? 30 : 10); + return afterTool + ? fauxAssistantMessage(`Answer for ${prompt}`) + : fauxAssistantMessage(fauxToolCall('work', { tag: prompt }, { id: `call_${prompt.at(-1)}` }), { + stopReason: 'toolUse', + }); +}; +const faux = fauxProvider({ provider: 'faux', models: [{ id: 'faux-model' }] }); +const models = createModels(); +models.setProvider(faux.provider); +faux.setResponses([respond, respond, respond, respond]); + +const registry = createRegistry(); +registry.install( + defineExtension({ + name: 'app', + tools: [ + defineTool({ + name: 'work', + description: 'Works for a while, then fails.', + parameters: Type.Object({ tag: Type.String() }), + execute: async args => { + await delay(args.tag.includes('A') ? 10 : 30); + throw new Error(`work failed for ${args.tag}`); + }, + }), + ], + }), +); + +const harness = await Harness.open(new MemoryStorage(), { models, registry }, context); +const model = { provider: 'faux', modelId: 'faux-model' }; +const a = await harness.createConversation({ ownership: { kind: 'ownerless' }, agent: { model } }, context); +const b = await harness.createConversation({ ownership: { kind: 'ownerless' }, agent: { model } }, context); + +// Each request has its own isolation scope, as it would in a server, with data of its own. +const ask = (conversation, request) => + Sentry.withIsolationScope(async isolationScope => { + isolationScope.setTag('request', request); + isolationScope.setConversationId(`scope-of-${request}`); + await (await conversation.submit({ type: 'input', content: request }, context)).wait(context); + }); +await Promise.all([ask(a, 'request A'), ask(b, 'request B')]); +await harness.close(context); diff --git a/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-interrupted.mjs b/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-interrupted.mjs new file mode 100644 index 000000000000..f0efbea32483 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-interrupted.mjs @@ -0,0 +1,114 @@ +import * as Sentry from '@sentry/node'; +import { BACKGROUND_CONTEXT } from '@earendil-works/chord/context'; +import { Type } from '@earendil-works/pi-ai'; +import { createModels } from '@earendil-works/pi-ai/models'; +import { fauxAssistantMessage, fauxProvider, fauxToolCall } from '@earendil-works/pi-ai/providers/faux'; +import { createRegistry, defineExtension, defineTool, Harness, MemoryStorage } from '@earendil-works/pi-durable'; + +const context = BACKGROUND_CONTEXT; + +// Six runs that do not end with an answer: +// 1. the model request fails (`stopReason` error), with retries off; +// 2. the provider throws, which faults the generation task; +// 3. the run is aborted while the `slow` tool waits; +// 4. the run is aborted while the model request is in flight; +// 5. the Harness closes while the `slow` tool waits; +// 6. in a second Harness, the model request fails and the Harness closes right after `wait()`. +const started = new Map(); +const toolStarted = job => new Promise(resolve => started.set(job, resolve)); + +const faux = fauxProvider({ provider: 'faux', models: [{ id: 'faux-model' }] }); +const realModels = createModels(); +realModels.setProvider(faux.provider); +// The faux provider turns a throwing response into an error message, so the throw of a broken +// provider is staged in front of pi-ai: the request after `breakNextRequest()` throws synchronously. +let brokenRequests = 0; +const breakNextRequest = () => brokenRequests++; +const models = new Proxy(realModels, { + get(target, property) { + const value = Reflect.get(target, property, target); + if (property === 'streamSimple') { + return (...args) => { + if (brokenRequests > 0) { + brokenRequests--; + throw new Error('provider broke'); + } + return value.apply(target, args); + }; + } + return typeof value === 'function' ? value.bind(target) : value; + }, +}); +faux.setResponses([ + fauxAssistantMessage('', { stopReason: 'error', errorMessage: 'provider exploded' }), + fauxAssistantMessage(fauxToolCall('slow', { job: 'abort' }, { id: 'call_abort' }), { stopReason: 'toolUse' }), + async (_transcript, options) => { + started.get('request')(); + await new Promise(resolve => options.signal.addEventListener('abort', resolve)); + return fauxAssistantMessage('partial', { stopReason: 'aborted' }); + }, + fauxAssistantMessage(fauxToolCall('slow', { job: 'close' }, { id: 'call_close' }), { stopReason: 'toolUse' }), + fauxAssistantMessage('', { stopReason: 'error', errorMessage: 'provider exploded again' }), +]); + +const registry = createRegistry(); +registry.install( + defineExtension({ + name: 'app', + tools: [ + defineTool({ + name: 'slow', + description: 'Waits until it is aborted.', + parameters: Type.Object({ job: Type.String() }), + execute: async (args, _api, toolContext) => { + started.get(args.job)(); + await new Promise((_, reject) => + toolContext.abortSignal.addEventListener('abort', () => reject(new Error('tool aborted'))), + ); + return { content: [] }; + }, + }), + ], + }), +); + +const harness = await Harness.open( + new MemoryStorage(), + { models, registry, settings: { retry: { enabled: false } } }, + context, +); +const root = await harness.root(context, { agent: { model: { provider: 'faux', modelId: 'faux-model' } } }); + +await (await root.submit({ type: 'input', content: 'Fail please.' }, context)).wait(context); +breakNextRequest(); +await (await root.submit({ type: 'input', content: 'Break please.' }, context)).wait(context); + +const abortRun = toolStarted('abort'); +const aborted = await root.submit({ type: 'input', content: 'Run slow, then abort.' }, context); +await abortRun; +await root.abort(context); +await aborted.wait(context); + +const requestRun = toolStarted('request'); +const abortedRequest = await root.submit({ type: 'input', content: 'Answer slowly, then abort.' }, context); +await requestRun; +await root.abort(context); +await abortedRequest.wait(context); + +const closeRun = toolStarted('close'); +await root.submit({ type: 'input', content: 'Run slow, then close.' }, context); +await closeRun; +await harness.close(context); + +const second = await Harness.open( + new MemoryStorage(), + { models, registry, settings: { retry: { enabled: false } } }, + context, +); +const secondRoot = await second.root(context, { agent: { model: { provider: 'faux', modelId: 'faux-model' } } }); +await (await secondRoot.submit({ type: 'input', content: 'Fail, then close.' }, context)).wait(context); +await second.close(context); + +// Everything captured during the runs is sent before this sentinel. +await Sentry.flush(2000); +Sentry.captureMessage('pi-durable interrupted done'); diff --git a/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-reports.mjs b/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-reports.mjs new file mode 100644 index 000000000000..50ff47c4f156 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-reports.mjs @@ -0,0 +1,70 @@ +import { BACKGROUND_CONTEXT } from '@earendil-works/chord/context'; +import { createModels } from '@earendil-works/pi-ai/models'; +import { fauxAssistantMessage, fauxProvider } from '@earendil-works/pi-ai/providers/faux'; +import { + createRegistry, + defineExtension, + defineTask, + GenerationTask, + Harness, + hook, + MemoryStorage, +} from '@earendil-works/pi-durable'; + +const context = BACKGROUND_CONTEXT; + +// Failures pi-durable does not propagate: a hook that throws (pi-durable reports it and keeps the +// run going) and a durable task whose phase throws (the scheduler faults the task). A compaction +// started while the conversation is idle runs outside any run. +const faux = fauxProvider({ provider: 'faux', models: [{ id: 'faux-model' }] }); +const models = createModels(); +models.setProvider(faux.provider); +faux.setResponses([ + fauxAssistantMessage('First answer with some detail.'), + fauxAssistantMessage('Second answer with more detail.'), + fauxAssistantMessage('Summary of the conversation.'), +]); + +const Charge = defineTask({ + name: 'app.charge', + version: 1, + initial: () => ({ phase: 'charge' }), + phases: { + charge: async () => { + throw new Error('card declined'); + }, + }, + abort: async (_task, runtime, taskContext) => { + await runtime.commit(() => ({ status: 'terminal', outcome: { status: 'aborted' } }), taskContext); + }, +}); + +const registry = createRegistry(); +registry.install( + defineExtension({ + name: 'app', + tasks: [Charge], + hooks: [ + hook(GenerationTask, { + afterResponse: () => { + throw new Error('afterResponse hook failed'); + }, + }), + ], + }), +); + +const harness = await Harness.open( + new MemoryStorage(), + { models, registry, settings: { compaction: { keepRecentTokens: 1 } } }, + context, +); +const root = await harness.root(context, { agent: { model: { provider: 'faux', modelId: 'faux-model' } } }); +await (await root.submit({ type: 'input', content: 'First question.' }, context)).wait(context); +await (await root.submit({ type: 'input', content: 'Second question.' }, context)).wait(context); + +const charge = await root.commit(tx => tx.createTask(Charge, {}, { ownership: { kind: 'conversation' } }), context); +await harness.waitForTask(charge, context); + +await harness.waitForTask(await root.compact(undefined, context), context); +await harness.close(context); diff --git a/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-restart-abort.mjs b/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-restart-abort.mjs new file mode 100644 index 000000000000..deb1ba445d95 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-restart-abort.mjs @@ -0,0 +1,57 @@ +import * as Sentry from '@sentry/node'; +import { mkdtemp } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { BACKGROUND_CONTEXT } from '@earendil-works/chord/context'; +import { createModels } from '@earendil-works/pi-ai/models'; +import { fauxAssistantMessage, fauxProvider } from '@earendil-works/pi-ai/providers/faux'; +import { createRegistry, Harness } from '@earendil-works/pi-durable'; +import { openNodeJsonlStorage } from '@earendil-works/pi-durable/storage/jsonl/node'; + +const context = BACKGROUND_CONTEXT; + +// The first Harness closes while a model request is in flight, which leaves the generation task in +// storage. A second Harness on the same storage aborts the conversation before anything resumes the +// task, so pi-durable runs only the abort handler of that task, in a Harness without a run for it. +let requestStarted; +const requestInFlight = new Promise(resolve => (requestStarted = resolve)); +const faux = fauxProvider({ provider: 'faux', models: [{ id: 'faux-model' }] }); +const models = createModels(); +models.setProvider(faux.provider); +faux.setResponses([ + async (_transcript, options) => { + requestStarted(); + await new Promise(resolve => options.signal.addEventListener('abort', resolve)); + return fauxAssistantMessage('partial', { stopReason: 'aborted' }); + }, +]); + +const registry = createRegistry(); +const directory = await mkdtemp(join(tmpdir(), 'pi-durable-restart-')); +const open = async () => Harness.open(await openNodeJsonlStorage(directory, context), { models, registry }, context); + +const first = await open(); +const root = await first.root(context, { agent: { model: { provider: 'faux', modelId: 'faux-model' } } }); +const submission = await root.submit({ type: 'input', content: 'Answer slowly.' }, context); +await requestInFlight; +await first.close(context); + +let restarted = false; +let runsStarted = 0; +Sentry.getClient().on('spanStart', span => { + if (restarted && Sentry.spanToJSON(span).name === 'invoke_agent') { + runsStarted++; + } +}); +restarted = true; + +const second = await open(); +await (await second.conversation(root.id, context)).abort(context); +const settled = await (await second.submission(submission.id, context)).wait(context); +await second.close(context); + +// Everything captured during the runs is sent before this sentinel. +await Sentry.flush(2000); +Sentry.captureMessage('pi-durable restart abort', { + extra: { status: settled.status, reason: settled.reason, runsStarted }, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-retention.mjs b/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-retention.mjs new file mode 100644 index 000000000000..dc84d6387ba9 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-retention.mjs @@ -0,0 +1,76 @@ +import * as Sentry from '@sentry/node'; +import { setFlagsFromString } from 'node:v8'; +import { runInNewContext } from 'node:vm'; +import { BACKGROUND_CONTEXT } from '@earendil-works/chord/context'; +import { Type } from '@earendil-works/pi-ai'; +import { createModels } from '@earendil-works/pi-ai/models'; +import { fauxAssistantMessage, fauxProvider, fauxToolCall } from '@earendil-works/pi-ai/providers/faux'; +import { createRegistry, defineExtension, defineTool, Harness, MemoryStorage } from '@earendil-works/pi-durable'; + +const context = BACKGROUND_CONTEXT; +const RUNS = 3; + +// Three runs that each call `spawn`, which creates a conversation it owns and returns without running +// it. The links to those conversations stay until the Harness closes; the traces must not. +const toolSpans = []; +Sentry.getClient().on('spanStart', span => { + if (Sentry.spanToJSON(span).name === 'execute_tool spawn') { + toolSpans.push(new WeakRef(span)); + } +}); + +const faux = fauxProvider({ provider: 'faux', models: [{ id: 'faux-model' }] }); +const models = createModels(); +models.setProvider(faux.provider); +faux.setResponses( + Array.from( + { length: RUNS * 2 }, + () => transcript => + transcript.messages.at(-1).role === 'toolResult' + ? fauxAssistantMessage('The subagent is ready.') + : fauxAssistantMessage(fauxToolCall('spawn', {}, { id: 'call_spawn' }), { stopReason: 'toolUse' }), + ), +); + +const registry = createRegistry(); +registry.install( + defineExtension({ + name: 'app', + tools: [ + defineTool({ + name: 'spawn', + description: 'Creates a subagent for later.', + parameters: Type.Object({}), + execute: async (_args, api, toolContext) => { + await api.commit( + tx => tx.createConversation({ ownership: { kind: 'task', taskId: api.taskId } }), + toolContext, + ); + return { content: [{ type: 'text', text: 'Subagent created.' }] }; + }, + }), + ], + }), +); + +const harness = await Harness.open(new MemoryStorage(), { models, registry }, context); +const root = await harness.root(context, { agent: { model: { provider: 'faux', modelId: 'faux-model' } } }); +for (let run = 1; run <= RUNS; run++) { + await (await root.submit({ type: 'input', content: `Run ${run}` }, context)).wait(context); +} +await harness.waitForIdle(context); + +// Sends the finished traces, which span streaming buffers until then. +await Sentry.flush(); +// A WeakRef is never cleared in the job that created it. +await new Promise(resolve => setImmediate(resolve)); +setFlagsFromString('--expose-gc'); +const gc = runInNewContext('gc'); +setFlagsFromString('--no-expose-gc'); +gc(); +gc(); + +Sentry.captureMessage('pi-durable retention', { + extra: { toolCalls: toolSpans.length, alive: toolSpans.filter(ref => ref.deref()).length }, +}); +await harness.close(context); diff --git a/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-subagent.mjs b/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-subagent.mjs new file mode 100644 index 000000000000..da80b508f252 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-subagent.mjs @@ -0,0 +1,121 @@ +import { BACKGROUND_CONTEXT } from '@earendil-works/chord/context'; +import { Type } from '@earendil-works/pi-ai'; +import { createModels } from '@earendil-works/pi-ai/models'; +import { fauxAssistantMessage, fauxProvider, fauxToolCall } from '@earendil-works/pi-ai/providers/faux'; +import { + AssistantEntry, + createRegistry, + defineExtension, + defineTool, + Harness, + MemoryStorage, +} from '@earendil-works/pi-durable'; + +const context = BACKGROUND_CONTEXT; + +// Three runs of the root conversation, each with one tool round: +// - `delegate` creates a conversation it owns and waits for its answer, the subagent pattern; +// - `fork_delegate` does the same with a fork of the current conversation; +// - `batch` runs the `inner` tool of its agent itself, passing its own api on. +// Each response is derived from the transcript, so the faux model answers every conversation. +const respond = transcript => { + const lastUser = [...transcript.messages].reverse().find(message => message.role === 'user'); + const text = + typeof lastUser.content === 'string' ? lastUser.content : lastUser.content.map(part => part.text).join(''); + const afterTool = transcript.messages.at(-1).role === 'toolResult'; + if (afterTool) { + return fauxAssistantMessage(`Done: ${text}`); + } + if (text === 'Delegate the weather question.') { + return fauxAssistantMessage(fauxToolCall('delegate', { task: 'Weather in Vienna?' }, { id: 'call_delegate' }), { + stopReason: 'toolUse', + }); + } + if (text === 'Fork a helper.') { + return fauxAssistantMessage(fauxToolCall('fork_delegate', {}, { id: 'call_fork' }), { stopReason: 'toolUse' }); + } + if (text === 'Run the batch.') { + return fauxAssistantMessage(fauxToolCall('batch', {}, { id: 'call_batch' }), { stopReason: 'toolUse' }); + } + return fauxAssistantMessage(`Answer: ${text}`); +}; +const faux = fauxProvider({ provider: 'faux', models: [{ id: 'faux-model' }] }); +const models = createModels(); +models.setProvider(faux.provider); +faux.setResponses(Array.from({ length: 12 }, () => respond)); + +const answerText = async (api, settled, toolContext) => { + const answer = await api.commit(tx => tx.entry(AssistantEntry, settled.answer), toolContext); + return (answer?.model?.[0]?.content ?? []) + .filter(part => part.type === 'text') + .map(part => part.text) + .join(''); +}; + +const registry = createRegistry(); +registry.install( + defineExtension({ + name: 'app', + tools: [ + defineTool({ + name: 'delegate', + description: 'Delegates a task to a subagent.', + parameters: Type.Object({ task: Type.String() }), + execute: async (args, api, toolContext) => { + const childId = await api.commit( + async tx => (await tx.createConversation({ ownership: { kind: 'task', taskId: api.taskId } })).id, + toolContext, + ); + const child = await api.conversation(childId, toolContext); + const settled = await ( + await child.submit({ type: 'input', content: args.task }, toolContext) + ).wait(toolContext); + return { content: [{ type: 'text', text: await answerText(api, settled, toolContext) }] }; + }, + }), + defineTool({ + name: 'fork_delegate', + description: 'Forks this conversation into a subagent.', + parameters: Type.Object({}), + execute: async (_args, api, toolContext) => { + const childId = await api.commit(async tx => { + const at = (await tx.scanEntries({ conversationId: api.conversationId }, 1)).items[0].id; + const fork = await tx.forkConversation(api.conversationId, at, { + ownership: { kind: 'task', taskId: api.taskId }, + }); + return fork.id; + }, toolContext); + const child = await api.conversation(childId, toolContext); + const settled = await ( + await child.submit({ type: 'input', content: 'Forked task.' }, toolContext) + ).wait(toolContext); + return { content: [{ type: 'text', text: await answerText(api, settled, toolContext) }] }; + }, + }), + defineTool({ + name: 'inner', + description: 'A plain tool.', + parameters: Type.Object({}), + execute: async () => ({ content: [{ type: 'text', text: 'INNER RESULT' }] }), + }), + defineTool({ + name: 'batch', + description: 'Runs the inner tool itself.', + parameters: Type.Object({}), + execute: async (_args, api, toolContext) => { + const inner = (await api.agent(toolContext)).tools.find(tool => tool.name === 'inner'); + await inner.execute({}, api, toolContext); + return { content: [{ type: 'text', text: 'OUTER RESULT' }] }; + }, + }), + ], + }), +); + +const harness = await Harness.open(new MemoryStorage(), { models, registry }, context); +const root = await harness.root(context, { agent: { model: { provider: 'faux', modelId: 'faux-model' } } }); +for (const content of ['Delegate the weather question.', 'Fork a helper.', 'Run the batch.']) { + await (await root.submit({ type: 'input', content }, context)).wait(context); +} +await harness.waitForIdle(context); +await harness.close(context); diff --git a/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-tools.mjs b/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-tools.mjs new file mode 100644 index 000000000000..fc42ebf2091f --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-tools.mjs @@ -0,0 +1,110 @@ +import * as Sentry from '@sentry/node'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { BACKGROUND_CONTEXT } from '@earendil-works/chord/context'; +import { Type } from '@earendil-works/pi-ai'; +import { createModels } from '@earendil-works/pi-ai/models'; +import { fauxAssistantMessage, fauxProvider, fauxToolCall } from '@earendil-works/pi-ai/providers/faux'; +import { + createRegistry, + defineExtension, + defineTool, + Harness, + hook, + MemoryStorage, + ToolTask, +} from '@earendil-works/pi-durable'; +import { NodeExecutionEnv } from '@earendil-works/pi-durable/env/node'; +import { CodingTools, createBashTool } from '@earendil-works/pi-durable/tools'; + +const context = BACKGROUND_CONTEXT; + +// One tool round, run one call at a time so the order of the error events is fixed: +// - `bash` is the built-in tool, registered by the app's own extension through its factory; it runs +// a command that exits non-zero, which it reports by throwing; +// - `read` is an app tool that replaces the built-in `read` by name, and throws; +// - `streamer` returns nothing, so its streamed output becomes the result; +// - `secret` returns a value an `afterTool` hook redacts before the model sees it; +// - `fail_now` is an app tool that throws. +const faux = fauxProvider({ provider: 'faux', models: [{ id: 'faux-model' }] }); +const models = createModels(); +models.setProvider(faux.provider); +faux.setResponses([ + fauxAssistantMessage( + [ + fauxToolCall('bash', { command: 'ls does-not-exist' }, { id: 'call_bash' }), + fauxToolCall('read', { path: 'notes.txt' }, { id: 'call_read' }), + fauxToolCall('streamer', {}, { id: 'call_streamer' }), + fauxToolCall('secret', {}, { id: 'call_secret' }), + fauxToolCall('fail_now', {}, { id: 'call_fail' }), + ], + { stopReason: 'toolUse' }, + ), + fauxAssistantMessage('Done.'), +]); + +const registry = createRegistry(); +registry.install(CodingTools); +registry.install( + defineExtension({ + name: 'app', + tools: [ + createBashTool(), + defineTool({ + name: 'read', + description: 'Reads notes.', + parameters: Type.Object({ path: Type.String() }), + execute: async () => { + throw new Error('app read failed'); + }, + }), + defineTool({ + name: 'streamer', + description: 'Streams its output.', + parameters: Type.Object({}), + execute: async (_args, api) => { + api.output('line 1\n'); + api.output('line 2\n'); + return {}; + }, + }), + defineTool({ + name: 'secret', + description: 'Returns a secret.', + parameters: Type.Object({}), + execute: async () => ({ content: [{ type: 'text', text: 'original secret value' }] }), + }), + defineTool({ + name: 'fail_now', + description: 'Always throws.', + parameters: Type.Object({}), + execute: async () => { + throw new Error('Intentional pi-durable tool failure'); + }, + }), + ], + hooks: [ + hook(ToolTask, { + afterTool: (call, result) => + call.name === 'secret' + ? { ...result, content: [{ type: 'text', text: 'redacted by afterTool' }] } + : undefined, + }), + ], + }), +); + +const cwd = mkdtempSync(join(tmpdir(), 'pi-durable-tools-')); +const harness = await Harness.open( + new MemoryStorage(), + { models, registry, env: () => new NodeExecutionEnv({ cwd }), settings: { toolExecution: 'sequential' } }, + context, +); +const root = await harness.root(context, { agent: { model: { provider: 'faux', modelId: 'faux-model' } } }); +await (await root.submit({ type: 'input', content: 'Run the tools.' }, context)).wait(context); +await harness.close(context); + +// Everything captured during the run is sent before this sentinel. +await Sentry.flush(2000); +Sentry.captureMessage('pi-durable tools done'); diff --git a/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario.mjs new file mode 100644 index 000000000000..42309d5ba2d4 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario.mjs @@ -0,0 +1,63 @@ +import * as Sentry from '@sentry/node'; +import { BACKGROUND_CONTEXT } from '@earendil-works/chord/context'; +import { Type } from '@earendil-works/pi-ai'; +import { createModels } from '@earendil-works/pi-ai/models'; +import { fauxAssistantMessage, fauxProvider, fauxToolCall } from '@earendil-works/pi-ai/providers/faux'; +import { + createRegistry, + defineExtension, + defineTool, + Harness, + MemoryStorage, + section, +} from '@earendil-works/pi-durable'; + +const context = BACKGROUND_CONTEXT; + +// `pi-ai`'s faux provider scripts model responses in-process. The first run calls two tools and then +// answers, the second run answers directly. +const faux = fauxProvider({ provider: 'faux', models: [{ id: 'faux-model' }] }); +const models = createModels(); +models.setProvider(faux.provider); +faux.setResponses([ + fauxAssistantMessage( + [fauxToolCall('get_weather', { city: 'Berlin' }, { id: 'call_1' }), fauxToolCall('broken', {}, { id: 'call_2' })], + { stopReason: 'toolUse' }, + ), + fauxAssistantMessage('It is 21 degrees and sunny in Berlin.'), + fauxAssistantMessage('Goodbye.'), +]); + +const registry = createRegistry(); +registry.install( + defineExtension({ + name: 'weather', + sections: [section('preamble', () => 'You are a weather assistant.', { tag: false })], + tools: [ + defineTool({ + name: 'get_weather', + description: 'Get the current weather for a city.', + parameters: Type.Object({ city: Type.String() }), + execute: async args => ({ content: [{ type: 'text', text: `It is 21 degrees and sunny in ${args.city}.` }] }), + }), + defineTool({ + name: 'broken', + description: 'Always fails.', + parameters: Type.Object({}), + execute: async () => { + throw new Error('broken tool'); + }, + }), + ], + }), +); + +// The Harness is opened and driven inside a request span. Its runs must still start traces of their +// own: the scheduler runs them later, and their trace must not depend on which request woke it. +await Sentry.startSpan({ name: 'pi-durable-request', op: 'http.server' }, async () => { + const harness = await Harness.open(new MemoryStorage(), { models, registry }, context); + const root = await harness.root(context, { agent: { model: { provider: 'faux', modelId: 'faux-model' } } }); + await (await root.submit({ type: 'input', content: 'What is the weather in Berlin?' }, context)).wait(context); + await (await root.submit({ type: 'input', content: 'Thanks!' }, context)).wait(context); + await harness.close(context); +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/pi-durable/test.ts b/dev-packages/node-integration-tests/suites/tracing/pi-durable/test.ts new file mode 100644 index 000000000000..be521b3d3d98 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/pi-durable/test.ts @@ -0,0 +1,747 @@ +import { + GEN_AI_CONVERSATION_ID, + GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_FINISH_REASONS, + GEN_AI_RESPONSE_STREAMING, + GEN_AI_SYSTEM_INSTRUCTIONS, + GEN_AI_TOOL_CALL_ARGUMENTS, + GEN_AI_TOOL_CALL_RESULT, + GEN_AI_TOOL_DEFINITIONS, + GEN_AI_TOOL_DESCRIPTION, + GEN_AI_TOOL_NAME, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + SENTRY_OP, + SENTRY_ORIGIN, + SENTRY_STATUS_MESSAGE, +} from '@sentry/conventions/attributes'; +import { GEN_AI_CHAT, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op'; +import { afterAll, expect } from 'vitest'; +import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; +import { conditionalTest, RUNTIME } from '../../../utils'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; + +// The pi packages declare `engines.node >= 22.19`, so they can't live in the package's root +// `devDependencies` (that would break `yarn install` on the 20.19 CI matrix). Install them per-suite +// instead, guarded by the `min: 22` skip below. +const PI_DURABLE_DEPENDENCIES = { + additionalDependencies: { + '@earendil-works/pi-durable': '^1.0.0', + '@earendil-works/pi-ai': '^1.0.0', + '@earendil-works/chord': '^1.0.0', + }, +}; + +const CONTENT_ATTRIBUTES = [ + GEN_AI_INPUT_MESSAGES, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_SYSTEM_INSTRUCTIONS, + GEN_AI_TOOL_DEFINITIONS, + GEN_AI_TOOL_CALL_ARGUMENTS, + GEN_AI_TOOL_CALL_RESULT, +]; + +conditionalTest({ min: 22 })('pi-durable integration', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + // The pi packages are ESM-only: their `exports` maps have no `require` condition. + if (mode === 'cjs') { + return; + } + + test('traces each run as its own invoke_agent trace with chat and execute_tool children', async () => { + let toolRunTraceId: string | undefined; + let brokenToolSpanId: string | undefined; + let errorTraceId: string | undefined; + let errorSpanId: string | undefined; + const conversationIds: unknown[] = []; + + await createRunner() + .unordered() + .expect({ + span: container => { + const spans = container.items; + const agent = spans.find(span => span.is_segment)!; + expect(agent.name).toBe('invoke_agent'); + expect(spans.filter(span => span.name === 'chat faux-model')).toHaveLength(2); + // One span per run, request and tool call: a tool wrapped twice would add a span. + expect(spans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.pi_durable')).toHaveLength( + 5, + ); + + expect(agent.attributes[SENTRY_OP]?.value).toBe(GEN_AI_INVOKE_AGENT); + expect(agent.attributes[SENTRY_ORIGIN]?.value).toBe('auto.ai.pi_durable'); + expect(agent.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('invoke_agent'); + const conversationId = agent.attributes[GEN_AI_CONVERSATION_ID]?.value; + // The root conversation's id is `1` in every pi-durable storage, so it is prefixed + // with a per-Harness id. + expect(conversationId).toMatch(/^[0-9a-f]{32}:1$/); + + const chats = spans + .filter(span => span.name === 'chat faux-model') + .sort((a, b) => a.start_timestamp - b.start_timestamp); + for (const chat of chats) { + expect(chat.parent_span_id).toBe(agent.span_id); + expect(chat.attributes[SENTRY_OP]?.value).toBe(GEN_AI_CHAT); + expect(chat.attributes[SENTRY_ORIGIN]?.value).toBe('auto.ai.pi_durable'); + expect(chat.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('faux'); + expect(chat.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('faux-model'); + expect(chat.attributes[GEN_AI_CONVERSATION_ID]?.value).toBe(conversationId); + expect(chat.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBeGreaterThan(0); + expect(chat.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBeGreaterThan(0); + // pi-durable sends the prompt and the tools as positional system messages. + expect(chat.attributes[GEN_AI_SYSTEM_INSTRUCTIONS]?.value).toBe('You are a weather assistant.'); + expect( + JSON.parse(String(chat.attributes[GEN_AI_TOOL_DEFINITIONS]?.value)).map( + (tool: { name: string }) => tool.name, + ), + ).toEqual(['get_weather', 'broken']); + } + + // Mapped from pi-ai's shape to the conventions: the tool calls and tool results the + // second request carries are what Sentry's conversation view renders. + expect(JSON.parse(String(chats[0]!.attributes[GEN_AI_INPUT_MESSAGES]?.value))).toEqual([ + { role: 'user', parts: [{ type: 'text', content: 'What is the weather in Berlin?' }] }, + ]); + const answerInput = JSON.parse(String(chats[1]!.attributes[GEN_AI_INPUT_MESSAGES]?.value)); + expect(answerInput.map((message: { role: string }) => message.role)).toEqual([ + 'user', + 'assistant', + 'tool', + 'tool', + ]); + expect(answerInput[1].parts).toEqual([ + { type: 'tool_call', id: 'call_1', name: 'get_weather', arguments: '{"city":"Berlin"}' }, + { type: 'tool_call', id: 'call_2', name: 'broken', arguments: '{}' }, + ]); + expect(answerInput[2].parts).toEqual([ + { + type: 'tool_call_response', + id: 'call_1', + name: 'get_weather', + result: 'It is 21 degrees and sunny in Berlin.', + }, + ]); + expect(chats.map(chat => chat.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value).sort()).toEqual([ + '["stop"]', + '["tool_call"]', + ]); + expect(chats.map(chat => chat.attributes[GEN_AI_OUTPUT_MESSAGES]?.value).join()).toContain( + 'It is 21 degrees and sunny in Berlin.', + ); + + const weather = spans.find(span => span.name === 'execute_tool get_weather')!; + expect(weather.parent_span_id).toBe(agent.span_id); + expect(weather.status).toBe('ok'); + expect(weather.attributes[SENTRY_OP]?.value).toBe(GEN_AI_EXECUTE_TOOL); + expect(weather.attributes[GEN_AI_TOOL_NAME]?.value).toBe('get_weather'); + expect(weather.attributes[GEN_AI_TOOL_DESCRIPTION]?.value).toBe('Get the current weather for a city.'); + expect(weather.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]?.value).toBe('call_1'); + expect(weather.attributes[GEN_AI_TOOL_CALL_ARGUMENTS]?.value).toBe('{"city":"Berlin"}'); + expect(weather.attributes[GEN_AI_TOOL_CALL_RESULT]?.value).toContain('sunny in Berlin'); + expect(weather.attributes[GEN_AI_CONVERSATION_ID]?.value).toBe(conversationId); + + const broken = spans.find(span => span.name === 'execute_tool broken')!; + expect(broken.parent_span_id).toBe(agent.span_id); + expect(broken.status).toBe('error'); + // The error result pi-durable gives the model, not the thrown error. + expect(broken.attributes[GEN_AI_TOOL_CALL_RESULT]?.value).toContain('[error] broken tool'); + + toolRunTraceId = agent.trace_id; + brokenToolSpanId = broken.span_id; + conversationIds.push(conversationId); + }, + }) + .expect({ + span: container => { + const spans = container.items; + const agent = spans.find(span => span.is_segment)!; + expect(agent.name).toBe('invoke_agent'); + expect(spans).toHaveLength(2); + + const chat = spans.find(span => span.name === 'chat faux-model')!; + expect(chat.parent_span_id).toBe(agent.span_id); + expect(chat.attributes[GEN_AI_OUTPUT_MESSAGES]?.value).toContain('Goodbye.'); + + conversationIds.push(agent.attributes[GEN_AI_CONVERSATION_ID]?.value); + }, + }) + .expect({ + span: container => { + // The request that opened the Harness and submitted both inputs holds no agent work. + expect(container.items).toHaveLength(1); + expect(container.items[0]!.name).toBe('pi-durable-request'); + }, + }) + .expect({ + event: event => { + const exception = event.exception?.values?.[0]; + expect(exception?.value).toBe('broken tool'); + expect(exception?.mechanism).toEqual({ type: 'auto.ai.pi_durable', handled: false }); + + errorTraceId = event.contexts?.trace?.trace_id; + errorSpanId = event.contexts?.trace?.span_id; + }, + }) + .start() + .completed(); + + // Both runs belong to the root conversation, so they share one conversation id. + expect(conversationIds).toHaveLength(2); + expect(conversationIds[0]).toBe(conversationIds[1]); + // The tool error is reported on the failing call's span, in the run's trace. + expect(errorTraceId).toBe(toolRunTraceId); + expect(errorSpanId).toBe(brokenToolSpanId); + }); + + test('records no message content when the client turns gen_ai recording off', async () => { + await createRunner() + .withEnv({ PI_DURABLE_RECORDING: 'off' }) + .unordered() + .expect({ + span: container => { + const spans = container.items; + expect(spans).toHaveLength(5); + for (const span of spans) { + for (const attribute of CONTENT_ATTRIBUTES) { + expect(span.attributes[attribute]).toBeUndefined(); + } + } + const weather = spans.find(span => span.name === 'execute_tool get_weather')!; + expect(weather.attributes[GEN_AI_TOOL_DESCRIPTION]?.value).toBe('Get the current weather for a city.'); + expect( + spans.find(span => span.name === 'chat faux-model')!.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value, + ).toBeGreaterThan(0); + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(2); + for (const span of container.items) { + for (const attribute of CONTENT_ATTRIBUTES) { + expect(span.attributes[attribute]).toBeUndefined(); + } + } + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(1); + expect(container.items[0]!.name).toBe('pi-durable-request'); + }, + }) + .expect({ + event: event => { + expect(event.exception?.values?.[0]?.value).toBe('broken tool'); + }, + }) + .start() + .completed(); + }); + }, + PI_DURABLE_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-tools.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + test('reports app tool failures but not failures of the built-in coding tools', async () => { + const appErrors: unknown[] = []; + + // Ordered, with spans ignored: a captured failure of the built-in `bash` tool would be an + // extra event before the sentinel, which the scenario sends once everything else is sent. + // The two app failures are processed concurrently, so they may arrive in either order. + await createRunner() + .ignore('span') + .expect({ + event: event => { + expect(['app read failed', 'Intentional pi-durable tool failure']).toContain( + event.exception?.values?.[0]?.value, + ); + appErrors.push(event.exception?.values?.[0]?.value); + }, + }) + .expect({ + event: event => { + expect(['app read failed', 'Intentional pi-durable tool failure']).toContain( + event.exception?.values?.[0]?.value, + ); + appErrors.push(event.exception?.values?.[0]?.value); + }, + }) + .expect({ + event: event => { + expect(event.message).toBe('pi-durable tools done'); + }, + }) + .start() + .completed(); + + expect(appErrors.sort()).toEqual(['Intentional pi-durable tool failure', 'app read failed']); + }); + + test('records the tool result the model receives', async () => { + let errorSpanId: string | undefined; + let failingToolSpanId: string | undefined; + + await createRunner() + .unordered() + .expect({ + event: event => { + const exception = event.exception?.values?.[0]; + expect(exception?.value).toBe('Intentional pi-durable tool failure'); + expect(exception?.mechanism).toEqual({ type: 'auto.ai.pi_durable', handled: false }); + errorSpanId = event.contexts?.trace?.span_id; + }, + }) + .expect({ + event: event => { + expect(event.exception?.values?.[0]?.value).toBe('app read failed'); + }, + }) + .expect({ + span: container => { + const spans = container.items; + + const bash = spans.find(span => span.name === 'execute_tool bash')!; + expect(bash.status).toBe('error'); + expect(bash.attributes[GEN_AI_TOOL_CALL_RESULT]?.value).toMatch(/Command exited with code \d+/); + + const read = spans.find(span => span.name === 'execute_tool read')!; + expect(read.status).toBe('error'); + expect(read.attributes[GEN_AI_TOOL_CALL_RESULT]?.value).toContain('app read failed'); + + // The streamed output becomes the result, since the tool returns no content. + const streamer = spans.find(span => span.name === 'execute_tool streamer')!; + expect(streamer.status).toBe('ok'); + expect(streamer.attributes[GEN_AI_TOOL_CALL_RESULT]?.value).toBe('line 1\nline 2\n'); + + const secret = spans.find(span => span.name === 'execute_tool secret')!; + expect(secret.attributes[GEN_AI_TOOL_CALL_RESULT]?.value).toBe('redacted by afterTool'); + + const failNow = spans.find(span => span.name === 'execute_tool fail_now')!; + expect(failNow.status).toBe('error'); + failingToolSpanId = failNow.span_id; + }, + }) + .start() + .completed(); + + expect(errorSpanId).toBe(failingToolSpanId); + }); + }, + PI_DURABLE_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-reports.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + test('captures failures pi-durable does not propagate, and tags idle compaction with its conversation', async () => { + let runConversationId: unknown; + let compactionConversationId: unknown; + + await createRunner() + .unordered() + .expect({ + event: event => { + const exception = event.exception?.values?.[0]; + expect(exception?.value).toBe('afterResponse hook failed'); + expect(exception?.mechanism).toEqual({ type: 'auto.ai.pi_durable', handled: false }); + }, + }) + .expect({ + event: event => { + const exception = event.exception?.values?.[0]; + expect(exception?.value).toBe('card declined'); + expect(exception?.mechanism).toEqual({ type: 'auto.ai.pi_durable', handled: false }); + }, + }) + .expect({ + span: container => { + const agent = container.items.find(span => span.is_segment)!; + expect(agent.name).toBe('invoke_agent'); + runConversationId = agent.attributes[GEN_AI_CONVERSATION_ID]?.value; + }, + }) + .expect({ + span: container => { + // A compaction started while idle belongs to no run, so its request is a trace of its own. + expect(container.items).toHaveLength(1); + const summary = container.items[0]!; + expect(summary.is_segment).toBe(true); + expect(summary.attributes[SENTRY_OP]?.value).toBe(GEN_AI_CHAT); + compactionConversationId = summary.attributes[GEN_AI_CONVERSATION_ID]?.value; + }, + }) + .start() + .completed(); + + expect(runConversationId).toMatch(/^[0-9a-f]{32}:1$/); + expect(compactionConversationId).toBe(runConversationId); + }); + }, + PI_DURABLE_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-subagent.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + test('nests subagent runs under the tool call that started them', async () => { + await createRunner() + .unordered() + .expect({ + span: container => { + const spans = container.items; + const parent = spans.find(span => span.is_segment)!; + const delegate = spans.find(span => span.name === 'execute_tool delegate')!; + expect(parent.name).toBe('invoke_agent'); + expect(delegate.parent_span_id).toBe(parent.span_id); + + // The subagent's run is a child of the delegating call, in the same trace, on its own + // conversation of the same Harness. + const child = spans.find(span => span.name === 'invoke_agent' && !span.is_segment)!; + expect(child.parent_span_id).toBe(delegate.span_id); + const parentConversation = String(parent.attributes[GEN_AI_CONVERSATION_ID]?.value); + const childConversation = String(child.attributes[GEN_AI_CONVERSATION_ID]?.value); + expect(childConversation).not.toBe(parentConversation); + expect(childConversation.split(':')[0]).toBe(parentConversation.split(':')[0]); + + const childChat = spans.find(span => span.parent_span_id === child.span_id)!; + expect(childChat.name).toBe('chat faux-model'); + expect(childChat.attributes[GEN_AI_CONVERSATION_ID]?.value).toBe(childConversation); + expect(delegate.attributes[GEN_AI_TOOL_CALL_RESULT]?.value).toBe('Answer: Weather in Vienna?'); + }, + }) + .expect({ + span: container => { + const spans = container.items; + const parent = spans.find(span => span.is_segment)!; + const fork = spans.find(span => span.name === 'execute_tool fork_delegate')!; + const child = spans.find(span => span.name === 'invoke_agent' && !span.is_segment)!; + expect(fork.parent_span_id).toBe(parent.span_id); + expect(child.parent_span_id).toBe(fork.span_id); + expect(fork.attributes[GEN_AI_TOOL_CALL_RESULT]?.value).toBe('Answer: Forked task.'); + }, + }) + .expect({ + span: container => { + const spans = container.items; + const batch = spans.find(span => span.name === 'execute_tool batch')!; + const inner = spans.find(span => span.name === 'execute_tool inner')!; + expect(batch.parent_span_id).toBe(spans.find(span => span.is_segment)!.span_id); + // A tool that runs another tool itself: the inner call ends with its own result. + expect(inner.parent_span_id).toBe(batch.span_id); + expect(inner.attributes[GEN_AI_TOOL_CALL_RESULT]?.value).toBe('INNER RESULT'); + expect(batch.attributes[GEN_AI_TOOL_CALL_RESULT]?.value).toBe('OUTER RESULT'); + }, + }) + .start() + .completed(); + }); + }, + PI_DURABLE_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-retention.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + // Bun does not implement `v8.setFlagsFromString`, which the scenario needs to force GC. + test.skipIf(RUNTIME === 'bun')( + 'does not keep the traces of tool calls alive for subagent conversations that never run', + async () => { + await createRunner() + .ignore('span') + .expect({ event: { message: 'pi-durable retention', extra: { toolCalls: 3, alive: 0 } } }) + .start() + .completed(); + }, + ); + }, + PI_DURABLE_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-restart-abort.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + test('starts no run for an abort handler in a Harness that never saw the run', async () => { + await createRunner() + .ignore('span') + .expect({ + event: { + message: 'pi-durable restart abort', + extra: { status: 'unanswered', reason: 'aborted', runsStarted: 0 }, + }, + }) + .start() + .completed(); + }); + }, + PI_DURABLE_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-interrupted.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + test('reports a provider that throws as a fault, and nothing for failed, aborted or closed runs', async () => { + await createRunner() + .ignore('span') + .expect({ + event: event => { + const exception = event.exception?.values?.[0]; + expect(exception?.value).toBe('provider broke'); + expect(exception?.mechanism).toEqual({ type: 'auto.ai.pi_durable', handled: false }); + }, + }) + .expect({ + event: event => { + expect(event.message).toBe('pi-durable interrupted done'); + }, + }) + .start() + .completed(); + }); + + test('ends every run that does not end with an answer', async () => { + await createRunner() + .unordered() + .expect({ + span: container => { + // 1. The model request failed: the run and the request are errors. + const chat = container.items.find(span => span.name === 'chat faux-model')!; + expect(chat.attributes[GEN_AI_INPUT_MESSAGES]?.value).toContain('Fail please.'); + expect(chat.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value).toBe('["error"]'); + expect(chat.status).toBe('error'); + expect(chat.attributes[SENTRY_STATUS_MESSAGE]?.value).toBe('internal_error'); + const agent = container.items.find(span => span.is_segment)!; + expect(agent.name).toBe('invoke_agent'); + expect(agent.status).toBe('error'); + expect(agent.attributes[SENTRY_STATUS_MESSAGE]?.value).toBe('model_error'); + }, + }) + .expect({ + span: container => { + // 2. The provider threw: the generation faulted. + const chat = container.items.find(span => span.name === 'chat faux-model')!; + expect(chat.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toBeUndefined(); + expect(chat.attributes[SENTRY_STATUS_MESSAGE]?.value).toBe('internal_error'); + const agent = container.items.find(span => span.is_segment)!; + expect(agent.attributes[SENTRY_STATUS_MESSAGE]?.value).toBe('internal_error'); + }, + }) + .expect({ + span: container => { + // 3. Aborted while the tool ran: cancelled, which streamed spans report as ok. + const slow = container.items.find(span => span.name === 'execute_tool slow')!; + expect(slow.attributes[GEN_AI_TOOL_CALL_ARGUMENTS]?.value).toBe('{"job":"abort"}'); + expect(slow.status).toBe('ok'); + expect(container.items.find(span => span.is_segment)!.status).toBe('ok'); + }, + }) + .expect({ + span: container => { + // 4. Aborted while the model request ran. + const chat = container.items.find(span => span.name === 'chat faux-model')!; + expect(chat.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value).toBe('["aborted"]'); + expect(container.items).toHaveLength(2); + }, + }) + .expect({ + span: container => { + // 5. The Harness closed while the tool ran: the run still ends, so the trace has a segment. + const slow = container.items.find(span => span.name === 'execute_tool slow')!; + expect(slow.attributes[GEN_AI_TOOL_CALL_ARGUMENTS]?.value).toBe('{"job":"close"}'); + const agent = container.items.find(span => span.is_segment)!; + expect(agent.name).toBe('invoke_agent'); + expect(slow.parent_span_id).toBe(agent.span_id); + }, + }) + .expect({ + span: container => { + // 6. The run failed, and the Harness closed right after `wait()`, before the commit that + // ended the run resolved: the run keeps its own status. + const chat = container.items.find(span => span.name === 'chat faux-model')!; + expect(chat.attributes[GEN_AI_INPUT_MESSAGES]?.value).toContain('Fail, then close.'); + const agent = container.items.find(span => span.is_segment)!; + expect(agent.status).toBe('error'); + expect(agent.attributes[SENTRY_STATUS_MESSAGE]?.value).toBe('model_error'); + }, + }) + .start() + .completed(); + }); + }, + PI_DURABLE_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-concurrent.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + test('keeps concurrent runs apart and out of the scopes of the requests that submitted them', async () => { + const runTraceIds = new Set(); + const errorTraceIds = new Set(); + + await createRunner() + .unordered() + .expect({ + span: container => { + const spans = container.items; + const agent = spans.find(span => span.is_segment)!; + expect(agent.name).toBe('invoke_agent'); + const chats = spans.filter(span => span.name === 'chat faux-model'); + expect(JSON.parse(String(chats[0]!.attributes[GEN_AI_INPUT_MESSAGES]?.value))[0].parts[0].content).toBe( + 'request A', + ); + // The run's own conversation id, not the one the request set on its scope. + const conversationId = agent.attributes[GEN_AI_CONVERSATION_ID]?.value; + expect(conversationId).toMatch(/^[0-9a-f]{32}:\d+$/); + for (const span of spans) { + expect(span.attributes[GEN_AI_CONVERSATION_ID]?.value).toBe(conversationId); + expect(span.attributes['request']).toBeUndefined(); + } + expect( + spans.find(span => span.name === 'execute_tool work')!.attributes[GEN_AI_TOOL_CALL_ARGUMENTS]?.value, + ).toBe('{"tag":"request A"}'); + runTraceIds.add(agent.trace_id); + }, + }) + .expect({ + span: container => { + const spans = container.items; + const agent = spans.find(span => span.is_segment)!; + expect(agent.name).toBe('invoke_agent'); + const chats = spans.filter(span => span.name === 'chat faux-model'); + expect(JSON.parse(String(chats[0]!.attributes[GEN_AI_INPUT_MESSAGES]?.value))[0].parts[0].content).toBe( + 'request B', + ); + const conversationId = agent.attributes[GEN_AI_CONVERSATION_ID]?.value; + expect(conversationId).toMatch(/^[0-9a-f]{32}:\d+$/); + for (const span of spans) { + expect(span.attributes[GEN_AI_CONVERSATION_ID]?.value).toBe(conversationId); + expect(span.attributes['request']).toBeUndefined(); + } + expect( + spans.find(span => span.name === 'execute_tool work')!.attributes[GEN_AI_TOOL_CALL_ARGUMENTS]?.value, + ).toBe('{"tag":"request B"}'); + runTraceIds.add(agent.trace_id); + }, + }) + .expect({ + event: event => { + expect(event.exception?.values?.[0]?.value).toBe('work failed for request A'); + expect(event.tags?.request).toBeUndefined(); + errorTraceIds.add(String(event.contexts?.trace?.trace_id)); + }, + }) + .expect({ + event: event => { + expect(event.exception?.values?.[0]?.value).toBe('work failed for request B'); + expect(event.tags?.request).toBeUndefined(); + errorTraceIds.add(String(event.contexts?.trace?.trace_id)); + }, + }) + .start() + .completed(); + + // Each tool error belongs to its run's trace. + expect(errorTraceIds).toEqual(runTraceIds); + }); + }, + PI_DURABLE_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-compaction.mjs', + 'instrument.mjs', + (createRunner, test, mode) => { + if (mode === 'cjs') { + return; + } + + test('keeps a compaction that pi-durable starts for a run inside that run', async () => { + await createRunner() + .unordered() + .expect({ + span: container => { + const spans = container.items; + const summary = spans.find(span => + String(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value).includes('Summary'), + )!; + const agent = spans.find(span => span.is_segment)!; + expect(agent.name).toBe('invoke_agent'); + expect(summary.parent_span_id).toBe(agent.span_id); + // pi-durable completes the summary without streaming; the run's own request streams. + expect(summary.attributes[GEN_AI_RESPONSE_STREAMING]).toBeUndefined(); + const answer = spans.find(span => + String(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value).includes('Third answer'), + )!; + expect(answer.parent_span_id).toBe(agent.span_id); + expect(answer.attributes[GEN_AI_RESPONSE_STREAMING]?.value).toBe(true); + }, + }) + .expect({ + span: container => expect(container.items.find(span => span.is_segment)!.name).toBe('invoke_agent'), + }) + .expect({ + span: container => expect(container.items.find(span => span.is_segment)!.name).toBe('invoke_agent'), + }) + .start() + .completed(); + }); + }, + PI_DURABLE_DEPENDENCIES, + ); +}); diff --git a/packages/astro/src/index.server.ts b/packages/astro/src/index.server.ts index ed0b1b540bb4..7c997a3de32f 100644 --- a/packages/astro/src/index.server.ts +++ b/packages/astro/src/index.server.ts @@ -102,6 +102,7 @@ export { langGraphIntegration, createFlueInstrumentation, mastraIntegration, + piDurableIntegration, mcpServerIntegration, SentryMastraExporter, parameterize, diff --git a/packages/aws-serverless/src/index.ts b/packages/aws-serverless/src/index.ts index 6f17bcc02c0c..d5aabe372bd0 100644 --- a/packages/aws-serverless/src/index.ts +++ b/packages/aws-serverless/src/index.ts @@ -67,6 +67,7 @@ export { langChainIntegration, langGraphIntegration, mastraIntegration, + piDurableIntegration, mcpServerIntegration, SentryMastraExporter, createFlueInstrumentation, diff --git a/packages/bun/src/index.ts b/packages/bun/src/index.ts index e96e052656cb..aa7b2cfc74bf 100644 --- a/packages/bun/src/index.ts +++ b/packages/bun/src/index.ts @@ -89,6 +89,7 @@ export { langChainIntegration, langGraphIntegration, mastraIntegration, + piDurableIntegration, mcpServerIntegration, SentryMastraExporter, createFlueInstrumentation, diff --git a/packages/deno/src/index.ts b/packages/deno/src/index.ts index 090eca082909..dd23caea66cc 100644 --- a/packages/deno/src/index.ts +++ b/packages/deno/src/index.ts @@ -140,6 +140,7 @@ export { langChainIntegration, langGraphIntegration, mastraIntegration, + piDurableIntegration, mcpServerIntegration, SentryMastraExporter, createFlueInstrumentation, diff --git a/packages/elysia/src/index.ts b/packages/elysia/src/index.ts index 17baec308ba1..659cb35cf870 100644 --- a/packages/elysia/src/index.ts +++ b/packages/elysia/src/index.ts @@ -69,6 +69,7 @@ export { langGraphIntegration, createFlueInstrumentation, mastraIntegration, + piDurableIntegration, mcpServerIntegration, SentryMastraExporter, modulesIntegration, diff --git a/packages/google-cloud-serverless/src/index.ts b/packages/google-cloud-serverless/src/index.ts index eb4fa0495540..828f77501a48 100644 --- a/packages/google-cloud-serverless/src/index.ts +++ b/packages/google-cloud-serverless/src/index.ts @@ -67,6 +67,7 @@ export { langChainIntegration, langGraphIntegration, mastraIntegration, + piDurableIntegration, mcpServerIntegration, SentryMastraExporter, createFlueInstrumentation, diff --git a/packages/node/src/index.ts b/packages/node/src/index.ts index 1a1105557d9f..0dbf9dc692b9 100644 --- a/packages/node/src/index.ts +++ b/packages/node/src/index.ts @@ -25,6 +25,7 @@ export { lruMemoizerIntegration, createFlueInstrumentation, mastraIntegration, + piDurableIntegration, mcpServerIntegration, SentryMastraExporter, mongoIntegration, diff --git a/packages/server-utils/src/ai/pi-ai/messages.ts b/packages/server-utils/src/ai/pi-ai/messages.ts index c5060cdb1181..1d9794a08a77 100644 --- a/packages/server-utils/src/ai/pi-ai/messages.ts +++ b/packages/server-utils/src/ai/pi-ai/messages.ts @@ -62,6 +62,80 @@ export function piAiFinishReason(stopReason: unknown): string | undefined { return stopReason === 'toolUse' ? 'tool_call' : stopReason; } +/** The request fields of a pi-ai `Context` that carry the system prompt and the tools. */ +export interface PiAiContext { + systemPrompt?: string; + messages?: unknown; + tools?: unknown; +} + +/** + * The system prompt a pi-ai request sends. pi-durable sends it as positional system messages with + * `sections` instead of `systemPrompt`, so the messages are replayed like pi-ai's own + * `getCurrentSystemPrompt` does: `content` is appended, `sections` are patched by name. + */ +export function piAiSystemInstructions(context: PiAiContext): string | undefined { + const content: string[] = []; + const sections = new Map(); + for (const message of piAiSystemMessages(context)) { + const text = piAiContentText(message.content); + if (text) { + content.push(text); + } + if (isObjectLike(message.sections)) { + for (const [name, value] of Object.entries(message.sections)) { + if (value === null) { + sections.delete(name); + } else if (typeof value === 'string') { + sections.set(name, value); + } + } + } + } + return [content.join('\n\n'), ...sections.values()].filter(Boolean).join('\n\n') || undefined; +} + +/** The tools a pi-ai request offers, with `toolsAdded` and `toolsRemoved` replayed like pi-ai's `getCurrentTools`. */ +export function piAiToolDefinitions(context: PiAiContext): unknown[] { + const tools = new Map(); + for (const message of piAiSystemMessages(context)) { + for (const tool of Array.isArray(message.toolsRemoved) ? message.toolsRemoved : []) { + if (isObjectLike(tool) && typeof tool.name === 'string') { + tools.delete(tool.name); + } + } + for (const tool of Array.isArray(message.toolsAdded) ? message.toolsAdded : []) { + if (isObjectLike(tool) && typeof tool.name === 'string') { + tools.set(tool.name, tool); + } + } + } + return [...tools.values()]; +} + +/** The system messages pi-ai sends, led by the one it builds from `systemPrompt` and `tools`. */ +function piAiSystemMessages(context: PiAiContext): Record[] { + const hasTools = Array.isArray(context.tools) && context.tools.length > 0; + const leading = + context.systemPrompt || hasTools + ? [{ role: 'system', content: context.systemPrompt ?? '', toolsAdded: context.tools }] + : []; + const messages = Array.isArray(context.messages) ? context.messages : []; + return [...leading, ...messages].filter( + (message): message is Record => isObjectLike(message) && message.role === 'system', + ); +} + +function piAiContentText(content: unknown): string { + if (typeof content === 'string') { + return content; + } + return piAiContentToParts(content) + .filter(part => part.type === 'text') + .map(part => part.content) + .join('\n'); +} + /** Text-only content as its text, joined the way pi-ai joins it; anything else as its mapped parts. */ export function piAiContentToString(content: unknown): string | undefined { const parts = piAiContentToParts(content); diff --git a/packages/server-utils/src/ai/pi-ai/providers.ts b/packages/server-utils/src/ai/pi-ai/providers.ts index 524afe79d573..96bc6149f620 100644 --- a/packages/server-utils/src/ai/pi-ai/providers.ts +++ b/packages/server-utils/src/ai/pi-ai/providers.ts @@ -2,15 +2,18 @@ import { _INTERNAL_shouldSkipAiProviderWrapping, _INTERNAL_skipAiProviderWrappin import { ANTHROPIC_AI_INTEGRATION_NAME } from '../anthropic-ai/constants'; import { GOOGLE_GENAI_INTEGRATION_NAME } from '../google-genai/constants'; import { OPENAI_INTEGRATION_NAME } from '../openai/constants'; +import { WORKERS_AI_INTEGRATION_NAME } from '../workers-ai/constants'; -// pi-ai sends its requests through the `openai`, `@anthropic-ai/sdk` and `@google/genai` clients. -// Left alone, those integrations report the same request a second time beside the `chat` span of -// the framework that sent it. Bedrock requests go through `@aws-sdk/client-bedrock-runtime`, which -// `awsIntegration` still reports; that one has no skip yet. +// pi-ai sends its requests through the `openai`, `@anthropic-ai/sdk` and `@google/genai` clients, and +// on Cloudflare through the Workers AI binding (`createAI()` of `agents/models/pi-ai`). Left alone, +// those integrations report the same request a second time beside the `chat` span of the framework +// that sent it. Bedrock requests go through `@aws-sdk/client-bedrock-runtime`, which `awsIntegration` +// still reports; that one has no skip yet. const PI_AI_PROVIDER_INTEGRATIONS = [ OPENAI_INTEGRATION_NAME, ANTHROPIC_AI_INTEGRATION_NAME, GOOGLE_GENAI_INTEGRATION_NAME, + WORKERS_AI_INTEGRATION_NAME, ]; /** diff --git a/packages/server-utils/src/ai/pi-durable/constants.ts b/packages/server-utils/src/ai/pi-durable/constants.ts new file mode 100644 index 000000000000..f266a250c24a --- /dev/null +++ b/packages/server-utils/src/ai/pi-durable/constants.ts @@ -0,0 +1,23 @@ +export const PI_DURABLE_INTEGRATION_NAME = 'PiDurable' as const; + +export const PI_DURABLE_ORIGIN = 'auto.ai.pi_durable'; + +/** Task kinds of the pi-durable built-in tasks. */ +export const PI_TASK = { + GENERATION: 'pi.generation', + TOOL: 'pi.tool', + COMPACTION: 'pi.compaction', +} as const; + +/** Kind of the built-in document that holds a conversation's run control. */ +export const PI_LIVE_DOC_KIND = 'pi.live'; + +/** Kind of the transcript entry that holds the tool result the model receives. */ +export const PI_TOOL_RESULT_ENTRY_KIND = 'pi.tool-result'; + +/** + * Cap on open run spans. A run span is removed when the run settles, but a run the process never + * sees settle (a crash, or a settlement the scheduler writes outside a task phase) would otherwise + * stay in the map for the lifetime of the process. + */ +export const MAX_TRACKED_PI_RUNS = 1000; diff --git a/packages/server-utils/src/ai/pi-durable/index.ts b/packages/server-utils/src/ai/pi-durable/index.ts new file mode 100644 index 000000000000..839967be3184 --- /dev/null +++ b/packages/server-utils/src/ai/pi-durable/index.ts @@ -0,0 +1,243 @@ +import type { Scope } from '@sentry/core'; +import { + addNonEnumerableProperty, + captureException, + getCurrentScope, + getDefaultIsolationScope, + SPAN_STATUS_ERROR, + startNewTrace, + withActiveSpan, +} from '@sentry/core'; +import type { GenAiOptions } from '../core/utils'; +import { skipPiAiProviderIntegrations } from '../pi-ai/providers'; +import { PI_DURABLE_ORIGIN, PI_TASK } from './constants'; +import { instrumentPiModels } from './models'; +import type { PiRuns } from './runs'; +import { createRuns, endRun, observeRunControl, startRun, toSentryConversationId } from './runs'; +import { instrumentTool, instrumentToolPhase } from './tools'; +import type { + PiHarness, + PiHarnessOptions, + PiPhaseHandler, + PiRegistryReader, + PiRegistrySnapshot, + PiTask, + PiTaskRuntime, + PiTool, +} from './types'; +import { withCleanScopes } from '../../utils/withCleanScopes'; +import { bound } from './utils'; + +export type PiDurableOptions = GenAiOptions; + +const INSTRUMENTED = Symbol.for('sentry.pi-durable.instrumented'); +const RUNS = Symbol.for('sentry.pi-durable.runs'); + +/** + * Return pi-durable `HarnessOptions` whose model requests, task phases and tool calls are traced. + * The caller's `models` and `registry` objects are wrapped, never modified. The result has the + * caller's options as prototype: pi-durable reads `settings`, `env` and `conversationCreated` at + * every use, so they stay live, getters included. + * + * Spans: + * - `gen_ai.invoke_agent` for each run, a new trace per run, or a child of the tool call that + * created the conversation for a subagent; + * - `gen_ai.chat` for each model request, under its run; + * - `gen_ai.execute_tool` for each tool call, under its run. + * + * The scheduler starts task phases from whichever async context last woke it, often an unrelated + * request. Every phase therefore runs in its run's own scopes and trace, never in the ones it + * inherits. + */ +export function instrumentPiDurableHarnessOptions( + harnessOptions: T, + options: PiDurableOptions = {}, +): T { + if (!harnessOptions || (harnessOptions as Record)[INSTRUMENTED]) { + return harnessOptions; + } + + const runs = createRuns(); + const own = (value: unknown): PropertyDescriptor => ({ value, enumerable: true, configurable: true, writable: true }); + const instrumented: T = Object.create(harnessOptions, { + ...(harnessOptions.models + ? { models: own(instrumentPiModels(harnessOptions.models, options, skipPiAiProviderIntegrations)) } + : {}), + ...(harnessOptions.registry ? { registry: own(instrumentRegistry(harnessOptions.registry, options, runs)) } : {}), + // pi-durable passes failures of extension code here, such as a throwing hook, and keeps going. + // Only an app that passed its own `onReport` gets a chance to handle them. + onReport: own((error: unknown) => { + captureException(error, { mechanism: { handled: !!harnessOptions.onReport, type: PI_DURABLE_ORIGIN } }); + harnessOptions.onReport?.(error); + }), + }); + addNonEnumerableProperty(instrumented, INSTRUMENTED, true); + addNonEnumerableProperty(instrumented, RUNS, runs); + return instrumented; +} + +/** + * End the open runs and tool calls of `harness` when it closes. Closing stops the scheduler without + * settling the inputs in flight, so nothing else would end their spans, and the maps that track + * them would keep the spans for the lifetime of the process. A run whose ending commit is still in + * flight ends with that commit's status. + */ +export function endRunsOnClose(harnessOptions: PiHarnessOptions, harness: unknown): void { + const runs = (harnessOptions as Record)[RUNS] as PiRuns | undefined; + const closable = harness as PiHarness | undefined; + if (!runs || typeof closable?.subscribeClose !== 'function') { + return; + } + closable.subscribeClose(() => { + for (const run of runs.active.values()) { + endRun(run, runs, run.pendingEnd ? run.pendingEnd.status : { code: SPAN_STATUS_ERROR, message: 'cancelled' }); + } + for (const calls of runs.toolCalls.values()) { + for (const call of calls) { + if (call.endTimestamp === undefined) { + call.span.setStatus({ code: SPAN_STATUS_ERROR, message: 'cancelled' }); + } + call.span.end(call.endTimestamp); + } + } + runs.toolCalls.clear(); + runs.owners.clear(); + }); +} + +function instrumentRegistry(registry: PiRegistryReader, options: PiDurableOptions, runs: PiRuns): PiRegistryReader { + // Cached so identity stays stable: the scheduler hands a task over to a new definition whenever + // `snapshot().task(kind)` returns a different object than the one it runs. + const snapshots = new WeakMap(); + const tasks = new WeakMap(); + const tools = new WeakMap(); + + const wrapTool = (tool: PiTool): PiTool => { + let wrapped = tools.get(tool); + if (!wrapped) { + wrapped = instrumentTool(tool, runs, options); + tools.set(tool, wrapped); + } + return wrapped; + }; + + const wrapTask = (task: PiTask | undefined): PiTask | undefined => { + if (!task?.definition?.phases) { + return task; + } + let wrapped = tasks.get(task); + if (!wrapped) { + const { definition } = task; + // pi-durable calls a phase on the `phases` object and `abort` on the definition; keep that + // receiver, so handlers that use `this` work. + const wrapPhase = + (phase: PiPhaseHandler, receiver: unknown, isAbort = false): PiPhaseHandler => + (taskRecord, runtime, context) => + runPhase( + definition.name, + observed => phase.call(receiver, taskRecord, observed, context), + runtime, + runs, + wrapTool, + isAbort, + ); + const phases: Record = {}; + for (const [name, phase] of Object.entries(definition.phases)) { + phases[name] = typeof phase === 'function' ? wrapPhase(phase, definition.phases) : phase; + } + wrapped = { + ...task, + definition: { + ...definition, + phases, + ...(typeof definition.abort === 'function' ? { abort: wrapPhase(definition.abort, definition, true) } : {}), + }, + }; + tasks.set(task, wrapped); + } + return wrapped; + }; + + const wrapSnapshot = (snapshot: PiRegistrySnapshot): PiRegistrySnapshot => { + let wrapped = snapshots.get(snapshot); + if (!wrapped) { + wrapped = new Proxy(snapshot, { + get(target, property) { + if (property === 'task') { + return (name: string) => wrapTask(target.task(name)); + } + if (property === 'tasks') { + return () => target.tasks().map(task => wrapTask(task)); + } + return bound(target, property); + }, + }); + snapshots.set(snapshot, wrapped); + } + return wrapped; + }; + + return new Proxy(registry, { + get(target, property) { + if (property === 'snapshot') { + return () => wrapSnapshot(target.snapshot()); + } + return bound(target, property); + }, + }); +} + +function runPhase( + kind: string, + phase: (runtime: PiTaskRuntime) => Promise, + runtime: PiTaskRuntime, + runs: PiRuns, + wrapTool: (tool: PiTool) => PiTool, + isAbort: boolean, +): Promise { + const { conversationId } = runtime; + const isRunWork = kind === PI_TASK.GENERATION || kind === PI_TASK.TOOL; + // A compaction joins the run it was started for. One started while idle has no run, and its + // `chat` span becomes the root of a trace of its own. An abort handler ends the run of its task, + // so it joins that run but never starts one. Tasks of extensions never join a run. + const run = + isRunWork || kind === PI_TASK.COMPACTION + ? (runs.active.get(conversationId) ?? (isRunWork && !isAbort ? startRun(conversationId, runs) : undefined)) + : undefined; + + let observed = runtime; + let finishToolPhase: (() => void) | undefined; + if (run && kind === PI_TASK.GENERATION) { + observed = observeRunControl(runtime, run, runs); + } else if (kind === PI_TASK.TOOL) { + const toolPhase = instrumentToolPhase(runtime, runs, wrapTool); + observed = toolPhase.runtime; + finishToolPhase = toolPhase.finish; + } + + const runInScope = async (scope: Scope): Promise => { + scope.setConversationId(toSentryConversationId(runs, conversationId)); + try { + return await phase(observed); + } catch (error) { + // The scheduler faults a task whose phase throws, and a faulted generation ends its run. A + // throw after an abort mark, or once the Harness closes, is not a fault. + if (!runtime.signal?.aborted) { + captureException(error, { mechanism: { handled: false, type: PI_DURABLE_ORIGIN } }); + if (run && kind === PI_TASK.GENERATION) { + endRun(run, runs, { code: SPAN_STATUS_ERROR, message: 'internal_error' }); + } + } + throw error; + } finally { + finishToolPhase?.(); + } + }; + + if (!run) { + return withCleanScopes(getDefaultIsolationScope().clone(), () => + startNewTrace(() => runInScope(getCurrentScope())), + ); + } + return withCleanScopes(run.isolationScope, () => withActiveSpan(run.span, runInScope)); +} diff --git a/packages/server-utils/src/ai/pi-durable/models.ts b/packages/server-utils/src/ai/pi-durable/models.ts new file mode 100644 index 000000000000..b50a5cdb0420 --- /dev/null +++ b/packages/server-utils/src/ai/pi-durable/models.ts @@ -0,0 +1,213 @@ +import type { Span } from '@sentry/core'; +import { + getCurrentScope, + getIsolationScope, + isThenable, + isURLObjectRelative, + parseStringToURLObject, + SPAN_STATUS_ERROR, + startSpanManual, + stringify, +} from '@sentry/core'; +import { + GEN_AI_CONVERSATION_ID, + GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MAX_TOKENS, + GEN_AI_REQUEST_MODEL, + GEN_AI_REQUEST_REASONING_LEVEL, + GEN_AI_REQUEST_TEMPERATURE, + GEN_AI_RESPONSE_FINISH_REASONS, + GEN_AI_RESPONSE_ID, + GEN_AI_RESPONSE_MODEL, + GEN_AI_RESPONSE_STREAMING, + GEN_AI_SYSTEM_INSTRUCTIONS, + GEN_AI_TOOL_DEFINITIONS, + SENTRY_ORIGIN, + SERVER_ADDRESS, + SERVER_PORT, +} from '@sentry/conventions/attributes'; +import { GEN_AI_CHAT } from '@sentry/conventions/op'; +import type { GenAiOptions } from '../core/utils'; +import { resolveAIRecordingOptions } from '../core/utils'; +import type { PiAiContext } from '../pi-ai/messages'; +import { + piAiAssistantMessageToGenAiMessage, + piAiFinishReason, + piAiMessagesToGenAiMessages, + piAiSystemInstructions, + piAiToolDefinitions, +} from '../pi-ai/messages'; +import { setPiAiUsageAttributes } from '../pi-ai/usage'; +import { PI_DURABLE_ORIGIN } from './constants'; +import type { PiAssistantMessage, PiEventStream, PiModel, PiModels, PiStreamOptions } from './types'; + +type ModelCall = (model: PiModel, request: unknown, options?: unknown) => unknown; + +const STREAMING_METHODS = new Set(['stream', 'streamSimple', 'streamDeferred']); +const COMPLETE_METHODS = new Set(['complete', 'completeSimple', 'fetchDeferred']); +// These fetch the answer of a parked request; their second argument is a handle, not the request. +const DEFERRED_METHODS = new Set(['streamDeferred', 'fetchDeferred']); + +/** + * Wrap a pi-ai `Models` so every model request it sends gets a `gen_ai.chat` span. + * + * The proxy hands every other member through bound to the original, so a `Models` instance whose + * methods read `this` keeps working, and the caller's own object is never modified. + */ +export function instrumentPiModels(models: T, options: GenAiOptions, onRequest: () => void): T { + const wrapped = new Map(); + + return new Proxy(models, { + get(target, property) { + const value: unknown = Reflect.get(target, property, target); + if (typeof value !== 'function') { + return value; + } + + const cached = wrapped.get(property); + if (cached) { + return cached; + } + + const original = (value as ModelCall).bind(target); + const method = String(property); + const streaming = STREAMING_METHODS.has(method); + const deferred = DEFERRED_METHODS.has(method); + const instrumented = + streaming || COMPLETE_METHODS.has(method) + ? (model: PiModel, request: unknown, callOptions?: unknown): unknown => { + onRequest(); + return traceModelRequest( + model, + deferred ? undefined : (request as PiAiContext), + deferred ? undefined : (callOptions as PiStreamOptions | undefined), + streaming, + options, + () => original(model, request, callOptions), + ); + } + : original; + + wrapped.set(property, instrumented); + return instrumented; + }, + }); +} + +function traceModelRequest( + model: PiModel, + context: PiAiContext | undefined, + callOptions: PiStreamOptions | undefined, + streaming: boolean, + options: GenAiOptions, + send: () => unknown, +): unknown { + // Read per request: `dataCollection.genAI` belongs to the current client, which can change. + const { recordInputs, recordOutputs } = resolveAIRecordingOptions(options); + const conversationId = + getCurrentScope().getScopeData().conversationId ?? getIsolationScope().getScopeData().conversationId; + + return startSpanManual( + { + name: model.id ? `chat ${model.id}` : 'chat', + op: GEN_AI_CHAT, + attributes: { + [SENTRY_ORIGIN]: PI_DURABLE_ORIGIN, + [GEN_AI_OPERATION_NAME]: 'chat', + ...(model.provider ? { [GEN_AI_PROVIDER_NAME]: model.provider } : {}), + ...(model.id ? { [GEN_AI_REQUEST_MODEL]: model.id } : {}), + ...(conversationId ? { [GEN_AI_CONVERSATION_ID]: conversationId } : {}), + ...getServerAttributes(model), + ...(streaming ? { [GEN_AI_RESPONSE_STREAMING]: true } : {}), + ...(callOptions?.temperature !== undefined ? { [GEN_AI_REQUEST_TEMPERATURE]: callOptions.temperature } : {}), + ...(callOptions?.maxTokens !== undefined ? { [GEN_AI_REQUEST_MAX_TOKENS]: callOptions.maxTokens } : {}), + ...(callOptions?.reasoning ? { [GEN_AI_REQUEST_REASONING_LEVEL]: callOptions.reasoning } : {}), + ...(recordInputs && context ? getRequestContentAttributes(context) : {}), + }, + }, + span => { + let result: unknown; + try { + result = send(); + } catch (error) { + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'internal_error' }); + span.end(); + throw error; + } + + // pi-ai never rejects a request: a provider failure or an abort settles as a message with + // `stopReason` `error` or `aborted`. The rejection handler only covers a broken provider. + let settled: unknown; + try { + settled = streaming ? (result as PiEventStream).result() : result; + } catch { + settled = undefined; + } + // A result of another shape must reach the caller unchanged, so its span ends without a response. + if (!isThenable(settled)) { + span.end(); + return result; + } + (settled as Promise).then( + message => endChatSpan(span, message, recordOutputs), + () => { + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'internal_error' }); + span.end(); + }, + ); + + return result; + }, + ); +} + +function getServerAttributes(model: PiModel): Record { + const url = model.baseUrl ? parseStringToURLObject(model.baseUrl) : undefined; + if (!url || isURLObjectRelative(url) || !url.hostname) { + return {}; + } + return { [SERVER_ADDRESS]: url.hostname, ...(url.port ? { [SERVER_PORT]: Number(url.port) } : {}) }; +} + +function getRequestContentAttributes(context: PiAiContext): Record { + const messages = piAiMessagesToGenAiMessages(context.messages); + const tools = piAiToolDefinitions(context); + return { + [GEN_AI_SYSTEM_INSTRUCTIONS]: piAiSystemInstructions(context), + [GEN_AI_INPUT_MESSAGES]: messages.length ? stringify(messages) : undefined, + [GEN_AI_TOOL_DEFINITIONS]: tools.length ? stringify(tools) : undefined, + }; +} + +function endChatSpan(span: Span, message: PiAssistantMessage, recordOutputs: boolean): void { + const responseModel = message.responseModel ?? message.model; + if (responseModel) { + span.setAttribute(GEN_AI_RESPONSE_MODEL, responseModel); + } + if (message.responseId) { + span.setAttribute(GEN_AI_RESPONSE_ID, message.responseId); + } + + const finishReason = piAiFinishReason(message.stopReason); + if (finishReason) { + span.setAttribute(GEN_AI_RESPONSE_FINISH_REASONS, stringify([finishReason])); + } + + const failed = message.stopReason === 'error'; + setPiAiUsageAttributes(span, message.usage, failed || message.stopReason === 'deferred'); + + const output = recordOutputs ? piAiAssistantMessageToGenAiMessage(message, finishReason) : undefined; + if (output) { + span.setAttribute(GEN_AI_OUTPUT_MESSAGES, stringify([output], String)); + } + + if (failed) { + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'internal_error' }); + } else if (message.stopReason === 'aborted') { + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'cancelled' }); + } + span.end(); +} diff --git a/packages/server-utils/src/ai/pi-durable/runs.ts b/packages/server-utils/src/ai/pi-durable/runs.ts new file mode 100644 index 000000000000..c0fd861f9b10 --- /dev/null +++ b/packages/server-utils/src/ai/pi-durable/runs.ts @@ -0,0 +1,245 @@ +import type { MaybeWeakRef, Scope, Span, SpanStatus } from '@sentry/core'; +import { + continueTrace, + debug, + derefWeakRef, + getDefaultIsolationScope, + getRootSpan, + SPAN_STATUS_ERROR, + startInactiveSpan, + startNewTrace, + uuid4, +} from '@sentry/core'; +import { GEN_AI_CONVERSATION_ID, GEN_AI_OPERATION_NAME, SENTRY_ORIGIN } from '@sentry/conventions/attributes'; +import { GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op'; +import { DEBUG_BUILD } from '../../debug-build'; +import { MAX_TRACKED_PI_RUNS, PI_DURABLE_ORIGIN, PI_LIVE_DOC_KIND } from './constants'; +import type { PiCommitChange, PiDocToken, PiLiveState, PiSettlement, PiTaskRuntime } from './types'; +import { withCleanScopes } from '../../utils/withCleanScopes'; +import { bound } from './utils'; + +// Unanswered reasons that mean the run was stopped, not that it failed. +const CANCELLED_REASONS = new Set(['aborted', 'reset']); + +/** + * One run: the work from an input to its answer. It spans several `pi.generation` tasks (run control + * moves to a new one after every tool round) and their `pi.tool` tasks, so it is tracked per + * conversation, which runs at most one run at a time. + */ +export interface PiRun { + conversationId: unknown; + span: Span; + isolationScope: Scope; + /** The run's first input submission, which identifies the run in `pi.live`. */ + firstInput?: unknown; + /** + * The status the run ends with, set while the commit that ends it is in flight. `harness.close()` + * can begin in that gap, because `wait()` resolves as soon as the commit settles the inputs. + */ + pendingEnd?: { status?: SpanStatus }; +} + +/** Open runs of one Harness, keyed by the pi-durable conversation id. */ +export interface PiRuns { + /** + * Random id of this Harness, prefixed to `gen_ai.conversation.id`. pi-durable numbers + * conversations per storage from 1, so the bare id would merge the root conversations of every + * Harness into one Sentry conversation. + */ + harnessId: string; + active: Map; + /** + * Tool calls that created a conversation they own, keyed by that conversation. The child's first + * run becomes a child of the call while the call's trace is in progress, and otherwise continues + * that trace with the call as parent. This is how a subagent's run joins the trace of the run that + * delegated to it. + */ + owners: Map; + /** + * Tool calls of running `pi.tool` phases, keyed by task id. Their spans end when the phase ends, + * after it has committed the result the model receives. + */ + toolCalls: Map; +} + +/** A tool call that created a conversation it owns. */ +export interface PiOwner { + /** + * The call's `execute_tool` span, held weakly: a conversation whose first run never starts must + * not keep the trace of the call alive. + */ + span: MaybeWeakRef; + /** The `sentry-trace` value of the span, used to continue its trace once that trace has ended. */ + sentryTrace: string; + /** The `baggage` value of the span, used together with `sentryTrace`. */ + baggage: string | undefined; +} + +/** One `execute()` of a tool. */ +export interface PiToolCall { + span: Span; + callId?: string; + recordOutputs: boolean; + /** Set once the span has a status other than ok, so the committed result does not overwrite it. */ + failed?: boolean; + /** When `execute()` settled, used as the span's end time. */ + endTimestamp?: number; +} + +export function createRuns(): PiRuns { + return { + harnessId: uuid4(), + active: new Map(), + owners: new Map(), + toolCalls: new Map(), + }; +} + +export function toSentryConversationId(runs: PiRuns, conversationId: unknown): string { + return `${runs.harnessId}:${String(conversationId)}`; +} + +/** Start a run of `conversationId`: a new trace, or a child of the tool call that owns the conversation. */ +export function startRun(conversationId: unknown, runs: PiRuns): PiRun { + if (runs.active.size >= MAX_TRACKED_PI_RUNS) { + const oldest = runs.active.keys().next().value; + runs.active.get(oldest)?.span.end(); + runs.active.delete(oldest); + } + + const owner = runs.owners.get(conversationId); + runs.owners.delete(conversationId); + const ownerSpan = derefWeakRef(owner?.span); + // Under a trace that has ended, the run would be split into one transaction per span when spans are + // not streamed. Continuing the trace instead also gives the same result whether or not GC ran. + const parentSpan = ownerSpan && getRootSpan(ownerSpan).isRecording() ? ownerSpan : undefined; + + const isolationScope = getDefaultIsolationScope().clone(); + const startRunSpan = (): Span => + startInactiveSpan({ + name: 'invoke_agent', + op: GEN_AI_INVOKE_AGENT, + ...(parentSpan ? { parentSpan } : {}), + attributes: { + [SENTRY_ORIGIN]: PI_DURABLE_ORIGIN, + [GEN_AI_OPERATION_NAME]: 'invoke_agent', + [GEN_AI_CONVERSATION_ID]: toSentryConversationId(runs, conversationId), + }, + }); + const span = withCleanScopes(isolationScope, () => { + if (parentSpan) { + return startRunSpan(); + } + return owner + ? continueTrace({ sentryTrace: owner.sentryTrace, baggage: owner.baggage }, startRunSpan) + : startNewTrace(startRunSpan); + }); + + const run: PiRun = { conversationId, span, isolationScope }; + runs.active.set(conversationId, run); + DEBUG_BUILD && debug.log(`[pi-durable] run of conversation ${String(conversationId)} started`); + return run; +} + +export function endRun(run: PiRun, runs: PiRuns, status?: SpanStatus): void { + if (runs.active.get(run.conversationId) !== run) { + return; + } + runs.active.delete(run.conversationId); + DEBUG_BUILD && debug.log(`[pi-durable] run of conversation ${String(run.conversationId)} ended`, status ?? 'ok'); + if (status) { + run.span.setStatus(status); + } + run.span.end(); +} + +/** + * Watch the generation task's commits for the end of its run. + * + * pi-durable keeps run control in the conversation's `pi.live` document: `run` names the task that + * owns the run and the run's inputs, it moves to the next generation after every tool round, and it + * is removed, in the same commit that settles the inputs, when the run ends. Every commit that moves + * or ends a run edits `pi.live` through `tx.doc()`, so reading that draft after the commit's change + * function returns shows whether the run survived the commit. + */ +export function observeRunControl(runtime: PiTaskRuntime, run: PiRun, runs: PiRuns): PiTaskRuntime { + const commit = (change: PiCommitChange, context: unknown): Promise => { + let ended = false; + let settlement: PiSettlement | undefined; + + const observedChange: PiCommitChange = async (tx, current) => { + // A commit can run its change again, so only the attempt that committed counts. + ended = false; + settlement = undefined; + let live: Promise | undefined; + const observedTx = new Proxy(tx, { + get(target, property) { + const value = bound(target, property); + if (typeof value !== 'function') { + return value; + } + if (property === 'doc') { + return (token: PiDocToken, id: unknown, ...rest: unknown[]) => { + const draft = value(token, id, ...rest); + if (token?.definition?.kind === PI_LIVE_DOC_KIND && id === run.conversationId) { + live = Promise.resolve(draft); + } + return draft; + }; + } + if (property === 'settleSubmission') { + return (id: unknown, submissionSettlement: PiSettlement, ...rest: unknown[]) => { + if (run.firstInput === undefined || id === run.firstInput) { + settlement = submissionSettlement; + } + return value(id, submissionSettlement, ...rest); + }; + } + return value; + }, + }); + + const result = await change(observedTx, current); + const draft = (await live) as PiLiveState | undefined; + if (draft) { + const firstInput = draft.run?.inputs?.[0]; + if (firstInput === undefined) { + ended = true; + } else if (run.firstInput === undefined) { + run.firstInput = firstInput; + } else if (firstInput !== run.firstInput) { + // The run ended and a queued input started the next one in the same commit. + ended = true; + } + } + run.pendingEnd = ended ? { status: toRunStatus(settlement) } : undefined; + return result; + }; + + return runtime.commit(observedChange, context).then( + () => { + if (ended) { + endRun(run, runs, toRunStatus(settlement)); + } + }, + error => { + run.pendingEnd = undefined; + throw error; + }, + ); + }; + + return new Proxy(runtime, { + get(target, property) { + return property === 'commit' ? commit : bound(target, property); + }, + }); +} + +function toRunStatus(settlement: PiSettlement | undefined): SpanStatus | undefined { + if (settlement?.status !== 'unanswered') { + return undefined; + } + const reason = settlement.reason ?? 'internal_error'; + return { code: SPAN_STATUS_ERROR, message: CANCELLED_REASONS.has(reason) ? 'cancelled' : reason }; +} diff --git a/packages/server-utils/src/ai/pi-durable/tools.ts b/packages/server-utils/src/ai/pi-durable/tools.ts new file mode 100644 index 000000000000..5d19f583dc14 --- /dev/null +++ b/packages/server-utils/src/ai/pi-durable/tools.ts @@ -0,0 +1,258 @@ +import type { Span } from '@sentry/core'; +import { + captureException, + isObjectLike, + makeWeakRef, + SPAN_STATUS_ERROR, + spanToBaggageHeader, + spanToTraceHeader, + startSpanManual, + stringify, + timestampInSeconds, +} from '@sentry/core'; +import { + GEN_AI_CONVERSATION_ID, + GEN_AI_OPERATION_NAME, + GEN_AI_TOOL_CALL_ARGUMENTS, + GEN_AI_TOOL_CALL_RESULT, + GEN_AI_TOOL_DESCRIPTION, + GEN_AI_TOOL_NAME, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import { GEN_AI_EXECUTE_TOOL } from '@sentry/conventions/op'; +import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../core/gen-ai-attributes'; +import type { GenAiOptions } from '../core/utils'; +import { resolveAIRecordingOptions } from '../core/utils'; +import { piAiContentToString } from '../pi-ai/messages'; +import { MAX_TRACKED_PI_RUNS, PI_DURABLE_ORIGIN, PI_TOOL_RESULT_ENTRY_KIND } from './constants'; +import type { PiRuns, PiToolCall } from './runs'; +import { toSentryConversationId } from './runs'; +import type { + PiAgent, + PiCommitChange, + PiEntryToken, + PiTaskRuntime, + PiTool, + PiToolExecutionApi, + PiToolResultMessage, +} from './types'; +import { bound } from './utils'; + +/** + * The tools pi-durable's `createBashTool`, `createReadTool`, `createEditTool` and `createWriteTool` + * factories return. They throw to report expected failures to the model, such as a command that + * exits non-zero, so their throws are not captured. Filled by the integration from the factories' + * channels, which also covers the tools an app registers in an extension of its own. + */ +const BUILT_IN_TOOLS = new WeakSet(); + +export function markBuiltInTool(tool: unknown): void { + if (isObjectLike(tool)) { + BUILT_IN_TOOLS.add(tool); + } +} + +/** + * Prepare the runtime of a `pi.tool` phase. The tool task resolves the called tool from its agent, + * so the runtime hands out traced tools, and it collects the result entries the phase commits. + * `finish` ends the phase's tool spans with the result the model receives, which pi-durable builds + * after `execute()` returns: from `api.output()`, through `afterTool` hooks and output limits. + */ +export function instrumentToolPhase( + runtime: PiTaskRuntime, + runs: PiRuns, + wrapTool: (tool: PiTool) => PiTool, +): { runtime: PiTaskRuntime; finish: () => void } { + const results = new Map(); + runs.toolCalls.set(runtime.taskId, []); + + const agent = (context: unknown): Promise => + runtime + .agent(context) + .then(resolved => (resolved?.tools ? { ...resolved, tools: resolved.tools.map(wrapTool) } : resolved)); + + const commit = (change: PiCommitChange, context: unknown): Promise => { + let committed: PiToolResultMessage[] = []; + const observedChange: PiCommitChange = (tx, current) => { + // A commit can run its change again, so only the attempt that committed counts. + committed = []; + return change(observeToolResultEntries(tx, committed), current); + }; + return runtime.commit(observedChange, context).then(() => { + for (const message of committed) { + results.set(message.toolCallId, message); + } + }); + }; + + return { + runtime: new Proxy(runtime, { + get(target, property) { + if (property === 'agent') { + return agent; + } + return property === 'commit' ? commit : bound(target, property); + }, + }), + finish: () => { + for (const call of runs.toolCalls.get(runtime.taskId) ?? []) { + finishToolCall(call, call.callId !== undefined ? results.get(call.callId) : undefined); + } + runs.toolCalls.delete(runtime.taskId); + }, + }; +} + +/** Wrap a tool so each call gets a `gen_ai.execute_tool` span. */ +export function instrumentTool(tool: PiTool, runs: PiRuns, options: GenAiOptions): PiTool { + const execute = (args: unknown, api: PiToolExecutionApi, context: unknown): Promise => { + const { recordInputs, recordOutputs } = resolveAIRecordingOptions(options); + const phaseCalls = runs.toolCalls.get(api?.taskId); + // A tool that runs another tool of its agent passes its own `api` on, so the inner call shares + // the call id. Only the outer call gets the committed result; the inner one ends with its own. + const calls = phaseCalls?.some(call => call.callId !== undefined && call.callId === api?.callId) + ? undefined + : phaseCalls; + + return startSpanManual( + { + name: `execute_tool ${tool.name}`, + op: GEN_AI_EXECUTE_TOOL, + attributes: { + [SENTRY_ORIGIN]: PI_DURABLE_ORIGIN, + [GEN_AI_OPERATION_NAME]: 'execute_tool', + [GEN_AI_TOOL_NAME]: tool.name, + ...(tool.description ? { [GEN_AI_TOOL_DESCRIPTION]: tool.description } : {}), + ...(api?.callId ? { [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: api.callId } : {}), + ...(api?.conversationId !== undefined + ? { [GEN_AI_CONVERSATION_ID]: toSentryConversationId(runs, api.conversationId) } + : {}), + ...(recordInputs && args !== undefined ? { [GEN_AI_TOOL_CALL_ARGUMENTS]: stringify(args) } : {}), + }, + }, + async span => { + const call: PiToolCall = { span, callId: api?.callId, recordOutputs }; + calls?.push(call); + try { + const result = await tool.execute(args, trackOwnedConversations(api, span, runs), context); + call.endTimestamp = timestampInSeconds(); + if (!calls) { + // No result entry follows a call outside a tool task phase, nor a nested call. + finishToolCall(call, { content: result?.content, isError: result?.isError }); + } + return result; + } catch (error) { + call.endTimestamp = timestampInSeconds(); + call.failed = true; + // pi-durable turns the throw into an error result for the model, so the app never gets a + // chance to handle it. An aborted call is not a failure. + if ((context as { abortSignal?: AbortSignal } | undefined)?.abortSignal?.aborted) { + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'cancelled' }); + } else { + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'internal_error' }); + if (!BUILT_IN_TOOLS.has(tool)) { + captureException(error, { mechanism: { handled: false, type: PI_DURABLE_ORIGIN } }); + } + } + if (!calls) { + span.end(call.endTimestamp); + } + throw error; + } + }, + ); + }; + + // The proxy's target is an empty object with the tool as prototype, not the tool itself, so a + // frozen tool does not break the proxy invariants. Members are read bound to the tool, so a + // method that reads a private field works. + return new Proxy(Object.create(tool) as PiTool, { + get(_target, property) { + return property === 'execute' ? execute : bound(tool, property); + }, + }); +} + +function finishToolCall(call: PiToolCall, message: PiToolResultMessage | undefined): void { + if (message) { + const result = call.recordOutputs ? piAiContentToString(message.content) : undefined; + if (result !== undefined) { + call.span.setAttribute(GEN_AI_TOOL_CALL_RESULT, result); + } + if (message.isError && !call.failed) { + call.span.setStatus({ code: SPAN_STATUS_ERROR, message: 'internal_error' }); + } + } + call.span.end(call.endTimestamp); +} + +function observeToolResultEntries(tx: object, committed: PiToolResultMessage[]): object { + return new Proxy(tx, { + get(target, property) { + const value = bound(target, property); + if (property !== 'appendEntry' || typeof value !== 'function') { + return value; + } + return ( + token: PiEntryToken, + conversationId: unknown, + draft: { model?: unknown[] } | undefined, + ...rest: unknown[] + ) => { + const message = token?.kind === PI_TOOL_RESULT_ENTRY_KIND ? draft?.model?.[0] : undefined; + if (isObjectLike(message)) { + committed.push(message); + } + return value(token, conversationId, draft, ...rest); + }; + }, + }); +} + +/** + * Record the conversations a tool call creates for itself, the subagent pattern: `api.commit()` with + * `tx.createConversation()` or `tx.forkConversation()` and `ownership: { kind: 'task', taskId: api.taskId }`. + */ +function trackOwnedConversations(api: PiToolExecutionApi, span: Span, runs: PiRuns): PiToolExecutionApi { + if (typeof api?.commit !== 'function') { + return api; + } + const commit = api.commit.bind(api); + + const trackCreatedConversations = (tx: object): object => + new Proxy(tx, { + get(target, property) { + const value = bound(target, property); + if ((property !== 'createConversation' && property !== 'forkConversation') || typeof value !== 'function') { + return value; + } + return async (...args: unknown[]) => { + const record = (await value(...args)) as { id?: unknown } | undefined; + const ownership = args.find( + (arg): arg is { ownership: { kind?: string } } => isObjectLike(arg) && isObjectLike(arg.ownership), + )?.ownership; + if (ownership?.kind === 'task' && record?.id !== undefined) { + if (runs.owners.size >= MAX_TRACKED_PI_RUNS) { + runs.owners.delete(runs.owners.keys().next().value); + } + runs.owners.set(record.id, { + span: makeWeakRef(span), + sentryTrace: spanToTraceHeader(span), + baggage: spanToBaggageHeader(span), + }); + } + return record; + }; + }, + }); + + return new Proxy(api, { + get(target, property) { + if (property !== 'commit') { + return bound(target, property); + } + return (change: (tx: object) => unknown, context: unknown) => + commit((tx: object) => change(trackCreatedConversations(tx)), context); + }, + }); +} diff --git a/packages/server-utils/src/ai/pi-durable/types.ts b/packages/server-utils/src/ai/pi-durable/types.ts new file mode 100644 index 000000000000..6244b18af1e5 --- /dev/null +++ b/packages/server-utils/src/ai/pi-durable/types.ts @@ -0,0 +1,141 @@ +/** + * Structural subsets of the `@earendil-works/pi-durable` and `@earendil-works/pi-ai` types the + * instrumentation reads. The SDK does not depend on either package, so only the fields used here + * are declared, and every one of them is treated as possibly absent at runtime. + */ + +import type { PiAiContext } from '../pi-ai/messages'; +import type { PiAiUsage } from '../pi-ai/usage'; + +export interface PiAssistantMessage { + role?: string; + content?: unknown[]; + provider?: string; + model?: string; + responseModel?: string; + responseId?: string; + usage?: PiAiUsage; + stopReason?: string; + errorMessage?: string; +} + +export interface PiModel { + id?: string; + provider?: string; + baseUrl?: string; +} + +export interface PiStreamOptions { + temperature?: number; + maxTokens?: number; + reasoning?: string; +} + +export interface PiEventStream { + result(): Promise; +} + +/** + * The `Models` methods that send a model request. Everything else passes through untouched. The + * deferred methods fetch the answer of a request the provider parked, so they take a handle in + * place of the request context. + */ +export interface PiModels { + stream?(model: PiModel, context: PiAiContext, options?: PiStreamOptions): PiEventStream; + streamSimple?(model: PiModel, context: PiAiContext, options?: PiStreamOptions): PiEventStream; + complete?(model: PiModel, context: PiAiContext, options?: PiStreamOptions): Promise; + completeSimple?(model: PiModel, context: PiAiContext, options?: PiStreamOptions): Promise; + streamDeferred?(model: PiModel, handle: unknown, options?: unknown): PiEventStream; + fetchDeferred?(model: PiModel, handle: unknown, options?: unknown): Promise; +} + +export interface PiToolExecutionResult { + content?: unknown[]; + isError?: boolean; +} + +export interface PiToolExecutionApi { + taskId?: unknown; + conversationId?: unknown; + callId?: string; + commit?(change: (tx: object) => unknown, context: unknown): Promise; +} + +export interface PiTool { + name: string; + description?: string; + execute(args: unknown, api: PiToolExecutionApi, context: unknown): Promise; +} + +export interface PiAgent { + tools?: readonly PiTool[]; +} + +/** The tool result the model receives, as pi-durable commits it in a `pi.tool-result` entry. */ +export interface PiToolResultMessage { + toolCallId?: unknown; + content?: unknown[]; + isError?: boolean; +} + +export interface PiDocToken { + definition?: { kind?: string }; +} + +export interface PiEntryToken { + kind?: string; +} + +export interface PiSettlement { + status?: string; + reason?: string; +} + +/** The `pi.live` document; `run` is present exactly while the conversation is busy. */ +export interface PiLiveState { + run?: { inputs?: unknown[] }; +} + +export type PiCommitChange = (tx: object, current: unknown) => unknown; + +/** pi-durable ids (conversations, tasks, submissions) are numbers, typed as `unknown` here. */ +export interface PiTaskRuntime { + readonly taskId: unknown; + readonly conversationId: unknown; + readonly signal?: AbortSignal; + agent(context: unknown): Promise; + commit(change: PiCommitChange, context: unknown): Promise; +} + +export type PiPhaseHandler = (task: unknown, runtime: PiTaskRuntime, context: unknown) => Promise; + +export interface PiTaskDefinition { + name: string; + phases: Record; + abort?: PiPhaseHandler; +} + +export interface PiTask { + definition: PiTaskDefinition; +} + +export interface PiRegistrySnapshot { + task(name: string): PiTask | undefined; + tasks(): readonly PiTask[]; +} + +export interface PiRegistryReader { + snapshot(): PiRegistrySnapshot; + subscribe(listener: () => void): () => void; +} + +export interface PiHarnessOptions { + models?: PiModels; + registry?: PiRegistryReader; + onReport?: (error: unknown) => void; +} + +export interface PiHarness { + /** Calls `listener` when `close()` begins. */ + subscribeClose?(listener: () => void): () => void; +} diff --git a/packages/server-utils/src/ai/pi-durable/utils.ts b/packages/server-utils/src/ai/pi-durable/utils.ts new file mode 100644 index 000000000000..0ae07b973277 --- /dev/null +++ b/packages/server-utils/src/ai/pi-durable/utils.ts @@ -0,0 +1,5 @@ +/** Read a member with `this` bound to the original, so getters and methods that use `this` keep working. */ +export function bound(target: object, property: PropertyKey): unknown { + const value: unknown = Reflect.get(target, property, target); + return typeof value === 'function' ? (value as (...args: unknown[]) => unknown).bind(target) : value; +} diff --git a/packages/server-utils/src/index.ts b/packages/server-utils/src/index.ts index c162dadb1629..446812f8eed5 100644 --- a/packages/server-utils/src/index.ts +++ b/packages/server-utils/src/index.ts @@ -50,6 +50,7 @@ export { langChainIntegration } from './integrations/langchain'; export { langGraphIntegration } from './integrations/langgraph'; export { createFlueInstrumentation } from './ai/flue'; export { flueIntegration } from './integrations/flue'; +export { piDurableIntegration } from './integrations/pi-durable'; export type { FlueOptions } from './ai/flue'; export { mastraIntegration } from './integrations/mastra'; export { mcpServerIntegration } from './integrations/mcp-server'; diff --git a/packages/server-utils/src/integrations/index.ts b/packages/server-utils/src/integrations/index.ts index df8d488b9d28..a09d590e1121 100644 --- a/packages/server-utils/src/integrations/index.ts +++ b/packages/server-utils/src/integrations/index.ts @@ -15,6 +15,7 @@ import { mongooseIntegration } from './mongoose'; import { lruMemoizerIntegration } from './lru-memoizer'; import { langChainIntegration } from './langchain'; import { langGraphIntegration } from './langgraph'; +import { piDurableIntegration } from './pi-durable'; import { mastraIntegration } from './mastra'; import { mcpServerIntegration } from './mcp-server'; import { vercelAIIntegration } from './vercel-ai'; @@ -59,6 +60,7 @@ export function getTracingIntegrations(): Integration[] { langChainIntegration(), langGraphIntegration(), mastraIntegration(), + piDurableIntegration(), vercelAIIntegration(), openAIIntegration(), anthropicAIIntegration(), diff --git a/packages/server-utils/src/integrations/pi-durable.ts b/packages/server-utils/src/integrations/pi-durable.ts new file mode 100644 index 000000000000..b5b115ec6c7b --- /dev/null +++ b/packages/server-utils/src/integrations/pi-durable.ts @@ -0,0 +1,76 @@ +import * as diagnosticsChannel from '../utils/diagnosticsChannel'; +import type { IntegrationFn } from '@sentry/core'; +import { debug, defineIntegration, isObjectLike } from '@sentry/core'; +import type { PiDurableOptions } from '../ai/pi-durable'; +import { endRunsOnClose, instrumentPiDurableHarnessOptions } from '../ai/pi-durable'; +import { PI_DURABLE_INTEGRATION_NAME } from '../ai/pi-durable/constants'; +import { markBuiltInTool } from '../ai/pi-durable/tools'; +import type { PiHarnessOptions } from '../ai/pi-durable/types'; +import { DEBUG_BUILD } from '../debug-build'; +import { CHANNELS } from '../orchestrion/channels'; +import { piDurableModuleNames } from '../orchestrion/config/pi-durable'; +import { invokeOrchestrionInstrumentation } from '../orchestrion/instrumentation'; + +interface HarnessOpenChannelContext { + arguments: unknown[]; + result?: unknown; +} + +interface CodingToolChannelContext { + result?: unknown; +} + +const _piDurableIntegration = ((options: PiDurableOptions = {}) => { + return { + name: PI_DURABLE_INTEGRATION_NAME, + setup(client) { + // Subscribing opens no span; spans open later, inside task phases. + invokeOrchestrionInstrumentation(client, piDurableModuleNames, instrumentPiDurable, [options], { + requiresTracingChannelBinding: false, + }); + }, + }; +}) satisfies IntegrationFn; + +function instrumentPiDurable(options: PiDurableOptions): void { + const harnessOpen = diagnosticsChannel.tracingChannel(CHANNELS.PI_DURABLE_HARNESS_OPEN); + + harnessOpen.start.subscribe(message => { + const args = (message as HarnessOpenChannelContext).arguments; + try { + if (args && isObjectLike(args[1])) { + args[1] = instrumentPiDurableHarnessOptions(args[1] as PiHarnessOptions, options); + } + } catch (error) { + DEBUG_BUILD && debug.error('[instrumentation:pi-durable] failed to instrument Harness options', error); + } + }); + + harnessOpen.asyncEnd.subscribe(message => { + const { arguments: args, result } = message as HarnessOpenChannelContext; + try { + if (args && isObjectLike(args[1])) { + endRunsOnClose(args[1], result); + } + } catch (error) { + DEBUG_BUILD && debug.error('[instrumentation:pi-durable] failed to observe Harness close', error); + } + }); + + diagnosticsChannel + .tracingChannel(CHANNELS.PI_DURABLE_CODING_TOOL) + .end.subscribe(message => markBuiltInTool((message as CodingToolChannelContext).result)); +} + +/** + * Diagnostics-channel-based integration for pi-durable (`@earendil-works/pi-durable` >= 1.0.0 < 2, + * whose API is experimental). Subscribes to the `orchestrion:@earendil-works/pi-durable:harnessOpen` + * channel injected into `Harness.open()` and to the `codingTool` channel injected into the built-in + * tool factories, so it requires the Sentry runtime hook or bundler plugin. + * + * Traces one `gen_ai.invoke_agent` span per run, with its model requests as `gen_ai.chat` and its + * tool calls as `gen_ai.execute_tool` children. From the first model request on, the `openai`, + * `@anthropic-ai/sdk`, `@google/genai` and Workers AI integrations stop reporting requests in the + * whole process, because pi-ai sends its requests through those clients. + */ +export const piDurableIntegration = defineIntegration(_piDurableIntegration); diff --git a/packages/server-utils/src/orchestrion/channels.ts b/packages/server-utils/src/orchestrion/channels.ts index a6ab33d4d4c6..4211c8c88d5c 100644 --- a/packages/server-utils/src/orchestrion/channels.ts +++ b/packages/server-utils/src/orchestrion/channels.ts @@ -27,6 +27,7 @@ import { mysqlChannels } from './config/mysql'; import { nestjsChannels } from './config/nestjs'; import { openaiChannels } from './config/openai'; import { pgChannels } from './config/pg'; +import { piDurableChannels } from './config/pi-durable'; import { postgresJsChannels } from './config/postgres'; import { prismaChannels } from './config/prisma'; import { redisChannels } from './config/redis'; @@ -82,6 +83,7 @@ export const CHANNELS = { ...nestjsChannels, ...openaiChannels, ...pgChannels, + ...piDurableChannels, ...postgresJsChannels, ...prismaChannels, ...redisChannels, diff --git a/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts b/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts index 4a133a2d89a7..683370c0fe61 100644 --- a/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts +++ b/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts @@ -49,6 +49,7 @@ export const CHANNEL_INTEGRATION_DEFINITIONS = [ { exportName: 'langGraphIntegration', modules: ['@langchain/langgraph'] }, { exportName: 'mastraIntegration', modules: ['@mastra/core'] }, { exportName: 'flueIntegration', modules: ['@flue/runtime'] }, + { exportName: 'piDurableIntegration', modules: ['@earendil-works/pi-durable'] }, { exportName: 'mcpServerIntegration', modules: ['@modelcontextprotocol/server', '@modelcontextprotocol/sdk'] }, { exportName: 'awsIntegration', modules: ['@aws-sdk/smithy-client', '@smithy/core', '@smithy/smithy-client'] }, { exportName: 'firebaseIntegration', modules: ['@firebase/firestore', 'firebase-functions'] }, diff --git a/packages/server-utils/src/orchestrion/config/index.ts b/packages/server-utils/src/orchestrion/config/index.ts index f8dded8a6a6d..2c6a7835db1a 100644 --- a/packages/server-utils/src/orchestrion/config/index.ts +++ b/packages/server-utils/src/orchestrion/config/index.ts @@ -23,6 +23,7 @@ import { langgraphConfig } from './langgraph'; import { lruMemoizerConfig } from './lru-memoizer'; import { flueConfig } from './flue'; import { mastraConfig } from './mastra'; +import { piDurableConfig } from './pi-durable'; import { mcpServerConfig } from './mcp-server'; import { mistralConfig } from './mistral'; import { mongodbConfig } from './mongodb'; @@ -88,6 +89,7 @@ export const SENTRY_INSTRUMENTATIONS: InstrumentationConfig[] = [ ...nestjsConfig, ...openaiConfig, ...pgConfig, + ...piDurableConfig, ...postgresJsConfig, ...prismaConfig, ...redisConfig, diff --git a/packages/server-utils/src/orchestrion/config/pi-durable.ts b/packages/server-utils/src/orchestrion/config/pi-durable.ts new file mode 100644 index 000000000000..c7dcaa902289 --- /dev/null +++ b/packages/server-utils/src/orchestrion/config/pi-durable.ts @@ -0,0 +1,35 @@ +import type { InstrumentationConfig } from '../apmTypes'; + +import { getModuleNames } from './module-names'; + +const module = { name: '@earendil-works/pi-durable', versionRange: '>=1.0.0 <2.0.0' }; + +// `Harness.open(storage, options, context)` is a method of the exported `Harness` object literal, the +// one entry point to a Harness. Its `options` carry the `models` and `registry` every model request +// and task phase goes through, so wrapping them at `start` covers the whole Harness. +export const piDurableConfig = [ + { + channelName: 'harnessOpen', + module: { ...module, filePath: 'dist/harness/harness.js' }, + functionQuery: { methodName: 'open', kind: 'Async' as const }, + }, + // The factories of the built-in coding tools. Their tools throw to report expected failures to the + // model, so the integration must know them wherever an app registers them. + ...[ + ['bash', 'createBashTool'], + ['read', 'createReadTool'], + ['edit', 'createEditTool'], + ['write', 'createWriteTool'], + ].map(([file, functionName]) => ({ + channelName: 'codingTool', + module: { ...module, filePath: `dist/tools/${file}.js` }, + functionQuery: { functionName: functionName as string, kind: 'Sync' as const }, + })), +] satisfies InstrumentationConfig[]; + +export const piDurableModuleNames = getModuleNames(piDurableConfig); + +export const piDurableChannels = { + PI_DURABLE_HARNESS_OPEN: 'orchestrion:@earendil-works/pi-durable:harnessOpen', + PI_DURABLE_CODING_TOOL: 'orchestrion:@earendil-works/pi-durable:codingTool', +} as const; diff --git a/packages/server-utils/src/utils/withCleanScopes.ts b/packages/server-utils/src/utils/withCleanScopes.ts new file mode 100644 index 000000000000..422cd2e7d31c --- /dev/null +++ b/packages/server-utils/src/utils/withCleanScopes.ts @@ -0,0 +1,20 @@ +import type { Scope } from '@sentry/core'; +import { getClient, getDefaultCurrentScope, withIsolationScope, withScope } from '@sentry/core'; + +/** + * Run `callback` with `isolationScope` and a fresh copy of the default current scope. For work that + * a library schedules from whichever async context last woke it: a forked scope would carry over the + * data of that context, such as the conversation id of another run. Only the client of that context + * is kept, because an SDK that initializes inside each request, such as `@sentry/cloudflare`, binds + * it to the request's scope alone. + */ +export function withCleanScopes(isolationScope: Scope, callback: () => T): T { + const client = getClient(); + return withIsolationScope(isolationScope, () => { + const scope = getDefaultCurrentScope().clone(); + if (client) { + scope.setClient(client); + } + return withScope(scope, callback); + }); +} diff --git a/packages/server-utils/test/ai/lib/tracing/pi-durable-retention.test.ts b/packages/server-utils/test/ai/lib/tracing/pi-durable-retention.test.ts new file mode 100644 index 000000000000..bc0059b7adb7 --- /dev/null +++ b/packages/server-utils/test/ai/lib/tracing/pi-durable-retention.test.ts @@ -0,0 +1,80 @@ +import { setFlagsFromString } from 'node:v8'; +import { runInNewContext } from 'node:vm'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import type { MaybeWeakRef, Span } from '@sentry/core'; +import { derefWeakRef, getMainCarrier, makeWeakRef, setCurrentClient, spanToJSON } from '@sentry/core'; +import { createRuns, startRun } from '../../../../src/ai/pi-durable/runs'; +import { instrumentTool } from '../../../../src/ai/pi-durable/tools'; +import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; + +// Own file on purpose: the other pi-durable tests keep every span that ends, which would keep the span +// this test needs to see collected. + +function collectGarbage(): void { + setFlagsFromString('--expose-gc'); + const gc = runInNewContext('gc') as () => void; + setFlagsFromString('--no-expose-gc'); + gc(); + gc(); +} + +describe('pi-durable memory retention', () => { + let client: TestClient; + + beforeEach(() => { + getMainCarrier().__SENTRY__ = undefined; + client = new TestClient( + getDefaultTestClientOptions({ dsn: 'https://public@dsn.ingest.sentry.io/1337', tracesSampleRate: 1 }), + ); + setCurrentClient(client); + client.init(); + }); + + afterEach(() => { + getMainCarrier().__SENTRY__ = undefined; + }); + + it('does not keep the span of a tool call alive for a conversation whose first run has not started', async () => { + const runs = createRuns(); + let call: MaybeWeakRef | undefined; + let traceId: string | undefined; + let spanId: string | undefined; + client.on('spanStart', span => { + if (!call) { + call = makeWeakRef(span); + ({ traceId, spanId } = span.spanContext()); + } + }); + const tool = instrumentTool( + { + name: 'delegate', + execute: async (_args, api) => { + await api.commit!( + tx => + (tx as { createConversation: (...args: unknown[]) => unknown }).createConversation({ + ownership: { kind: 'task', taskId: 7 }, + }), + undefined, + ); + return { content: [{ type: 'text', text: 'Subagent created.' }] }; + }, + }, + runs, + {}, + ); + + await tool.execute( + {}, + { taskId: 7, callId: 'call_1', commit: async change => change({ createConversation: async () => ({ id: 42 }) }) }, + undefined, + ); + // A WeakRef is never cleared in the job that created it. + await new Promise(resolve => setTimeout(resolve, 0)); + collectGarbage(); + + expect(derefWeakRef(call)).toBeUndefined(); + const run = spanToJSON(startRun(42, runs).span); + expect(run.trace_id).toBe(traceId); + expect(run.parent_span_id).toBe(spanId); + }); +}); diff --git a/packages/server-utils/test/ai/lib/tracing/pi-durable.test.ts b/packages/server-utils/test/ai/lib/tracing/pi-durable.test.ts new file mode 100644 index 000000000000..c2b8f6e9b68e --- /dev/null +++ b/packages/server-utils/test/ai/lib/tracing/pi-durable.test.ts @@ -0,0 +1,1048 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import type { Event, Span } from '@sentry/core'; +import { + _INTERNAL_clearAiProviderSkips, + _INTERNAL_shouldSkipAiProviderWrapping, + _INTERNAL_skipAiProviderWrapping, + getCurrentScope, + getMainCarrier, + getRootSpan, + setCurrentClient, + spanToJSON, + spanToStaticSpanJSON, + withScope, +} from '@sentry/core'; +import { + GEN_AI_CONVERSATION_ID, + GEN_AI_INPUT_MESSAGES, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MAX_TOKENS, + GEN_AI_REQUEST_MODEL, + GEN_AI_REQUEST_REASONING_LEVEL, + GEN_AI_REQUEST_TEMPERATURE, + GEN_AI_RESPONSE_FINISH_REASONS, + GEN_AI_RESPONSE_ID, + GEN_AI_RESPONSE_MODEL, + GEN_AI_RESPONSE_STREAMING, + GEN_AI_SYSTEM_INSTRUCTIONS, + GEN_AI_TOOL_CALL_ARGUMENTS, + GEN_AI_TOOL_CALL_RESULT, + GEN_AI_TOOL_DEFINITIONS, + GEN_AI_TOOL_DESCRIPTION, + GEN_AI_TOOL_NAME, + GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_REASONING_OUTPUT_TOKENS, + GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, + SENTRY_ORIGIN, + SENTRY_STATUS_MESSAGE, + SERVER_ADDRESS, + SERVER_PORT, +} from '@sentry/conventions/attributes'; +import { GEN_AI_CHAT, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op'; +import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../src/ai/core/gen-ai-attributes'; +import { endRunsOnClose, instrumentPiDurableHarnessOptions } from '../../../../src/ai/pi-durable'; +import { MAX_TRACKED_PI_RUNS } from '../../../../src/ai/pi-durable/constants'; +import type { PiRuns } from '../../../../src/ai/pi-durable/runs'; +import { createRuns, startRun } from '../../../../src/ai/pi-durable/runs'; +import { instrumentTool, markBuiltInTool } from '../../../../src/ai/pi-durable/tools'; +import type { + PiCommitChange, + PiEventStream, + PiHarnessOptions, + PiLiveState, + PiModels, + PiRegistryReader, + PiSettlement, + PiTask, + PiTaskRuntime, + PiTool, + PiToolExecutionApi, + PiToolExecutionResult, +} from '../../../../src/ai/pi-durable/types'; +import { ANTHROPIC_AI_INTEGRATION_NAME } from '../../../../src/ai/anthropic-ai/constants'; +import { GOOGLE_GENAI_INTEGRATION_NAME } from '../../../../src/ai/google-genai/constants'; +import { OPENAI_INTEGRATION_NAME } from '../../../../src/ai/openai/constants'; +import { instrumentWorkersAiClient } from '../../../../src/ai/workers-ai'; +import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; + +const LIVE_DOC = { definition: { kind: 'pi.live' } }; +const MODEL = { id: 'faux-model', provider: 'faux' }; +const ANSWER = { role: 'assistant', content: [{ type: 'text', text: 'Paris.' }], stopReason: 'stop' }; + +describe('instrumentPiDurableHarnessOptions', () => { + let endedSpans: Span[]; + let events: Event[]; + let client: TestClient; + + beforeEach(() => { + _INTERNAL_clearAiProviderSkips(); + getMainCarrier().__SENTRY__ = undefined; + client = new TestClient( + getDefaultTestClientOptions({ dsn: 'https://public@dsn.ingest.sentry.io/1337', tracesSampleRate: 1 }), + ); + setCurrentClient(client); + client.init(); + + endedSpans = []; + events = []; + client.on('spanEnd', span => endedSpans.push(span)); + // The root spans of these tests are sent as transaction events; keep only the error events. + TestClient.sendEventCalled = event => { + if (event.type !== 'transaction') { + events.push(event); + } + }; + }); + + afterEach(() => { + TestClient.sendEventCalled = undefined; + _INTERNAL_clearAiProviderSkips(); + getMainCarrier().__SENTRY__ = undefined; + }); + + it('wraps the options once and leaves the caller objects untouched', () => { + const models = { completeSimple: async () => ({}) }; + const options: PiHarnessOptions = { models }; + + const instrumented = instrumentPiDurableHarnessOptions(options); + + expect(instrumented).not.toBe(options); + expect(instrumented.models).not.toBe(models); + expect(options.models).toBe(models); + expect(instrumentPiDurableHarnessOptions(instrumented)).toBe(instrumented); + }); + + // pi-durable reads `settings`, `env` and `conversationCreated` at every use, so getters and later + // assignments on the caller's options must reach it. + it('keeps the options the caller changes later live', () => { + let env = 'first'; + const options = { models: {}, settings: { toolExecution: 'sequential' } } as PiHarnessOptions & { + settings: unknown; + env?: string; + }; + Object.defineProperty(options, 'env', { get: () => env, enumerable: true }); + + const instrumented = instrumentPiDurableHarnessOptions(options); + options.settings = { toolExecution: 'parallel' }; + env = 'second'; + + expect(instrumented.settings).toEqual({ toolExecution: 'parallel' }); + expect(instrumented.env).toBe('second'); + }); + + it('hands members other than model requests through, bound to the original', () => { + class Models { + private _provider = 'faux'; + public getProvider(): string { + return this._provider; + } + } + + const instrumented = instrumentPiDurableHarnessOptions({ models: new Models() as PiModels }); + + expect((instrumented.models as unknown as Models).getProvider()).toBe('faux'); + expect(endedSpans).toHaveLength(0); + }); + + it('traces a streamed model request and skips the provider SDK integrations', async () => { + const message = { + ...ANSWER, + model: 'faux-model', + usage: { input: 10, output: 2, totalTokens: 12 }, + }; + const models: PiModels = { streamSimple: () => ({ result: async () => message }) }; + const instrumented = instrumentPiDurableHarnessOptions({ models }); + + await instrumented.models!.streamSimple!(MODEL, { + messages: [{ role: 'user', content: 'Capital of France?' }], + }).result(); + + expect(_INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME)).toBe(true); + const chat = spanToStaticSpanJSON(endedSpans[0]!); + expect(chat.description).toBe('chat faux-model'); + expect(chat.op).toBe(GEN_AI_CHAT); + expect(chat.data).toMatchObject({ + [GEN_AI_PROVIDER_NAME]: 'faux', + [GEN_AI_REQUEST_MODEL]: 'faux-model', + [GEN_AI_RESPONSE_STREAMING]: true, + [GEN_AI_RESPONSE_FINISH_REASONS]: '["stop"]', + [GEN_AI_USAGE_INPUT_TOKENS]: 10, + [GEN_AI_USAGE_OUTPUT_TOKENS]: 2, + }); + expect(chat.data[GEN_AI_INPUT_MESSAGES]).toBe( + '[{"role":"user","parts":[{"type":"text","content":"Capital of France?"}]}]', + ); + expect(chat.data[GEN_AI_OUTPUT_MESSAGES]).toBe( + '[{"role":"assistant","parts":[{"type":"text","content":"Paris."}],"finish_reason":"stop"}]', + ); + }); + + it.each([ + ['stream', true], + ['streamSimple', true], + ['streamDeferred', true], + ['complete', false], + ['completeSimple', false], + ['fetchDeferred', false], + ])('traces %s with the request options and the response identity', async (method, streaming) => { + const message = { ...ANSWER, responseModel: 'faux-model-2026', responseId: 'resp_1' }; + const models = { + [method]: streaming ? () => ({ result: async () => message }) : async () => message, + } as PiModels; + const instrumented = instrumentPiDurableHarnessOptions({ models }); + + const result = (instrumented.models as Record unknown>)[method]!( + { ...MODEL, baseUrl: 'https://api.example.com:8443/v1' }, + { messages: [{ role: 'user', content: 'Hi.' }] }, + { temperature: 0.2, maxTokens: 50, reasoning: 'low' }, + ); + await (streaming ? (result as PiEventStream).result() : result); + + const chat = spanToJSON(endedSpans[0]!); + expect(chat.name).toBe('chat faux-model'); + expect(chat.attributes[GEN_AI_RESPONSE_STREAMING]).toBe(streaming ? true : undefined); + expect(chat.attributes[GEN_AI_RESPONSE_MODEL]).toBe('faux-model-2026'); + expect(chat.attributes[GEN_AI_RESPONSE_ID]).toBe('resp_1'); + expect(chat.attributes[SERVER_ADDRESS]).toBe('api.example.com'); + expect(chat.attributes[SERVER_PORT]).toBe(8443); + // The deferred methods take a handle, not a request, so there is nothing to record. + const deferred = method.endsWith('Deferred'); + expect(chat.attributes[GEN_AI_REQUEST_TEMPERATURE]).toBe(deferred ? undefined : 0.2); + expect(chat.attributes[GEN_AI_REQUEST_MAX_TOKENS]).toBe(deferred ? undefined : 50); + expect(chat.attributes[GEN_AI_REQUEST_REASONING_LEVEL]).toBe(deferred ? undefined : 'low'); + expect(chat.attributes[GEN_AI_INPUT_MESSAGES]).toBe( + deferred ? undefined : '[{"role":"user","parts":[{"type":"text","content":"Hi."}]}]', + ); + }); + + it('sets the conversation id of the scope on the chat span itself', async () => { + const models: PiModels = { completeSimple: async () => ANSWER }; + const instrumented = instrumentPiDurableHarnessOptions({ models }); + + await withScope(async scope => { + scope.setConversationId('harness:7'); + await instrumented.models!.completeSimple!(MODEL, { messages: [] }); + }); + + expect(spanToJSON(endedSpans[0]!).attributes[GEN_AI_CONVERSATION_ID]).toBe('harness:7'); + }); + + it('counts the cached tokens of a response into its input tokens', async () => { + const message = { + ...ANSWER, + usage: { input: 37, output: 2, cacheRead: 52, cacheWrite: 38, reasoning: 1, totalTokens: 129 }, + }; + const models: PiModels = { completeSimple: async () => message }; + const instrumented = instrumentPiDurableHarnessOptions({ models }); + + await instrumented.models!.completeSimple!(MODEL, { messages: [] }); + + expect(spanToJSON(endedSpans[0]!).attributes).toMatchObject({ + [GEN_AI_USAGE_INPUT_TOKENS]: 127, + [GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]: 52, + [GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]: 38, + [GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]: 1, + [GEN_AI_USAGE_TOTAL_TOKENS]: 129, + }); + }); + + // The provider parks the request and pi-durable fetches the answer later: only the fetch is billed. + it('records no usage for a request the provider parked, and the usage of its answer', async () => { + const parked = { + role: 'assistant', + content: [], + stopReason: 'deferred', + usage: { input: 0, output: 0, totalTokens: 0 }, + }; + const answer = { ...ANSWER, usage: { input: 8, output: 2, totalTokens: 10 } }; + const models: PiModels = { completeSimple: async () => parked, fetchDeferred: async () => answer }; + const instrumented = instrumentPiDurableHarnessOptions({ models }); + + await instrumented.models!.completeSimple!(MODEL, { messages: [] }); + await instrumented.models!.fetchDeferred!(MODEL, { id: 'handle_1' }); + + const [parkedChat, answerChat] = endedSpans.map(span => spanToJSON(span).attributes); + expect(parkedChat![GEN_AI_RESPONSE_FINISH_REASONS]).toBe('["deferred"]'); + expect(parkedChat![GEN_AI_USAGE_TOTAL_TOKENS]).toBeUndefined(); + expect(answerChat![GEN_AI_USAGE_TOTAL_TOKENS]).toBe(10); + }); + + it('marks a failed request with a fixed status message and maps a tool-calling stop', async () => { + const failed = { + role: 'assistant', + content: [], + stopReason: 'error', + errorMessage: '401 invalid x-api-key for user@example.com', + usage: { input: 0, output: 0, totalTokens: 0 }, + }; + const toolCall = { + role: 'assistant', + content: [{ type: 'toolCall', id: 'call_1', name: 'read', arguments: { path: 'a.txt' } }], + stopReason: 'toolUse', + }; + const models: PiModels = { completeSimple: async () => failed, complete: async () => toolCall }; + const instrumented = instrumentPiDurableHarnessOptions({ models }); + + await instrumented.models!.completeSimple!(MODEL, { messages: [] }); + await instrumented.models!.complete!(MODEL, { messages: [] }); + + const [chatFailed, chatToolCall] = endedSpans.map(span => spanToJSON(span)); + expect(chatFailed!.status).toBe('error'); + expect(chatFailed!.attributes[SENTRY_STATUS_MESSAGE]).toBe('internal_error'); + expect(JSON.stringify(chatFailed!.attributes)).not.toContain('x-api-key'); + expect(chatFailed!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toBeUndefined(); + expect(chatToolCall!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toBe('["tool_call"]'); + expect(chatToolCall!.attributes[GEN_AI_OUTPUT_MESSAGES]).toContain('"finish_reason":"tool_call"'); + }); + + it('marks a rejected or throwing request as errored and keeps the error', async () => { + const models: PiModels = { + completeSimple: async () => { + throw new Error('boom'); + }, + stream: () => { + throw new Error('sync boom'); + }, + }; + const instrumented = instrumentPiDurableHarnessOptions({ models }); + + await expect(instrumented.models!.completeSimple!(MODEL, { messages: [] })).rejects.toThrow('boom'); + expect(() => instrumented.models!.stream!(MODEL, { messages: [] })).toThrow('sync boom'); + + expect(endedSpans.map(span => spanToJSON(span).attributes[SENTRY_STATUS_MESSAGE])).toEqual([ + 'internal_error', + 'internal_error', + ]); + }); + + it('returns a result of another shape unchanged and ends its span', () => { + const stream = { events: [] }; + const models: PiModels = { + stream: () => stream as unknown as PiEventStream, + completeSimple: (() => 'parked') as unknown as PiModels['completeSimple'], + }; + const instrumented = instrumentPiDurableHarnessOptions({ models }); + + expect(instrumented.models!.stream!(MODEL, { messages: [] })).toBe(stream); + expect(instrumented.models!.completeSimple!(MODEL, { messages: [] })).toBe('parked'); + + expect(endedSpans.map(span => spanToJSON(span).name)).toEqual(['chat faux-model', 'chat faux-model']); + }); + + it('reads the system prompt and the tools from the positional system messages', async () => { + const models: PiModels = { + streamSimple: () => ({ result: async () => ({ role: 'assistant', content: [], stopReason: 'stop' }) }), + }; + const instrumented = instrumentPiDurableHarnessOptions({ models }); + + // pi-durable never sets `systemPrompt` or `tools`: it sends prompt sections and tool changes as + // system messages, and each request replays them. + await instrumented.models!.streamSimple!(MODEL, { + messages: [ + { + role: 'system', + content: '', + sections: { preamble: 'You help.' }, + toolsAdded: [{ name: 'read' }, { name: 'bash' }], + }, + { role: 'user', content: 'Hi.' }, + { role: 'system', content: '', sections: { mode: 'Plan only.' }, toolsRemoved: [{ name: 'bash' }] }, + ], + }).result(); + + const chat = spanToStaticSpanJSON(endedSpans[0]!); + expect(chat.data[GEN_AI_SYSTEM_INSTRUCTIONS]).toBe('You help.\n\nPlan only.'); + expect(chat.data[GEN_AI_TOOL_DEFINITIONS]).toBe('[{"name":"read"}]'); + expect(chat.data[GEN_AI_INPUT_MESSAGES]).toBe('[{"role":"user","parts":[{"type":"text","content":"Hi."}]}]'); + }); + + describe('provider skip', () => { + const models: PiModels = { completeSimple: async () => ANSWER }; + + it('skips the provider integrations on the first request, not when the options are wrapped', async () => { + const instrumented = instrumentPiDurableHarnessOptions({ models }); + expect(_INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME)).toBe(false); + + await instrumented.models!.completeSimple!(MODEL, { messages: [] }); + + expect(_INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME)).toBe(true); + expect(_INTERNAL_shouldSkipAiProviderWrapping(ANTHROPIC_AI_INTEGRATION_NAME)).toBe(true); + expect(_INTERNAL_shouldSkipAiProviderWrapping(GOOGLE_GENAI_INTEGRATION_NAME)).toBe(true); + }); + + // The registry is reset per client, so a one-shot call would be undone by the next `init()`. + it('applies the skip again after the registry is cleared', async () => { + const instrumented = instrumentPiDurableHarnessOptions({ models }); + await instrumented.models!.completeSimple!(MODEL, { messages: [] }); + _INTERNAL_clearAiProviderSkips(); + expect(_INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME)).toBe(false); + + await instrumented.models!.completeSimple!(MODEL, { messages: [] }); + + expect(_INTERNAL_shouldSkipAiProviderWrapping(OPENAI_INTEGRATION_NAME)).toBe(true); + }); + + it('applies the skip to every provider when only some are already registered', async () => { + _INTERNAL_skipAiProviderWrapping([OPENAI_INTEGRATION_NAME]); + const instrumented = instrumentPiDurableHarnessOptions({ models }); + + await instrumented.models!.completeSimple!(MODEL, { messages: [] }); + + expect(_INTERNAL_shouldSkipAiProviderWrapping(ANTHROPIC_AI_INTEGRATION_NAME)).toBe(true); + expect(_INTERNAL_shouldSkipAiProviderWrapping(GOOGLE_GENAI_INTEGRATION_NAME)).toBe(true); + }); + + // `createAI()` of `agents/models/pi-ai` sends pi-ai requests through the Workers AI binding. + it('reports a request sent through the Workers AI binding once', async () => { + const ai = instrumentWorkersAiClient({ + run: async (_model: string, _inputs: unknown, _options: unknown) => new Response('{}'), + gateway: () => ({}), + toMarkdown: async () => [], + }); + const models: PiModels = { + completeSimple: async () => { + await ai.run('@cf/moonshotai/kimi-k2.7-code', { messages: [] }, { returnRawResponse: true }); + return ANSWER; + }, + }; + const instrumented = instrumentPiDurableHarnessOptions({ models }); + + await instrumented.models!.completeSimple!(MODEL, { messages: [] }); + + expect(endedSpans.map(span => spanToStaticSpanJSON(span).description)).toEqual(['chat faux-model']); + }); + }); + + describe('content recording', () => { + const tool: PiTool = { + name: 'get_weather', + description: 'Get the weather.', + execute: async () => ({ content: [{ type: 'text', text: 'sunny' }] }), + }; + const message = { + role: 'assistant', + content: [{ type: 'toolCall', id: 'call_1', name: 'get_weather', arguments: { city: 'Berlin' } }], + stopReason: 'toolUse', + }; + const context = { + messages: [ + { role: 'system', content: '', sections: { preamble: 'You help.' }, toolsAdded: [{ name: 'get_weather' }] }, + { role: 'user', content: 'Weather in Berlin?' }, + ], + }; + + it('records nothing when inputs and outputs are off', async () => { + const options = { recordInputs: false, recordOutputs: false }; + const instrumented = instrumentPiDurableHarnessOptions( + { models: { completeSimple: async () => message } as PiModels }, + options, + ); + + await instrumented.models!.completeSimple!(MODEL, context); + await instrumentTool(tool, createRuns(), options).execute({ city: 'Berlin' }, { callId: 'call_1' }, undefined); + + const [chat, execute] = endedSpans.map(span => spanToJSON(span).attributes); + expect(Object.keys(chat!)).not.toContain(GEN_AI_INPUT_MESSAGES); + expect(Object.keys(chat!)).not.toContain(GEN_AI_OUTPUT_MESSAGES); + expect(Object.keys(chat!)).not.toContain(GEN_AI_SYSTEM_INSTRUCTIONS); + expect(Object.keys(chat!)).not.toContain(GEN_AI_TOOL_DEFINITIONS); + expect(Object.keys(execute!)).not.toContain(GEN_AI_TOOL_CALL_ARGUMENTS); + expect(Object.keys(execute!)).not.toContain(GEN_AI_TOOL_CALL_RESULT); + expect(execute![GEN_AI_TOOL_DESCRIPTION]).toBe('Get the weather.'); + }); + + it('follows the data collection settings of the current client', async () => { + const strict = new TestClient( + getDefaultTestClientOptions({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + tracesSampleRate: 1, + dataCollection: { genAI: { inputs: false, outputs: false } }, + }), + ); + setCurrentClient(strict); + strict.init(); + strict.on('spanEnd', span => endedSpans.push(span)); + const models: PiModels = { completeSimple: async () => message }; + const instrumented = instrumentPiDurableHarnessOptions({ models }); + + await instrumented.models!.completeSimple!(MODEL, context); + await instrumentTool(tool, createRuns(), {}).execute({ city: 'Berlin' }, { callId: 'call_1' }, undefined); + + const recorded = endedSpans.flatMap(span => Object.keys(spanToJSON(span).attributes)); + expect(recorded).not.toContain(GEN_AI_INPUT_MESSAGES); + expect(recorded).not.toContain(GEN_AI_OUTPUT_MESSAGES); + expect(recorded).not.toContain(GEN_AI_SYSTEM_INSTRUCTIONS); + expect(recorded).not.toContain(GEN_AI_TOOL_DEFINITIONS); + expect(recorded).not.toContain(GEN_AI_TOOL_CALL_ARGUMENTS); + expect(recorded).not.toContain(GEN_AI_TOOL_CALL_RESULT); + }); + }); + + it('captures the failures pi-durable reports and still hands them to the caller', async () => { + const reported: unknown[] = []; + const options: PiHarnessOptions = {}; + const instrumented = instrumentPiDurableHarnessOptions(options); + options.onReport = error => reported.push(error); + const error = new Error('afterResponse hook failed'); + + instrumented.onReport!(error); + await client.flush(); + + expect(reported).toEqual([error]); + expect(client.event?.exception?.values?.[0]?.value).toBe('afterResponse hook failed'); + expect(client.event?.exception?.values?.[0]?.mechanism).toEqual({ type: 'auto.ai.pi_durable', handled: true }); + }); + + it('reports the failures as unhandled when the app passed no onReport', async () => { + const options: PiHarnessOptions = {}; + const instrumented = instrumentPiDurableHarnessOptions(options); + + instrumented.onReport!(new Error('section render failed')); + await client.flush(); + + expect(client.event?.exception?.values?.[0]?.mechanism).toEqual({ type: 'auto.ai.pi_durable', handled: false }); + }); + + it('returns the same wrapped task for the same definition in every snapshot', () => { + const generation: PiTask = { definition: { name: 'pi.generation', phases: { prepare: async () => undefined } } }; + const registry: PiRegistryReader = { + // A new snapshot object per call, as pi-durable publishes one per registry change. + snapshot: () => ({ task: () => generation, tasks: () => [generation] }), + subscribe: () => () => undefined, + }; + + const instrumented = instrumentPiDurableHarnessOptions({ registry }).registry!; + + const first = instrumented.snapshot().task('pi.generation'); + expect(first).not.toBe(generation); + expect(instrumented.snapshot().task('pi.generation')).toBe(first); + expect(instrumented.snapshot().tasks()[0]).toBe(first); + }); + + // pi-durable calls a phase on the `phases` object and `abort` on the definition. + it('calls task phases and abort handlers with their own receiver', async () => { + const definition = { + name: 'app.job', + phases: { + run(this: { helper(): Promise }) { + return this.helper(); + }, + helper: async () => 'helped', + }, + abort(this: { name: string }) { + return Promise.resolve(this.name); + }, + }; + const task = { definition } as unknown as PiTask; + const registry: PiRegistryReader = { + snapshot: () => ({ task: () => task, tasks: () => [task] }), + subscribe: () => () => undefined, + }; + const runtime: PiTaskRuntime = { + taskId: 1, + conversationId: 1, + agent: async () => ({}), + commit: async () => undefined, + }; + + const wrapped = instrumentPiDurableHarnessOptions({ registry }).registry!.snapshot().task('app.job')!; + + await expect(wrapped.definition.phases.run!(undefined, runtime, undefined)).resolves.toBe('helped'); + await expect(wrapped.definition.abort!(undefined, runtime, undefined)).resolves.toBe('app.job'); + }); + + describe('tools', () => { + it('runs a frozen tool and a tool that reads a private field', async () => { + const frozen: PiTool = Object.freeze({ + name: 'frozen', + execute: async () => ({ content: [{ type: 'text', text: 'ok' }] }), + }); + class PrefixTool { + #prefix = 'pre'; + public name = 'prefixed'; + public prepareArguments(args: string): string { + return `${this.#prefix}:${args}`; + } + public async execute(): Promise { + return { content: [{ type: 'text', text: this.#prefix }] }; + } + } + const runs = createRuns(); + + const wrappedFrozen = instrumentTool(frozen, runs, {}); + const wrappedPrefix = instrumentTool(new PrefixTool() as unknown as PiTool, runs, {}); + + expect(wrappedFrozen.name).toBe('frozen'); + await expect(wrappedFrozen.execute({}, {}, undefined)).resolves.toEqual({ + content: [{ type: 'text', text: 'ok' }], + }); + expect((wrappedPrefix as unknown as PrefixTool).prepareArguments('x')).toBe('pre:x'); + await expect(wrappedPrefix.execute({}, {}, undefined)).resolves.toEqual({ + content: [{ type: 'text', text: 'pre' }], + }); + expect(endedSpans.map(span => spanToJSON(span).name)).toEqual(['execute_tool frozen', 'execute_tool prefixed']); + }); + + it('ends a call outside a tool phase with its own result', async () => { + const runs = createRuns(); + const tool: PiTool = { + name: 'get_weather', + description: 'Get the weather.', + execute: async () => ({ content: [{ type: 'text', text: 'sunny' }] }), + }; + + await instrumentTool(tool, runs, {}).execute( + { city: 'Berlin' }, + { callId: 'call_1', conversationId: 4 }, + undefined, + ); + + const execute = spanToJSON(endedSpans[0]!); + expect(execute.name).toBe('execute_tool get_weather'); + expect(execute.status).toBe('ok'); + expect(execute.attributes).toMatchObject({ + [SENTRY_OP]: GEN_AI_EXECUTE_TOOL, + [SENTRY_ORIGIN]: 'auto.ai.pi_durable', + [GEN_AI_TOOL_NAME]: 'get_weather', + [GEN_AI_TOOL_DESCRIPTION]: 'Get the weather.', + [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: 'call_1', + [GEN_AI_CONVERSATION_ID]: `${runs.harnessId}:4`, + [GEN_AI_TOOL_CALL_ARGUMENTS]: '{"city":"Berlin"}', + [GEN_AI_TOOL_CALL_RESULT]: 'sunny', + }); + }); + + it.each([ + ['no content blocks', []], + ['an empty text block', [{ type: 'text', text: '' }]], + ])('records an empty tool result given as %s', async (_label, content) => { + const tool: PiTool = { name: 'clear_cache', execute: async () => ({ content }) }; + + await instrumentTool(tool, createRuns(), {}).execute({}, { callId: 'call_1' }, undefined); + + expect(spanToJSON(endedSpans[0]!).attributes[GEN_AI_TOOL_CALL_RESULT]).toBe(''); + }); + + it('marks error results and throws, and captures a throw unless the tool is built in', async () => { + const runs = createRuns(); + const erroring: PiTool = { + name: 'erroring', + execute: async () => ({ isError: true, content: [{ type: 'text', text: 'not found' }] }), + }; + const throwing: PiTool = { + name: 'throwing', + execute: async () => { + throw new Error('tool failed'); + }, + }; + const builtIn: PiTool = { + name: 'bash', + execute: async () => { + throw new Error('Command exited with code 1'); + }, + }; + markBuiltInTool(builtIn); + + await instrumentTool(erroring, runs, {}).execute({}, {}, undefined); + await expect(instrumentTool(throwing, runs, {}).execute({}, {}, undefined)).rejects.toThrow('tool failed'); + await expect(instrumentTool(builtIn, runs, {}).execute({}, {}, undefined)).rejects.toThrow('code 1'); + await client.flush(); + + const [errored, thrown, bash] = endedSpans.map(span => spanToJSON(span)); + expect(errored!.attributes[SENTRY_STATUS_MESSAGE]).toBe('internal_error'); + expect(errored!.attributes[GEN_AI_TOOL_CALL_RESULT]).toBe('not found'); + expect(thrown!.attributes[SENTRY_STATUS_MESSAGE]).toBe('internal_error'); + expect(bash!.attributes[SENTRY_STATUS_MESSAGE]).toBe('internal_error'); + expect(events.map(event => event.exception?.values?.[0]?.value)).toEqual(['tool failed']); + expect(events[0]?.exception?.values?.[0]?.mechanism).toEqual({ type: 'auto.ai.pi_durable', handled: false }); + }); + + it('links a conversation to the failed tool call that created it', async () => { + const runs = createRuns(); + const tool = instrumentTool( + { + name: 'delegate', + execute: async (_args, api) => { + await api.commit!( + tx => + (tx as { createConversation: (...args: unknown[]) => unknown }).createConversation({ + ownership: { kind: 'task', taskId: 7 }, + }), + undefined, + ); + throw new Error('delegation failed'); + }, + }, + runs, + {}, + ); + const api: PiToolExecutionApi = { + taskId: 7, + callId: 'call_1', + commit: async change => change({ createConversation: async () => ({ id: 42 }) }), + }; + + await expect(tool.execute({}, api, undefined)).rejects.toThrow('delegation failed'); + + const call = spanToJSON(endedSpans[0]!); + expect(spanToJSON(startRun(42, runs).span)).toMatchObject({ + trace_id: call.trace_id, + parent_span_id: call.span_id, + }); + }); + + it('marks an aborted call as cancelled and captures nothing', async () => { + const controller = new AbortController(); + const tool: PiTool = { + name: 'slow', + execute: async () => { + controller.abort(); + throw new Error('tool aborted'); + }, + }; + + await expect( + instrumentTool(tool, createRuns(), {}).execute({}, {}, { abortSignal: controller.signal }), + ).rejects.toThrow('tool aborted'); + await client.flush(); + + expect(spanToStaticSpanJSON(endedSpans[0]!).status).toBe('cancelled'); + expect(events).toHaveLength(0); + }); + + it('ends a nested tool call with its own result and links a forked conversation to the call', async () => { + const runs = createRuns(); + runs.toolCalls.set(7, []); + const inner = instrumentTool( + { name: 'inner', execute: async () => ({ content: [{ type: 'text', text: 'INNER' }] }) }, + runs, + {}, + ); + const outer = instrumentTool( + { + name: 'outer', + execute: async (_args, api) => { + await inner.execute({}, api, undefined); + await api.commit!( + tx => + (tx as { forkConversation: (...args: unknown[]) => unknown }).forkConversation(1, 3, { + ownership: { kind: 'task', taskId: 7 }, + }), + undefined, + ); + return { content: [{ type: 'text', text: 'OUTER' }] }; + }, + }, + runs, + {}, + ); + const api: PiToolExecutionApi = { + taskId: 7, + callId: 'call_1', + commit: async change => change({ forkConversation: async () => ({ id: 42 }) }), + }; + + await outer.execute({}, api, undefined); + + expect(endedSpans.map(span => spanToJSON(span).name)).toEqual(['execute_tool inner']); + expect(spanToJSON(endedSpans[0]!).attributes[GEN_AI_TOOL_CALL_RESULT]).toBe('INNER'); + // The outer call waits for the result entry its phase commits. + const [outerCall] = runs.toolCalls.get(7)!; + expect(outerCall!.span.spanContext().spanId).toBe(spanToJSON(endedSpans[0]!).parent_span_id); + expect(spanToJSON(startRun(42, runs).span).parent_span_id).toBe(outerCall!.span.spanContext().spanId); + }); + + it('starts the first run of a conversation as a root once the trace of the call that created it has ended', async () => { + const runs = createRuns(); + const tool = instrumentTool( + { + name: 'delegate', + execute: async (_args, api) => { + await api.commit!( + tx => + (tx as { createConversation: (...args: unknown[]) => unknown }).createConversation({ + ownership: { kind: 'task', taskId: 7 }, + }), + undefined, + ); + return { content: [{ type: 'text', text: 'Subagent created.' }] }; + }, + }, + runs, + {}, + ); + const api: PiToolExecutionApi = { + taskId: 7, + callId: 'call_1', + commit: async change => change({ createConversation: async () => ({ id: 42 }) }), + }; + await tool.execute({}, api, undefined); + + const run = startRun(42, runs).span; + + const call = spanToJSON(endedSpans[0]!); + expect(spanToJSON(run)).toMatchObject({ trace_id: call.trace_id, parent_span_id: call.span_id }); + expect(getRootSpan(run)).toBe(run); + }); + }); + + describe('runs', () => { + /** A generation phase that commits each of `states` as the `pi.live` document, in order. */ + function generationPhase(states: (PiLiveState | { settle: PiSettlement })[]): PiTask { + return { + definition: { + name: 'pi.generation', + phases: { + request: async (_task, runtime) => { + for (const state of states) { + await runtime.commit(async tx => { + const draft = (await (tx as { doc: (...args: unknown[]) => Promise }).doc(LIVE_DOC, 1))!; + if ('settle' in state) { + (tx as { settleSubmission: (...args: unknown[]) => void }).settleSubmission(5, state.settle); + delete draft.run; + } else { + draft.run = state.run; + } + return undefined; + }, undefined); + } + }, + }, + }, + }; + } + + function harnessFor(task: PiTask): PiHarnessOptions { + const registry: PiRegistryReader = { + snapshot: () => ({ task: () => task, tasks: () => [task] }), + subscribe: () => () => undefined, + }; + return instrumentPiDurableHarnessOptions({ registry }); + } + + async function runGeneration(harness: PiHarnessOptions): Promise { + const live: PiLiveState = {}; + const tx = { doc: async () => live, settleSubmission: () => undefined }; + const runtime: PiTaskRuntime = { + taskId: 3, + conversationId: 1, + agent: async () => ({}), + commit: async (change: PiCommitChange) => { + await change(tx, undefined); + }, + }; + await harness.registry!.snapshot().task('pi.generation')!.definition.phases.request!( + undefined, + runtime, + undefined, + ); + } + + function runSpans(): ReturnType[] { + return endedSpans + .map(span => spanToJSON(span)) + .filter(span => span.attributes[SENTRY_OP] === GEN_AI_INVOKE_AGENT); + } + + it('keeps the run open while run control moves to the next generation', async () => { + await runGeneration(harnessFor(generationPhase([{ run: { inputs: [5] } }, { run: { inputs: [5, 6] } }]))); + + expect(runSpans()).toHaveLength(0); + }); + + it.each([ + ['model_error', 'error', 'model_error'], + ['no_model', 'error', 'no_model'], + ['aborted', 'ok', undefined], + ['reset', 'ok', undefined], + [undefined, 'error', 'internal_error'], + ])('ends an unanswered run with reason %s as %s', async (reason, status, message) => { + await runGeneration( + harnessFor(generationPhase([{ run: { inputs: [5] } }, { settle: { status: 'unanswered', reason } }])), + ); + + const runs = runSpans(); + expect(runs).toHaveLength(1); + expect(runs[0]!.status).toBe(status); + expect(runs[0]!.attributes[SENTRY_STATUS_MESSAGE]).toBe(message); + expect(runs[0]!.attributes[GEN_AI_CONVERSATION_ID]).toMatch(/^[0-9a-f]{32}:1$/); + }); + + // `@sentry/cloudflare` initializes the SDK inside each request, so the client is bound to the + // scope of that request and never to the default scope. + it('traces a run when only the scope that starts the phase has the client', async () => { + getCurrentScope().setClient(undefined); + const harness = harnessFor(generationPhase([{ run: { inputs: [5] } }, { settle: { status: 'done' } }])); + + await withScope(async scope => { + scope.setClient(client); + await runGeneration(harness); + }); + + expect(runSpans()).toHaveLength(1); + }); + + it('ends the run when a queued input starts the next run in the same commit', async () => { + await runGeneration(harnessFor(generationPhase([{ run: { inputs: [5] } }, { run: { inputs: [7] } }]))); + + const runs = runSpans(); + expect(runs).toHaveLength(1); + expect(runs[0]!.status).toBe('ok'); + }); + + it('starts no run for an abort handler whose task has no run', async () => { + const task: PiTask = { + definition: { + name: 'pi.generation', + phases: {}, + abort: async (_task, runtime) => + runtime.commit(async tx => { + await (tx as { doc: (...args: unknown[]) => Promise }).doc(LIVE_DOC, 1); + return undefined; + }, undefined), + }, + }; + const harness = harnessFor(task); + const runtime: PiTaskRuntime = { + taskId: 3, + conversationId: 1, + agent: async () => ({}), + commit: async (change: PiCommitChange) => { + await change({ doc: async () => ({}) }, undefined); + }, + }; + + await harness.registry!.snapshot().task('pi.generation')!.definition.abort!(undefined, runtime, undefined); + + expect(runSpans()).toEqual([]); + expect((harness as unknown as Record)[Symbol.for('sentry.pi-durable.runs')]!.active.size).toBe(0); + }); + + it('gives every Harness its own conversation id prefix', async () => { + const phase = generationPhase([{ run: { inputs: [5] } }, { settle: { status: 'done' } }]); + const first = harnessFor(phase); + + await runGeneration(first); + await runGeneration(first); + await runGeneration(harnessFor(phase)); + + const ids = runSpans().map(run => String(run.attributes[GEN_AI_CONVERSATION_ID])); + expect(ids).toHaveLength(3); + expect(ids[0]).toBe(ids[1]); + expect(ids[2]).not.toBe(ids[0]); + expect(ids.every(id => id.endsWith(':1'))).toBe(true); + }); + + it('ends the open runs when the Harness closes', async () => { + const harness = harnessFor(generationPhase([{ run: { inputs: [5] } }])); + let onClose: (() => void) | undefined; + endRunsOnClose(harness, { subscribeClose: (listener: () => void) => ((onClose = listener), () => undefined) }); + + await runGeneration(harness); + expect(runSpans()).toHaveLength(0); + + onClose!(); + + expect(runSpans()).toHaveLength(1); + expect(spanToStaticSpanJSON(endedSpans[0]!).status).toBe('cancelled'); + }); + + it('ends the tool calls in flight and forgets subagent links when the Harness closes', async () => { + const harness = harnessFor(generationPhase([])); + const runs = (harness as unknown as Record)[Symbol.for('sentry.pi-durable.runs')]!; + let onClose: (() => void) | undefined; + endRunsOnClose(harness, { subscribeClose: (listener: () => void) => ((onClose = listener), () => undefined) }); + let delegated: () => void = () => undefined; + const delegating = new Promise(resolve => (delegated = resolve)); + const tool = instrumentTool( + { + name: 'delegate', + execute: async (_args, api) => { + await api.commit!( + tx => + (tx as { createConversation: (...args: unknown[]) => unknown }).createConversation({ + ownership: { kind: 'task', taskId: 7 }, + }), + undefined, + ); + delegated(); + return new Promise(() => undefined); + }, + }, + runs, + {}, + ); + runs.toolCalls.set(7, []); + void tool.execute( + {}, + { + taskId: 7, + callId: 'call_1', + commit: async change => change({ createConversation: async () => ({ id: 42 }) }), + }, + undefined, + ); + await delegating; + + onClose!(); + + expect(endedSpans.map(span => spanToStaticSpanJSON(span).status)).toEqual(['cancelled']); + expect(spanToJSON(startRun(42, runs).span).parent_span_id).toBeUndefined(); + }); + + // `wait()` resolves once the ending commit settles the inputs, before `commit()` resolves, so an + // app that closes right after `wait()` closes inside that gap. + it('ends a run with its own settlement when the Harness closes before the ending commit resolves', async () => { + const harness = harnessFor( + generationPhase([{ run: { inputs: [5] } }, { settle: { status: 'unanswered', reason: 'model_error' } }]), + ); + let onClose: (() => void) | undefined; + endRunsOnClose(harness, { subscribeClose: (listener: () => void) => ((onClose = listener), () => undefined) }); + const live: PiLiveState = {}; + const tx = { doc: async () => live, settleSubmission: () => undefined }; + let commits = 0; + const runtime: PiTaskRuntime = { + taskId: 3, + conversationId: 1, + agent: async () => ({}), + commit: async (change: PiCommitChange) => { + await change(tx, undefined); + commits++; + if (commits === 2) { + onClose!(); + } + }, + }; + + await harness.registry!.snapshot().task('pi.generation')!.definition.phases.request!( + undefined, + runtime, + undefined, + ); + + const runs = runSpans(); + expect(runs).toHaveLength(1); + expect(runs[0]!.status).toBe('error'); + expect(runs[0]!.attributes[SENTRY_STATUS_MESSAGE]).toBe('model_error'); + }); + + // A run the process never sees settle would otherwise stay in the map for the process lifetime. + it('ends the oldest run span when the run tracker overflows', () => { + const runs = createRuns(); + + for (let conversation = 0; conversation <= MAX_TRACKED_PI_RUNS; conversation++) { + startRun(conversation, runs); + } + + expect(runs.active.size).toBe(MAX_TRACKED_PI_RUNS); + expect(runs.active.has(0)).toBe(false); + expect(endedSpans).toHaveLength(1); + expect(spanToJSON(endedSpans[0]!).attributes[GEN_AI_CONVERSATION_ID]).toBe(`${runs.harnessId}:0`); + }); + }); +}); diff --git a/packages/server-utils/test/ai/lib/utils/pi-ai-messages.test.ts b/packages/server-utils/test/ai/lib/utils/pi-ai-messages.test.ts index 131b1043ba77..4cb99d1af65c 100644 --- a/packages/server-utils/test/ai/lib/utils/pi-ai-messages.test.ts +++ b/packages/server-utils/test/ai/lib/utils/pi-ai-messages.test.ts @@ -4,6 +4,8 @@ import { piAiContentToString, piAiFinishReason, piAiMessagesToGenAiMessages, + piAiSystemInstructions, + piAiToolDefinitions, } from '../../../../src/ai/pi-ai/messages'; describe('convert pi-ai messages to gen_ai messages', () => { @@ -136,6 +138,28 @@ describe('convert pi-ai messages to gen_ai messages', () => { expect(piAiAssistantMessageToGenAiMessage({ role: 'assistant', content: [] }, 'stop')).toBeUndefined(); }); + it('replays positional system messages into the current system prompt and tools', () => { + const context = { + systemPrompt: 'Base prompt.', + tools: [{ name: 'read' }], + messages: [ + { + role: 'system', + content: '', + sections: { preamble: 'You help.', mode: 'Edit.' }, + toolsAdded: [{ name: 'bash' }], + }, + { role: 'user', content: 'Hi.' }, + { role: 'system', content: '', sections: { mode: null }, toolsRemoved: [{ name: 'read' }] }, + { role: 'system', content: 'Appended.', sections: { mode: 'Plan only.' } }, + ], + }; + + expect(piAiSystemInstructions(context)).toBe('Base prompt.\n\nAppended.\n\nYou help.\n\nPlan only.'); + expect(piAiToolDefinitions(context)).toStrictEqual([{ name: 'bash' }]); + expect(piAiSystemInstructions({ messages: [{ role: 'user', content: 'Hi.' }] })).toBeUndefined(); + }); + it('renders text-only content as text and other content as mapped parts', () => { expect( piAiContentToString([ From b70f6bb6d634da88e82f15c0568b02dcaba6bd39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Peer=20St=C3=B6cklmair?= Date: Thu, 8 Oct 2026 15:49:03 +0300 Subject: [PATCH 40/47] test(e2e): Add a node-pi-durable end-to-end application (#24994) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an E2E app for the pi-durable instrumentation. It covers what the node integration test cannot: the packed SDK loaded with `--import`, real model requests through OpenRouter, SQLite storage, a subagent, and a run that resumes after the server crashes during a tool call. A small supervisor restarts the server for that case, as a process manager would. pi-ai sends `anthropic/claude-haiku-4.5` through `@anthropic-ai/sdk`, so the first test also checks that each model request is reported once and that the provider's HTTP call nests inside its `chat` span. The app is optional with a `latest` variant, because `E2E_OPENROUTER_API_KEY` is only available in the optional E2E job. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../node-pi-durable/.gitignore | 5 + .../node-pi-durable/instrument.mjs | 8 + .../node-pi-durable/package.json | 42 ++++ .../node-pi-durable/playwright.config.mjs | 10 + .../node-pi-durable/src/server.mjs | 162 ++++++++++++++ .../node-pi-durable/src/start.mjs | 35 +++ .../node-pi-durable/start-event-proxy.mjs | 6 + .../node-pi-durable/tests/pi-durable.test.ts | 199 ++++++++++++++++++ .../node-pi-durable/tests/utils.ts | 47 +++++ 9 files changed, 514 insertions(+) create mode 100644 dev-packages/e2e-tests/test-applications/node-pi-durable/.gitignore create mode 100644 dev-packages/e2e-tests/test-applications/node-pi-durable/instrument.mjs create mode 100644 dev-packages/e2e-tests/test-applications/node-pi-durable/package.json create mode 100644 dev-packages/e2e-tests/test-applications/node-pi-durable/playwright.config.mjs create mode 100644 dev-packages/e2e-tests/test-applications/node-pi-durable/src/server.mjs create mode 100644 dev-packages/e2e-tests/test-applications/node-pi-durable/src/start.mjs create mode 100644 dev-packages/e2e-tests/test-applications/node-pi-durable/start-event-proxy.mjs create mode 100644 dev-packages/e2e-tests/test-applications/node-pi-durable/tests/pi-durable.test.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-pi-durable/tests/utils.ts diff --git a/dev-packages/e2e-tests/test-applications/node-pi-durable/.gitignore b/dev-packages/e2e-tests/test-applications/node-pi-durable/.gitignore new file mode 100644 index 000000000000..6054d28d34f9 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-pi-durable/.gitignore @@ -0,0 +1,5 @@ +node_modules +.data +results.junit.xml +test-results +playwright-report diff --git a/dev-packages/e2e-tests/test-applications/node-pi-durable/instrument.mjs b/dev-packages/e2e-tests/test-applications/node-pi-durable/instrument.mjs new file mode 100644 index 000000000000..b774d9306950 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-pi-durable/instrument.mjs @@ -0,0 +1,8 @@ +import * as Sentry from '@sentry/node'; + +Sentry.init({ + environment: 'qa', + dsn: process.env.E2E_TEST_DSN, + tunnel: 'http://localhost:3031/', // proxy server + tracesSampleRate: 1.0, +}); diff --git a/dev-packages/e2e-tests/test-applications/node-pi-durable/package.json b/dev-packages/e2e-tests/test-applications/node-pi-durable/package.json new file mode 100644 index 000000000000..baadb930e355 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-pi-durable/package.json @@ -0,0 +1,42 @@ +{ + "name": "node-pi-durable", + "version": "0.0.0", + "private": true, + "type": "module", + "scripts": { + "start": "node src/start.mjs", + "clean": "npx rimraf node_modules .data pnpm-lock.yaml", + "test:build": "pnpm install", + "test:build-latest": "pnpm install && pnpm add @earendil-works/pi-durable@latest @earendil-works/pi-ai@latest @earendil-works/chord@latest", + "test:assert": "OPENROUTER_API_KEY=$E2E_OPENROUTER_API_KEY playwright test" + }, + "dependencies": { + "@earendil-works/chord": "1.0.0", + "@earendil-works/pi-ai": "1.0.0", + "@earendil-works/pi-durable": "1.0.0", + "@sentry/node": "file:../../packed/sentry-node-packed.tgz" + }, + "devDependencies": { + "@playwright/test": "~1.63.0", + "@sentry-internal/test-utils": "link:../../../test-utils", + "@sentry/conventions": "0.26.0", + "@sentry/core": "file:../../packed/sentry-core-packed.tgz", + "@types/node": "24.x" + }, + "engines": { + "node": "24.x" + }, + "volta": { + "node": "24.15.0", + "extends": "../../package.json" + }, + "sentryTest": { + "optional": true, + "optionalVariants": [ + { + "build-command": "pnpm test:build-latest", + "label": "node-pi-durable (latest)" + } + ] + } +} diff --git a/dev-packages/e2e-tests/test-applications/node-pi-durable/playwright.config.mjs b/dev-packages/e2e-tests/test-applications/node-pi-durable/playwright.config.mjs new file mode 100644 index 000000000000..ad0caf7da2c9 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-pi-durable/playwright.config.mjs @@ -0,0 +1,10 @@ +import { getPlaywrightConfig } from '@sentry-internal/test-utils'; + +const config = getPlaywrightConfig( + { startCommand: 'pnpm start' }, + // Each test drives real OpenRouter tool-calling turns, and one restarts the server mid-run, which + // does not fit the default 30s timeout when the provider is slow. + { timeout: 120_000 }, +); + +export default config; diff --git a/dev-packages/e2e-tests/test-applications/node-pi-durable/src/server.mjs b/dev-packages/e2e-tests/test-applications/node-pi-durable/src/server.mjs new file mode 100644 index 000000000000..79892991f03d --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-pi-durable/src/server.mjs @@ -0,0 +1,162 @@ +import { mkdirSync, writeFileSync } from 'node:fs'; +import { createServer } from 'node:http'; +import { fileURLToPath } from 'node:url'; +import { BACKGROUND_CONTEXT } from '@earendil-works/chord/context'; +import { Type } from '@earendil-works/pi-ai'; +import { createModels } from '@earendil-works/pi-ai/models'; +import { openrouterProvider } from '@earendil-works/pi-ai/providers/openrouter'; +import { + AssistantEntry, + createRegistry, + defineExtension, + defineTool, + Harness, + section, +} from '@earendil-works/pi-durable'; +import { openNodeSqliteStorage } from '@earendil-works/pi-durable/storage/sqlite/node'; +import * as Sentry from '@sentry/node'; + +const context = BACKGROUND_CONTEXT; +const MODEL = { provider: 'openrouter', modelId: 'anthropic/claude-haiku-4.5' }; +const CRASH_EXIT_CODE = 75; + +if (!process.env.OPENROUTER_API_KEY) { + throw new Error('OPENROUTER_API_KEY is not set (E2E_OPENROUTER_API_KEY in the e2e-tests .env)'); +} + +const dataDir = fileURLToPath(new URL('../.data/', import.meta.url)); +mkdirSync(dataDir, { recursive: true }); + +const models = createModels(); +models.setProvider(openrouterProvider()); // reads OPENROUTER_API_KEY + +const tools = [ + defineTool({ + name: 'get_weather', + description: 'Get the current weather for a city.', + parameters: Type.Object({ city: Type.String() }), + // The manual span should nest under the SDK's `execute_tool` span. + execute: async args => + Sentry.startSpan({ name: 'resolve-weather', attributes: { 'weather.city': args.city } }, () => ({ + content: [{ type: 'text', text: `It is 21 degrees and sunny in ${args.city}.` }], + })), + }), + defineTool({ + name: 'fail_now', + description: 'Always throws an error. Call this when the user asks to trigger a failure.', + parameters: Type.Object({}), + execute: async () => { + throw new Error('Intentional pi-durable tool failure'); + }, + }), + defineTool({ + name: 'crash_once', + description: 'Runs one step of a job. Call this when the user asks for crash_once.', + parameters: Type.Object({ job: Type.String() }), + // Replay-safe, so pi-durable reruns the call after the crash instead of failing it. + replay: 'safe', + execute: async args => { + // The marker is created once, atomically; its presence means the first attempt already crashed. + try { + writeFileSync(`${dataDir}crashed-${args.job}`, 'crashed once', { flag: 'wx' }); + } catch { + return { content: [{ type: 'text', text: `Step of job ${args.job} completed.` }] }; + } + process.exit(CRASH_EXIT_CODE); + }, + }), + defineTool({ + name: 'delegate', + description: 'Delegate a self-contained task to a subagent and get its answer back.', + parameters: Type.Object({ task: Type.String() }), + replay: 'safe', + execute: async (args, api, toolContext) => { + const childId = await api.commit(async tx => { + const existing = (await tx.scanConversations({ ownerTaskId: api.taskId }, 1)).items[0]; + if (existing !== undefined) return existing.id; + return (await tx.createConversation({ ownership: { kind: 'task', taskId: api.taskId } })).id; + }, toolContext); + const child = await api.conversation(childId, toolContext); + const request = { type: 'input', content: args.task, requestId: `delegate:${api.taskId}` }; + const settled = await (await child.submit(request, toolContext)).wait(toolContext); + if (settled.status !== 'done') { + return { isError: true, content: [{ type: 'text', text: `Subagent ended: ${settled.reason}` }] }; + } + const answer = await api.commit(tx => tx.entry(AssistantEntry, settled.answer), toolContext); + const text = (answer?.model?.[0]?.content ?? []) + .filter(part => part.type === 'text') + .map(part => part.text) + .join(''); + return { content: [{ type: 'text', text }] }; + }, + }), +]; + +const registry = createRegistry(); +registry.install( + defineExtension({ + name: 'e2e', + sections: [ + section('preamble', () => 'You are a test assistant. Use the tools exactly as asked. Keep answers short.', { + tag: false, + }), + ], + tools, + }), +); + +const storage = await openNodeSqliteStorage(`${dataDir}pi.sqlite`); +const harness = await Harness.open(storage, { models, registry }, context); +// Continue whatever the previous process left unfinished, such as a run interrupted by `crash_once`. +harness.resume(); + +async function readJson(req) { + let body = ''; + for await (const chunk of req) body += chunk; + return body ? JSON.parse(body) : {}; +} + +async function handle(req, res) { + const url = new URL(req.url, 'http://localhost'); + const parts = url.pathname.split('/').filter(Boolean); + + if (req.method === 'POST' && url.pathname === '/conversations') { + const conversation = await harness.createConversation( + { ownership: { kind: 'ownerless' }, agent: { model: MODEL } }, + context, + ); + return { id: conversation.id }; + } + + // Returns as soon as the input is admitted. The run continues in the background, possibly in a + // later process, so clients poll `GET /submissions/:id` for the outcome. + if (req.method === 'POST' && parts[0] === 'conversations' && parts[2] === 'messages') { + const conversation = await harness.conversation(Number(parts[1]), context); + if (!conversation) return undefined; + const { content } = await readJson(req); + const submission = await conversation.submit({ type: 'input', content }, context); + return { submissionId: submission.id }; + } + + if (req.method === 'GET' && parts[0] === 'submissions') { + const submission = await harness.submission(Number(parts[1]), context); + if (!submission) return undefined; + const record = await submission.status(context); + return { status: record.status, reason: record.reason, detail: record.detail }; + } + + return undefined; +} + +createServer((req, res) => { + handle(req, res).then( + result => { + res.writeHead(result ? 200 : 404, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(result ?? { error: 'not found' })); + }, + error => { + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: String(error) })); + }, + ); +}).listen(3030); diff --git a/dev-packages/e2e-tests/test-applications/node-pi-durable/src/start.mjs b/dev-packages/e2e-tests/test-applications/node-pi-durable/src/start.mjs new file mode 100644 index 000000000000..55406c81b107 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-pi-durable/src/start.mjs @@ -0,0 +1,35 @@ +import { spawn } from 'node:child_process'; +import { rmSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +// The `crash_once` tool exits the server mid-call with this code to simulate a crash. Restarting the +// server on the same storage, as a process manager would, is what lets the run resume. +const CRASH_EXIT_CODE = 75; + +const appDir = fileURLToPath(new URL('..', import.meta.url)); +rmSync(new URL('../.data', import.meta.url), { recursive: true, force: true }); + +let server; + +function startServer() { + server = spawn(process.execPath, ['--import', './instrument.mjs', 'src/server.mjs'], { + cwd: appDir, + stdio: 'inherit', + }); + server.on('exit', code => { + if (code === CRASH_EXIT_CODE) { + startServer(); + } else { + process.exit(code ?? 1); + } + }); +} + +for (const signal of ['SIGINT', 'SIGTERM']) { + process.on(signal, () => { + server.kill(signal); + process.exit(0); + }); +} + +startServer(); diff --git a/dev-packages/e2e-tests/test-applications/node-pi-durable/start-event-proxy.mjs b/dev-packages/e2e-tests/test-applications/node-pi-durable/start-event-proxy.mjs new file mode 100644 index 000000000000..dd9d4d2b9da8 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-pi-durable/start-event-proxy.mjs @@ -0,0 +1,6 @@ +import { startEventProxyServer } from '@sentry-internal/test-utils'; + +startEventProxyServer({ + port: 3031, + proxyServerName: 'node-pi-durable', +}); diff --git a/dev-packages/e2e-tests/test-applications/node-pi-durable/tests/pi-durable.test.ts b/dev-packages/e2e-tests/test-applications/node-pi-durable/tests/pi-durable.test.ts new file mode 100644 index 000000000000..b00011096879 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-pi-durable/tests/pi-durable.test.ts @@ -0,0 +1,199 @@ +import { expect, test } from '@playwright/test'; +import { + GEN_AI_CONVERSATION_ID, + GEN_AI_COST_TOTAL_TOKENS, + GEN_AI_INPUT_MESSAGES, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_FINISH_REASONS, + GEN_AI_TOOL_CALL_ARGUMENTS, + GEN_AI_TOOL_CALL_RESULT, + GEN_AI_TOOL_NAME, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + SENTRY_ORIGIN, + SERVER_ADDRESS, +} from '@sentry/conventions/attributes'; +import { GEN_AI_CHAT, GEN_AI_INVOKE_AGENT, HTTP_CLIENT, HTTP_SERVER } from '@sentry/conventions/op'; +import { collectStreamedSpans, getSpanOp, waitForError } from '@sentry-internal/test-utils'; +import { createConversation, sendMessage } from './utils'; + +const APP = 'node-pi-durable'; + +test('traces a run as invoke_agent with chat, execute_tool and provider spans', async ({ baseURL }) => { + const conversationId = await createConversation(baseURL!); + // A run is its own trace, and its `invoke_agent` segment ends last. + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => + spansOfTrace.some( + span => + span.is_segment && String(span.attributes[GEN_AI_CONVERSATION_ID]?.value).endsWith(`:${conversationId}`), + ) && spansOfTrace.some(span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'get_weather'), + ); + + const settled = await sendMessage( + baseURL!, + conversationId, + 'What is the weather in Vienna? Use the get_weather tool.', + ); + expect(settled.status, settled.detail).toBe('done'); + + const spans = await spansPromise; + const agent = spans.find(span => span.is_segment)!; + const chats = spans.filter(span => getSpanOp(span) === GEN_AI_CHAT); + // The manual span of the tool is the one way to pick the call that ran, should the model call twice. + const manualSpan = spans.find(span => span.name === 'resolve-weather')!; + const tool = spans.find(span => span.span_id === manualSpan.parent_span_id)!; + const providerCalls = spans.filter(span => getSpanOp(span) === HTTP_CLIENT); + + expect(getSpanOp(agent)).toBe(GEN_AI_INVOKE_AGENT); + expect(agent.attributes[SENTRY_ORIGIN]?.value).toBe('auto.ai.pi_durable'); + // The request that submitted the input is a trace of its own: the scheduler runs the work later. + expect(spans.some(span => getSpanOp(span) === HTTP_SERVER)).toBe(false); + // pi-ai sends the requests through `@anthropic-ai/sdk`, whose own integration must stay out so + // each request is reported once. + expect(spans.filter(span => String(span.attributes[SENTRY_ORIGIN]?.value).startsWith('auto.ai.'))).toEqual( + spans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.pi_durable'), + ); + + // One tool-calling response, then the answer. + expect(chats.length).toBeGreaterThanOrEqual(2); + for (const chat of chats) { + expect(chat.parent_span_id).toBe(agent.span_id); + expect(chat.attributes[SENTRY_ORIGIN]?.value).toBe('auto.ai.pi_durable'); + expect(chat.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('openrouter'); + expect(chat.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('anthropic/claude-haiku-4.5'); + // pi-durable retries a provider error inside the run; such a request has no usage and no HTTP span + // of its own to assert on. + if (chat.status === 'ok') { + expect(typeof chat.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe('number'); + expect(typeof chat.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe('number'); + expect(typeof chat.attributes[GEN_AI_COST_TOTAL_TOKENS]?.value).toBe('number'); + expect(providerCalls.some(providerCall => providerCall.parent_span_id === chat.span_id)).toBe(true); + } + } + + // The request after the tool round carries the tool call and its result in the conventions shape. + const answerInput = JSON.parse( + String( + chats.find(chat => chat.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["stop"]')!.attributes[ + GEN_AI_INPUT_MESSAGES + ]?.value, + ), + ) as { role: string; parts: { type: string }[] }[]; + expect(answerInput.map(message => message.role)).toContain('tool'); + expect(answerInput.flatMap(message => message.parts.map(part => part.type))).toEqual( + expect.arrayContaining(['tool_call', 'tool_call_response']), + ); + + expect(tool.attributes[GEN_AI_TOOL_NAME]?.value).toBe('get_weather'); + expect(tool.parent_span_id).toBe(agent.span_id); + expect(tool.status).toBe('ok'); + expect(tool.attributes[GEN_AI_TOOL_CALL_ARGUMENTS]?.value).toContain('Vienna'); + expect(tool.attributes[GEN_AI_TOOL_CALL_RESULT]?.value).toContain('21 degrees and sunny in'); + + // The provider's HTTP calls nest inside the `chat` span that sent them. + expect(providerCalls.length).toBeGreaterThan(0); + for (const providerCall of providerCalls) { + expect(chats.map(chat => chat.span_id)).toContain(providerCall.parent_span_id); + expect(providerCall.attributes[SERVER_ADDRESS]?.value).toBe('openrouter.ai'); + } +}); + +test('reports a throwing tool as an error on its execute_tool span', async ({ baseURL }) => { + const conversationId = await createConversation(baseURL!); + const errorPromise = waitForError( + APP, + event => event.exception?.values?.[0]?.value === 'Intentional pi-durable tool failure', + ); + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => + spansOfTrace.some( + span => + span.is_segment && String(span.attributes[GEN_AI_CONVERSATION_ID]?.value).endsWith(`:${conversationId}`), + ) && spansOfTrace.some(span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'fail_now'), + ); + + const settled = await sendMessage(baseURL!, conversationId, 'Call the fail_now tool, then tell me what happened.'); + expect(settled.status, settled.detail).toBe('done'); + + const [error, spans] = await Promise.all([errorPromise, spansPromise]); + // The error is reported on the span of the call that threw. + const tool = spans.find(span => span.span_id === error.contexts?.trace?.span_id)!; + + expect(tool.attributes[GEN_AI_TOOL_NAME]?.value).toBe('fail_now'); + expect(tool.status).toBe('error'); + expect(tool.trace_id).toBe(error.contexts?.trace?.trace_id); + expect(error.exception?.values?.[0]?.mechanism).toEqual({ type: 'auto.ai.pi_durable', handled: false }); +}); + +test('nests a subagent run under the tool call that delegated to it', async ({ baseURL }) => { + const conversationId = await createConversation(baseURL!); + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => + spansOfTrace.some( + span => + span.is_segment && String(span.attributes[GEN_AI_CONVERSATION_ID]?.value).endsWith(`:${conversationId}`), + ) && spansOfTrace.some(span => !span.is_segment && getSpanOp(span) === GEN_AI_INVOKE_AGENT), + ); + + const settled = await sendMessage( + baseURL!, + conversationId, + 'Use the delegate tool to ask a subagent: "What is 2 + 2? Reply with only the number." Then reply with its answer.', + ); + expect(settled.status, settled.detail).toBe('done'); + + const spans = await spansPromise; + const agent = spans.find(span => span.is_segment)!; + const delegate = spans.find(span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'delegate')!; + const subagent = spans.find(span => !span.is_segment && getSpanOp(span) === GEN_AI_INVOKE_AGENT)!; + const subagentChat = spans.find(span => getSpanOp(span) === GEN_AI_CHAT && span.parent_span_id === subagent.span_id); + + expect(delegate.parent_span_id).toBe(agent.span_id); + expect(subagent.parent_span_id).toBe(delegate.span_id); + expect(subagentChat).toBeDefined(); + + // The subagent has a conversation of its own, in the same Harness. + const [agentHarness, agentConversation] = String(agent.attributes[GEN_AI_CONVERSATION_ID]?.value).split(':'); + const [subagentHarness, subagentConversation] = String(subagent.attributes[GEN_AI_CONVERSATION_ID]?.value).split(':'); + expect(subagentHarness).toBe(agentHarness); + expect(subagentConversation).not.toBe(agentConversation); +}); + +test('resumes a run in a new trace after the server crashes during a tool call', async ({ baseURL }) => { + const conversationId = await createConversation(baseURL!); + // The tool span of the crashed process never ends, so a finished `crash_once` span can only come + // from the rerun in the restarted server. + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => + spansOfTrace.some( + span => + span.is_segment && String(span.attributes[GEN_AI_CONVERSATION_ID]?.value).endsWith(`:${conversationId}`), + ) && spansOfTrace.some(span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'crash_once' && span.status === 'ok'), + ); + + const settled = await sendMessage( + baseURL!, + conversationId, + 'Run the crash_once tool for job "nightly", then reply with the word DONE.', + ); + expect(settled.status, settled.detail).toBe('done'); + + const spans = await spansPromise; + const agent = spans.find(span => span.is_segment)!; + const tool = spans.find(span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'crash_once')!; + const chats = spans.filter(span => getSpanOp(span) === GEN_AI_CHAT); + const answer = chats.find(span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["stop"]'); + + expect(getSpanOp(agent)).toBe(GEN_AI_INVOKE_AGENT); + expect(tool.parent_span_id).toBe(agent.span_id); + expect(answer?.parent_span_id).toBe(agent.span_id); + // The request that called the tool ran in the crashed process, so this trace starts with the rerun + // of the tool. A run that never crashed would start with that request. + expect(chats.every(chat => chat.start_timestamp >= tool.start_timestamp)).toBe(true); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-pi-durable/tests/utils.ts b/dev-packages/e2e-tests/test-applications/node-pi-durable/tests/utils.ts new file mode 100644 index 000000000000..8f2754169431 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-pi-durable/tests/utils.ts @@ -0,0 +1,47 @@ +import { expect } from '@playwright/test'; + +export async function createConversation(baseURL: string): Promise { + const res = await fetch(`${baseURL}/conversations`, { method: 'POST' }); + expect(res.status).toBe(200); + return ((await res.json()) as { id: number }).id; +} + +/** + * Send a message and wait until pi-durable settles it. + * + * The server only admits the input; the run continues in the background, and in the crash test in a + * second server process. Polling the submission until it settles keeps each test to its own run, and + * failed requests while the server restarts are retried. + */ +export async function sendMessage( + baseURL: string, + conversationId: number, + content: string, +): Promise<{ status: string; reason?: string; detail?: string }> { + const res = await fetch(`${baseURL}/conversations/${conversationId}/messages`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ content }), + }); + expect(res.status).toBe(200); + const { submissionId } = (await res.json()) as { submissionId: number }; + + const deadline = Date.now() + 90_000; + while (Date.now() < deadline) { + try { + const submission = (await (await fetch(`${baseURL}/submissions/${submissionId}`)).json()) as { + status: string; + reason?: string; + detail?: string; + }; + if (submission.status === 'done' || submission.status === 'unanswered') { + return submission; + } + } catch { + // The server is restarting. + } + await new Promise(resolve => setTimeout(resolve, 250)); + } + + throw new Error(`Submission ${submissionId} did not settle within 90s`); +} From 966e1ff0aa4d9b59e8dd7a5e6ac112175ddc6525 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Peer=20St=C3=B6cklmair?= Date: Thu, 8 Oct 2026 15:49:03 +0300 Subject: [PATCH 41/47] feat(cloudflare): Skip spans for the pi_ tables of PiHarness (#25028) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Agents SDK `PiHarness` ([changelog](https://developers.cloudflare.com/changelog/post/2026-10-02-pi-harness/)) keeps the state of pi-durable in the SQLite database of the Durable Object, in tables with the `pi_` prefix of its session store. Each statement became a `db.query` span: one run with one tool call had 189 of them next to its 7 agent spans, and the request that submitted the prompt had 120 more. These statements are framework bookkeeping, like the ones on the `cf_` tables of `agents`, so they are now skipped the same way, and `durableObjectSqlSpanAllowlist: [/^pi_/]` brings them back. The cost is the same as for `cf_`: a user table whose name starts with `pi_` also loses its spans until it is on the allowlist. Names such as `api_keys` or `pipelines` are not affected. Before: [196 spans](https://sentry-sdks.sentry.io/explore/traces/trace/7da432828f034a709be522a7de5bd4e2/), after: [7 spans](https://sentry-sdks.sentry.io/explore/traces/trace/0a8c909bd53a4d61b78f98af21fc1ebb/) for the same prompt. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- packages/cloudflare/src/client.ts | 12 +++++---- .../instrumentations/instrumentSqlStorage.ts | 2 +- .../cloudflare/src/utils/internalSqlQuery.ts | 19 ++++++++------ .../test/instrumentSqlStorage.test.ts | 25 ++++++++++++++++++- .../test/utils/internalSqlQuery.test.ts | 2 ++ 5 files changed, 46 insertions(+), 14 deletions(-) diff --git a/packages/cloudflare/src/client.ts b/packages/cloudflare/src/client.ts index 1a968cfe66b2..3b4f591d5e22 100644 --- a/packages/cloudflare/src/client.ts +++ b/packages/cloudflare/src/client.ts @@ -371,12 +371,14 @@ interface BaseCloudflareOptions { rpcTracePropagationBindings?: TracePropagationTargets; /** - * Table names that should stay instrumented even though they match the reserved `cf_` prefix used - * by Durable Object frameworks (`agents`, `partyserver`, ...) for their internal SQLite tables. + * Table names that should stay instrumented even though they match a reserved prefix used by + * Durable Object frameworks for their internal SQLite tables: `cf_` (`agents`, `partyserver`, ...) + * and `pi_` (pi-durable in the `PiHarness` of `agents`). * - * By default, `exec` queries against `cf_`-prefixed tables are treated as framework noise and no - * `db.query` span is created for them. If one of your own tables happens to use this prefix, add it - * here to opt it back into instrumentation. Entries are matched against each table name in the + * By default, `exec` queries against tables with these prefixes are treated as framework noise and + * no `db.query` span is created for them. If one of your own tables happens to use such a prefix, + * add it here to opt it back into instrumentation, or add `/^pi_/` to see the statements of + * pi-durable. Entries are matched against each table name in the * query summary — strings must match exactly, while regular expressions give you prefix/pattern * matching. * diff --git a/packages/cloudflare/src/instrumentations/instrumentSqlStorage.ts b/packages/cloudflare/src/instrumentations/instrumentSqlStorage.ts index 276cd0be5f8d..c34dab1214ba 100644 --- a/packages/cloudflare/src/instrumentations/instrumentSqlStorage.ts +++ b/packages/cloudflare/src/instrumentations/instrumentSqlStorage.ts @@ -37,7 +37,7 @@ export function instrumentSqlStorage(sql: SqlStorage): SqlStorage { if (childSpanWillNotBeRecorded() && !mayTargetCloudflareInternalTable(query)) { // This span is never sent, so skip the costly sanitize and summary // steps. We still start the span so it records its dropped span - // outcome. A query that may target a `cf_` table takes the full path, + // outcome. A query that may target a `cf_` or `pi_` table takes the full path, // because an internal query must start no span. return startSpan({ name: 'exec', attributes: SPAN_ATTRIBUTES }, callOriginal); } diff --git a/packages/cloudflare/src/utils/internalSqlQuery.ts b/packages/cloudflare/src/utils/internalSqlQuery.ts index 70d45a5af9d8..4ea558850527 100644 --- a/packages/cloudflare/src/utils/internalSqlQuery.ts +++ b/packages/cloudflare/src/utils/internalSqlQuery.ts @@ -9,8 +9,10 @@ import { stringMatchesSomePattern } from '@sentry/core'; * between framework versions, so we match the reserved prefix rather than an enumerated list. * * The `cf_` prefix is a reserved convention for framework-managed tables, so user tables should not - * use it. In case a user table does collide with the prefix, the `durableObjectSqlSpanAllowlist` - * option lets them opt those tables back into instrumentation. + * use it. The same goes for `pi_`: the `PiHarness` of `agents` keeps the state of pi-durable in + * tables with that prefix, and one run writes close to 200 statements. In case a user table does + * collide with a prefix, the `durableObjectSqlSpanAllowlist` option lets them opt those tables back + * into instrumentation. * * The check operates on the query summary produced by `getSqlQuerySummary` (`{operation} {table} ...`, * the same value used as the span name), so table targets are already isolated from the rest of the @@ -38,14 +40,16 @@ export function targetsCloudflareInternalTable( } /** - * Returns `false` when the raw `query` has no word that starts with `cf_`, so it cannot target a - * Cloudflare internal table. Needs no sanitizing, which makes it cheap enough to run on every query. + * Returns `false` when the raw `query` has no word that starts with `cf_` or `pi_`, so it cannot + * target a Cloudflare internal table. Needs no sanitizing, which makes it cheap enough to run on + * every query. */ export function mayTargetCloudflareInternalTable(query: string): boolean { - return CF_PREFIX_RE.test(query); + return INTERNAL_PREFIX_RE.test(query); } -const CF_PREFIX_RE = /\bcf_/i; +const INTERNAL_TABLE_PREFIXES = ['cf_', 'pi_']; +const INTERNAL_PREFIX_RE = /\b(?:cf|pi)_/i; // `CREATE [UNIQUE] INDEX [IF NOT EXISTS] ON ` — the IF EXISTS shape mirrors DDL_RE // in @sentry/core. @@ -53,7 +57,8 @@ const CREATE_INDEX_TABLE_RE = /^\s*CREATE\s+(?:UNIQUE\s+)?INDEX(?:\s+IF\s+(?:NOT\s+)?EXISTS)?\s+[^\s(,;)]+\s+ON\s+(?
[^\s(,;)]+)/i; function isCloudflareInternalTable(table: string, allowlist?: Array): boolean { - if (!table.toLowerCase().startsWith('cf_')) { + const name = table.toLowerCase(); + if (!INTERNAL_TABLE_PREFIXES.some(prefix => name.startsWith(prefix))) { return false; } diff --git a/packages/cloudflare/test/instrumentSqlStorage.test.ts b/packages/cloudflare/test/instrumentSqlStorage.test.ts index edf590f84a0f..7d0092bfcfed 100644 --- a/packages/cloudflare/test/instrumentSqlStorage.test.ts +++ b/packages/cloudflare/test/instrumentSqlStorage.test.ts @@ -218,6 +218,26 @@ describe('instrumentSqlStorage', () => { }); }); + // The Agents SDK `PiHarness` keeps the state of pi-durable in its own tables, all with the `pi_` + // prefix of its session store. + describe('pi-durable tables (pi_ prefix) are skipped', () => { + it.each([ + ['SELECT', 'SELECT record FROM pi_conversations WHERE id = ?'], + [ + 'upsert', + `INSERT INTO pi_tasks (id, conversation_id, kind, status, record) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET status = excluded.status, record = excluded.record`, + ], + ['INSERT OR IGNORE', 'INSERT OR IGNORE INTO pi_record_ids (id, record_type) VALUES (?, ?)'], + ['UPDATE', 'UPDATE pi_durable_metadata SET next_id = ?, next_seq = ? WHERE singleton = ?'], + ['DELETE', 'DELETE FROM pi_document_revisions WHERE document_id = ?'], + ['CREATE TABLE', 'CREATE TABLE pi_durable_schema (version INTEGER NOT NULL) STRICT'], + ['CREATE INDEX', 'CREATE INDEX pi_tasks_by_status ON pi_tasks (status, id)'], + ])('skips %s', (_label, query) => { + expect(execCreatesSpan(query)).toBe(false); + }); + }); + describe('user queries stay instrumented', () => { it.each([ ['SELECT', 'SELECT * FROM users WHERE id = ?'], @@ -228,6 +248,8 @@ describe('instrumentSqlStorage', () => { ['CREATE INDEX', 'CREATE INDEX idx_name ON users (name)'], ['table with cf in the middle', 'SELECT * FROM my_cf_table'], ['table starting with cfg', 'SELECT * FROM cfg_settings'], + ['table with pi_ in the middle', 'SELECT * FROM api_keys'], + ['table starting with pi', 'SELECT * FROM pipelines'], ['INSERT OR REPLACE', 'INSERT OR REPLACE INTO users (id, name) VALUES (?, ?)'], ['REPLACE INTO', 'REPLACE INTO sessions (id, token) VALUES (?, ?)'], ['UPDATE OR IGNORE', 'UPDATE OR IGNORE products SET price = ? WHERE id = ?'], @@ -241,10 +263,11 @@ describe('instrumentSqlStorage', () => { }); }); - describe('durableObjectSqlSpanAllowlist (opt a cf_ table back into instrumentation)', () => { + describe('durableObjectSqlSpanAllowlist (opt a cf_ or pi_ table back into instrumentation)', () => { it.each([ ['exact string', 'SELECT * FROM cf_my_table', ['cf_my_table']], ['regex', 'SELECT * FROM cf_reports_daily', [/^cf_reports_/]], + ['regex for the pi-durable tables', 'SELECT record FROM pi_conversations WHERE id = ?', [/^pi_/]], ['upsert target', 'INSERT OR REPLACE INTO cf_my_table (id) VALUES (?)', ['cf_my_table']], ['CREATE INDEX target', 'CREATE INDEX idx_mine ON cf_my_table (id)', ['cf_my_table']], ])('instruments an allowlisted table matched by %s', (_label, query, allowlist) => { diff --git a/packages/cloudflare/test/utils/internalSqlQuery.test.ts b/packages/cloudflare/test/utils/internalSqlQuery.test.ts index 0c9d6c641dde..7b15a5d6d737 100644 --- a/packages/cloudflare/test/utils/internalSqlQuery.test.ts +++ b/packages/cloudflare/test/utils/internalSqlQuery.test.ts @@ -31,6 +31,7 @@ describe('mayTargetCloudflareInternalTable', () => { ['a quoted cf_ table', 'SELECT * FROM "cf_agents_state"'], ['a schema-qualified cf_ table', 'SELECT * FROM main.cf_agents_state'], ['a cf_ table in a CREATE INDEX ON clause', 'CREATE INDEX idx_agents_state_id ON cf_agents_state (id)'], + ['a pi_ table', 'SELECT record FROM pi_tasks WHERE id = ?'], ])('returns true for %s', (_label, query) => { expect(mayTargetCloudflareInternalTable(query)).toBe(true); }); @@ -39,6 +40,7 @@ describe('mayTargetCloudflareInternalTable', () => { ['a user table', 'SELECT * FROM users WHERE id = ?'], ['a table with cf in the middle', 'SELECT * FROM my_cf_table'], ['a table starting with cfg', 'SELECT * FROM cfg_settings'], + ['a table with pi_ in the middle', 'SELECT * FROM api_keys'], ])('returns false for %s', (_label, query) => { expect(mayTargetCloudflareInternalTable(query)).toBe(false); }); From 221dd8647ffa32414d0be8ed514b3616ddf42053 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Peer=20St=C3=B6cklmair?= Date: Thu, 8 Oct 2026 15:49:03 +0300 Subject: [PATCH 42/47] test(e2e): Add a cloudflare-pi-durable end-to-end application (#25022) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cloudflare now hosts pi-durable in Durable Objects through the Agents SDK `PiHarness` ([changelog](https://developers.cloudflare.com/changelog/post/2026-10-02-pi-harness/)). This app runs pi-durable that way, with `@sentry/cloudflare/vite` as the only Sentry setup. It checks that a prompt becomes an `invoke_agent` trace with `chat`, `execute_tool` and provider `http.client` spans, that a throwing tool is reported on its span, and that a run resumes in a new trace after the Durable Object resets during a tool call. The reset uses `ctx.abort()`, which drops the object like an eviction; the next request starts it again, and `PiHarness` resumes pi from its SQLite state. Like `node-pi-durable` and `cloudflare-think`, it calls a real model through OpenRouter, so it is optional. The changelog example uses Workers AI (`createAI({ binding: env.AI })`), but the AI binding needs a Cloudflare account even in local dev, so that path has a unit test in #24993 instead. This app found a bug that #24993 now fixes: `@sentry/cloudflare` initializes the SDK inside each request, so the client is on the scope of that request only. The task phases ran on a copy of the default scope, had no client, and dropped every pi-durable span and error. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../cloudflare-pi-durable/package.json | 42 +++++ .../playwright.config.ts | 13 ++ .../cloudflare-pi-durable/src/assistant.ts | 96 ++++++++++++ .../cloudflare-pi-durable/src/env.d.ts | 9 ++ .../cloudflare-pi-durable/src/index.ts | 9 ++ .../src/instrument.server.ts | 10 ++ .../start-event-proxy.mjs | 6 + .../tests/pi-durable.test.ts | 147 ++++++++++++++++++ .../cloudflare-pi-durable/tests/utils.ts | 38 +++++ .../cloudflare-pi-durable/tsconfig.json | 20 +++ .../cloudflare-pi-durable/vite.config.mts | 7 + .../cloudflare-pi-durable/wrangler.toml | 18 +++ 12 files changed, 415 insertions(+) create mode 100644 dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/package.json create mode 100644 dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/playwright.config.ts create mode 100644 dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/src/assistant.ts create mode 100644 dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/src/env.d.ts create mode 100644 dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/src/index.ts create mode 100644 dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/src/instrument.server.ts create mode 100644 dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/start-event-proxy.mjs create mode 100644 dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/tests/pi-durable.test.ts create mode 100644 dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/tests/utils.ts create mode 100644 dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/tsconfig.json create mode 100644 dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/vite.config.mts create mode 100644 dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/wrangler.toml diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/package.json b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/package.json new file mode 100644 index 000000000000..5f172858bcf4 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/package.json @@ -0,0 +1,42 @@ +{ + "name": "cloudflare-pi-durable", + "version": "0.0.0", + "private": true, + "scripts": { + "dev": "vite dev", + "build": "vite build", + "preview": "vite preview --port 38789", + "typecheck": "tsc --noEmit", + "test:build": "pnpm install && pnpm build", + "test:build-latest": "pnpm install && pnpm add agents@latest @earendil-works/pi-durable@latest @earendil-works/pi-ai@latest && pnpm build", + "test:assert": "playwright test" + }, + "dependencies": { + "@earendil-works/pi-ai": "1.0.0", + "@earendil-works/pi-durable": "1.0.0", + "@sentry/cloudflare": "file:../../packed/sentry-cloudflare-packed.tgz", + "agents": "0.26.0" + }, + "devDependencies": { + "@cloudflare/vite-plugin": "1.57.2", + "@cloudflare/workers-types": "^4.20260426.0", + "@playwright/test": "~1.63.0", + "@sentry-internal/test-utils": "link:../../../test-utils", + "@sentry/conventions": "0.26.0", + "typescript": "^5.5.2", + "vite": "7.3.5", + "wrangler": "^4.136.2" + }, + "volta": { + "extends": "../../package.json" + }, + "sentryTest": { + "optional": true, + "optionalVariants": [ + { + "build-command": "pnpm test:build-latest", + "label": "cloudflare-pi-durable (latest)" + } + ] + } +} diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/playwright.config.ts b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/playwright.config.ts new file mode 100644 index 000000000000..1b9be61d8261 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/playwright.config.ts @@ -0,0 +1,13 @@ +import { getPlaywrightConfig } from '@sentry-internal/test-utils'; + +const config = getPlaywrightConfig( + { + startCommand: 'pnpm preview', + port: 38789, + }, + // Each test drives real OpenRouter tool-calling turns, and one resets the Durable Object mid-run, + // which does not fit the default 30s timeout when the provider is slow. + { timeout: 120_000 }, +); + +export default config; diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/src/assistant.ts b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/src/assistant.ts new file mode 100644 index 000000000000..bd0d25a8c5b3 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/src/assistant.ts @@ -0,0 +1,96 @@ +import { Type } from '@earendil-works/pi-ai'; +import { createModels } from '@earendil-works/pi-ai/models'; +import { openrouterProvider } from '@earendil-works/pi-ai/providers/openrouter'; +import { createRegistry, defineExtension, defineTool, Harness, section } from '@earendil-works/pi-durable'; +import * as Sentry from '@sentry/cloudflare'; +import { Agent } from 'agents'; +import { PiHarness } from 'agents/harness/pi'; + +/** + * pi-durable hosted by the Agents SDK `PiHarness`: pi keeps its state in this object's SQLite + * database, and the harness reopens pi when the object restarts. Nothing wires Sentry into + * pi-durable by hand, the Vite plugin injects the channel into `Harness.open()`. + */ +export class Assistant extends Agent { + public registry = createRegistry(); + + public harness = new PiHarness({ + harness: ({ storage, context }) => { + // pi-ai reads provider keys from `process.env` by default, so the Worker hands over its secret. + const models = createModels({ + authContext: { + env: async name => (name === 'OPENROUTER_API_KEY' ? this.env.E2E_OPENROUTER_API_KEY : undefined), + fileExists: async () => false, + }, + }); + models.setProvider(openrouterProvider()); + return Harness.open(storage, { models, registry: this.registry }, context); + }, + defaults: { model: { provider: 'openrouter', id: 'anthropic/claude-haiku-4.5' } }, + }); + + public constructor(ctx: DurableObjectState, env: Env) { + super(ctx, env); + this.registry.install( + defineExtension({ + name: 'e2e', + sections: [ + section('preamble', () => 'You are a test assistant. Use the tools exactly as asked. Keep answers short.', { + tag: false, + }), + ], + tools: [ + defineTool({ + name: 'get_weather', + description: 'Get the current weather for a city.', + parameters: Type.Object({ city: Type.String() }), + // The manual span should nest under the SDK's `execute_tool` span. + execute: async args => + Sentry.startSpan({ name: 'resolve-weather', attributes: { 'weather.city': args.city } }, () => ({ + content: [{ type: 'text', text: `It is 21 degrees and sunny in ${args.city}.` }], + })), + }), + defineTool({ + name: 'fail_now', + description: 'Always throws an error. Call this when the user asks to trigger a failure.', + parameters: Type.Object({}), + execute: async () => { + throw new Error('Intentional pi-durable tool failure'); + }, + }), + defineTool({ + name: 'crash_once', + description: 'Runs one step of a job. Call this when the user asks for crash_once.', + parameters: Type.Object({ job: Type.String() }), + // Replay-safe, so pi-durable reruns the call after the reset instead of failing it. + replay: 'safe', + execute: async args => { + const marker = `crashed:${args.job}`; + if (await this.ctx.storage.get(marker)) { + return { content: [{ type: 'text', text: `Step of job ${args.job} completed.` }] }; + } + await this.ctx.storage.put(marker, true); + await this.ctx.storage.sync(); + // Drops the object mid-call, like an eviction. The next request starts it again. + this.ctx.abort('crash_once resets the Durable Object'); + throw new Error('The Durable Object did not reset'); + }, + }), + ], + }), + ); + this.lifecycle.use(this.harness); + } + + public async onRequest(request: Request): Promise { + if (request.method === 'GET') { + const operationId = new URL(request.url).searchParams.get('operation') ?? ''; + const { status, reason, text } = await this.harness.wait(operationId); + return Response.json({ status, reason, text }); + } + + const { message, operationId } = await request.json<{ message: string; operationId?: string }>(); + const { status, reason, text } = await this.harness.prompt(message, { operationId }); + return Response.json({ status, reason, text }); + } +} diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/src/env.d.ts b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/src/env.d.ts new file mode 100644 index 000000000000..a088008cd966 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/src/env.d.ts @@ -0,0 +1,9 @@ +declare namespace Cloudflare { + interface Env { + E2E_TEST_DSN: string; + E2E_OPENROUTER_API_KEY: string; + Assistant: DurableObjectNamespace; + } +} + +interface Env extends Cloudflare.Env {} diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/src/index.ts b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/src/index.ts new file mode 100644 index 000000000000..756f31007b6b --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/src/index.ts @@ -0,0 +1,9 @@ +import { routeAgentRequest } from 'agents'; + +export { Assistant } from './assistant'; + +export default { + async fetch(request, env): Promise { + return (await routeAgentRequest(request, env)) ?? new Response('Not found', { status: 404 }); + }, +} satisfies ExportedHandler; diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/src/instrument.server.ts b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/src/instrument.server.ts new file mode 100644 index 000000000000..4bd8e8b9fdc0 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/src/instrument.server.ts @@ -0,0 +1,10 @@ +import { defineCloudflareOptions } from '@sentry/cloudflare'; + +// The Sentry Vite plugin picks this file up by convention, next to the worker entry named in +// wrangler's `main`, and hands its default export to `withSentry`. +export default defineCloudflareOptions((env: Env) => ({ + dsn: env.E2E_TEST_DSN, + environment: 'qa', + tunnel: 'http://localhost:3031/', + tracesSampleRate: 1.0, +})); diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/start-event-proxy.mjs b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/start-event-proxy.mjs new file mode 100644 index 000000000000..bdcf0b3d4465 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/start-event-proxy.mjs @@ -0,0 +1,6 @@ +import { startEventProxyServer } from '@sentry-internal/test-utils'; + +startEventProxyServer({ + port: 3031, + proxyServerName: 'cloudflare-pi-durable', +}); diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/tests/pi-durable.test.ts b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/tests/pi-durable.test.ts new file mode 100644 index 000000000000..bf2a4092188d --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/tests/pi-durable.test.ts @@ -0,0 +1,147 @@ +import { expect, test } from '@playwright/test'; +import { + GEN_AI_CONVERSATION_ID, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_FINISH_REASONS, + GEN_AI_TOOL_CALL_ARGUMENTS, + GEN_AI_TOOL_CALL_RESULT, + GEN_AI_TOOL_NAME, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + SENTRY_ORIGIN, + SERVER_ADDRESS, +} from '@sentry/conventions/attributes'; +import { DB_QUERY, GEN_AI_CHAT, GEN_AI_INVOKE_AGENT, HTTP_CLIENT, HTTP_SERVER } from '@sentry/conventions/op'; +import { collectStreamedSpans, getSpanOp, waitForError } from '@sentry-internal/test-utils'; +import { newAgentId, prompt, submitPrompt, waitForOperation } from './utils'; + +const APP = 'cloudflare-pi-durable'; + +test('traces a PiHarness prompt as invoke_agent with chat, execute_tool and provider spans', async ({ baseURL }) => { + // A run is its own trace, and its `invoke_agent` segment ends last. Only this test calls get_weather. + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => + spansOfTrace.some(span => span.is_segment && getSpanOp(span) === GEN_AI_INVOKE_AGENT) && + spansOfTrace.some(span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'get_weather'), + ); + + const answer = await prompt( + baseURL!, + newAgentId('weather'), + 'What is the weather in Vienna? Use the get_weather tool.', + ); + expect(answer.status, answer.reason).toBe('done'); + + const spans = await spansPromise; + const agent = spans.find(span => span.is_segment)!; + const chats = spans.filter(span => getSpanOp(span) === GEN_AI_CHAT); + // The manual span of the tool is the one way to pick the call that ran, should the model call twice. + const manualSpan = spans.find(span => span.name === 'resolve-weather')!; + const tool = spans.find(span => span.span_id === manualSpan.parent_span_id)!; + const providerCalls = spans.filter(span => getSpanOp(span) === HTTP_CLIENT); + + expect(agent.attributes[SENTRY_ORIGIN]?.value).toBe('auto.ai.pi_durable'); + // The PiHarness root session is pi conversation 1, in a Harness of its own per Durable Object. + expect(agent.attributes[GEN_AI_CONVERSATION_ID]?.value).toMatch(/^[0-9a-f]{32}:1$/); + // The request that submitted the prompt is a trace of its own: the scheduler runs the work. + expect(spans.some(span => getSpanOp(span) === HTTP_SERVER)).toBe(false); + // pi-durable keeps its state in the `pi_` tables of `PiHarness`, whose statements start no span. + expect(spans.filter(span => getSpanOp(span) === DB_QUERY && /\bpi_/.test(String(span.name)))).toEqual([]); + // pi-ai sends the requests through `@anthropic-ai/sdk`, whose own integration must stay out so + // each request is reported once. + expect(spans.filter(span => String(span.attributes[SENTRY_ORIGIN]?.value).startsWith('auto.ai.'))).toEqual( + spans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.pi_durable'), + ); + + // One tool-calling response, then the answer. + expect(chats.length).toBeGreaterThanOrEqual(2); + for (const chat of chats) { + expect(chat.parent_span_id).toBe(agent.span_id); + expect(chat.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('openrouter'); + expect(chat.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('anthropic/claude-haiku-4.5'); + expect(chat.attributes[GEN_AI_CONVERSATION_ID]?.value).toBe(agent.attributes[GEN_AI_CONVERSATION_ID]?.value); + // pi-durable retries a provider error inside the run; such a request has no usage and no HTTP span + // of its own to assert on. + if (chat.status === 'ok') { + expect(typeof chat.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe('number'); + expect(typeof chat.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe('number'); + expect(providerCalls.some(providerCall => providerCall.parent_span_id === chat.span_id)).toBe(true); + } + } + + expect(tool.attributes[GEN_AI_TOOL_NAME]?.value).toBe('get_weather'); + expect(tool.parent_span_id).toBe(agent.span_id); + expect(tool.status).toBe('ok'); + expect(tool.attributes[GEN_AI_TOOL_CALL_ARGUMENTS]?.value).toContain('Vienna'); + expect(tool.attributes[GEN_AI_TOOL_CALL_RESULT]?.value).toContain('21 degrees and sunny in'); + + // The provider's HTTP calls nest inside the `chat` span that sent them. + expect(providerCalls.length).toBeGreaterThan(0); + for (const providerCall of providerCalls) { + expect(chats.map(chat => chat.span_id)).toContain(providerCall.parent_span_id); + expect(providerCall.attributes[SERVER_ADDRESS]?.value).toBe('openrouter.ai'); + } +}); + +test('reports a throwing tool as an error on its execute_tool span', async ({ baseURL }) => { + const errorPromise = waitForError( + APP, + event => event.exception?.values?.[0]?.value === 'Intentional pi-durable tool failure', + ); + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => + spansOfTrace.some(span => span.is_segment && getSpanOp(span) === GEN_AI_INVOKE_AGENT) && + spansOfTrace.some(span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'fail_now'), + ); + + const answer = await prompt(baseURL!, newAgentId('fail'), 'Call the fail_now tool, then tell me what happened.'); + expect(answer.status, answer.reason).toBe('done'); + + const [error, spans] = await Promise.all([errorPromise, spansPromise]); + // The error is reported on the span of the call that threw. + const tool = spans.find(span => span.span_id === error.contexts?.trace?.span_id)!; + + expect(tool.attributes[GEN_AI_TOOL_NAME]?.value).toBe('fail_now'); + expect(tool.status).toBe('error'); + expect(tool.trace_id).toBe(error.contexts?.trace?.trace_id); + expect(error.exception?.values?.[0]?.mechanism).toEqual({ type: 'auto.ai.pi_durable', handled: false }); +}); + +test('resumes a run in a new trace after the Durable Object resets during a tool call', async ({ baseURL }) => { + const agentId = newAgentId('reset'); + const operationId = crypto.randomUUID(); + // The tool span of the reset object never ends, so a finished `crash_once` span can only come + // from the rerun after the restart. + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => + spansOfTrace.some(span => span.is_segment && getSpanOp(span) === GEN_AI_INVOKE_AGENT) && + spansOfTrace.some(span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'crash_once' && span.status === 'ok'), + ); + + // The reset fails the request that submitted the prompt, but pi has stored the input already. + await submitPrompt( + baseURL!, + agentId, + 'Run the crash_once tool for job "nightly", then reply with the word DONE.', + operationId, + ); + const answer = await waitForOperation(baseURL!, agentId, operationId); + expect(answer.status, answer.reason).toBe('done'); + + const spans = await spansPromise; + const agent = spans.find(span => span.is_segment)!; + const tool = spans.find(span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'crash_once')!; + const chats = spans.filter(span => getSpanOp(span) === GEN_AI_CHAT); + const finalAnswer = chats.find(span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["stop"]'); + + expect(getSpanOp(agent)).toBe(GEN_AI_INVOKE_AGENT); + expect(tool.parent_span_id).toBe(agent.span_id); + expect(finalAnswer?.parent_span_id).toBe(agent.span_id); + // The request that called the tool ran before the reset, so this trace starts with the rerun of the + // tool. A run that never reset would start with that request. + expect(chats.every(chat => chat.start_timestamp >= tool.start_timestamp)).toBe(true); +}); diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/tests/utils.ts b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/tests/utils.ts new file mode 100644 index 000000000000..0e23ca86301e --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/tests/utils.ts @@ -0,0 +1,38 @@ +import { expect } from '@playwright/test'; + +export type Operation = { status: string; reason?: string; text?: string }; + +/** + * A Durable Object name nothing has used yet. The name is the `:name` path segment + * `routeAgentRequest` maps to an object, so each test gets its own SQLite database and pi Harness. + */ +export function newAgentId(prefix: string): string { + return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; +} + +/** Submit a prompt to the root session. The response arrives once pi settles it. */ +export function submitPrompt( + baseURL: string, + agentId: string, + message: string, + operationId?: string, +): Promise { + return fetch(`${baseURL}/agents/assistant/${agentId}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ message, operationId }), + }); +} + +export async function prompt(baseURL: string, agentId: string, message: string): Promise { + const res = await submitPrompt(baseURL, agentId, message); + expect(res.status).toBe(200); + return (await res.json()) as Operation; +} + +/** Wait until pi settles an operation. The request starts the object again if it was reset. */ +export async function waitForOperation(baseURL: string, agentId: string, operationId: string): Promise { + const res = await fetch(`${baseURL}/agents/assistant/${agentId}?operation=${operationId}`); + expect(res.status).toBe(200); + return (await res.json()) as Operation; +} diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/tsconfig.json b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/tsconfig.json new file mode 100644 index 000000000000..40863136da68 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "es2022", + "lib": ["es2022"], + "module": "es2022", + "moduleResolution": "Bundler", + "resolveJsonModule": true, + "allowJs": true, + "checkJs": false, + "noEmit": true, + "isolatedModules": true, + "allowSyntheticDefaultImports": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipLibCheck": true, + "types": ["@cloudflare/workers-types"] + }, + "exclude": ["tests"], + "include": ["src/**/*.ts"] +} diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/vite.config.mts b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/vite.config.mts new file mode 100644 index 000000000000..005f4448f6cb --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/vite.config.mts @@ -0,0 +1,7 @@ +import { cloudflare } from '@cloudflare/vite-plugin'; +import { sentryCloudflareVitePlugin } from '@sentry/cloudflare/vite'; +import { defineConfig } from 'vite'; + +export default defineConfig({ + plugins: [cloudflare(), sentryCloudflareVitePlugin()], +}); diff --git a/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/wrangler.toml b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/wrangler.toml new file mode 100644 index 000000000000..1b2207bd862f --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/cloudflare-pi-durable/wrangler.toml @@ -0,0 +1,18 @@ +#:schema node_modules/wrangler/config-schema.json +name = "cloudflare-pi-durable" +main = "src/index.ts" +compatibility_date = "2026-09-15" +compatibility_flags = ["nodejs_compat"] + +# `vite preview` takes no `--var` flag, so the secrets are declared here and read from +# the process environment the Playwright web server inherits. +[secrets] +required = ["E2E_TEST_DSN", "E2E_OPENROUTER_API_KEY"] + +[[durable_objects.bindings]] +name = "Assistant" +class_name = "Assistant" + +[[migrations]] +tag = "v1" +new_sqlite_classes = ["Assistant"] From 5bb9a6832c37236240cce9766658ab4894e0458e Mon Sep 17 00:00:00 2001 From: Andrei <168741329+andreiborza@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:50:13 +0200 Subject: [PATCH 43/47] feat(server-utils): Map Mastra classifier evaluations to gen_ai.evaluate spans (#25044) ## What Mastra classifier evaluations (Jev calls through Mastra's `Classifier`, its processors and scorers) now show up as `gen_ai.evaluate` spans with model, provider and token usage. When gen_ai recording is on, they also record the evaluated state, questions and answers. ## Why These calls skip the AI SDK hook, and our Mastra exporter dropped their spans, so they were invisible. Closes: #25031 --------- Co-authored-by: Claude Opus 5.5 --- .../mastra/scenario-classifier-error.mjs | 48 +++++ .../tracing/mastra/scenario-classifier.mjs | 51 ++++++ .../suites/tracing/mastra/test.ts | 117 +++++++++++- packages/server-utils/src/ai/core/utils.ts | 10 ++ .../src/ai/mastra/classifier-evaluation.ts | 58 ++++++ .../server-utils/src/ai/mastra/constants.ts | 9 +- packages/server-utils/src/ai/mastra/index.ts | 66 +++++-- packages/server-utils/src/ai/mastra/types.ts | 15 ++ packages/server-utils/src/ai/mastra/utils.ts | 28 ++- .../server-utils/src/ai/typesafe/index.ts | 11 +- .../server-utils/src/ai/workers-ai/utils.ts | 9 +- .../src/integrations/mastra-classifier.ts | 73 ++++++++ .../server-utils/src/integrations/mastra.ts | 2 + .../src/orchestrion/config/mastra.ts | 21 ++- .../ai/lib/tracing/mastra-exporter.test.ts | 167 +++++++++++++----- 15 files changed, 600 insertions(+), 85 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier-error.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier.mjs create mode 100644 packages/server-utils/src/ai/mastra/classifier-evaluation.ts create mode 100644 packages/server-utils/src/integrations/mastra-classifier.ts diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier-error.mjs b/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier-error.mjs new file mode 100644 index 000000000000..7a70b22c7527 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier-error.mjs @@ -0,0 +1,48 @@ +import * as Sentry from '@sentry/node'; +import { Mastra } from '@mastra/core'; +import { Classifier } from '@mastra/core/classifier'; +import { Observability } from '@mastra/observability'; +import { SentryMastraExporter } from '@sentry/node'; + +// Inlined `EvaluationModelV4` mock: the ESM/CJS runner copies only this file into its temp dir. +const evaluationModel = { + specificationVersion: 'v4', + provider: 'typesafe-ai.evaluation', + modelId: 'jev-latest', + supportedQuestionTypes: ['choice', 'score', 'boolean'], + async doEvaluate() { + throw new Error('evaluation failed'); + }, +}; + +async function run() { + const classifier = new Classifier({ + id: 'request-classifier', + model: evaluationModel, + questions: { urgent: { type: 'boolean', instructions: 'Does this request need an immediate response?' } }, + }); + + const mastra = new Mastra({ + classifiers: { classifier }, + logger: false, + observability: new Observability({ + configs: { + default: { + serviceName: 'mastra-test', + exporters: [new SentryMastraExporter()], + }, + }, + }), + }); + + await Sentry.startSpan({ op: 'function', name: 'mastra-test' }, async () => { + await mastra + .getClassifier('classifier') + .evaluate({ state: { message: 'I was charged twice.' }, maxRetries: 0 }) + .catch(() => undefined); + }); + + await mastra.observability.shutdown(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier.mjs b/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier.mjs new file mode 100644 index 000000000000..e4e5efb15074 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-classifier.mjs @@ -0,0 +1,51 @@ +import * as Sentry from '@sentry/node'; +import { Mastra } from '@mastra/core'; +import { Classifier } from '@mastra/core/classifier'; +import { Observability } from '@mastra/observability'; +import { SentryMastraExporter } from '@sentry/node'; + +// Inlined `EvaluationModelV4` mock: the ESM/CJS runner copies only this file into its temp dir. +const evaluationModel = { + specificationVersion: 'v4', + provider: 'typesafe-ai.evaluation', + modelId: 'jev-latest', + supportedQuestionTypes: ['choice', 'score', 'boolean'], + async doEvaluate() { + return { + answers: { urgent: { type: 'boolean', probability: 0.9 } }, + usage: { inputTokens: 30, outputTokens: 2 }, + warnings: [], + }; + }, +}; + +async function run() { + const classifier = new Classifier({ + id: 'request-classifier', + model: evaluationModel, + questions: { urgent: { type: 'boolean', instructions: 'Does this request need an immediate response?' } }, + }); + + const mastra = new Mastra({ + classifiers: { classifier }, + logger: false, + observability: new Observability({ + configs: { + default: { + serviceName: 'mastra-test', + exporters: [new SentryMastraExporter()], + }, + }, + }), + }); + + await Sentry.startSpan({ op: 'function', name: 'mastra-test' }, async () => { + await mastra + .getClassifier('classifier') + .evaluate({ state: { message: 'I was charged twice.', apiKey: 'sk-test-123' } }); + }); + + await mastra.observability.shutdown(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts index 281362c074f4..d26ea2281428 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/test.ts @@ -20,6 +20,7 @@ import { SENTRY_ORIGIN, URL_FULL, } from '@sentry/conventions/attributes'; +import { GEN_AI_CHAT, GEN_AI_EVALUATE, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op'; import { afterAll, expect } from 'vitest'; import { conditionalTest } from '../../../utils'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; @@ -43,6 +44,14 @@ const MASTRA_NESTING_DEPENDENCIES = { }, }; +// `Classifier` (the `@mastra/core/classifier` entry) ships in newer `@mastra/core` releases only. +const MASTRA_CLASSIFIER_DEPENDENCIES = { + additionalDependencies: { + '@mastra/core': '1.74.0', + '@mastra/observability': '1.18.3', + }, +}; + conditionalTest({ min: 22 })('Mastra integration', () => { afterAll(() => { cleanupChildProcesses(); @@ -63,7 +72,7 @@ conditionalTest({ min: 22 })('Mastra integration', () => { const agentSpan = spans.find(span => span.name === 'invoke_agent weather_agent')!; expect(agentSpan.status).toBe('ok'); - expect(agentSpan.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(agentSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_INVOKE_AGENT); expect(agentSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra'); expect(agentSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('invoke_agent'); expect(agentSpan.attributes[GEN_AI_AGENT_NAME].value).toBe('weather_agent'); @@ -75,7 +84,7 @@ conditionalTest({ min: 22 })('Mastra integration', () => { const chatSpan = spans.find(span => span.name === 'chat gpt-4o-mini')!; expect(chatSpan.status).toBe('ok'); - expect(chatSpan.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(chatSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_CHAT); expect(chatSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra'); expect(chatSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); expect(chatSpan.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gpt-4o-mini'); @@ -157,7 +166,7 @@ conditionalTest({ min: 22 })('Mastra integration', () => { const toolSpan = spans.find(span => span.name === 'execute_tool get_weather')!; expect(toolSpan.status).toBe('ok'); - expect(toolSpan.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); + expect(toolSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EXECUTE_TOOL); expect(toolSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra'); expect(toolSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('execute_tool'); expect(toolSpan.attributes[GEN_AI_TOOL_NAME].value).toBe('get_weather'); @@ -196,7 +205,7 @@ conditionalTest({ min: 22 })('Mastra integration', () => { expect(spans.map(span => span.name)).toEqual(['invoke_agent math_workflow']); const workflowSpan = spans[0]!; - expect(workflowSpan.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(workflowSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_INVOKE_AGENT); expect(workflowSpan.attributes[GEN_AI_PIPELINE_NAME].value).toBe('math_workflow'); }, }) @@ -207,6 +216,98 @@ conditionalTest({ min: 22 })('Mastra integration', () => { MASTRA_DEPENDENCIES, ); + createEsmAndCjsTests( + __dirname, + 'scenario-classifier.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('maps a classifier evaluation to a gen_ai.evaluate span', async () => { + await createRunner() + .expect({ + span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra'); + expect(spans.map(span => span.name)).toEqual(['evaluate jev-latest']); + + const evaluateSpan = spans[0]!; + expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE); + expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.mastra'); + expect(evaluateSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('evaluate'); + expect(evaluateSpan.attributes[GEN_AI_REQUEST_MODEL].value).toBe('jev-latest'); + expect(evaluateSpan.attributes[GEN_AI_PROVIDER_NAME].value).toBe('typesafe-ai.evaluation'); + expect(evaluateSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(30); + expect(evaluateSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(2); + expect(evaluateSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(32); + expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); + expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); + }, + }) + .start() + .completed(); + }); + }, + MASTRA_CLASSIFIER_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-classifier.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('records the evaluated state, questions and answers when genAI recording is on', async () => { + await createRunner() + .expect({ + span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); + const evaluateSpan = container.items.find(span => span.name === 'evaluate jev-latest')!; + expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ + { + type: 'evaluation', + state: { message: 'I was charged twice.', apiKey: '[REDACTED]' }, + questions: { + urgent: { type: 'boolean', instructions: 'Does this request need an immediate response?' }, + }, + }, + ]); + expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([ + { type: 'evaluation', answers: { urgent: { type: 'boolean', probability: 0.9 } } }, + ]); + }, + }) + .start() + .completed(); + }); + }, + MASTRA_CLASSIFIER_DEPENDENCIES, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-classifier-error.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('ends a failed classifier evaluation span with an error status', async () => { + await createRunner() + .ignore('event') + .expect({ + span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'mastra-test')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra'); + expect(spans.map(span => span.name)).toEqual(['evaluate jev-latest']); + + const evaluateSpan = spans[0]!; + expect(evaluateSpan.status).toBe('error'); + expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value).toContain('I was charged twice.'); + expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); + }, + }) + .start() + .completed(); + }); + }, + MASTRA_CLASSIFIER_DEPENDENCIES, + ); + createEsmAndCjsTests( __dirname, 'scenario-auto.mjs', @@ -302,14 +403,14 @@ conditionalTest({ min: 22 })('Mastra integration', () => { ); expect(loadSpans.length).toBeGreaterThan(0); const cacheGetParentIds = loadSpans.map(span => span.parent_span_id); - const chat = container.items.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.chat')!; + const chat = container.items.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_CHAT)!; expect(chat).toBeDefined(); const chatSpanId = chat.span_id; const executeTool = container.items.find( span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'count_items', )!; - expect(executeTool.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); + expect(executeTool.attributes[SENTRY_OP]?.value).toBe(GEN_AI_EXECUTE_TOOL); const executeToolSpanId = executeTool.span_id; // The `executeWithContext` bridge makes the exporter spans active during the real work, so the @@ -355,14 +456,14 @@ conditionalTest({ min: 22 })('Mastra integration', () => { ); expect(loadSpans.length).toBeGreaterThan(0); const cacheGetParentIds = loadSpans.map(span => span.parent_span_id); - const chat = container.items.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.chat')!; + const chat = container.items.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_CHAT)!; expect(chat).toBeDefined(); const chatSpanId = chat.span_id; const executeTool = container.items.find( span => span.attributes[GEN_AI_TOOL_NAME]?.value === 'count_items', )!; - expect(executeTool.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); + expect(executeTool.attributes[SENTRY_OP]?.value).toBe(GEN_AI_EXECUTE_TOOL); const executeToolSpanId = executeTool.span_id; expect(chatSpanId).toBeDefined(); diff --git a/packages/server-utils/src/ai/core/utils.ts b/packages/server-utils/src/ai/core/utils.ts index 49f76fb18998..93949ec40eca 100644 --- a/packages/server-utils/src/ai/core/utils.ts +++ b/packages/server-utils/src/ai/core/utils.ts @@ -136,6 +136,16 @@ export function getTokenUsageAttributes( return attributes; } +/** Serialize the `state` and `questions` of an evaluation request (TypeSafe, Workers AI, Mastra classifiers). */ +export function getEvaluationInputMessages(request: Record): string | undefined { + return stringify([{ type: 'evaluation', state: request.state, questions: request.questions }]); +} + +/** Serialize the `answers` of an evaluation result (TypeSafe, Workers AI, Mastra classifiers). */ +export function getEvaluationOutputMessages(answers: unknown): string | undefined { + return stringify([{ type: 'evaluation', answers }]); +} + /** One assistant turn for {@link setOutputMessagesAttribute}. */ export interface GenAiOutputMessage { /** The message's text content, already flattened out of any content-part array. */ diff --git a/packages/server-utils/src/ai/mastra/classifier-evaluation.ts b/packages/server-utils/src/ai/mastra/classifier-evaluation.ts new file mode 100644 index 000000000000..952748b89512 --- /dev/null +++ b/packages/server-utils/src/ai/mastra/classifier-evaluation.ts @@ -0,0 +1,58 @@ +import type { Span, SpanTimeInput } from '@sentry/core'; +import type { MastraExportedSpan, MastraSpanOutputProcessor } from './types'; + +/** + * Links a `Classifier.evaluate()` call to the Sentry span the exporter opens for its + * `classifier_evaluation` span. Mastra's span carries no input or output, so the integration adds + * them from the call. Mastra ends its span before `evaluate()` returns the answers, so the exporter + * leaves the Sentry span open and the integration ends it once the call settles. + */ +export interface ClassifierEvaluationCall { + span?: Span; + /** Mastra's span and the processors of its observability instance, to filter the call's data. */ + mastraSpan?: MastraExportedSpan; + spanOutputProcessors?: MastraSpanOutputProcessor[]; + /** The recording options of the exporter that opened the span, so the call's data follows them. */ + recordInputs?: boolean; + recordOutputs?: boolean; + /** Mastra's end time, set when Mastra ended its span before the call settled. */ + endTime?: SpanTimeInput; + settled?: boolean; +} + +// Mastra emits `span_started` synchronously inside `evaluate()`, before its first `await`, so at +// most one call is ever in this window. +let startingCall: ClassifierEvaluationCall | undefined; + +export function setStartingClassifierEvaluation(call: ClassifierEvaluationCall | undefined): void { + startingCall = call; +} + +export function takeStartingClassifierEvaluation(): ClassifierEvaluationCall | undefined { + const call = startingCall; + startingCall = undefined; + return call; +} + +/** + * Run the call's data through Mastra's span output processors (for example its default + * `SensitiveDataFilter`), as Mastra does for the input and output of its own spans. Returns `undefined` + * when a processor drops the span or throws, so unfiltered data is never recorded. + */ +export function processClassifierEvaluationData( + call: ClassifierEvaluationCall, + data: { input?: unknown; output?: unknown }, +): { input?: unknown; output?: unknown } | undefined { + let span: MastraExportedSpan | undefined = { ...call.mastraSpan, ...data } as MastraExportedSpan; + for (const processor of call.spanOutputProcessors ?? []) { + try { + span = processor.process(span); + } catch { + return undefined; + } + if (!span) { + return undefined; + } + } + return span; +} diff --git a/packages/server-utils/src/ai/mastra/constants.ts b/packages/server-utils/src/ai/mastra/constants.ts index 26a05910c45d..ab55142a2bf5 100644 --- a/packages/server-utils/src/ai/mastra/constants.ts +++ b/packages/server-utils/src/ai/mastra/constants.ts @@ -1,4 +1,10 @@ -import { GEN_AI_CHAT, GEN_AI_EMBEDDINGS, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op'; +import { + GEN_AI_CHAT, + GEN_AI_EMBEDDINGS, + GEN_AI_EVALUATE, + GEN_AI_EXECUTE_TOOL, + GEN_AI_INVOKE_AGENT, +} from '@sentry/conventions/op'; import type { MastraSpanType } from './types'; export const MASTRA_INTEGRATION_NAME = 'Mastra' as const; @@ -36,6 +42,7 @@ export const SPAN_TYPE_OPS: Readonly = new Set([ diff --git a/packages/server-utils/src/ai/mastra/index.ts b/packages/server-utils/src/ai/mastra/index.ts index 0bfcdd712777..7943275fe6bd 100644 --- a/packages/server-utils/src/ai/mastra/index.ts +++ b/packages/server-utils/src/ai/mastra/index.ts @@ -7,6 +7,7 @@ import { LRUMap, SPAN_STATUS_ERROR, startInactiveSpan, + timestampInSeconds, } from '@sentry/core'; import { GEN_AI_RESPONSE_MODEL, SENTRY_OP, SENTRY_ORIGIN } from '@sentry/conventions/attributes'; import { DEBUG_BUILD } from '../../debug-build'; @@ -19,6 +20,7 @@ import { getUsageAttributes, isExportedSpanType, mergeUsageAttributes, + type AttributeRecordingOptions, type SpanAttributes, } from './utils'; import { @@ -29,8 +31,17 @@ import { MAX_TRACKED_MASTRA_SPANS, MODEL_SPAN_TYPES, } from './constants'; +import type { ClassifierEvaluationCall } from './classifier-evaluation'; +import { takeStartingClassifierEvaluation } from './classifier-evaluation'; import { registerMastraSpan, unregisterMastraSpan } from './span-registry'; -import type { MastraExportedSpan, MastraObservabilityExporter, MastraSpanType, MastraTracingEvent } from './types'; +import type { + MastraExportedSpan, + MastraExporterInitOptions, + MastraObservabilityExporter, + MastraSpanOutputProcessor, + MastraSpanType, + MastraTracingEvent, +} from './types'; export type MastraExporterOptions = GenAiOptions; @@ -38,6 +49,7 @@ interface TrackedSpan { span: Span; spanType: MastraSpanType; usage: SpanAttributes; + evaluation?: ClassifierEvaluationCall; } const FLUSH_TIMEOUT_MS = 2000; @@ -64,11 +76,17 @@ export class SentryMastraExporter implements MastraObservabilityExporter { */ private readonly _skipped = new LRUMap(MAX_TRACKED_MASTRA_SPANS); private readonly _options: MastraExporterOptions; + private _spanOutputProcessors: MastraSpanOutputProcessor[] = []; public constructor(options: MastraExporterOptions = {}) { this._options = options; } + /** Called by Mastra with the config of the observability instance this exporter belongs to. */ + public init(options: MastraExporterInitOptions): void { + this._spanOutputProcessors = options.config?.spanOutputProcessors ?? []; + } + /** Mastra's interface is async; the work is synchronous. */ public async exportTracingEvent(event: MastraTracingEvent): Promise { if (!getClient()) { @@ -117,20 +135,22 @@ export class SentryMastraExporter implements MastraObservabilityExporter { return; } + // Per event: the exporter can be constructed before `Sentry.init()`, when `dataCollection.genAI` does not exist yet. + const recordingOptions = resolveAIRecordingOptions(this._options); switch (event.type) { case 'span_started': - this._onSpanStarted(span); + this._onSpanStarted(span, recordingOptions); break; case 'span_updated': - this._onSpanUpdated(span); + this._onSpanUpdated(span, recordingOptions); break; case 'span_ended': - this._onSpanEnded(span); + this._onSpanEnded(span, recordingOptions); break; } } - private _onSpanStarted(span: MastraExportedSpan): void { + private _onSpanStarted(span: MastraExportedSpan, recordingOptions: AttributeRecordingOptions): void { const parentId = this._resolveParentId(span.parentSpanId); const parentSpan = parentId ? this._spans.get(parentId)?.span : undefined; const activeSpan = getActiveSpan(); @@ -141,13 +161,24 @@ export class SentryMastraExporter implements MastraObservabilityExporter { // Prefer the Mastra parent so the tree stays together; else the active request span. parentSpan: parentSpan ?? activeSpan, attributes: { - ...this._attributesFor(span), + ...getSpanAttributes(span, recordingOptions), [SENTRY_OP]: getOperation(span.type)?.op, [SENTRY_ORIGIN]: MASTRA_ORIGIN, }, }); - this._trackSpan(span.id, { span: sentrySpan, spanType: span.type, usage: {} }); + const evaluation = span.type === 'classifier_evaluation' ? takeStartingClassifierEvaluation() : undefined; + if (evaluation) { + Object.assign(evaluation, { + span: sentrySpan, + mastraSpan: span, + spanOutputProcessors: this._spanOutputProcessors, + recordInputs: recordingOptions.recordInputs, + recordOutputs: recordingOptions.recordOutputs, + }); + } + + this._trackSpan(span.id, { span: sentrySpan, spanType: span.type, usage: {}, evaluation }); } /** Track a started span, ending any Sentry span that would otherwise be dropped without `end()`. */ @@ -174,14 +205,14 @@ export class SentryMastraExporter implements MastraObservabilityExporter { return this._spans.remove(id); } - private _onSpanUpdated(span: MastraExportedSpan): void { + private _onSpanUpdated(span: MastraExportedSpan, recordingOptions: AttributeRecordingOptions): void { const tracked = this._spans.get(span.id); if (tracked) { - tracked.span.setAttributes(this._attributesFor(span)); + tracked.span.setAttributes(getSpanAttributes(span, recordingOptions)); } } - private _onSpanEnded(span: MastraExportedSpan): void { + private _onSpanEnded(span: MastraExportedSpan, recordingOptions: AttributeRecordingOptions): void { const tracked = this._spans.get(span.id); if (!tracked) { DEBUG_BUILD && debug.warn(`[Mastra] no Sentry span open for ended span ${span.id} (${span.name})`); @@ -189,7 +220,7 @@ export class SentryMastraExporter implements MastraObservabilityExporter { } const { span: sentrySpan } = tracked; - sentrySpan.setAttributes(this._attributesFor(span)); + sentrySpan.setAttributes(getSpanAttributes(span, recordingOptions)); sentrySpan.updateName(getSpanName(span)); if (MODEL_SPAN_TYPES.has(span.type)) { @@ -201,7 +232,13 @@ export class SentryMastraExporter implements MastraObservabilityExporter { sentrySpan.setStatus({ code: SPAN_STATUS_ERROR, message: span.errorInfo.message }); } - sentrySpan.end(span.endTime); + const { evaluation } = tracked; + if (evaluation && !evaluation.settled) { + // The integration adds the call's input and answers once `evaluate()` settles, then ends the span. + evaluation.endTime = span.endTime ?? timestampInSeconds(); + } else { + sentrySpan.end(span.endTime); + } this._removeTracked(span.id); } @@ -240,9 +277,4 @@ export class SentryMastraExporter implements MastraObservabilityExporter { } return undefined; } - - private _attributesFor(span: MastraExportedSpan): ReturnType { - // Per event: the exporter can be constructed before `Sentry.init()`, when `dataCollection.genAI` does not exist yet. - return getSpanAttributes(span, resolveAIRecordingOptions(this._options)); - } } diff --git a/packages/server-utils/src/ai/mastra/types.ts b/packages/server-utils/src/ai/mastra/types.ts index 35d125644346..5de6ad91efcc 100644 --- a/packages/server-utils/src/ai/mastra/types.ts +++ b/packages/server-utils/src/ai/mastra/types.ts @@ -15,6 +15,7 @@ export type MastraSpanType = | 'provider_tool_call' | 'client_tool_call' | 'rag_embedding' + | 'classifier_evaluation' | (string & {}); export type MastraTracingEventType = 'span_started' | 'span_updated' | 'span_ended'; @@ -58,6 +59,8 @@ export interface MastraSpanAttributes { stopSequences?: string[]; seed?: number; }; + // classifier_evaluation + modelId?: string; // tool_call / mcp_tool_call / provider_tool_call / client_tool_call toolDescription?: string; // agent_run / workflow_run @@ -69,6 +72,7 @@ export interface MastraSpanAttributes { export interface MastraExportedSpan { id: string; + traceId?: string; parentSpanId?: string; /** Mastra's own name; unused for Sentry span names. */ name: string; @@ -91,6 +95,17 @@ export interface MastraTracingEvent { exportedSpan: MastraExportedSpan; } +/** Mastra's `SpanOutputProcessor`, such as its default `SensitiveDataFilter`. */ +export interface MastraSpanOutputProcessor { + name: string; + process(span: MastraExportedSpan): MastraExportedSpan | undefined; +} + +/** Subset of what Mastra passes to `ObservabilityExporter.init()`. */ +export interface MastraExporterInitOptions { + config?: { spanOutputProcessors?: MastraSpanOutputProcessor[] }; +} + /** Subset of Mastra's `ObservabilityExporter`. The rest of that interface is optional. */ export interface MastraObservabilityExporter { name: string; diff --git a/packages/server-utils/src/ai/mastra/utils.ts b/packages/server-utils/src/ai/mastra/utils.ts index 0cdb20e1b80f..5fa9f1b62316 100644 --- a/packages/server-utils/src/ai/mastra/utils.ts +++ b/packages/server-utils/src/ai/mastra/utils.ts @@ -61,14 +61,21 @@ export function getOperation(spanType: MastraSpanType): { op: string; operationN */ export function getSpanName(span: MastraExportedSpan): string { const operationName = getOperation(span.type)?.operationName ?? span.type; - const identifier = - MODEL_SPAN_TYPES.has(span.type) || span.type === 'rag_embedding' - ? span.attributes?.model - : (span.entityName ?? span.entityId); + const identifier = getSpanIdentifier(span); return identifier ? `${operationName} ${identifier}` : operationName; } +function getSpanIdentifier(span: MastraExportedSpan): string | undefined { + if (span.type === 'classifier_evaluation') { + return span.attributes?.modelId; + } + if (MODEL_SPAN_TYPES.has(span.type) || span.type === 'rag_embedding') { + return span.attributes?.model; + } + return span.entityName ?? span.entityId; +} + function serialize(value: unknown): string | undefined { if (value === undefined || value === null) { return undefined; @@ -157,6 +164,10 @@ export function getSpanAttributes(span: MastraExportedSpan, options: AttributeRe addToolAttributes(attributes, span, options); } + if (span.type === 'classifier_evaluation') { + addEvaluationAttributes(attributes, span); + } + if (span.errorInfo) { attributes[ERROR_TYPE] = span.errorInfo.name ?? span.errorInfo.id; } @@ -243,6 +254,15 @@ function addToolAttributes( } } +/** Mastra's `Classifier` sets no input or output on this span, so only the model and usage are known. */ +function addEvaluationAttributes(attributes: SpanAttributes, span: MastraExportedSpan): void { + const attrs = span.attributes ?? {}; + + attributes[GEN_AI_REQUEST_MODEL] = attrs.modelId; + attributes[GEN_AI_PROVIDER_NAME] = attrs.provider; + Object.assign(attributes, getUsageAttributes(attrs.usage)); +} + function responseText(output: unknown): string | undefined { if (typeof output === 'string') { return output; diff --git a/packages/server-utils/src/ai/typesafe/index.ts b/packages/server-utils/src/ai/typesafe/index.ts index 9552e5e96a4d..99280c6843ad 100644 --- a/packages/server-utils/src/ai/typesafe/index.ts +++ b/packages/server-utils/src/ai/typesafe/index.ts @@ -12,9 +12,9 @@ import { } from '@sentry/conventions/attributes'; import { GEN_AI_EVALUATE } from '@sentry/conventions/op'; import type { Span, SpanAttributes } from '@sentry/core'; -import { isObjectLike, SPAN_STATUS_ERROR, startInactiveSpan, stringify, withActiveSpan } from '@sentry/core'; +import { isObjectLike, SPAN_STATUS_ERROR, startInactiveSpan, withActiveSpan } from '@sentry/core'; import type { GenAiOptions } from '../core/utils'; -import { resolveAIRecordingOptions } from '../core/utils'; +import { getEvaluationInputMessages, getEvaluationOutputMessages, resolveAIRecordingOptions } from '../core/utils'; import { TYPESAFE_ORIGIN, TYPESAFE_PROVIDER_NAME } from './constants'; /** @@ -54,11 +54,6 @@ function getRequestAttributes( }; } -/** Serialize the `state` and `questions` of an evaluation request. Also used for TypeSafe models on Workers AI. */ -export function getEvaluationInputMessages(request: Record): string | undefined { - return stringify([{ type: 'evaluation', state: request.state, questions: request.questions }]); -} - /** * Add the response model, token usage and (optionally) the answers of a `systemOne` result. * Also used for TypeSafe models on Workers AI. @@ -86,7 +81,7 @@ export function addResponseAttributes(span: Span, result: unknown, recordOutputs } if (recordOutputs && result.answers !== undefined) { - span.setAttribute(GEN_AI_OUTPUT_MESSAGES, stringify([{ type: 'evaluation', answers: result.answers }])); + span.setAttribute(GEN_AI_OUTPUT_MESSAGES, getEvaluationOutputMessages(result.answers)); } } diff --git a/packages/server-utils/src/ai/workers-ai/utils.ts b/packages/server-utils/src/ai/workers-ai/utils.ts index 66dd3d567c8c..73da7883b12e 100644 --- a/packages/server-utils/src/ai/workers-ai/utils.ts +++ b/packages/server-utils/src/ai/workers-ai/utils.ts @@ -20,8 +20,13 @@ import { GEN_AI_CHAT, GEN_AI_EMBEDDINGS, GEN_AI_EVALUATE } from '@sentry/convent import { isObjectLike, stringify } from '@sentry/core'; import type { Span, SpanAttributeValue } from '@sentry/core'; import { GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../core/gen-ai-attributes'; -import { extractSystemInstructions, getTokenUsageAttributes, setOutputMessagesAttribute } from '../core/utils'; -import { addResponseAttributes as addEvaluateResponseAttributes, getEvaluationInputMessages } from '../typesafe'; +import { + extractSystemInstructions, + getEvaluationInputMessages, + getTokenUsageAttributes, + setOutputMessagesAttribute, +} from '../core/utils'; +import { addResponseAttributes as addEvaluateResponseAttributes } from '../typesafe'; // Re-exported so `workers-ai/streaming.ts` keeps importing it from this module. export { setOutputMessagesAttribute }; import { WORKERS_AI_ORIGIN, WORKERS_AI_PROVIDER_NAME } from './constants'; diff --git a/packages/server-utils/src/integrations/mastra-classifier.ts b/packages/server-utils/src/integrations/mastra-classifier.ts new file mode 100644 index 000000000000..d25434bb2c9d --- /dev/null +++ b/packages/server-utils/src/integrations/mastra-classifier.ts @@ -0,0 +1,73 @@ +import { GEN_AI_INPUT_MESSAGES, GEN_AI_OUTPUT_MESSAGES } from '@sentry/conventions/attributes'; +import { isObjectLike } from '@sentry/core'; +import type { ClassifierEvaluationCall } from '../ai/mastra/classifier-evaluation'; +import { processClassifierEvaluationData, setStartingClassifierEvaluation } from '../ai/mastra/classifier-evaluation'; +import { getEvaluationInputMessages, getEvaluationOutputMessages } from '../ai/core/utils'; +import { CHANNELS } from '../orchestrion/channels'; +import { safeChannelCallback } from '../tracing-channel'; +import * as diagnosticsChannel from '../utils/diagnosticsChannel'; + +interface ClassifierEvaluateChannelContext { + // `evaluate({ state, questions? })`; `self` is the `Classifier`. + arguments: unknown[]; + self?: unknown; + result?: unknown; +} + +/** + * Add the evaluated state, questions and answers of a `Classifier.evaluate()` call to the exporter's + * `classifier_evaluation` span, which Mastra leaves without input or output. + */ +export function recordClassifierEvaluations(): void { + const channel = diagnosticsChannel.tracingChannel( + CHANNELS.MASTRA_CLASSIFIER_EVALUATE, + ); + const calls = new WeakMap(); + + channel.start.subscribe(message => { + safeChannelCallback(() => { + const call: ClassifierEvaluationCall = {}; + calls.set(message as object, call); + setStartingClassifierEvaluation(call); + }); + }); + // `evaluate()` has reached its first `await`, so Mastra has already started its span. + channel.end.subscribe(() => { + safeChannelCallback(() => setStartingClassifierEvaluation(undefined)); + }); + channel.asyncEnd.subscribe(message => { + safeChannelCallback(() => { + const call = calls.get(message as object); + if (call) { + finishClassifierEvaluation(call, message as ClassifierEvaluateChannelContext); + } + }); + }); +} + +function finishClassifierEvaluation(call: ClassifierEvaluationCall, message: ClassifierEvaluateChannelContext): void { + call.settled = true; + const { span, recordInputs, recordOutputs } = call; + if (!span) { + return; + } + + const params = isObjectLike(message.arguments[0]) ? message.arguments[0] : {}; + // Questions given to the constructor take precedence over the ones passed to `evaluate()`, as in Mastra. + const questions = (isObjectLike(message.self) ? message.self.questions : undefined) ?? params.questions; + const data = processClassifierEvaluationData(call, { + input: recordInputs ? { state: params.state, questions } : undefined, + output: recordOutputs && isObjectLike(message.result) ? message.result.answers : undefined, + }); + + if (isObjectLike(data?.input)) { + span.setAttribute(GEN_AI_INPUT_MESSAGES, getEvaluationInputMessages(data.input)); + } + if (data?.output !== undefined) { + span.setAttribute(GEN_AI_OUTPUT_MESSAGES, getEvaluationOutputMessages(data.output)); + } + + if (call.endTime) { + span.end(call.endTime); + } +} diff --git a/packages/server-utils/src/integrations/mastra.ts b/packages/server-utils/src/integrations/mastra.ts index 9856ef51651d..1f541fe83303 100644 --- a/packages/server-utils/src/integrations/mastra.ts +++ b/packages/server-utils/src/integrations/mastra.ts @@ -24,6 +24,7 @@ import type { MastraObservabilityExporter } from '../ai/mastra/types'; import { DEBUG_BUILD } from '../debug-build'; import { CHANNELS } from '../orchestrion/channels'; import { mastraModuleNames } from '../orchestrion/config/mastra'; +import { recordClassifierEvaluations } from './mastra-classifier'; import { invokeOrchestrionInstrumentation } from '../orchestrion/instrumentation'; import { bindSpanToChannelStore, safeChannelCallback } from '../tracing-channel'; @@ -96,6 +97,7 @@ function instrumentExporter(options: MastraOptions): void { }); captureExecuteWithContextErrors(); + recordClassifierEvaluations(); } /** diff --git a/packages/server-utils/src/orchestrion/config/mastra.ts b/packages/server-utils/src/orchestrion/config/mastra.ts index dfa6ee151cbf..3e5c0d53c658 100644 --- a/packages/server-utils/src/orchestrion/config/mastra.ts +++ b/packages/server-utils/src/orchestrion/config/mastra.ts @@ -41,11 +41,30 @@ const mastraContextConfig: InstrumentationConfig[] = ['executeWithContext', 'exe }), ); -export const mastraConfig = [...mastraConstructorConfig, ...mastraContextConfig] satisfies InstrumentationConfig[]; +// Mastra's `classifier_evaluation` span carries no input or output, so we read them from the +// `Classifier.evaluate()` call. `Classifier` lives in the stable, unhashed `classifier` entry. +const mastraClassifierConfig: InstrumentationConfig[] = [ + { + channelName: 'mastraClassifierEvaluate', + module: { + name: '@mastra/core', + versionRange: '>=1.63.2 <2.0.0', + filePath: /^dist\/classifier\/index\.(?:cjs|mjs|js)$/, + }, + functionQuery: { className: 'Classifier', methodName: 'evaluate', kind: 'Async' }, + }, +]; + +export const mastraConfig = [ + ...mastraConstructorConfig, + ...mastraContextConfig, + ...mastraClassifierConfig, +] satisfies InstrumentationConfig[]; export const mastraModuleNames = getModuleNames(mastraConfig); export const mastraChannels = { MASTRA_CONSTRUCTOR: 'orchestrion:@mastra/core:mastraConstructor', MASTRA_EXECUTE_WITH_CONTEXT: 'orchestrion:@mastra/core:mastraExecuteWithContext', + MASTRA_CLASSIFIER_EVALUATE: 'orchestrion:@mastra/core:mastraClassifierEvaluate', } as const; diff --git a/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts b/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts index 5265272ffa0f..844cc9e82514 100644 --- a/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts @@ -1,4 +1,37 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { + ERROR_TYPE, + GEN_AI_AGENT_NAME, + GEN_AI_CONVERSATION_ID, + GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_PIPELINE_NAME, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_REQUEST_STOP_SEQUENCES, + GEN_AI_RESPONSE_MODEL, + GEN_AI_RESPONSE_TEXT, + GEN_AI_RESPONSE_TOOL_CALLS, + GEN_AI_SYSTEM_INSTRUCTIONS, + GEN_AI_TOOL_CALL_ARGUMENTS, + GEN_AI_TOOL_NAME, + GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_REASONING_OUTPUT_TOKENS, + GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import { + GEN_AI_CHAT, + GEN_AI_EMBEDDINGS, + GEN_AI_EVALUATE, + GEN_AI_EXECUTE_TOOL, + GEN_AI_INVOKE_AGENT, +} from '@sentry/conventions/op'; import type { Span } from '@sentry/core'; import { _INTERNAL_clearAiProviderSkips, @@ -8,6 +41,8 @@ import { spanToStaticSpanJSON, } from '@sentry/core'; import { SentryMastraExporter } from '../../../../src/ai/mastra'; +import type { ClassifierEvaluationCall } from '../../../../src/ai/mastra/classifier-evaluation'; +import { setStartingClassifierEvaluation } from '../../../../src/ai/mastra/classifier-evaluation'; import type { MastraExportedSpan, MastraSpanType, MastraTracingEvent } from '../../../../src/ai/mastra/types'; import { OPENAI_INTEGRATION_NAME } from '../../../../src/ai/openai/constants'; import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; @@ -81,11 +116,11 @@ describe('SentryMastraExporter', () => { const json = spanToStaticSpanJSON(endedSpans[0]!); expect(json.description).toBe('invoke_agent weather_agent'); - expect(json.data['sentry.op']).toBe('gen_ai.invoke_agent'); - expect(json.data['sentry.origin']).toBe('auto.ai.mastra'); - expect(json.data['gen_ai.operation.name']).toBe('invoke_agent'); - expect(json.data['gen_ai.agent.name']).toBe('weather_agent'); - expect(json.data['gen_ai.pipeline.name']).toBe('weather_agent'); + expect(json.data[SENTRY_OP]).toBe(GEN_AI_INVOKE_AGENT); + expect(json.data[SENTRY_ORIGIN]).toBe('auto.ai.mastra'); + expect(json.data[GEN_AI_OPERATION_NAME]).toBe('invoke_agent'); + expect(json.data[GEN_AI_AGENT_NAME]).toBe('weather_agent'); + expect(json.data[GEN_AI_PIPELINE_NAME]).toBe('weather_agent'); }); it('names generation spans `chat {model}` and maps usage to the current conventions', async () => { @@ -108,17 +143,61 @@ describe('SentryMastraExporter', () => { const json = spanToStaticSpanJSON(endedSpans[0]!); expect(json.description).toBe('chat gpt-5'); - expect(json.data['sentry.op']).toBe('gen_ai.chat'); - expect(json.data['gen_ai.provider.name']).toBe('openai'); - expect(json.data['gen_ai.usage.input_tokens']).toBe(10); - expect(json.data['gen_ai.usage.output_tokens']).toBe(4); - expect(json.data['gen_ai.usage.total_tokens']).toBe(14); - expect(json.data['gen_ai.usage.reasoning.output_tokens']).toBe(3); - expect(json.data['gen_ai.usage.cache_read.input_tokens']).toBe(2); - expect(json.data['gen_ai.usage.cache_creation.input_tokens']).toBe(5); + expect(json.data[SENTRY_OP]).toBe(GEN_AI_CHAT); + expect(json.data[GEN_AI_PROVIDER_NAME]).toBe('openai'); + expect(json.data[GEN_AI_USAGE_INPUT_TOKENS]).toBe(10); + expect(json.data[GEN_AI_USAGE_OUTPUT_TOKENS]).toBe(4); + expect(json.data[GEN_AI_USAGE_TOTAL_TOKENS]).toBe(14); + expect(json.data[GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]).toBe(3); + expect(json.data[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]).toBe(2); + expect(json.data[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]).toBe(5); expect(json.data['gen_ai.usage.reasoning_tokens']).toBeUndefined(); - expect(json.data['gen_ai.response.text']).toBeUndefined(); - expect(json.data['gen_ai.response.tool_calls']).toBeUndefined(); + expect(json.data[GEN_AI_RESPONSE_TEXT]).toBeUndefined(); + expect(json.data[GEN_AI_RESPONSE_TOOL_CALLS]).toBeUndefined(); + }); + + it('maps classifier evaluations to `evaluate {model}` spans', async () => { + const attributes = { classifierId: 'sentiment', modelId: 'jev-1', provider: 'typesafe.evaluation' }; + const span = makeSpan({ id: 'eval-1', type: 'classifier_evaluation', name: "classifier evaluate: 'sentiment'" }); + await run( + started({ ...span, attributes }), + ended({ ...span, attributes: { ...attributes, usage: { inputTokens: 30, outputTokens: 2 } } }), + ); + + const json = spanToStaticSpanJSON(endedSpans[0]!); + expect(json.description).toBe('evaluate jev-1'); + expect(json.data).toMatchObject({ + [SENTRY_OP]: GEN_AI_EVALUATE, + [SENTRY_ORIGIN]: 'auto.ai.mastra', + [GEN_AI_OPERATION_NAME]: 'evaluate', + [GEN_AI_REQUEST_MODEL]: 'jev-1', + [GEN_AI_PROVIDER_NAME]: 'typesafe.evaluation', + [GEN_AI_USAGE_INPUT_TOKENS]: 30, + [GEN_AI_USAGE_OUTPUT_TOKENS]: 2, + [GEN_AI_USAGE_TOTAL_TOKENS]: 32, + }); + }); + + it('leaves a classifier evaluation span open for the call to end once it settles', async () => { + const call: ClassifierEvaluationCall = {}; + const span = makeSpan({ id: 'eval-1', type: 'classifier_evaluation', attributes: { modelId: 'jev-1' } }); + + setStartingClassifierEvaluation(call); + await run(started(span), ended(span)); + + expect(endedSpans).toHaveLength(0); + expect(call.endTime).toEqual(span.endTime); + expect(spanToStaticSpanJSON(call.span!).description).toBe('evaluate jev-1'); + }); + + it('passes its own recording options to the classifier evaluation call', async () => { + exporter = new SentryMastraExporter({ recordInputs: false, recordOutputs: false }); + const call: ClassifierEvaluationCall = {}; + + setStartingClassifierEvaluation(call); + await run(started(makeSpan({ id: 'eval-1', type: 'classifier_evaluation' }))); + + expect(call).toMatchObject({ recordInputs: false, recordOutputs: false }); }); it('records the agent-level prompt and response as gen_ai messages', async () => { @@ -131,10 +210,10 @@ describe('SentryMastraExporter', () => { await run(started(span), ended(span)); const json = spanToStaticSpanJSON(endedSpans[0]!); - expect(json.data['gen_ai.input.messages']).toBe('[{"role":"user","content":"hi"}]'); - expect(json.data['gen_ai.output.messages']).toBe('{"text":"hello"}'); - expect(json.data['gen_ai.response.text']).toBe('hello'); - expect(json.data['gen_ai.system_instructions']).toBe('be brief'); + expect(json.data[GEN_AI_INPUT_MESSAGES]).toBe('[{"role":"user","content":"hi"}]'); + expect(json.data[GEN_AI_OUTPUT_MESSAGES]).toBe('{"text":"hello"}'); + expect(json.data[GEN_AI_RESPONSE_TEXT]).toBe('hello'); + expect(json.data[GEN_AI_SYSTEM_INSTRUCTIONS]).toBe('be brief'); }); it.each([ @@ -150,7 +229,7 @@ describe('SentryMastraExporter', () => { }); await run(started(span), ended(span)); - expect(spanToStaticSpanJSON(endedSpans[0]!).data['gen_ai.response.text']).toBe(expected); + expect(spanToStaticSpanJSON(endedSpans[0]!).data[GEN_AI_RESPONSE_TEXT]).toBe(expected); }); it.each([ @@ -158,28 +237,28 @@ describe('SentryMastraExporter', () => { 'workflow_run', { type: 'workflow_run' as const, entityName: 'math_workflow' }, 'invoke_agent math_workflow', - 'gen_ai.invoke_agent', + GEN_AI_INVOKE_AGENT, ], [ 'rag_embedding', { type: 'rag_embedding' as const, attributes: { model: 'text-embedding-3' } }, 'embeddings text-embedding-3', - 'gen_ai.embeddings', + GEN_AI_EMBEDDINGS, ], [ 'mcp_tool_call', { type: 'mcp_tool_call' as const, entityName: 'search' }, 'execute_tool search', - 'gen_ai.execute_tool', + GEN_AI_EXECUTE_TOOL, ], - ['entityId when entityName is missing', { entityId: 'agent-42' }, 'invoke_agent agent-42', 'gen_ai.invoke_agent'], + ['entityId when entityName is missing', { entityId: 'agent-42' }, 'invoke_agent agent-42', GEN_AI_INVOKE_AGENT], ])('names a %s span', async (_label, overrides, description, op) => { const span = makeSpan(overrides); await run(started(span), ended(span)); const json = spanToStaticSpanJSON(endedSpans[0]!); expect(json.description).toBe(description); - expect(json.data['sentry.op']).toBe(op); + expect(json.data[SENTRY_OP]).toBe(op); }); it('omits inputs and outputs when recording is disabled', async () => { @@ -188,8 +267,8 @@ describe('SentryMastraExporter', () => { await run(started(span), ended(span)); const json = spanToStaticSpanJSON(endedSpans[0]!); - expect(json.data['gen_ai.input.messages']).toBeUndefined(); - expect(json.data['gen_ai.output.messages']).toBeUndefined(); + expect(json.data[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); + expect(json.data[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); expect(JSON.stringify(json.data)).not.toContain('pii'); }); @@ -226,9 +305,9 @@ describe('SentryMastraExporter', () => { const chatJson = json.find(span => span.description === 'chat gpt-5'); expect(toolJson).toBeDefined(); expect(chatJson).toBeDefined(); - expect(toolJson!.data['sentry.op']).toBe('gen_ai.execute_tool'); - expect(toolJson!.data['gen_ai.tool.name']).toBe('get_weather'); - expect(toolJson!.data['gen_ai.tool.call.arguments']).toBe('{"city":"Berlin"}'); + expect(toolJson!.data[SENTRY_OP]).toBe(GEN_AI_EXECUTE_TOOL); + expect(toolJson!.data[GEN_AI_TOOL_NAME]).toBe('get_weather'); + expect(toolJson!.data[GEN_AI_TOOL_CALL_ARGUMENTS]).toBe('{"city":"Berlin"}'); expect(toolJson!.parent_span_id).toBe(chatJson!.span_id); }); @@ -251,10 +330,10 @@ describe('SentryMastraExporter', () => { .map(span => spanToStaticSpanJSON(span)) .find(span => span.description === 'invoke_agent agent'); expect(agentJson).toBeDefined(); - expect(agentJson!.data['gen_ai.usage.input_tokens']).toBe(10); - expect(agentJson!.data['gen_ai.usage.output_tokens']).toBe(4); - expect(agentJson!.data['gen_ai.usage.total_tokens']).toBe(14); - expect(agentJson!.data['gen_ai.response.model']).toBe('gpt-5-2026'); + expect(agentJson!.data[GEN_AI_USAGE_INPUT_TOKENS]).toBe(10); + expect(agentJson!.data[GEN_AI_USAGE_OUTPUT_TOKENS]).toBe(4); + expect(agentJson!.data[GEN_AI_USAGE_TOTAL_TOKENS]).toBe(14); + expect(agentJson!.data[GEN_AI_RESPONSE_MODEL]).toBe('gpt-5-2026'); }); it('sums usage across multiple generations onto the parent agent', async () => { @@ -285,11 +364,11 @@ describe('SentryMastraExporter', () => { .map(span => spanToStaticSpanJSON(span)) .find(span => span.description === 'invoke_agent agent'); expect(agentJson).toBeDefined(); - expect(agentJson!.data['gen_ai.usage.input_tokens']).toBe(40); - expect(agentJson!.data['gen_ai.usage.output_tokens']).toBe(12); - expect(agentJson!.data['gen_ai.usage.total_tokens']).toBe(52); - expect(agentJson!.data['gen_ai.usage.cache_read.input_tokens']).toBe(3); - expect(agentJson!.data['gen_ai.response.model']).toBe('gpt-5-2026'); + expect(agentJson!.data[GEN_AI_USAGE_INPUT_TOKENS]).toBe(40); + expect(agentJson!.data[GEN_AI_USAGE_OUTPUT_TOKENS]).toBe(12); + expect(agentJson!.data[GEN_AI_USAGE_TOTAL_TOKENS]).toBe(52); + expect(agentJson!.data[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]).toBe(3); + expect(agentJson!.data[GEN_AI_RESPONSE_MODEL]).toBe('gpt-5-2026'); }); it('marks the span errored without capturing a reconstructed exception', async () => { @@ -298,7 +377,7 @@ describe('SentryMastraExporter', () => { const json = spanToStaticSpanJSON(endedSpans[0]!); expect(json.status).toBe('internal_error'); - expect(json.data['error.type']).toBe('ToolError'); + expect(json.data[ERROR_TYPE]).toBe('ToolError'); }); it.each([ @@ -334,7 +413,7 @@ describe('SentryMastraExporter', () => { const json = spanToStaticSpanJSON(endedSpans[0]!); expect(Object.keys(json.data).filter(key => key.startsWith('mastra.'))).toEqual([]); - expect(json.data['gen_ai.conversation.id']).toBe('thread-9'); + expect(json.data[GEN_AI_CONVERSATION_ID]).toBe('thread-9'); }); it.each([ @@ -348,14 +427,14 @@ describe('SentryMastraExporter', () => { const span = makeSpan({ entityName: 'agent', ...overrides }); await run(started(span), ended(span)); - expect(spanToStaticSpanJSON(endedSpans[0]!).data['gen_ai.conversation.id']).toBe(expected); + expect(spanToStaticSpanJSON(endedSpans[0]!).data[GEN_AI_CONVERSATION_ID]).toBe(expected); }); it('does not use runId as gen_ai.conversation.id', async () => { const span = makeSpan({ entityName: 'agent', metadata: { runId: 'run-3' } }); await run(started(span), ended(span)); - expect(spanToStaticSpanJSON(endedSpans[0]!).data['gen_ai.conversation.id']).toBeUndefined(); + expect(spanToStaticSpanJSON(endedSpans[0]!).data[GEN_AI_CONVERSATION_ID]).toBeUndefined(); }); it('falls back to the bare operation name when there is no identifier', async () => { @@ -411,7 +490,7 @@ describe('SentryMastraExporter', () => { }); await run(started(span), ended(span)); - expect(spanToStaticSpanJSON(endedSpans[0]!).data['gen_ai.request.stop_sequences']).toEqual(['\n\n', 'END']); + expect(spanToStaticSpanJSON(endedSpans[0]!).data[GEN_AI_REQUEST_STOP_SEQUENCES]).toEqual(['\n\n', 'END']); }); // Parentless dropped spans used to be stored under a falsy sentinel that `LRUMap.remove` never From 2fdcb23ad5b6aa8a714a980cdaf04e148a62ef82 Mon Sep 17 00:00:00 2001 From: Andrei <168741329+andreiborza@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:50:13 +0200 Subject: [PATCH 44/47] test(e2e): Add Mastra Jev classifier test to node-mastra (#25115) ## What Adds a node-mastra e2e test that runs a real Jev evaluation through a Mastra `Classifier` (via OpenRouter) and checks the `gen_ai.evaluate` span. ## Why Covers the classifier instrumentation from #25044 against the real provider, not only a mock model. --- .../node-mastra/package.json | 5 +- .../src/mastra/agents/weather-agent.ts | 11 +--- .../mastra/classifiers/ticket-classifier.ts | 17 +++++ .../node-mastra/src/mastra/index.ts | 15 ++++- .../node-mastra/src/mastra/openrouter.ts | 10 +++ .../node-mastra/tests/classifier.test.ts | 65 +++++++++++++++++++ 6 files changed, 110 insertions(+), 13 deletions(-) create mode 100644 dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/classifiers/ticket-classifier.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/openrouter.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts diff --git a/dev-packages/e2e-tests/test-applications/node-mastra/package.json b/dev-packages/e2e-tests/test-applications/node-mastra/package.json index 20340b7e81a6..592a00860591 100644 --- a/dev-packages/e2e-tests/test-applications/node-mastra/package.json +++ b/dev-packages/e2e-tests/test-applications/node-mastra/package.json @@ -15,11 +15,11 @@ "test:dev": "TEST_ENV=development playwright test" }, "dependencies": { - "@mastra/core": "~1.65.0", + "@mastra/core": "~1.74.0", "@mastra/libsql": "~1.22.4", "@mastra/memory": "~1.28.3", "@mastra/observability": "~1.17.6", - "@openrouter/ai-sdk-provider": "~3.0.0", + "@openrouter/ai-sdk-provider": "~3.1.0", "@sentry/node": "file:../../packed/sentry-node-packed.tgz", "ai": "~7.0.97", "dataloader": "~2.2.3", @@ -28,6 +28,7 @@ "devDependencies": { "@playwright/test": "~1.56.0", "@sentry-internal/test-utils": "link:../../../test-utils", + "@sentry/conventions": "0.26.0", "@types/node": "24.x", "mastra": "~1.28.0", "typescript": "~5.9.0" diff --git a/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/agents/weather-agent.ts b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/agents/weather-agent.ts index 8b33b4cc4d5e..2b56591e4ec5 100644 --- a/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/agents/weather-agent.ts +++ b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/agents/weather-agent.ts @@ -1,19 +1,10 @@ -import { createOpenRouter } from '@openrouter/ai-sdk-provider'; import { Agent } from '@mastra/core/agent'; import { Memory } from '@mastra/memory'; import { LibSQLStore } from '@mastra/libsql'; import { countItemsTool } from '../tools/count-items.js'; import { failNowTool } from '../tools/fail-now.js'; import { getWeatherTool } from '../tools/get-weather.js'; - -const apiKey = process.env.E2E_OPENROUTER_API_KEY; -if (!apiKey) { - throw new Error('E2E_OPENROUTER_API_KEY is not set'); -} - -// Call OpenRouter directly (rather than the default Vercel AI Gateway) so the -// e2e test needs only a single OpenRouter key, reusing `E2E_OPENROUTER_API_KEY`. -const openrouter = createOpenRouter({ apiKey }); +import { openrouter } from '../openrouter.js'; // An in-memory libsql store is enough for the test: Mastra requires a storage // provider before `generate(..., { memory: { thread, resource } })` is accepted, diff --git a/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/classifiers/ticket-classifier.ts b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/classifiers/ticket-classifier.ts new file mode 100644 index 000000000000..6a66654bb7e9 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/classifiers/ticket-classifier.ts @@ -0,0 +1,17 @@ +import { Classifier } from '@mastra/core/classifier'; +import { openrouter } from '../openrouter.js'; + +export const TICKET_CLASSIFIER = 'ticketClassifier'; + +// A Jev evaluation through OpenRouter's Decisions API. `Classifier` calls the model's `doEvaluate()` +// directly, so its span comes from the Mastra exporter, not from the Vercel AI integration. +export const ticketClassifier = new Classifier({ + id: TICKET_CLASSIFIER, + model: openrouter.evaluationModel('typesafe/jev-1.13'), + questions: { + is_bug: { + type: 'boolean', + instructions: 'Is the customer reporting a software defect?', + }, + }, +}); diff --git a/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/index.ts b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/index.ts index b6f2dd975b39..c5528dbd6ba9 100644 --- a/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/index.ts +++ b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/index.ts @@ -2,6 +2,7 @@ import { Mastra } from '@mastra/core'; import { registerApiRoute } from '@mastra/core/server'; import { LibSQLStore } from '@mastra/libsql'; import { WEATHER_AGENT, weatherAgent } from './agents/weather-agent.js'; +import { TICKET_CLASSIFIER, ticketClassifier } from './classifiers/ticket-classifier.js'; // The agent is driven through Mastra's built-in `POST /api/agents/:id/generate` // endpoint (see tests/utils.ts). `dataloader` (orchestrion-instrumented) is exercised @@ -19,12 +20,24 @@ const manualRoute = registerApiRoute('/manual-route', { }, }); +// Evaluates the registered classifier outside of any agent run. Registration lets Mastra start a root +// `classifier_evaluation` span when no Mastra span is active. +const classifyRoute = registerApiRoute('/classify', { + method: 'POST', + async handler(c) { + const { ticket } = await c.req.json<{ ticket: string }>(); + const result = await ticketClassifier.evaluate({ state: { ticket } }); + return c.json({ answers: result.answers }); + }, +}); + export const mastra = new Mastra({ agents: { [WEATHER_AGENT]: weatherAgent }, + classifiers: { [TICKET_CLASSIFIER]: ticketClassifier }, storage: new LibSQLStore({ id: 'mastra-storage', url: ':memory:' }), server: { port: 4111, - apiRoutes: [manualRoute], + apiRoutes: [manualRoute, classifyRoute], }, // No `bundler.externals` override: `mastra build` defaults to externalizing all non-workspace deps, // so Hono (and the other instrumented modules) stay unbundled and the `--import` orchestrion hook diff --git a/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/openrouter.ts b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/openrouter.ts new file mode 100644 index 000000000000..b04e3e9e5358 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/openrouter.ts @@ -0,0 +1,10 @@ +import { createOpenRouter } from '@openrouter/ai-sdk-provider'; + +const apiKey = process.env.E2E_OPENROUTER_API_KEY; +if (!apiKey) { + throw new Error('E2E_OPENROUTER_API_KEY is not set'); +} + +// Call OpenRouter directly (rather than the default Vercel AI Gateway) so the +// e2e test needs only a single OpenRouter key, reusing `E2E_OPENROUTER_API_KEY`. +export const openrouter = createOpenRouter({ apiKey }); diff --git a/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts b/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts new file mode 100644 index 000000000000..6eee9bb68359 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts @@ -0,0 +1,65 @@ +import { expect, test } from '@playwright/test'; +import { + GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_USAGE_INPUT_TOKENS, + SENTRY_ORIGIN, + URL_FULL, +} from '@sentry/conventions/attributes'; +import { GEN_AI_EVALUATE, HTTP_SERVER } from '@sentry/conventions/op'; +import { collectStreamedSpans, getSpanOp, SerializedStreamedSpan } from '@sentry-internal/test-utils'; + +const APP = 'node-mastra'; +const TICKET = 'My checkout page shows a blank screen after I click Pay.'; + +const isEvaluateSpan = (span: SerializedStreamedSpan): boolean => getSpanOp(span) === GEN_AI_EVALUATE; +const isClassifyServerSpan = (span: SerializedStreamedSpan): boolean => + getSpanOp(span) === HTTP_SERVER && String(span.attributes?.[URL_FULL]?.value ?? '').includes('/classify'); + +test('captures a Mastra classifier (Jev) evaluation as a gen_ai.evaluate span', async ({ baseURL }) => { + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => spansOfTrace.some(isEvaluateSpan) && spansOfTrace.some(isClassifyServerSpan), + ); + + const res = await fetch(`${baseURL}/classify`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ ticket: TICKET }), + }); + expect(res.status).toBe(200); + await res.json(); + + const spans = await spansPromise; + const serverSpan = spans.find(isClassifyServerSpan)!; + + // Only the Mastra exporter reports the evaluation: `Classifier` calls `doEvaluate()` directly, so the + // Vercel AI integration does not see it. + const evaluateSpans = spans.filter(isEvaluateSpan); + expect(evaluateSpans.map(span => ({ name: span.name, origin: span.attributes?.[SENTRY_ORIGIN]?.value }))).toEqual([ + { name: 'evaluate typesafe/jev-1.13', origin: 'auto.ai.mastra' }, + ]); + + const evaluateSpan = evaluateSpans[0]!; + expect(evaluateSpan.parent_span_id).toBe(serverSpan.span_id); + expect(evaluateSpan.attributes?.[GEN_AI_OPERATION_NAME]?.value).toBe('evaluate'); + expect(evaluateSpan.attributes?.[GEN_AI_REQUEST_MODEL]?.value).toBe('typesafe/jev-1.13'); + expect(evaluateSpan.attributes?.[GEN_AI_PROVIDER_NAME]?.value).toBe('openrouter'); + expect(evaluateSpan.attributes?.[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBeGreaterThan(0); + expect(JSON.parse(String(evaluateSpan.attributes?.[GEN_AI_INPUT_MESSAGES]?.value))).toEqual([ + { + type: 'evaluation', + state: { ticket: TICKET }, + questions: { + is_bug: { type: 'boolean', instructions: 'Is the customer reporting a software defect?' }, + }, + }, + ]); + // The probability comes from the live model, so only its shape is stable. + expect(JSON.parse(String(evaluateSpan.attributes?.[GEN_AI_OUTPUT_MESSAGES]?.value))).toEqual([ + { type: 'evaluation', answers: { is_bug: { type: 'boolean', probability: expect.any(Number) } } }, + ]); +}); From e99e6e1f766c324e5c1d06e331b97639454d769a Mon Sep 17 00:00:00 2001 From: Charly Gomez Date: Thu, 8 Oct 2026 15:00:17 +0200 Subject: [PATCH 45/47] feat(remix): Propagate the server trace to Remix 3 page loads (#25145) Browser page loads in Remix 3 started a new trace. The server middleware now adds `sentry-trace` and `baggage` entries to the `Server-Timing` header of HTML responses, ahead of the route entry, the same channel the Remix 2 SDK and Nitro use. The browser SDK already reads both off the navigation timing entry for page loads, so no client change is needed. A response a shared cache may store (`public`, `s-maxage`, or a positive `max-age` without `private`/`no-store`) carries the route entry only: a cached document would otherwise hand one request's trace to every later page load, the ISR problem the Next.js SDK works around on the client. The e2e test asserts that the page load span shares the server span's trace id and has it as parent. Fixes #25137 --------- Co-authored-by: Claude Fable 5.1 --- .../remix-v3/tests/client.test.ts | 19 +++ packages/remix/src/v3/server/middleware.ts | 98 +++++++++-- packages/remix/test/v3/middleware.test.ts | 153 +++++++++++++++++- 3 files changed, 257 insertions(+), 13 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/remix-v3/tests/client.test.ts b/dev-packages/e2e-tests/test-applications/remix-v3/tests/client.test.ts index 70f77fb7627f..393a3e8e3195 100644 --- a/dev-packages/e2e-tests/test-applications/remix-v3/tests/client.test.ts +++ b/dev-packages/e2e-tests/test-applications/remix-v3/tests/client.test.ts @@ -15,6 +15,25 @@ test('sends a pageload span', async ({ page }) => { expect(span.attributes?.['sentry.segment.name.source']?.value).toBe('route'); }); +test('continues the server trace on a page load', async ({ page }) => { + const serverSpanPromise = waitForStreamedSpan( + APP_NAME, + span => + getSpanOp(span) === 'http.server' && span.is_segment === true && span.attributes?.['http.route']?.value === '/', + ); + const pageloadPromise = waitForStreamedSpan( + APP_NAME, + span => getSpanOp(span) === 'pageload' && span.is_segment === true, + ); + + await page.goto('/'); + + const [serverSpan, pageload] = await Promise.all([serverSpanPromise, pageloadPromise]); + // The document's response carried the trace in `Server-Timing`, which the browser SDK picks up. + expect(pageload.trace_id).toBe(serverSpan.trace_id); + expect(pageload.parent_span_id).toBe(serverSpan.span_id); +}); + test('parameterizes a page load on a route with params', async ({ page }) => { const spanPromise = waitForStreamedSpan(APP_NAME, span => getSpanOp(span) === 'pageload' && span.is_segment === true); diff --git a/packages/remix/src/v3/server/middleware.ts b/packages/remix/src/v3/server/middleware.ts index 7ae9dc989362..f2ce59c48f01 100644 --- a/packages/remix/src/v3/server/middleware.ts +++ b/packages/remix/src/v3/server/middleware.ts @@ -4,6 +4,7 @@ import { getIsolationScope, getRootSpan, getSpanStatusFromHttpCode, + getTraceData, INTERNAL_setSegmentNameSourceIfSegment, type Scope, updateSpanName, @@ -44,9 +45,7 @@ export function sentryRemixMiddleware(matcher: MatcherLike): MiddlewareLike { } setResponseStatus(response); - if (route) { - addRouteHeader(response, context.request, route); - } + addServerTimingHeaders(response, context.request, route); return response; }; @@ -70,19 +69,96 @@ function applyRoute(isolationScope: Scope, route: string, method: string): void } /** - * Tells the browser SDK which route served a document, so it can name its page load and navigation - * spans after the pattern. The browser has no route table of its own. Only HTML responses carry it, - * which is what document loads and the runtime's frame fetches ask for. + * What the browser SDK needs from the server on a document: the trace to continue, so a page load + * joins the request's trace, and the route that served it, so spans are named after the pattern. + * The browser reads both off the navigation timing entry. Only HTML responses carry them, which is + * what document loads and the runtime's frame fetches ask for. */ -function addRouteHeader(response: Response, request: Request, route: string): void { +function addServerTimingHeaders(response: Response, request: Request, route: string | undefined): void { if (!request.headers.get('accept')?.includes('text/html')) { return; } - try { - response.headers.append('Server-Timing', formatRouteTiming(route)); - } catch { - // Immutable headers, e.g. a response passed through from `fetch()`. + + const entries: string[] = []; + // A shared cache would hand this request's trace to every later page load, so a cacheable response + // carries the route only. The route is the same for every request to it. + if (!isSharedCacheable(response)) { + const traceData = getTraceData(); + if (traceData['sentry-trace']) { + entries.push(`sentry-trace;desc="${traceData['sentry-trace']}"`); + } + if (traceData.baggage) { + entries.push(`baggage;desc="${traceData.baggage}"`); + } + } + if (route) { + entries.push(formatRouteTiming(route)); + } + + for (const entry of entries) { + try { + response.headers.append('Server-Timing', entry); + } catch { + // Immutable headers, e.g. a response passed through from `fetch()`. + return; + } + } +} + +// Cache-control fields a CDN reads instead of `Cache-Control`. +const CDN_CACHE_CONTROL_HEADERS = [ + 'cdn-cache-control', + 'cloudflare-cdn-cache-control', + 'vercel-cdn-cache-control', + 'surrogate-control', +]; + +/** + * Whether a cache in front of the app may store this response and serve it to other users. + * + * A CDN that reads a targeted field ignores `Cache-Control`; a cache without targeted support reads + * `Cache-Control`. Any of them storing the response is enough. + */ +function isSharedCacheable(response: Response): boolean { + for (const name of CDN_CACHE_CONTROL_HEADERS) { + const value = response.headers.get(name); + if (value !== null && sharedCachingVerdict(value.toLowerCase()) === true) { + return true; + } + } + + const verdict = sharedCachingVerdict(response.headers.get('cache-control')?.toLowerCase() ?? ''); + if (verdict !== undefined) { + return verdict; + } + // Without a lifetime in `Cache-Control`, a cache falls back to `Expires`. + const expires = response.headers.get('expires'); + return expires !== null && Date.parse(expires) > Date.now(); +} + +/** Whether these directives let a shared cache reuse the response, or `undefined` when they say nothing. */ +function sharedCachingVerdict(directives: string): boolean | undefined { + if (/\b(?:no-store|private)\b/.test(directives)) { + return false; + } + // A stale copy is served even after a zero lifetime. + if (/\b(?:stale-while-revalidate|stale-if-error)\s*=\s*[1-9]/.test(directives)) { + return true; + } + // `s-maxage` overrides `max-age` for shared caches. A zero lifetime means the cache revalidates + // every time, so it never serves this response to anyone else. + const sharedMaxAge = directives.match(/\bs-maxage\s*=\s*(\d+)/)?.[1]; + if (sharedMaxAge !== undefined) { + return Number(sharedMaxAge) > 0; + } + if (/\bpublic\b/.test(directives)) { + return true; + } + const maxAge = directives.match(/\bmax-age\s*=\s*(\d+)/)?.[1]; + if (maxAge !== undefined) { + return Number(maxAge) > 0; } + return undefined; } function setResponseStatus(response: Response): void { diff --git a/packages/remix/test/v3/middleware.test.ts b/packages/remix/test/v3/middleware.test.ts index 88c8f1fc42db..3bc7031300bf 100644 --- a/packages/remix/test/v3/middleware.test.ts +++ b/packages/remix/test/v3/middleware.test.ts @@ -1,4 +1,13 @@ -import { describe, expect, it } from 'vitest'; +import type * as SentryCore from '@sentry/core'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +// Trace data as the SDK would produce it for the active request; empty when nothing is active. +const getTraceData = vi.hoisted(() => vi.fn<() => { 'sentry-trace'?: string; baggage?: string }>(() => ({}))); + +vi.mock('@sentry/core', async importOriginal => ({ + ...(await importOriginal()), + getTraceData, +})); import { sentryRemixMiddleware } from '../../src/v3/server/middleware'; import type { MatcherLike, RequestContextLike } from '../../src/v3/types'; @@ -16,6 +25,146 @@ function contextFor(url: string, headers: Record = {}): RequestC } describe('sentryRemixMiddleware', () => { + beforeEach(() => { + getTraceData.mockReturnValue({}); + }); + + it('propagates the trace to HTML responses through Server-Timing, ahead of the route', async () => { + getTraceData.mockReturnValue({ 'sentry-trace': 'abc-def-1', baggage: 'sentry-trace_id=abc' }); + const middleware = sentryRemixMiddleware(matcherFor('/users/:id')); + + const response = await middleware( + contextFor('http://x/users/1', { accept: 'text/html' }), + async () => new Response(''), + ); + + expect(response.headers.get('server-timing')).toBe( + 'sentry-trace;desc="abc-def-1", baggage;desc="sentry-trace_id=abc", sentry-route;desc="/users/:id"', + ); + }); + + it('propagates the trace even when no route matched', async () => { + getTraceData.mockReturnValue({ 'sentry-trace': 'abc-def-1' }); + const middleware = sentryRemixMiddleware(matcherFor(undefined)); + + const response = await middleware( + contextFor('http://x/nope', { accept: 'text/html' }), + async () => new Response(''), + ); + + expect(response.headers.get('server-timing')).toBe('sentry-trace;desc="abc-def-1"'); + }); + + it.each([ + ['public', 'public, max-age=60'], + ['s-maxage', 's-maxage=300'], + ['max-age', 'max-age=60'], + ])('leaves the trace off a response a shared cache may store (%s), keeping the route', async (_why, cacheControl) => { + getTraceData.mockReturnValue({ 'sentry-trace': 'abc-def-1', baggage: 'sentry-trace_id=abc' }); + const middleware = sentryRemixMiddleware(matcherFor('/')); + + const response = await middleware( + contextFor('http://x/', { accept: 'text/html' }), + async () => new Response('', { headers: { 'cache-control': cacheControl } }), + ); + + expect(response.headers.get('server-timing')).toBe('sentry-route;desc="/"'); + }); + + it.each([ + ['stale-while-revalidate after a zero lifetime', { 'cache-control': 'max-age=0, stale-while-revalidate=60' }], + ['stale-if-error after a zero shared lifetime', { 'cache-control': 's-maxage=0, stale-if-error=600' }], + [ + 'CDN-Cache-Control over a private Cache-Control', + { 'cache-control': 'private', 'cdn-cache-control': 'max-age=3600' }, + ], + [ + 'Vercel-CDN-Cache-Control over no-store', + { 'cache-control': 'no-store', 'vercel-cdn-cache-control': 's-maxage=3600' }, + ], + ['Cloudflare-CDN-Cache-Control', { 'cloudflare-cdn-cache-control': 'public, max-age=60' }], + ['Surrogate-Control', { 'surrogate-control': 'max-age=3600' }], + ])('leaves the trace off a response a CDN or stale-serving cache may reuse (%s)', async (_why, headers) => { + getTraceData.mockReturnValue({ 'sentry-trace': 'abc-def-1' }); + const middleware = sentryRemixMiddleware(matcherFor('/')); + + const response = await middleware( + contextFor('http://x/', { accept: 'text/html' }), + async () => new Response('', { headers }), + ); + + expect(response.headers.get('server-timing')).toBe('sentry-route;desc="/"'); + }); + + it('propagates the trace when the CDN field forbids caching and nothing else allows it', async () => { + getTraceData.mockReturnValue({ 'sentry-trace': 'abc-def-1' }); + const middleware = sentryRemixMiddleware(matcherFor('/')); + + const response = await middleware( + contextFor('http://x/', { accept: 'text/html' }), + async () => new Response('', { headers: { 'cdn-cache-control': 'max-age=0' } }), + ); + + expect(response.headers.get('server-timing')).toContain('sentry-trace;desc="abc-def-1"'); + }); + + it('leaves the trace off a response with a future Expires and no Cache-Control', async () => { + getTraceData.mockReturnValue({ 'sentry-trace': 'abc-def-1' }); + const middleware = sentryRemixMiddleware(matcherFor('/')); + + const response = await middleware( + contextFor('http://x/', { accept: 'text/html' }), + async () => new Response('', { headers: { expires: new Date(Date.now() + 60_000).toUTCString() } }), + ); + + expect(response.headers.get('server-timing')).toBe('sentry-route;desc="/"'); + }); + + it('propagates the trace when max-age=0 overrides a future Expires', async () => { + getTraceData.mockReturnValue({ 'sentry-trace': 'abc-def-1' }); + const middleware = sentryRemixMiddleware(matcherFor('/')); + + const response = await middleware( + contextFor('http://x/', { accept: 'text/html' }), + async () => + new Response('', { + headers: { 'cache-control': 'max-age=0', expires: new Date(Date.now() + 60_000).toUTCString() }, + }), + ); + + expect(response.headers.get('server-timing')).toContain('sentry-trace;desc="abc-def-1"'); + }); + + it('propagates the trace when Expires is in the past', async () => { + getTraceData.mockReturnValue({ 'sentry-trace': 'abc-def-1' }); + const middleware = sentryRemixMiddleware(matcherFor('/')); + + const response = await middleware( + contextFor('http://x/', { accept: 'text/html' }), + async () => new Response('', { headers: { expires: 'Thu, 01 Jan 1970 00:00:00 GMT' } }), + ); + + expect(response.headers.get('server-timing')).toContain('sentry-trace;desc="abc-def-1"'); + }); + + it.each([ + ['private', 'private, max-age=60'], + ['no-store', 'no-store'], + ['max-age=0', 'max-age=0, must-revalidate'], + ['s-maxage=0', 'max-age=3600, s-maxage=0'], + ['no cache header', undefined], + ])('propagates the trace on a response only this user can get back (%s)', async (_why, cacheControl) => { + getTraceData.mockReturnValue({ 'sentry-trace': 'abc-def-1' }); + const middleware = sentryRemixMiddleware(matcherFor('/')); + + const response = await middleware( + contextFor('http://x/', { accept: 'text/html' }), + async () => new Response('', { headers: cacheControl ? { 'cache-control': cacheControl } : {} }), + ); + + expect(response.headers.get('server-timing')).toContain('sentry-trace;desc="abc-def-1"'); + }); + it('reports the route on HTML responses through Server-Timing', async () => { const middleware = sentryRemixMiddleware(matcherFor('/users/:id')); @@ -60,7 +209,7 @@ describe('sentryRemixMiddleware', () => { expect(response.headers.has('server-timing')).toBe(false); }); - it('adds nothing when no route matched', async () => { + it('adds nothing when no route matched and no trace is active', async () => { const middleware = sentryRemixMiddleware(matcherFor(undefined)); const response = await middleware( From 53ae9968cce44a0e40efd78e841a0ba3fd3fe0a0 Mon Sep 17 00:00:00 2001 From: JPeer264 Date: Thu, 8 Oct 2026 15:22:38 +0200 Subject: [PATCH 46/47] meta(changelog): Update changelog for 11.6.0 --- CHANGELOG.md | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b6d793fae6c..21394e7db5b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,84 @@ - "You miss 100 percent of the chances you don't take. — Wayne Gretzky" — Michael Scott -- **feat(server-utils)** Add instrumentation for pi-durable ([#24993](https://github.com/getsentry/sentry-javascript/pull/24993/)) +## 11.6.0 -The new `piDurableIntegration` traces agents built with [pi-durable](https://github.com/earendil-works/pi/tree/main/packages/durable): each run, with its model requests and tool calls. +### Important Changes + +- **feat(nextjs): Add `@sentry/nextjs/cloudflare` ([#24998](https://github.com/getsentry/sentry-javascript/pull/24998))** + + The new `@sentry/nextjs/cloudflare` entry point exports `withSentry` for the Worker entry of a Next.js app on Cloudflare Workers, for example `.open-next/worker.js` of OpenNext. It is `withSentry` of `@sentry/cloudflare` with the Next.js server handling added, so a Next.js app on Workers gets the same spans as on Node.js. + + ```ts + import * as Sentry from '@sentry/nextjs/cloudflare'; + import handler from './.open-next/worker.js'; + + export default Sentry.withSentry( + env => ({ + dsn: env.SENTRY_DSN, + tracesSampleRate: 1.0, + }), + handler, + ); + ``` + +- **feat(server-utils): Add instrumentation for pi-durable ([#24993](https://github.com/getsentry/sentry-javascript/pull/24993))** + + The new `piDurableIntegration` traces agents built with [pi-durable](https://github.com/earendil-works/pi/tree/main/packages/durable): each run, with its model requests and tool calls. + +- **fix(core): Correct sleep clock drift on telemetry timestamps ([#23054](https://github.com/getsentry/sentry-javascript/pull/23054))** + + When a device sleeps, the `performance.now()` clock of the browser stops. Before this change, timestamps of spans and other telemetry recorded after the sleep could be too early. The SDK now detects this drift and corrects its time origin, also for spans created from performance entries. Span end times are now computed from the `performance.now()` duration since the span start, like in OpenTelemetry ([#24903](https://github.com/getsentry/sentry-javascript/pull/24903)). + +### Other Changes + +- feat(cloudflare): Skip spans for the pi_ tables of PiHarness ([#25028](https://github.com/getsentry/sentry-javascript/pull/25028)) +- feat(remix): Propagate the server trace to Remix 3 page loads ([#25145](https://github.com/getsentry/sentry-javascript/pull/25145)) +- feat(server-utils): Map Mastra classifier evaluations to gen_ai.evaluate spans ([#25044](https://github.com/getsentry/sentry-javascript/pull/25044)) +- feat(server-utils): Record LangChain TypeSafeClassifier runs as gen_ai.evaluate spans ([#25120](https://github.com/getsentry/sentry-javascript/pull/25120)) +- fix(browser): Check for clock drift on page lifecycle events ([#25091](https://github.com/getsentry/sentry-javascript/pull/25091)) +- fix(core): Exclude attachments from internal exception events ([#25160](https://github.com/getsentry/sentry-javascript/pull/25160)) +- fix(effect): Convert Effect span times to the Sentry clock ([#24972](https://github.com/getsentry/sentry-javascript/pull/24972)) +- fix(effect): Detect HTTP spans named after the request method ([#25159](https://github.com/getsentry/sentry-javascript/pull/25159)) +- fix(node): Register Vercel keep-alive listeners once ([#25146](https://github.com/getsentry/sentry-javascript/pull/25146)) +- fix(replay): Place CLS and INP web vitals at the right time ([#24990](https://github.com/getsentry/sentry-javascript/pull/24990)) +- fix(server-utils): Record LangGraph span I/O for non-MessagesAnnotation state ([#23315](https://github.com/getsentry/sentry-javascript/pull/23315)) +- fix(sveltekit): Upload source maps with SvelteKit 3 ([#25139](https://github.com/getsentry/sentry-javascript/pull/25139)) + +
+ Internal Changes + +- chore: Add external contributor to CHANGELOG.md ([#25136](https://github.com/getsentry/sentry-javascript/pull/25136)) +- chore(deps-dev): Bump next from 15.5.24 to 15.5.27 ([#25171](https://github.com/getsentry/sentry-javascript/pull/25171)) +- chore(nitro): Remove stale bundler-plugin-core rollup external ([#25149](https://github.com/getsentry/sentry-javascript/pull/25149)) +- feat(cloudflare): Wrap vinext Worker entries with `@sentry/nextjs/cloudflare` ([#25000](https://github.com/getsentry/sentry-javascript/pull/25000)) +- feat(deps): Bump next from 16.2.11 to 16.3.8 in /dev-packages/e2e-tests/test-applications/nextjs-sourcemaps ([#25170](https://github.com/getsentry/sentry-javascript/pull/25170)) +- fix(deps): Bump @graphql-tools/utils to 12.0.3 ([#25162](https://github.com/getsentry/sentry-javascript/pull/25162)) +- ref(eslint-plugin-sdk): Improve `no-unsafe-random-apis` rule ([#25123](https://github.com/getsentry/sentry-javascript/pull/25123)) +- test(cloudflare): Port the remaining static suites to span streaming ([#25112](https://github.com/getsentry/sentry-javascript/pull/25112)) +- test(cloudflare): Widen the wrangler readiness retry window in integration tests ([#24925](https://github.com/getsentry/sentry-javascript/pull/24925)) +- test(e2e): Add a cloudflare-pi-durable end-to-end application ([#25022](https://github.com/getsentry/sentry-javascript/pull/25022)) +- test(e2e): Add a node-pi-durable end-to-end application ([#24994](https://github.com/getsentry/sentry-javascript/pull/24994)) +- test(e2e): Add Mastra Jev classifier test to node-mastra ([#25115](https://github.com/getsentry/sentry-javascript/pull/25115)) +- test(e2e): Wait for the soft navigation entry before hiding the page ([#25127](https://github.com/getsentry/sentry-javascript/pull/25127)) +- test(nextjs): Run nextjs-16 on Bun, Deno and Cloudflare ([#24999](https://github.com/getsentry/sentry-javascript/pull/24999)) +- test(node): Add streamed span collection helpers ([#25131](https://github.com/getsentry/sentry-javascript/pull/25131)) +- test(node): Cover Jev calls inside createAgent and LangGraph ([#25132](https://github.com/getsentry/sentry-javascript/pull/25132)) +- test(node): Port AMQP integration tests to span streaming ([#25158](https://github.com/getsentry/sentry-javascript/pull/25158)) +- test(node): Port Anthropic integration tests to span streaming ([#25082](https://github.com/getsentry/sentry-javascript/pull/25082)) +- test(node): Port Bedrock tests to span streaming ([#25086](https://github.com/getsentry/sentry-javascript/pull/25086)) +- test(node): Port Google GenAI tests to span streaming ([#25083](https://github.com/getsentry/sentry-javascript/pull/25083)) +- test(node): Port Knex integration tests to span streaming ([#25154](https://github.com/getsentry/sentry-javascript/pull/25154)) +- test(node): Port LangChain tests to span streaming ([#25084](https://github.com/getsentry/sentry-javascript/pull/25084)) +- test(node): Port LangGraph tests to span streaming ([#25085](https://github.com/getsentry/sentry-javascript/pull/25085)) +- test(node): Port Mastra tests to span streaming ([#25087](https://github.com/getsentry/sentry-javascript/pull/25087)) +- test(node): Port MySQL integration tests to span streaming ([#25148](https://github.com/getsentry/sentry-javascript/pull/25148)) +- test(node): Port OpenAI integration tests to span streaming ([#25081](https://github.com/getsentry/sentry-javascript/pull/25081)) +- test(node): Port pool and DataLoader integration tests to span streaming ([#25156](https://github.com/getsentry/sentry-javascript/pull/25156)) +- test(node): Port Tedious integration tests to span streaming ([#25155](https://github.com/getsentry/sentry-javascript/pull/25155)) +- test(node): Port Vercel AI v5-v7 tests to span streaming ([#25090](https://github.com/getsentry/sentry-javascript/pull/25090)) + +
Work in this release was contributed by @zkasuran. Thank you for your contribution! From 3605c5051320e6a13dc9e9573996d3a4bbb4221a Mon Sep 17 00:00:00 2001 From: JPeer264 Date: Thu, 8 Oct 2026 15:59:07 +0200 Subject: [PATCH 47/47] chore(size-limit): Bump @sentry/node size limits --- .size-limit.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.size-limit.js b/.size-limit.js index fcb14bb45828..6e085d4f6e52 100644 --- a/.size-limit.js +++ b/.size-limit.js @@ -419,7 +419,7 @@ module.exports = [ import: createImport('init'), ignore: [...builtinModules, ...nodePrefixedBuiltinModules], gzip: true, - limit: '150 KB', + limit: '151 KB', disablePlugins: ['@size-limit/esbuild'], }, { @@ -465,7 +465,7 @@ module.exports = [ path: 'packages/node/build/esm/index.js', import: createImport('init'), gzip: true, - limit: '128 KB', + limit: '130 KB', disablePlugins: ['@size-limit/esbuild'], ignore: [...builtinModules, ...nodePrefixedBuiltinModules], modifyWebpackConfig: function (config) {