From 3cd34006796aab20eb5d37b628c7c7c7468c6e40 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2026 11:15:53 +0000 Subject: [PATCH 1/3] feat(server-utils): Decide gen_ai content recording per call for Mastra, TypeSafe, Workers AI and Bedrock recordInputs and recordOutputs of mastraIntegration, typesafeIntegration, instrumentTypeSafeClient, instrumentWorkersAiClient and awsIntegration (Bedrock Runtime) can be functions that decide per call. They run once when the call starts, in the caller's async context, and the end and response paths reuse that decision. Mastra decides per span when the exporter receives its start, so its functions run at export time. Co-Authored-By: Francesco Gringl-Novy <2411343+mydea@users.noreply.github.com> --- packages/server-utils/src/ai/mastra/index.ts | 40 +++-- .../server-utils/src/ai/typesafe/index.ts | 19 ++- .../server-utils/src/ai/workers-ai/index.ts | 24 +-- .../server-utils/src/ai/workers-ai/types.ts | 13 +- .../src/integrations/aws-sdk/index.ts | 4 +- .../aws-sdk/services/ServicesExtensions.ts | 4 +- .../aws-sdk/services/bedrock-runtime.ts | 50 ++++-- .../server-utils/src/integrations/typesafe.ts | 22 ++- .../ai/lib/tracing/mastra-exporter.test.ts | 54 ++++++- .../test/ai/lib/tracing/workers-ai.test.ts | 82 ++++++++++ .../test/integrations/aws-sdk-bedrock.test.ts | 127 +++++++++++++-- .../test/integrations/typesafe.test.ts | 149 ++++++++++++++++++ 12 files changed, 521 insertions(+), 67 deletions(-) create mode 100644 packages/server-utils/test/integrations/typesafe.test.ts diff --git a/packages/server-utils/src/ai/mastra/index.ts b/packages/server-utils/src/ai/mastra/index.ts index 7943275fe6bd..887a548d9091 100644 --- a/packages/server-utils/src/ai/mastra/index.ts +++ b/packages/server-utils/src/ai/mastra/index.ts @@ -11,8 +11,8 @@ import { } from '@sentry/core'; import { GEN_AI_RESPONSE_MODEL, SENTRY_OP, SENTRY_ORIGIN } from '@sentry/conventions/attributes'; import { DEBUG_BUILD } from '../../debug-build'; -import type { GenAiOptions } from '../core/utils'; -import { resolveAIRecordingOptions } from '../core/utils'; +import type { GenAiCallOptions } from '../core/utils'; +import { resolveGenAiCallRecording } from '../core/utils'; import { getOperation, getSpanAttributes, @@ -43,12 +43,23 @@ import type { MastraTracingEvent, } from './types'; -export type MastraExporterOptions = GenAiOptions; +/** The span that `recordInputs`/`recordOutputs` decide on. */ +export interface MastraCallInfo { + /** The Mastra span as exported, when it starts. */ + span: MastraExportedSpan; +} + +/** + * `recordInputs`/`recordOutputs` functions run when the exporter receives the start of a Mastra span, + * not in the async context of the code that started it. + */ +export type MastraExporterOptions = GenAiCallOptions; interface TrackedSpan { span: Span; spanType: MastraSpanType; usage: SpanAttributes; + recordingOptions: AttributeRecordingOptions; evaluation?: ClassifierEvaluationCall; } @@ -135,22 +146,23 @@ export class SentryMastraExporter implements MastraObservabilityExporter { return; } - // Per event: the exporter can be constructed before `Sentry.init()`, when `dataCollection.genAI` does not exist yet. - const recordingOptions = resolveAIRecordingOptions(this._options); switch (event.type) { case 'span_started': - this._onSpanStarted(span, recordingOptions); + this._onSpanStarted(span); break; case 'span_updated': - this._onSpanUpdated(span, recordingOptions); + this._onSpanUpdated(span); break; case 'span_ended': - this._onSpanEnded(span, recordingOptions); + this._onSpanEnded(span); break; } } - private _onSpanStarted(span: MastraExportedSpan, recordingOptions: AttributeRecordingOptions): void { + private _onSpanStarted(span: MastraExportedSpan): void { + // Per span, not in the constructor: the exporter can be constructed before `Sentry.init()`, when + // `dataCollection.genAI` does not exist yet. + const recordingOptions = resolveGenAiCallRecording(this._options, { span }); const parentId = this._resolveParentId(span.parentSpanId); const parentSpan = parentId ? this._spans.get(parentId)?.span : undefined; const activeSpan = getActiveSpan(); @@ -178,7 +190,7 @@ export class SentryMastraExporter implements MastraObservabilityExporter { }); } - this._trackSpan(span.id, { span: sentrySpan, spanType: span.type, usage: {}, evaluation }); + this._trackSpan(span.id, { span: sentrySpan, spanType: span.type, usage: {}, recordingOptions, evaluation }); } /** Track a started span, ending any Sentry span that would otherwise be dropped without `end()`. */ @@ -205,21 +217,21 @@ export class SentryMastraExporter implements MastraObservabilityExporter { return this._spans.remove(id); } - private _onSpanUpdated(span: MastraExportedSpan, recordingOptions: AttributeRecordingOptions): void { + private _onSpanUpdated(span: MastraExportedSpan): void { const tracked = this._spans.get(span.id); if (tracked) { - tracked.span.setAttributes(getSpanAttributes(span, recordingOptions)); + tracked.span.setAttributes(getSpanAttributes(span, tracked.recordingOptions)); } } - private _onSpanEnded(span: MastraExportedSpan, recordingOptions: AttributeRecordingOptions): void { + private _onSpanEnded(span: MastraExportedSpan): void { const tracked = this._spans.get(span.id); if (!tracked) { DEBUG_BUILD && debug.warn(`[Mastra] no Sentry span open for ended span ${span.id} (${span.name})`); return; } - const { span: sentrySpan } = tracked; + const { span: sentrySpan, recordingOptions } = tracked; sentrySpan.setAttributes(getSpanAttributes(span, recordingOptions)); sentrySpan.updateName(getSpanName(span)); diff --git a/packages/server-utils/src/ai/typesafe/index.ts b/packages/server-utils/src/ai/typesafe/index.ts index 7bffea993239..9fa991fb78b4 100644 --- a/packages/server-utils/src/ai/typesafe/index.ts +++ b/packages/server-utils/src/ai/typesafe/index.ts @@ -13,10 +13,19 @@ import { import { GEN_AI_DECIDE } from '@sentry/conventions/op'; import type { Span, SpanAttributes } from '@sentry/core'; import { isObjectLike, SPAN_STATUS_ERROR, startInactiveSpan, withActiveSpan } from '@sentry/core'; -import type { GenAiOptions } from '../core/utils'; -import { getEvaluationInputMessages, getEvaluationOutputMessages, resolveAIRecordingOptions } from '../core/utils'; +import type { GenAiCallOptions } from '../core/utils'; +import { getEvaluationInputMessages, getEvaluationOutputMessages, resolveGenAiCallRecording } from '../core/utils'; import { TYPESAFE_ORIGIN, TYPESAFE_PROVIDER_NAME } from './constants'; +/** The call that `recordInputs`/`recordOutputs` decide on. */ +export interface TypeSafeCallInfo { + /** The request, the first argument of `systemOne` as passed. */ + request: unknown; +} + +/** Options for the TypeSafe integration. */ +export type TypeSafeOptions = GenAiCallOptions; + /** * Start the span for a `systemOne(request)` call. The request model falls back to the client's * `defaultModel`, the same way the SDK resolves it. @@ -119,7 +128,7 @@ export function onSystemOneResponse( /** * Instrument a TypeSafe client (`@typesafe-ai/sdk`) with Sentry tracing. */ -export function instrumentTypeSafeClient(client: T, options?: GenAiOptions): T { +export function instrumentTypeSafeClient(client: T, options?: TypeSafeOptions): T { return new Proxy(client, { get(target, prop) { // Read with the real client as receiver: the SDK keeps its API key in a private class field. @@ -140,9 +149,9 @@ function instrumentSystemOne( systemOne: (...args: unknown[]) => unknown, client: object, args: unknown[], - options: GenAiOptions | undefined, + options: TypeSafeOptions | undefined, ): unknown { - const { recordInputs, recordOutputs } = resolveAIRecordingOptions(options); + const { recordInputs, recordOutputs } = resolveGenAiCallRecording(options, { request: args[0] }); const span = startDecideSpan(args[0], client, recordInputs); const endWithError = (): void => { span.setStatus({ code: SPAN_STATUS_ERROR, message: 'internal_error' }); diff --git a/packages/server-utils/src/ai/workers-ai/index.ts b/packages/server-utils/src/ai/workers-ai/index.ts index ff62c38a7048..940b43b90ff6 100644 --- a/packages/server-utils/src/ai/workers-ai/index.ts +++ b/packages/server-utils/src/ai/workers-ai/index.ts @@ -1,7 +1,7 @@ import { SENTRY_OP } from '@sentry/conventions/attributes'; import { getClient, hasSpanStreamingEnabled, SPAN_STATUS_ERROR, startSpan, startSpanManual } from '@sentry/core'; import type { Span } from '@sentry/core'; -import { isReadableStream, resolveAIRecordingOptions } from '../core/utils'; +import { isReadableStream, resolveGenAiCallRecording } from '../core/utils'; import { shouldSkipAiProviderSpan } from '../pi-ai/providers'; import { WORKERS_AI_INTEGRATION_NAME } from './constants'; import { instrumentWorkersAiStream } from './streaming'; @@ -20,7 +20,7 @@ import { function instrumentRun( originalRun: (...args: unknown[]) => Promise, context: unknown, - options: WorkersAiOptions & Required>, + options: WorkersAiOptions, ): (...args: unknown[]) => Promise { return function instrumentedRun(...args: unknown[]): Promise { if (shouldSkipAiProviderSpan(WORKERS_AI_INTEGRATION_NAME)) { @@ -30,6 +30,12 @@ function instrumentRun( const [model, inputs, runOptions] = args as [unknown, unknown, Record | undefined]; const operationName = getOperationName(model, inputs); + const { recordInputs, recordOutputs } = resolveGenAiCallRecording(options, { + operation: operationName, + model, + inputs, + options: runOptions, + }); const requestAttributes = extractRequestAttributes(model, inputs, operationName); const modelName = typeof model === 'string' && model ? model : 'unknown'; const client = getClient(); @@ -71,17 +77,17 @@ function instrumentRun( return handleError(error); } - if (options.recordInputs) { + if (recordInputs) { addRequestAttributes(span, inputs, operationName); } return originalResult.then(result => { if (isReadableStream(result)) { - return instrumentWorkersAiStream(result, span, options.recordOutputs); + return instrumentWorkersAiStream(result, span, recordOutputs); } // The model did not actually return a stream — finalize the span eagerly. - addResponseAttributes(span, result, options.recordOutputs, operationName); + addResponseAttributes(span, result, recordOutputs, operationName); span.end(); return result; }, handleError); @@ -91,13 +97,13 @@ function instrumentRun( return startSpan(spanConfig, (span: Span) => { const originalResult = originalRun.apply(context, args) as Promise; - if (options.recordInputs) { + if (recordInputs) { addRequestAttributes(span, inputs, operationName); } return originalResult.then(result => { if (!returnsRawResponse) { - addResponseAttributes(span, result, options.recordOutputs, operationName); + addResponseAttributes(span, result, recordOutputs, operationName); } return result; }); @@ -121,14 +127,12 @@ function instrumentRun( * ``` */ export function instrumentWorkersAiClient(client: T, options?: WorkersAiOptions): T { - const resolvedOptions = resolveAIRecordingOptions(options); - const instrumented = new Proxy(client, { get(target: object, prop: string | symbol, receiver: unknown): unknown { const value = Reflect.get(target, prop, receiver); if (prop === 'run' && typeof value === 'function') { - return instrumentRun(value as (...args: unknown[]) => Promise, target, resolvedOptions); + return instrumentRun(value as (...args: unknown[]) => Promise, target, options ?? {}); } // Bind passed-through functions to the original target to preserve `this` (e.g. private fields). diff --git a/packages/server-utils/src/ai/workers-ai/types.ts b/packages/server-utils/src/ai/workers-ai/types.ts index a9209990981c..0208f326db8e 100644 --- a/packages/server-utils/src/ai/workers-ai/types.ts +++ b/packages/server-utils/src/ai/workers-ai/types.ts @@ -1,7 +1,16 @@ -import type { GenAiOptions } from '../core/utils'; +import type { GenAiCallOptions } from '../core/utils'; + +/** The call that `recordInputs`/`recordOutputs` decide on: the arguments of `run()` as passed. */ +export interface WorkersAiCallInfo { + /** The gen_ai operation of the call, such as `chat` or `embeddings`. */ + operation: string; + model: unknown; + inputs: unknown; + options: Record | undefined; +} /** Options for the Workers AI integration. */ -export type WorkersAiOptions = GenAiOptions; +export type WorkersAiOptions = GenAiCallOptions; /** * The token usage reported by Workers AI text generation models. diff --git a/packages/server-utils/src/integrations/aws-sdk/index.ts b/packages/server-utils/src/integrations/aws-sdk/index.ts index 1088f7881aee..49d909e5d31a 100644 --- a/packages/server-utils/src/integrations/aws-sdk/index.ts +++ b/packages/server-utils/src/integrations/aws-sdk/index.ts @@ -10,7 +10,6 @@ import { SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; import { RPC } from '@sentry/conventions/op'; -import type { GenAiOptions } from '../../ai/core/utils'; import type { OrchestrionChannelContext } from '../../orchestrion/types'; import { CHANNELS } from '../../orchestrion/channels'; import { awsSdkModuleNames } from '../../orchestrion/config/aws-sdk'; @@ -19,6 +18,7 @@ import type { TracingChannelLifeCycleOptions } from '../../tracing-channel'; import { bindTracingChannelToSpan, safeChannelCallback } from '../../tracing-channel'; import { AWS_SDK_ORIGIN_INTEGRATION_NAME } from './constants'; import { ServicesExtensions } from './services'; +import type { BedrockRuntimeOptions } from './services/bedrock-runtime'; import type { NormalizedRequest, NormalizedResponse, RequestMetadata } from './types'; import { extractAttributesFromNormalizedRequest, normalizeV3Request, removeSuffixFromStringIfExists } from './utils'; @@ -65,7 +65,7 @@ function setMetadataAttributes(span: Span, metadata: Record | undef } // `recordInputs` and `recordOutputs` apply to Bedrock Runtime calls, the only AWS calls that carry prompts and responses. -export type AwsIntegrationOptions = GenAiOptions; +export type AwsIntegrationOptions = BedrockRuntimeOptions; const _awsIntegration = ((options?: AwsIntegrationOptions) => { const servicesExtensions = new ServicesExtensions(options); diff --git a/packages/server-utils/src/integrations/aws-sdk/services/ServicesExtensions.ts b/packages/server-utils/src/integrations/aws-sdk/services/ServicesExtensions.ts index a322ab457820..d359607dc407 100644 --- a/packages/server-utils/src/integrations/aws-sdk/services/ServicesExtensions.ts +++ b/packages/server-utils/src/integrations/aws-sdk/services/ServicesExtensions.ts @@ -1,6 +1,6 @@ import type { Span } from '@sentry/core'; -import type { GenAiOptions } from '../../../ai/core/utils'; import type { NormalizedRequest, NormalizedResponse, RequestMetadata } from '../types'; +import type { BedrockRuntimeOptions } from './bedrock-runtime'; import { BedrockRuntimeServiceExtension } from './bedrock-runtime'; import { DynamodbServiceExtension } from './dynamodb'; import { KinesisServiceExtension } from './kinesis'; @@ -17,7 +17,7 @@ export class ServicesExtensions implements ServiceExtension { // registered extension still get the base rpc span from the subscriber. private _services: Map; - public constructor(genAiOptions?: GenAiOptions) { + public constructor(genAiOptions?: BedrockRuntimeOptions) { this._services = new Map([ ['SecretsManager', new SecretsManagerServiceExtension()], ['SFN', new StepFunctionsServiceExtension()], diff --git a/packages/server-utils/src/integrations/aws-sdk/services/bedrock-runtime.ts b/packages/server-utils/src/integrations/aws-sdk/services/bedrock-runtime.ts index e822b5945bdc..dd910fc977e5 100644 --- a/packages/server-utils/src/integrations/aws-sdk/services/bedrock-runtime.ts +++ b/packages/server-utils/src/integrations/aws-sdk/services/bedrock-runtime.ts @@ -13,13 +13,13 @@ import { GEN_AI_USAGE_OUTPUT_TOKENS, } from '@sentry/conventions/attributes'; import { GEN_AI_CHAT, GEN_AI_GENERATE_CONTENT } from '@sentry/conventions/op'; -import type { GenAiOptions, StreamResponseState } from '../../../ai/core/utils'; +import type { GenAiCallOptions, StreamResponseState } from '../../../ai/core/utils'; import type { GenAiMessagePart } from '../../../ai/types'; import { endStreamSpan, getErrorStatus, getTokenUsageAttributes, - resolveAIRecordingOptions, + resolveGenAiCallRecording, } from '../../../ai/core/utils'; import { isInsidePiAiSpan } from '../../../ai/pi-ai/providers'; import { DEBUG_BUILD } from '../../../debug-build'; @@ -28,7 +28,7 @@ import { GEN_AI_OPERATION_NAME_VALUE_GENERATE_CONTENT, GEN_AI_SYSTEM_VALUE_AWS_BEDROCK, } from '../constants'; -import type { NormalizedRequest, NormalizedResponse } from '../types'; +import type { CommandInput, NormalizedRequest, NormalizedResponse } from '../types'; import { appendStreamPart, contentBlocksToParts, @@ -66,15 +66,38 @@ type BedrockStreamState = StreamResponseState & { parts?: GenAiMessagePart[] }; const textDecoder = new TextDecoder(); +/** The call that `recordInputs`/`recordOutputs` decide on: a Bedrock Runtime command as sent. */ +export interface BedrockRuntimeCallInfo { + /** The command name, such as `Converse` or `InvokeModelWithResponseStream`. */ + commandName: string; + /** The command input as passed. */ + input: CommandInput; +} + +/** Options for the Bedrock Runtime instrumentation of the AWS integration. */ +export type BedrockRuntimeOptions = GenAiCallOptions; + export class BedrockRuntimeServiceExtension implements ServiceExtension { // Requests pi-ai sends, which its own `gen_ai.chat` span already reports. They keep the plain AWS // client span, without the model call on it. private _piAiRequests: WeakSet = new WeakSet(); - public constructor(private readonly _options: GenAiOptions = {}) {} + // Decided when the request starts, in the caller's async context, and reused by the response hooks. + private _recordOutputs: WeakMap = new WeakMap(); + + public constructor(private readonly _options: BedrockRuntimeOptions = {}) {} - private _recordingOptions(): Required { - return resolveAIRecordingOptions(this._options); + private _resolveRecording(request: NormalizedRequest): boolean { + const { recordInputs, recordOutputs } = resolveGenAiCallRecording(this._options, { + commandName: request.commandName, + input: request.commandInput, + }); + this._recordOutputs.set(request, recordOutputs); + return recordInputs; + } + + private _shouldRecordOutputs(response: NormalizedResponse): boolean { + return this._recordOutputs.get(response.request) ?? false; } public requestPreSpanHook(request: NormalizedRequest): RequestMetadata { @@ -133,6 +156,8 @@ export class BedrockRuntimeServiceExtension implements ServiceExtension { [GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_CHAT, }; + const recordInputs = this._resolveRecording(request); + const modelId = request.commandInput.modelId; if (modelId) { spanAttributes[GEN_AI_REQUEST_MODEL] = modelId; @@ -158,7 +183,7 @@ export class BedrockRuntimeServiceExtension implements ServiceExtension { } } - if (this._recordingOptions().recordInputs) { + if (recordInputs) { Object.assign(spanAttributes, getConverseInputAttributes(request.commandInput)); } @@ -177,6 +202,7 @@ export class BedrockRuntimeServiceExtension implements ServiceExtension { [GEN_AI_OPERATION_NAME]: GEN_AI_OPERATION_NAME_VALUE_GENERATE_CONTENT, }; + const recordInputs = this._resolveRecording(request); const modelId = request.commandInput?.modelId; if (modelId) { spanAttributes[GEN_AI_REQUEST_MODEL] = modelId; @@ -287,7 +313,7 @@ export class BedrockRuntimeServiceExtension implements ServiceExtension { } } - if (this._recordingOptions().recordInputs) { + if (recordInputs) { Object.assign(spanAttributes, getInvokeModelInputAttributes(modelId, requestBody)); } } @@ -304,7 +330,7 @@ export class BedrockRuntimeServiceExtension implements ServiceExtension { const { stopReason, usage, output } = response.data; setStopReason(span, stopReason); - if (this._recordingOptions().recordOutputs) { + if (this._shouldRecordOutputs(response)) { setOutputMessages(span, contentBlocksToParts(output?.message?.content)); } span.setAttributes( @@ -332,7 +358,7 @@ export class BedrockRuntimeServiceExtension implements ServiceExtension { response.data.stream as AsyncIterable, span, response.request.abortSignal, - this._recordingOptions().recordOutputs, + this._shouldRecordOutputs(response), processConverseStreamEvent, ); } @@ -400,7 +426,7 @@ export class BedrockRuntimeServiceExtension implements ServiceExtension { span.setAttribute(GEN_AI_USAGE_OUTPUT_TOKENS, Math.ceil(responseBody.outputs[0].text.length / 6)); } } - if (this._recordingOptions().recordOutputs) { + if (this._shouldRecordOutputs(response)) { setOutputMessages(span, getInvokeModelOutputParts(responseBody)); } } @@ -424,7 +450,7 @@ export class BedrockRuntimeServiceExtension implements ServiceExtension { stream as AsyncIterable<{ chunk?: { bytes?: Uint8Array } }>, span, response.request.abortSignal, - this._recordingOptions().recordOutputs, + this._shouldRecordOutputs(response), (item, state) => { if (processChunk) { const parsedChunk = parseChunk(item?.chunk?.bytes); diff --git a/packages/server-utils/src/integrations/typesafe.ts b/packages/server-utils/src/integrations/typesafe.ts index 70f2b39c020b..d07563d0e497 100644 --- a/packages/server-utils/src/integrations/typesafe.ts +++ b/packages/server-utils/src/integrations/typesafe.ts @@ -1,8 +1,8 @@ import * as diagnosticsChannel from '../utils/diagnosticsChannel'; import type { IntegrationFn } from '@sentry/core'; import { defineIntegration } from '@sentry/core'; -import type { GenAiOptions } from '../ai/core/utils'; -import { resolveAIRecordingOptions } from '../ai/core/utils'; +import { resolveGenAiCallRecording } from '../ai/core/utils'; +import type { TypeSafeOptions } from '../ai/typesafe'; import { addResponseAttributes, onSystemOneResponse, startDecideSpan } from '../ai/typesafe'; import { TYPESAFE_INTEGRATION_NAME } from '../ai/typesafe/constants'; import type { OrchestrionChannelContext } from '../orchestrion/types'; @@ -11,7 +11,10 @@ import { typesafeModuleNames } from '../orchestrion/config/typesafe'; import { invokeOrchestrionInstrumentation } from '../orchestrion/instrumentation'; import { bindTracingChannelToSpan } from '../tracing-channel'; -const _typesafeIntegration = ((options: GenAiOptions = {}) => { +// The recording decision is made once, when the span starts, in the caller's async context. +type TypeSafeChannelContext = OrchestrionChannelContext & { _sentryRecordOutputs?: boolean }; + +const _typesafeIntegration = ((options: TypeSafeOptions = {}) => { return { name: TYPESAFE_INTEGRATION_NAME, setup(client) { @@ -20,14 +23,19 @@ const _typesafeIntegration = ((options: GenAiOptions = {}) => { }; }) satisfies IntegrationFn; -function instrumentTypeSafe(options: GenAiOptions): void { +function instrumentTypeSafe(options: TypeSafeOptions): void { bindTracingChannelToSpan( - diagnosticsChannel.tracingChannel(CHANNELS.TYPESAFE_SYSTEM_ONE), - data => startDecideSpan(data.arguments?.[0], data.self, resolveAIRecordingOptions(options).recordInputs), + diagnosticsChannel.tracingChannel(CHANNELS.TYPESAFE_SYSTEM_ONE), + data => { + const request = data.arguments?.[0]; + const { recordInputs, recordOutputs } = resolveGenAiCallRecording(options, { request }); + data._sentryRecordOutputs = recordOutputs; + return startDecideSpan(request, data.self, recordInputs); + }, { beforeSpanEnd: (span, data) => { if (!('error' in data)) { - addResponseAttributes(span, data.result, resolveAIRecordingOptions(options).recordOutputs); + addResponseAttributes(span, data.result, !!data._sentryRecordOutputs); } }, deferSpanEnd: ({ data, end }) => diff --git a/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts b/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts index d2b897a59a55..d56e2df36cad 100644 --- a/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { ERROR_TYPE, GEN_AI_AGENT_NAME, @@ -40,6 +40,7 @@ import { setCurrentClient, spanToStaticSpanJSON, } from '@sentry/core'; +import type { MastraCallInfo } from '../../../../src/ai/mastra'; import { SentryMastraExporter } from '../../../../src/ai/mastra'; import type { ClassifierEvaluationCall } from '../../../../src/ai/mastra/classifier-evaluation'; import { setStartingClassifierEvaluation } from '../../../../src/ai/mastra/classifier-evaluation'; @@ -272,6 +273,57 @@ describe('SentryMastraExporter', () => { expect(JSON.stringify(json.data)).not.toContain('pii'); }); + describe('per-span recording options', () => { + function recorded(span: Span): { inputs: boolean; outputs: boolean } { + const { data } = spanToStaticSpanJSON(span); + return { inputs: GEN_AI_INPUT_MESSAGES in data, outputs: GEN_AI_OUTPUT_MESSAGES in data }; + } + + const isPublicAgent = ({ span }: MastraCallInfo): boolean => span.entityName === 'public_agent'; + + it('decides for each Mastra span from the span at export time', async () => { + const calls: MastraCallInfo[] = []; + exporter = new SentryMastraExporter({ + recordInputs: call => { + calls.push(call); + return isPublicAgent(call); + }, + recordOutputs: isPublicAgent, + }); + const publicSpan = makeSpan({ id: 'a', entityName: 'public_agent', input: 'hi', output: { text: 'hello' } }); + const privateSpan = makeSpan({ id: 'b', entityName: 'private_agent', input: 'hi', output: { text: 'hello' } }); + + await run(started(publicSpan), ended(publicSpan), started(privateSpan), ended(privateSpan)); + + expect(calls[0]).toEqual({ span: publicSpan }); + expect(endedSpans.map(recorded)).toEqual([ + { inputs: true, outputs: true }, + { inputs: false, outputs: false }, + ]); + }); + + it('decides inputs and outputs separately, once per span, and keeps the decision until the span ends', async () => { + const recordOutputs = vi.fn().mockReturnValueOnce(true).mockReturnValue(false); + exporter = new SentryMastraExporter({ recordInputs: () => false, recordOutputs }); + const span = makeSpan({ entityName: 'agent', input: 'hi', output: { text: 'hello' } }); + + await run(started(span), { type: 'span_updated', exportedSpan: span }, ended(span)); + + expect(recordOutputs).toHaveBeenCalledTimes(1); + expect(endedSpans.map(recorded)).toEqual([{ inputs: false, outputs: true }]); + }); + + it('passes the per-span decision to the classifier evaluation call', async () => { + exporter = new SentryMastraExporter({ recordInputs: () => true, recordOutputs: () => false }); + const call: ClassifierEvaluationCall = {}; + + setStartingClassifierEvaluation(call); + await run(started(makeSpan({ id: 'eval-1', type: 'classifier_evaluation' }))); + + expect(call).toMatchObject({ recordInputs: true, recordOutputs: false }); + }); + }); + it('re-parents a tool onto the generation across a dropped model_step', async () => { const agent = makeSpan({ id: 'agent-1', entityName: 'agent' }); const gen = makeSpan({ diff --git a/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts b/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts index 5b1f8738b7a3..6613bd40d48c 100644 --- a/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts @@ -17,13 +17,17 @@ import { import { GEN_AI_DECIDE } from '@sentry/conventions/op'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { + getCurrentScope, getMainCarrier, SEMANTIC_ATTRIBUTE_SENTRY_SAMPLE_RATE, setCurrentClient, spanToStaticSpanJSON, + withScope, } from '@sentry/core'; import type { DataCollection, Span } from '@sentry/core'; +import { setAsyncLocalStorageAsyncContextStrategy } from '../../../../src/async-context'; import { instrumentWorkersAiClient } from '../../../../src/ai/workers-ai'; +import type { WorkersAiCallInfo } from '../../../../src/ai/workers-ai/types'; import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; const MODEL = '@cf/meta/llama-3.1-8b-instruct'; @@ -163,6 +167,84 @@ describe('instrumentWorkersAiClient', () => { expect(endedSpans).toHaveLength(1); expect(spanToStaticSpanJSON(endedSpans[0]!).data).toEqual(expected); }); + + describe('per-call recording options', () => { + const messages = [ + { role: 'system', content: 'Answer in one word.' }, + { role: 'user', content: 'Capital of France?' }, + ]; + + function recorded(span: Span): { inputs: boolean; outputs: boolean } { + const { data } = spanToStaticSpanJSON(span); + return { inputs: GEN_AI_INPUT_MESSAGES in data, outputs: GEN_AI_OUTPUT_MESSAGES in data }; + } + + it('decides for each call from the run() arguments', async () => { + const endedSpans = setupClient(); + const calls: WorkersAiCallInfo[] = []; + const decide = (call: WorkersAiCallInfo): boolean => { + calls.push(call); + return (call.inputs as { privacy?: string }).privacy === 'public'; + }; + const client = { run: vi.fn().mockResolvedValue({ response: 'Paris' }) }; + const instrumented = instrumentWorkersAiClient(client, { recordInputs: decide, recordOutputs: decide }); + + const publicInputs = { messages, privacy: 'public' }; + const runOptions = { gateway: { id: 'my-gateway' } }; + await instrumented.run(MODEL, publicInputs, runOptions); + await instrumented.run(MODEL, { messages, privacy: 'private' }); + + expect(calls[0]).toEqual({ operation: 'chat', model: MODEL, inputs: publicInputs, options: runOptions }); + expect(endedSpans.map(recorded)).toEqual([ + { inputs: true, outputs: true }, + { inputs: false, outputs: false }, + ]); + }); + + it("runs the functions in the caller's async context", async () => { + setAsyncLocalStorageAsyncContextStrategy(); + const endedSpans = setupClient(); + const decide = (): boolean => getCurrentScope().getScopeData().tags.privacy === 'public'; + const client = { run: vi.fn().mockResolvedValue({ response: 'Paris' }) }; + const instrumented = instrumentWorkersAiClient(client, { recordInputs: decide, recordOutputs: decide }); + + await withScope(async scope => { + scope.setTag('privacy', 'public'); + await instrumented.run(MODEL, { messages }); + }); + await withScope(async scope => { + scope.setTag('privacy', 'private'); + await instrumented.run(MODEL, { messages }); + }); + + expect(endedSpans.map(recorded)).toEqual([ + { inputs: true, outputs: true }, + { inputs: false, outputs: false }, + ]); + }); + + it('decides inputs and outputs separately, once per call, for streams too', async () => { + const endedSpans = setupClient(); + const recordOutputs = vi.fn(() => true); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('data: {"response":"Paris"}\n\ndata: [DONE]\n\n')); + controller.close(); + }, + }); + const client = { run: vi.fn().mockResolvedValue(stream) }; + const instrumented = instrumentWorkersAiClient(client, { recordInputs: () => false, recordOutputs }); + + const result = (await instrumented.run(MODEL, { messages, stream: true })) as ReadableStream; + const reader = result.getReader(); + while (!(await reader.read()).done) { + // drain + } + + expect(recordOutputs).toHaveBeenCalledTimes(1); + expect(endedSpans.map(recorded)).toEqual([{ inputs: false, outputs: true }]); + }); + }); }); it('records TypeSafe models as decide spans, like the TypeSafe integration', async () => { diff --git a/packages/server-utils/test/integrations/aws-sdk-bedrock.test.ts b/packages/server-utils/test/integrations/aws-sdk-bedrock.test.ts index a5ef64146989..7e47d29286df 100644 --- a/packages/server-utils/test/integrations/aws-sdk-bedrock.test.ts +++ b/packages/server-utils/test/integrations/aws-sdk-bedrock.test.ts @@ -17,11 +17,14 @@ import { } from '@sentry/conventions/attributes'; import { GEN_AI_GENERATE_CONTENT } from '@sentry/conventions/op'; import type { Span } from '@sentry/core'; -import { setCurrentClient, spanToJSON, startInactiveSpan } from '@sentry/core'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import type { GenAiOptions } from '../../src/ai/core/utils'; +import { getCurrentScope, setCurrentClient, spanToJSON, startInactiveSpan, withScope } from '@sentry/core'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { + BedrockRuntimeCallInfo, + BedrockRuntimeOptions, +} from '../../src/integrations/aws-sdk/services/bedrock-runtime'; import { BedrockRuntimeServiceExtension } from '../../src/integrations/aws-sdk/services/bedrock-runtime'; -import type { NormalizedResponse, RequestMetadata } from '../../src/integrations/aws-sdk/types'; +import type { NormalizedRequest, RequestMetadata } from '../../src/integrations/aws-sdk/types'; import { getDefaultTestClientOptions, TestClient } from '../mocks/client'; const CLAUDE_MODEL_ID = 'anthropic.claude-3-5-sonnet-20240620-v1:0'; @@ -36,7 +39,7 @@ const CLAUDE_REQUEST_BODY = { function preSpanHook( commandName: string, commandInput: Record, - options?: GenAiOptions, + options?: BedrockRuntimeOptions, ): RequestMetadata { return new BedrockRuntimeServiceExtension(options).requestPreSpanHook({ serviceName: 'BedrockRuntime', @@ -258,14 +261,12 @@ function respond( commandInput: Record, data: unknown, abortSignal?: AbortSignal, - options?: GenAiOptions, + options?: BedrockRuntimeOptions, ): void { - const response: NormalizedResponse = { - data, - request: { serviceName: 'BedrockRuntime', commandName, commandInput, abortSignal }, - requestId: 'b0a5e5a2-2c2b-4f56-9d0e-8c7c2d3f4a10', - }; - new BedrockRuntimeServiceExtension(options).responseHook(response, span); + const extension = new BedrockRuntimeServiceExtension(options); + const request: NormalizedRequest = { serviceName: 'BedrockRuntime', commandName, commandInput, abortSignal }; + extension.requestPreSpanHook(request); + extension.responseHook({ data, request, requestId: 'b0a5e5a2-2c2b-4f56-9d0e-8c7c2d3f4a10' }, span); } async function drain(stream: AsyncIterable): Promise { @@ -859,3 +860,105 @@ describe('BedrockRuntimeServiceExtension output recording', () => { }, ); }); + +describe('BedrockRuntimeServiceExtension per-call recording options', () => { + const CONVERSE_RESPONSE = { output: { message: { role: 'assistant', content: [{ text: 'Paris.' }] } } }; + + beforeEach(() => { + const client = new TestClient(getDefaultTestClientOptions({ tracesSampleRate: 1 })); + setCurrentClient(client); + client.init(); + }); + + afterEach(() => { + setCurrentClient(undefined as never); + }); + + function converse( + extension: BedrockRuntimeServiceExtension, + commandInput: Record, + ): { inputs: boolean; outputs: boolean } { + const request: NormalizedRequest = { serviceName: 'BedrockRuntime', commandName: 'Converse', commandInput }; + const { spanAttributes } = extension.requestPreSpanHook(request); + const span = startInactiveSpan({ name: 'bedrock' }); + extension.responseHook({ data: CONVERSE_RESPONSE, request, requestId: 'id' }, span); + return { + inputs: GEN_AI_INPUT_MESSAGES in spanAttributes!, + outputs: GEN_AI_OUTPUT_MESSAGES in spanToJSON(span).attributes, + }; + } + + const input = (privacy: string): Record => ({ + modelId: 'amazon.nova-pro-v1:0', + messages: [{ role: 'user', content: [{ text: 'Capital of France?' }] }], + requestMetadata: { privacy }, + }); + + const isPublic = (call: BedrockRuntimeCallInfo): boolean => call.input.requestMetadata.privacy === 'public'; + + it('decides for each call from the command name and input', () => { + const calls: BedrockRuntimeCallInfo[] = []; + const recordInputs = (call: BedrockRuntimeCallInfo): boolean => { + calls.push(call); + return isPublic(call); + }; + const extension = new BedrockRuntimeServiceExtension({ recordInputs, recordOutputs: isPublic }); + + const publicInput = input('public'); + const results = [converse(extension, publicInput), converse(extension, input('private'))]; + + expect(calls[0]).toEqual({ commandName: 'Converse', input: publicInput }); + expect(results).toEqual([ + { inputs: true, outputs: true }, + { inputs: false, outputs: false }, + ]); + }); + + it("runs the functions in the caller's async context", () => { + const decide = (): boolean => getCurrentScope().getScopeData().tags.privacy === 'public'; + const extension = new BedrockRuntimeServiceExtension({ recordInputs: decide, recordOutputs: decide }); + + const results = ['public', 'private'].map(privacy => + withScope(scope => { + scope.setTag('privacy', privacy); + return converse(extension, input('')); + }), + ); + + expect(results).toEqual([ + { inputs: true, outputs: true }, + { inputs: false, outputs: false }, + ]); + }); + + it('decides inputs and outputs separately, once, and keeps the decision for the response', () => { + const recordOutputs = vi.fn().mockReturnValueOnce(true).mockReturnValue(false); + const extension = new BedrockRuntimeServiceExtension({ recordInputs: () => false, recordOutputs }); + + expect(converse(extension, input(''))).toEqual({ inputs: false, outputs: true }); + expect(recordOutputs).toHaveBeenCalledTimes(1); + }); + + it('keeps the request-time decision until a stream is drained', async () => { + const recordOutputs = vi.fn().mockReturnValueOnce(true).mockReturnValue(false); + const extension = new BedrockRuntimeServiceExtension({ recordOutputs }); + const request: NormalizedRequest = { + serviceName: 'BedrockRuntime', + commandName: 'ConverseStream', + commandInput: input(''), + }; + extension.requestPreSpanHook(request); + const span = startInactiveSpan({ name: 'bedrock' }); + const data = { + stream: (async function* () { + yield { contentBlockDelta: { contentBlockIndex: 0, delta: { text: 'Paris.' } } }; + })(), + }; + + extension.responseHook({ data, request, requestId: 'id' }, span); + await drain(data.stream); + + expect(recordOutputs).toHaveBeenCalledTimes(1); + expect(spanToJSON(span).attributes).toHaveProperty(GEN_AI_OUTPUT_MESSAGES); + }); +}); diff --git a/packages/server-utils/test/integrations/typesafe.test.ts b/packages/server-utils/test/integrations/typesafe.test.ts new file mode 100644 index 000000000000..3911fb6e2576 --- /dev/null +++ b/packages/server-utils/test/integrations/typesafe.test.ts @@ -0,0 +1,149 @@ +import { tracingChannel } from 'node:diagnostics_channel'; +import type { Span } from '@sentry/core'; +import { + getCurrentScope, + GLOBAL_OBJ, + setAsyncContextStrategy, + setCurrentClient, + spanToJSON, + withScope, +} from '@sentry/core'; +import { GEN_AI_INPUT_MESSAGES, GEN_AI_OUTPUT_MESSAGES } from '@sentry/conventions/attributes'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { setAsyncLocalStorageAsyncContextStrategy } from '../../src/async-context'; +import type { TypeSafeCallInfo } from '../../src/ai/typesafe'; +import { instrumentTypeSafeClient } from '../../src/ai/typesafe'; +import { typesafeIntegration } from '../../src/integrations/typesafe'; +import { CHANNELS } from '../../src/orchestrion/channels'; +import { getDefaultTestClientOptions, TestClient } from '../mocks/client'; + +type Decide = (call: TypeSafeCallInfo) => boolean | undefined; + +const RESULT = { + model: 'jev-1', + answers: { is_urgent: { type: 'noul', noul: 0.97 } }, + usage: { input_tokens: 10, output_tokens: 5 }, +}; + +const request = (privacy: string): Record => ({ + state: 'Help!', + questions: { is_urgent: { type: 'noul', instructions: 'Urgent?' } }, + metadata: { privacy }, +}); + +describe('TypeSafe per-call recording options', () => { + let endedSpans: Span[]; + let decideInputs: Decide; + let decideOutputs: Decide; + + function useClient(): void { + const client = new TestClient( + getDefaultTestClientOptions({ dsn: 'https://public@dsn.ingest.sentry.io/1337', tracesSampleRate: 1 }), + ); + setCurrentClient(client); + client.init(); + client.on('spanEnd', span => endedSpans.push(span)); + } + + function recorded(span: Span): { inputs: boolean; outputs: boolean } { + const { attributes } = spanToJSON(span); + return { inputs: GEN_AI_INPUT_MESSAGES in attributes, outputs: GEN_AI_OUTPUT_MESSAGES in attributes }; + } + + const isPublic = (call: TypeSafeCallInfo): boolean => + (call.request as { metadata: { privacy: string } }).metadata.privacy === 'public'; + + beforeAll(() => { + setAsyncLocalStorageAsyncContextStrategy(); + GLOBAL_OBJ.__SENTRY_ORCHESTRION__ = { runtime: ['@typesafe-ai/sdk'] }; + endedSpans = []; + useClient(); + // The channel stays subscribed for the whole file, so each test swaps the decision functions. + typesafeIntegration({ + recordInputs: call => decideInputs(call), + recordOutputs: call => decideOutputs(call), + }).setup!(getCurrentScope().getClient()!); + }); + + afterAll(() => { + setAsyncContextStrategy(undefined); + delete GLOBAL_OBJ.__SENTRY_ORCHESTRION__; + }); + + beforeEach(() => { + endedSpans = []; + decideInputs = () => true; + decideOutputs = () => true; + useClient(); + }); + + function systemOne(params: Record): unknown { + return tracingChannel(CHANNELS.TYPESAFE_SYSTEM_ONE).traceSync(() => RESULT, { arguments: [params], self: {} }); + } + + it('decides for each call from its request', () => { + const calls: TypeSafeCallInfo[] = []; + decideInputs = call => { + calls.push(call); + return isPublic(call); + }; + decideOutputs = isPublic; + + const publicRequest = request('public'); + systemOne(publicRequest); + systemOne(request('private')); + + expect(calls[0]).toEqual({ request: publicRequest }); + expect(endedSpans.map(recorded)).toEqual([ + { inputs: true, outputs: true }, + { inputs: false, outputs: false }, + ]); + }); + + it("runs the functions in the caller's async context", async () => { + decideInputs = () => getCurrentScope().getScopeData().tags.privacy === 'public'; + decideOutputs = decideInputs; + + await withScope(async scope => { + scope.setTag('privacy', 'public'); + systemOne(request('')); + }); + await withScope(async scope => { + scope.setTag('privacy', 'private'); + systemOne(request('')); + }); + + expect(endedSpans.map(recorded)).toEqual([ + { inputs: true, outputs: true }, + { inputs: false, outputs: false }, + ]); + }); + + it('decides inputs and outputs separately', () => { + decideInputs = () => false; + decideOutputs = () => true; + + systemOne(request('')); + + expect(endedSpans.map(recorded)).toEqual([{ inputs: false, outputs: true }]); + }); + + it('instrumentTypeSafeClient decides per call, once, and keeps the decision until the response', async () => { + const recordOutputs = vi.fn(isPublic); + const response = new Response(JSON.stringify(RESULT)); + const client = { + systemOne: () => ({ asResponse: () => Promise.resolve(response.clone()) }), + }; + const instrumented = instrumentTypeSafeClient(client, { recordInputs: isPublic, recordOutputs }); + + instrumented.systemOne(request('public')); + instrumented.systemOne(request('private')); + await vi.waitFor(() => expect(endedSpans).toHaveLength(2)); + + expect(recordOutputs).toHaveBeenCalledTimes(2); + expect(endedSpans.map(recorded)).toEqual([ + { inputs: true, outputs: true }, + { inputs: false, outputs: false }, + ]); + }); +}); From a1d63ef6c1fb91ae7f937a949e808c1c6f5a5f13 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2026 12:50:03 +0000 Subject: [PATCH 2/3] test(node): Cover per-call gen_ai recording for Mastra, TypeSafe and Bedrock with node integration tests Replaces the per-call recording unit tests for Mastra, TypeSafe and Bedrock with one node integration test each. Workers AI has no node suite, so it keeps one focused unit test. Co-Authored-By: Francesco Gringl-Novy <2411343+mydea@users.noreply.github.com> --- .../instrument-with-recording-functions.mjs | 15 ++ .../bedrock/scenario-recording-functions.mjs | 104 ++++++++++++ .../suites/aws-serverless/bedrock/test.ts | 33 ++++ .../tracing/mastra/core/observability/test.ts | 45 +++++- .../instrument-with-recording-functions.mjs | 16 ++ .../mastra/scenario-recording-functions.mjs | 77 +++++++++ .../instrument-with-recording-functions.mjs | 15 ++ .../typesafe/scenario-recording-functions.mjs | 53 +++++++ .../suites/tracing/typesafe/test.ts | 32 ++++ .../ai/lib/tracing/mastra-exporter.test.ts | 54 +------ .../test/ai/lib/tracing/workers-ai.test.ts | 115 ++++++-------- .../test/integrations/aws-sdk-bedrock.test.ts | 111 +------------ .../test/integrations/typesafe.test.ts | 149 ------------------ 13 files changed, 440 insertions(+), 379 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/aws-serverless/bedrock/instrument-with-recording-functions.mjs create mode 100644 dev-packages/node-integration-tests/suites/aws-serverless/bedrock/scenario-recording-functions.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/mastra/instrument-with-recording-functions.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/mastra/scenario-recording-functions.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-with-recording-functions.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/typesafe/scenario-recording-functions.mjs delete mode 100644 packages/server-utils/test/integrations/typesafe.test.ts diff --git a/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/instrument-with-recording-functions.mjs b/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/instrument-with-recording-functions.mjs new file mode 100644 index 000000000000..eb58d7e81c4e --- /dev/null +++ b/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/instrument-with-recording-functions.mjs @@ -0,0 +1,15 @@ +import * as Sentry from '@sentry/aws-serverless'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + integrations: [ + Sentry.awsIntegration({ + recordInputs: ({ input }) => input.requestMetadata?.privacy === 'public', + recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public', + }), + ], + transport: loggingTransport, +}); diff --git a/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/scenario-recording-functions.mjs b/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/scenario-recording-functions.mjs new file mode 100644 index 000000000000..3e0131f303a7 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/scenario-recording-functions.mjs @@ -0,0 +1,104 @@ +import * as Sentry from '@sentry/aws-serverless'; +import { BedrockRuntimeClient, ConverseCommand, ConverseStreamCommand } from '@aws-sdk/client-bedrock-runtime'; +// Force the HTTP/1 handler so `nock` can intercept the request instead of hitting real AWS. +import { NodeHttpHandler } from '@smithy/node-http-handler'; +import { EventStreamCodec } from '@smithy/core/event-streams'; +import nock from 'nock'; +import { Buffer } from 'node:buffer'; + +nock.disableNetConnect(); + +const region = 'us-east-1'; +const credentials = { accessKeyId: 'aws-test-key', secretAccessKey: 'aws-test-secret' }; +const host = `https://bedrock-runtime.${region}.amazonaws.com`; + +// Bedrock streams responses in the binary AWS event stream format, not JSON. The SDK decodes and +// checks each message (headers, payload, CRC32), so the mocked response must use the same format. +// The SDK's own codec encodes the messages. It needs UTF-8 helpers to encode the header strings. +// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTSelectObjectAppendix.html +const codec = new EventStreamCodec( + bytes => Buffer.from(bytes).toString('utf8'), + text => Buffer.from(text), +); + +// The codec needs a type for each header value. All Bedrock headers are strings. +function encodeMessage(headers, payload) { + return codec.encode({ + headers: Object.fromEntries(Object.entries(headers).map(([name, value]) => [name, { type: 'string', value }])), + body: Buffer.from(JSON.stringify(payload)), + }); +} + +// The SDK reads `:event-type` to find the stream member (for example `messageStart` or `chunk`). +function event(eventType, payload) { + return encodeMessage( + { ':message-type': 'event', ':event-type': eventType, ':content-type': 'application/json' }, + payload, + ); +} + +function mockStream(path, messages) { + nock(host).post(path).reply(200, Buffer.concat(messages), { 'content-type': 'application/vnd.amazon.eventstream' }); +} + +function createClient() { + return new BedrockRuntimeClient({ region, credentials, maxAttempts: 1, requestHandler: new NodeHttpHandler() }); +} + +function converse(modelId, privacy) { + nock(host) + .post(/\/model\/.*\/converse$/) + .reply( + 200, + JSON.stringify({ + output: { message: { role: 'assistant', content: [{ text: 'Hello from Bedrock' }] } }, + stopReason: 'end_turn', + usage: { inputTokens: 12, outputTokens: 8, totalTokens: 20 }, + }), + { 'content-type': 'application/json' }, + ); + + return createClient().send( + new ConverseCommand({ + modelId, + messages: [{ role: 'user', content: [{ text: 'Hello' }] }], + requestMetadata: { privacy }, + }), + ); +} + +async function run() { + await Sentry.startSpan({ name: 'Test Transaction' }, async () => { + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'public'); + await converse('amazon.nova-pro-v1:0', 'public'); + }); + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'private'); + await converse('amazon.nova-lite-v1:0', 'private'); + + mockStream(/\/model\/.*\/converse-stream$/, [ + event('messageStart', { role: 'assistant' }), + event('contentBlockDelta', { contentBlockIndex: 0, delta: { text: 'Hello from Bedrock' } }), + event('contentBlockStop', { contentBlockIndex: 0 }), + event('messageStop', { stopReason: 'end_turn' }), + ]); + const pending = createClient().send( + new ConverseStreamCommand({ + modelId: 'amazon.nova-micro-v1:0', + messages: [{ role: 'user', content: [{ text: 'Hello' }] }], + requestMetadata: { privacy: 'public' }, + }), + ); + // Changing the tag after the call started must not change its outputs decision. + Sentry.getCurrentScope().setTag('privacy', 'public'); + const response = await pending; + for await (const _ of response.stream) { + // drain + } + }); + }); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/test.ts b/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/test.ts index d021a54859b3..34af30e6fcd1 100644 --- a/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/test.ts +++ b/dev-packages/node-integration-tests/suites/aws-serverless/bedrock/test.ts @@ -140,4 +140,37 @@ describe('awsIntegration - Bedrock', () => { await createTestRunner().ignore('event').expect({ span: assertStreamSpans }).start().completed(); }); }); + + createEsmAndCjsTests( + __dirname, + 'scenario-recording-functions.mjs', + 'instrument-with-recording-functions.mjs', + (createTestRunner, test) => { + test('decides content recording per call when the options are functions', { timeout: 90_000 }, async () => { + await createTestRunner() + .ignore('event') + .expect({ + span: container => { + const spans = container.items.filter(span => span.attributes[GEN_AI_REQUEST_MODEL]); + const recordedByModel = Object.fromEntries( + spans.map(span => [ + span.attributes[GEN_AI_REQUEST_MODEL]?.value, + { + inputs: GEN_AI_INPUT_MESSAGES in span.attributes, + outputs: GEN_AI_OUTPUT_MESSAGES in span.attributes, + }, + ]), + ); + expect(recordedByModel).toEqual({ + 'amazon.nova-pro-v1:0': { inputs: true, outputs: true }, + 'amazon.nova-lite-v1:0': { inputs: false, outputs: false }, + 'amazon.nova-micro-v1:0': { inputs: true, outputs: false }, + }); + }, + }) + .start() + .completed(); + }); + }, + ); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/core/observability/test.ts b/dev-packages/node-integration-tests/suites/tracing/mastra/core/observability/test.ts index 909db4b6a1c3..398d2c685ccc 100644 --- a/dev-packages/node-integration-tests/suites/tracing/mastra/core/observability/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/core/observability/test.ts @@ -1,4 +1,11 @@ -import { SENTRY_OP, SENTRY_ORIGIN } from '@sentry/conventions/attributes'; +import { + GEN_AI_AGENT_NAME, + GEN_AI_INPUT_MESSAGES, + GEN_AI_OUTPUT_MESSAGES, + SENTRY_OP, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import { GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op'; import { afterAll, expect, describe } from 'vitest'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../../utils/runner'; import { supports } from '../../../../../utils'; @@ -74,4 +81,40 @@ describe.runIf(supports({ min: 22 }))('Mastra integration', () => { }, MASTRA_DEPENDENCIES, ); + + createEsmAndCjsTests( + join(__dirname, '../..'), + 'scenario-recording-functions.mjs', + 'instrument-with-recording-functions.mjs', + (createRunner, test) => { + test('decides content recording per span when the options are functions', async () => { + await createRunner() + .expect({ + span: container => { + const agentSpans = container.items.filter( + span => + span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra' && + span.attributes[SENTRY_OP]?.value === GEN_AI_INVOKE_AGENT, + ); + const recordedByAgent = Object.fromEntries( + agentSpans.map(span => [ + span.attributes[GEN_AI_AGENT_NAME]?.value, + { + inputs: GEN_AI_INPUT_MESSAGES in span.attributes, + outputs: GEN_AI_OUTPUT_MESSAGES in span.attributes, + }, + ]), + ); + expect(recordedByAgent).toEqual({ + public_agent: { inputs: true, outputs: true }, + private_agent: { inputs: false, outputs: true }, + }); + }, + }) + .start() + .completed(); + }); + }, + MASTRA_DEPENDENCIES, + ); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/instrument-with-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/mastra/instrument-with-recording-functions.mjs new file mode 100644 index 000000000000..9a3ff05db63c --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/instrument-with-recording-functions.mjs @@ -0,0 +1,16 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + integrations: [ + Sentry.mastraIntegration({ + recordInputs: ({ span }) => span.entityName === 'public_agent', + // Mastra sets `output` only when the span ends, so this records outputs only if decided at start. + recordOutputs: ({ span }) => span.output === undefined, + }), + ], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-recording-functions.mjs new file mode 100644 index 000000000000..c9ad1f7dfb1f --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/mastra/scenario-recording-functions.mjs @@ -0,0 +1,77 @@ +import express from 'express'; +import * as Sentry from '@sentry/node'; +import { Mastra } from '@mastra/core'; +import { Agent } from '@mastra/core/agent'; + +// Inlined OpenAI-compatible mock: the ESM/CJS runner copies only this file into its temp dir. +function startMockProvider(responses) { + const app = express(); + app.use(express.json()); + + let call = 0; + app.post('/v1/chat/completions', (req, res) => { + const response = responses[Math.min(call, responses.length - 1)]; + call++; + res.json({ + id: response.id, + object: 'chat.completion', + created: 1, + model: req.body.model, + choices: [ + { + index: 0, + finish_reason: response.toolCalls ? 'tool_calls' : 'stop', + message: { + role: 'assistant', + content: response.content ?? null, + ...(response.toolCalls ? { tool_calls: response.toolCalls } : {}), + }, + }, + ], + usage: response.usage, + }); + }); + + const server = app.listen(0); + return { + url: `http://localhost:${server.address().port}/v1`, + close: () => server.close(), + }; +} + +const provider = startMockProvider([ + { + id: 'chatcmpl-1', + content: 'It is 22C in Berlin.', + usage: { prompt_tokens: 12, completion_tokens: 7, total_tokens: 19 }, + }, +]); + +async function run() { + const model = { id: 'openai/gpt-4o-mini', url: provider.url, apiKey: 'test' }; + const agents = { + public_agent: new Agent({ + id: 'public_agent', + name: 'public_agent', + instructions: 'You report the weather.', + model, + }), + private_agent: new Agent({ + id: 'private_agent', + name: 'private_agent', + instructions: 'You report the weather.', + model, + }), + }; + const mastra = new Mastra({ agents, logger: false }); + + await Sentry.startSpan({ op: 'function', name: 'mastra-test' }, async () => { + await mastra.getAgent('public_agent').generate('What is the weather in Berlin?'); + await mastra.getAgent('private_agent').generate('What is the weather in Berlin?'); + }); + + await mastra.observability.shutdown(); + provider.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-with-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-with-recording-functions.mjs new file mode 100644 index 000000000000..1b2d75a3bc9e --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-with-recording-functions.mjs @@ -0,0 +1,15 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + integrations: [ + Sentry.typesafeIntegration({ + recordInputs: ({ request }) => request?.model?.startsWith('public'), + recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public', + }), + ], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/scenario-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/typesafe/scenario-recording-functions.mjs new file mode 100644 index 000000000000..3f46d3f7ed45 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/scenario-recording-functions.mjs @@ -0,0 +1,53 @@ +import * as Sentry from '@sentry/node'; +import { noul, TypeSafeClient } from '@typesafe-ai/sdk'; +import express from 'express'; + +function startMockServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/systemone', (req, res) => { + res.json({ + model: req.body.model, + answers: { authIssue: { type: 'noul', noul: 0.98 } }, + usage: { input_tokens: 275, output_tokens: 20 }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockServer(); + const client = new TypeSafeClient({ + apiKey: 'mock-api-key', + baseURL: `http://localhost:${server.address().port}`, + retry: { maxRetries: 0 }, + }); + const request = { state: 'I cannot log in.', questions: { authIssue: noul('Is there a login problem?') } }; + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'public'); + await client.systemOne({ ...request, model: 'public' }); + }); + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'private'); + await client.systemOne({ ...request, model: 'private' }); + + // Changing the tag after the call started must not change its outputs decision. + const pending = client.systemOne({ ...request, model: 'public-late' }); + Sentry.getCurrentScope().setTag('privacy', 'public'); + await pending; + }); + }); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts b/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts index 56e7840fe759..96eb7011f72c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts @@ -135,4 +135,36 @@ describe('TypeSafe integration', () => { }); }); }); + + createEsmAndCjsTests( + __dirname, + 'scenario-recording-functions.mjs', + 'instrument-with-recording-functions.mjs', + (createRunner, test) => { + test('decides content recording per call when the options are functions', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE); + const recordedByModel = Object.fromEntries( + spans.map(span => [ + span.attributes[GEN_AI_REQUEST_MODEL]?.value, + { + inputs: GEN_AI_INPUT_MESSAGES in span.attributes, + outputs: GEN_AI_OUTPUT_MESSAGES in span.attributes, + }, + ]), + ); + expect(recordedByModel).toEqual({ + public: { inputs: true, outputs: true }, + private: { inputs: false, outputs: false }, + 'public-late': { inputs: true, outputs: false }, + }); + }, + }) + .start() + .completed(); + }); + }, + ); }); diff --git a/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts b/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts index d56e2df36cad..d2b897a59a55 100644 --- a/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/mastra-exporter.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { ERROR_TYPE, GEN_AI_AGENT_NAME, @@ -40,7 +40,6 @@ import { setCurrentClient, spanToStaticSpanJSON, } from '@sentry/core'; -import type { MastraCallInfo } from '../../../../src/ai/mastra'; import { SentryMastraExporter } from '../../../../src/ai/mastra'; import type { ClassifierEvaluationCall } from '../../../../src/ai/mastra/classifier-evaluation'; import { setStartingClassifierEvaluation } from '../../../../src/ai/mastra/classifier-evaluation'; @@ -273,57 +272,6 @@ describe('SentryMastraExporter', () => { expect(JSON.stringify(json.data)).not.toContain('pii'); }); - describe('per-span recording options', () => { - function recorded(span: Span): { inputs: boolean; outputs: boolean } { - const { data } = spanToStaticSpanJSON(span); - return { inputs: GEN_AI_INPUT_MESSAGES in data, outputs: GEN_AI_OUTPUT_MESSAGES in data }; - } - - const isPublicAgent = ({ span }: MastraCallInfo): boolean => span.entityName === 'public_agent'; - - it('decides for each Mastra span from the span at export time', async () => { - const calls: MastraCallInfo[] = []; - exporter = new SentryMastraExporter({ - recordInputs: call => { - calls.push(call); - return isPublicAgent(call); - }, - recordOutputs: isPublicAgent, - }); - const publicSpan = makeSpan({ id: 'a', entityName: 'public_agent', input: 'hi', output: { text: 'hello' } }); - const privateSpan = makeSpan({ id: 'b', entityName: 'private_agent', input: 'hi', output: { text: 'hello' } }); - - await run(started(publicSpan), ended(publicSpan), started(privateSpan), ended(privateSpan)); - - expect(calls[0]).toEqual({ span: publicSpan }); - expect(endedSpans.map(recorded)).toEqual([ - { inputs: true, outputs: true }, - { inputs: false, outputs: false }, - ]); - }); - - it('decides inputs and outputs separately, once per span, and keeps the decision until the span ends', async () => { - const recordOutputs = vi.fn().mockReturnValueOnce(true).mockReturnValue(false); - exporter = new SentryMastraExporter({ recordInputs: () => false, recordOutputs }); - const span = makeSpan({ entityName: 'agent', input: 'hi', output: { text: 'hello' } }); - - await run(started(span), { type: 'span_updated', exportedSpan: span }, ended(span)); - - expect(recordOutputs).toHaveBeenCalledTimes(1); - expect(endedSpans.map(recorded)).toEqual([{ inputs: false, outputs: true }]); - }); - - it('passes the per-span decision to the classifier evaluation call', async () => { - exporter = new SentryMastraExporter({ recordInputs: () => true, recordOutputs: () => false }); - const call: ClassifierEvaluationCall = {}; - - setStartingClassifierEvaluation(call); - await run(started(makeSpan({ id: 'eval-1', type: 'classifier_evaluation' }))); - - expect(call).toMatchObject({ recordInputs: true, recordOutputs: false }); - }); - }); - it('re-parents a tool onto the generation across a dropped model_step', async () => { const agent = makeSpan({ id: 'agent-1', entityName: 'agent' }); const gen = makeSpan({ diff --git a/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts b/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts index 6613bd40d48c..ac20396846d1 100644 --- a/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/workers-ai.test.ts @@ -168,82 +168,61 @@ describe('instrumentWorkersAiClient', () => { expect(spanToStaticSpanJSON(endedSpans[0]!).data).toEqual(expected); }); - describe('per-call recording options', () => { - const messages = [ - { role: 'system', content: 'Answer in one word.' }, - { role: 'user', content: 'Capital of France?' }, - ]; - - function recorded(span: Span): { inputs: boolean; outputs: boolean } { - const { data } = spanToStaticSpanJSON(span); - return { inputs: GEN_AI_INPUT_MESSAGES in data, outputs: GEN_AI_OUTPUT_MESSAGES in data }; - } - - it('decides for each call from the run() arguments', async () => { - const endedSpans = setupClient(); - const calls: WorkersAiCallInfo[] = []; - const decide = (call: WorkersAiCallInfo): boolean => { - calls.push(call); - return (call.inputs as { privacy?: string }).privacy === 'public'; - }; - const client = { run: vi.fn().mockResolvedValue({ response: 'Paris' }) }; - const instrumented = instrumentWorkersAiClient(client, { recordInputs: decide, recordOutputs: decide }); - - const publicInputs = { messages, privacy: 'public' }; - const runOptions = { gateway: { id: 'my-gateway' } }; - await instrumented.run(MODEL, publicInputs, runOptions); - await instrumented.run(MODEL, { messages, privacy: 'private' }); - - expect(calls[0]).toEqual({ operation: 'chat', model: MODEL, inputs: publicInputs, options: runOptions }); - expect(endedSpans.map(recorded)).toEqual([ - { inputs: true, outputs: true }, - { inputs: false, outputs: false }, - ]); + // Workers AI has no node integration suite (it runs on Cloudflare), so this covers the per-call options here. + it('decides content recording per call when the options are functions', async () => { + setAsyncLocalStorageAsyncContextStrategy(); + const endedSpans = setupClient(); + const messages = [{ role: 'user', content: 'Capital of France?' }]; + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('data: {"response":"Paris"}\n\ndata: [DONE]\n\n')); + controller.close(); + }, }); - - it("runs the functions in the caller's async context", async () => { - setAsyncLocalStorageAsyncContextStrategy(); - const endedSpans = setupClient(); - const decide = (): boolean => getCurrentScope().getScopeData().tags.privacy === 'public'; - const client = { run: vi.fn().mockResolvedValue({ response: 'Paris' }) }; - const instrumented = instrumentWorkersAiClient(client, { recordInputs: decide, recordOutputs: decide }); - - await withScope(async scope => { - scope.setTag('privacy', 'public'); - await instrumented.run(MODEL, { messages }); - }); - await withScope(async scope => { - scope.setTag('privacy', 'private'); - await instrumented.run(MODEL, { messages }); - }); - - expect(endedSpans.map(recorded)).toEqual([ - { inputs: true, outputs: true }, - { inputs: false, outputs: false }, - ]); + const client = { + run: vi.fn((_model: string, inputs: { stream?: boolean }) => + Promise.resolve(inputs.stream ? stream : { response: 'Paris' }), + ), + }; + const recordInputs = vi.fn(({ inputs }: WorkersAiCallInfo) => (inputs as { user?: string }).user === 'public'); + const instrumented = instrumentWorkersAiClient(client, { + recordInputs, + recordOutputs: () => getCurrentScope().getScopeData().tags.privacy === 'public', }); - it('decides inputs and outputs separately, once per call, for streams too', async () => { - const endedSpans = setupClient(); - const recordOutputs = vi.fn(() => true); - const stream = new ReadableStream({ - start(controller) { - controller.enqueue(new TextEncoder().encode('data: {"response":"Paris"}\n\ndata: [DONE]\n\n')); - controller.close(); - }, - }); - const client = { run: vi.fn().mockResolvedValue(stream) }; - const instrumented = instrumentWorkersAiClient(client, { recordInputs: () => false, recordOutputs }); - - const result = (await instrumented.run(MODEL, { messages, stream: true })) as ReadableStream; - const reader = result.getReader(); + await withScope(async scope => { + scope.setTag('privacy', 'public'); + await instrumented.run(MODEL, { messages, user: 'public' }, { gateway: { id: 'gw' } }); + }); + await withScope(async scope => { + scope.setTag('privacy', 'private'); + await instrumented.run(MODEL, { messages, user: 'private' }); + + // Changing the tag after the call started must not change its outputs decision. + const result = instrumented.run(MODEL, { messages, user: 'public', stream: true }); + scope.setTag('privacy', 'public'); + const reader = ((await result) as ReadableStream).getReader(); while (!(await reader.read()).done) { // drain } + }); - expect(recordOutputs).toHaveBeenCalledTimes(1); - expect(endedSpans.map(recorded)).toEqual([{ inputs: false, outputs: true }]); + expect(recordInputs).toHaveBeenCalledWith({ + operation: 'chat', + model: MODEL, + inputs: { messages, user: 'public' }, + options: { gateway: { id: 'gw' } }, }); + expect( + endedSpans.map(span => { + const { data } = spanToStaticSpanJSON(span); + return { inputs: GEN_AI_INPUT_MESSAGES in data, outputs: GEN_AI_OUTPUT_MESSAGES in data }; + }), + ).toEqual([ + { inputs: true, outputs: true }, + { inputs: false, outputs: false }, + { inputs: true, outputs: false }, + ]); }); }); diff --git a/packages/server-utils/test/integrations/aws-sdk-bedrock.test.ts b/packages/server-utils/test/integrations/aws-sdk-bedrock.test.ts index 7e47d29286df..c44aa35a9b97 100644 --- a/packages/server-utils/test/integrations/aws-sdk-bedrock.test.ts +++ b/packages/server-utils/test/integrations/aws-sdk-bedrock.test.ts @@ -17,12 +17,9 @@ import { } from '@sentry/conventions/attributes'; import { GEN_AI_GENERATE_CONTENT } from '@sentry/conventions/op'; import type { Span } from '@sentry/core'; -import { getCurrentScope, setCurrentClient, spanToJSON, startInactiveSpan, withScope } from '@sentry/core'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import type { - BedrockRuntimeCallInfo, - BedrockRuntimeOptions, -} from '../../src/integrations/aws-sdk/services/bedrock-runtime'; +import { setCurrentClient, spanToJSON, startInactiveSpan } from '@sentry/core'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import type { BedrockRuntimeOptions } from '../../src/integrations/aws-sdk/services/bedrock-runtime'; import { BedrockRuntimeServiceExtension } from '../../src/integrations/aws-sdk/services/bedrock-runtime'; import type { NormalizedRequest, RequestMetadata } from '../../src/integrations/aws-sdk/types'; import { getDefaultTestClientOptions, TestClient } from '../mocks/client'; @@ -860,105 +857,3 @@ describe('BedrockRuntimeServiceExtension output recording', () => { }, ); }); - -describe('BedrockRuntimeServiceExtension per-call recording options', () => { - const CONVERSE_RESPONSE = { output: { message: { role: 'assistant', content: [{ text: 'Paris.' }] } } }; - - beforeEach(() => { - const client = new TestClient(getDefaultTestClientOptions({ tracesSampleRate: 1 })); - setCurrentClient(client); - client.init(); - }); - - afterEach(() => { - setCurrentClient(undefined as never); - }); - - function converse( - extension: BedrockRuntimeServiceExtension, - commandInput: Record, - ): { inputs: boolean; outputs: boolean } { - const request: NormalizedRequest = { serviceName: 'BedrockRuntime', commandName: 'Converse', commandInput }; - const { spanAttributes } = extension.requestPreSpanHook(request); - const span = startInactiveSpan({ name: 'bedrock' }); - extension.responseHook({ data: CONVERSE_RESPONSE, request, requestId: 'id' }, span); - return { - inputs: GEN_AI_INPUT_MESSAGES in spanAttributes!, - outputs: GEN_AI_OUTPUT_MESSAGES in spanToJSON(span).attributes, - }; - } - - const input = (privacy: string): Record => ({ - modelId: 'amazon.nova-pro-v1:0', - messages: [{ role: 'user', content: [{ text: 'Capital of France?' }] }], - requestMetadata: { privacy }, - }); - - const isPublic = (call: BedrockRuntimeCallInfo): boolean => call.input.requestMetadata.privacy === 'public'; - - it('decides for each call from the command name and input', () => { - const calls: BedrockRuntimeCallInfo[] = []; - const recordInputs = (call: BedrockRuntimeCallInfo): boolean => { - calls.push(call); - return isPublic(call); - }; - const extension = new BedrockRuntimeServiceExtension({ recordInputs, recordOutputs: isPublic }); - - const publicInput = input('public'); - const results = [converse(extension, publicInput), converse(extension, input('private'))]; - - expect(calls[0]).toEqual({ commandName: 'Converse', input: publicInput }); - expect(results).toEqual([ - { inputs: true, outputs: true }, - { inputs: false, outputs: false }, - ]); - }); - - it("runs the functions in the caller's async context", () => { - const decide = (): boolean => getCurrentScope().getScopeData().tags.privacy === 'public'; - const extension = new BedrockRuntimeServiceExtension({ recordInputs: decide, recordOutputs: decide }); - - const results = ['public', 'private'].map(privacy => - withScope(scope => { - scope.setTag('privacy', privacy); - return converse(extension, input('')); - }), - ); - - expect(results).toEqual([ - { inputs: true, outputs: true }, - { inputs: false, outputs: false }, - ]); - }); - - it('decides inputs and outputs separately, once, and keeps the decision for the response', () => { - const recordOutputs = vi.fn().mockReturnValueOnce(true).mockReturnValue(false); - const extension = new BedrockRuntimeServiceExtension({ recordInputs: () => false, recordOutputs }); - - expect(converse(extension, input(''))).toEqual({ inputs: false, outputs: true }); - expect(recordOutputs).toHaveBeenCalledTimes(1); - }); - - it('keeps the request-time decision until a stream is drained', async () => { - const recordOutputs = vi.fn().mockReturnValueOnce(true).mockReturnValue(false); - const extension = new BedrockRuntimeServiceExtension({ recordOutputs }); - const request: NormalizedRequest = { - serviceName: 'BedrockRuntime', - commandName: 'ConverseStream', - commandInput: input(''), - }; - extension.requestPreSpanHook(request); - const span = startInactiveSpan({ name: 'bedrock' }); - const data = { - stream: (async function* () { - yield { contentBlockDelta: { contentBlockIndex: 0, delta: { text: 'Paris.' } } }; - })(), - }; - - extension.responseHook({ data, request, requestId: 'id' }, span); - await drain(data.stream); - - expect(recordOutputs).toHaveBeenCalledTimes(1); - expect(spanToJSON(span).attributes).toHaveProperty(GEN_AI_OUTPUT_MESSAGES); - }); -}); diff --git a/packages/server-utils/test/integrations/typesafe.test.ts b/packages/server-utils/test/integrations/typesafe.test.ts deleted file mode 100644 index 3911fb6e2576..000000000000 --- a/packages/server-utils/test/integrations/typesafe.test.ts +++ /dev/null @@ -1,149 +0,0 @@ -import { tracingChannel } from 'node:diagnostics_channel'; -import type { Span } from '@sentry/core'; -import { - getCurrentScope, - GLOBAL_OBJ, - setAsyncContextStrategy, - setCurrentClient, - spanToJSON, - withScope, -} from '@sentry/core'; -import { GEN_AI_INPUT_MESSAGES, GEN_AI_OUTPUT_MESSAGES } from '@sentry/conventions/attributes'; -import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; -import { setAsyncLocalStorageAsyncContextStrategy } from '../../src/async-context'; -import type { TypeSafeCallInfo } from '../../src/ai/typesafe'; -import { instrumentTypeSafeClient } from '../../src/ai/typesafe'; -import { typesafeIntegration } from '../../src/integrations/typesafe'; -import { CHANNELS } from '../../src/orchestrion/channels'; -import { getDefaultTestClientOptions, TestClient } from '../mocks/client'; - -type Decide = (call: TypeSafeCallInfo) => boolean | undefined; - -const RESULT = { - model: 'jev-1', - answers: { is_urgent: { type: 'noul', noul: 0.97 } }, - usage: { input_tokens: 10, output_tokens: 5 }, -}; - -const request = (privacy: string): Record => ({ - state: 'Help!', - questions: { is_urgent: { type: 'noul', instructions: 'Urgent?' } }, - metadata: { privacy }, -}); - -describe('TypeSafe per-call recording options', () => { - let endedSpans: Span[]; - let decideInputs: Decide; - let decideOutputs: Decide; - - function useClient(): void { - const client = new TestClient( - getDefaultTestClientOptions({ dsn: 'https://public@dsn.ingest.sentry.io/1337', tracesSampleRate: 1 }), - ); - setCurrentClient(client); - client.init(); - client.on('spanEnd', span => endedSpans.push(span)); - } - - function recorded(span: Span): { inputs: boolean; outputs: boolean } { - const { attributes } = spanToJSON(span); - return { inputs: GEN_AI_INPUT_MESSAGES in attributes, outputs: GEN_AI_OUTPUT_MESSAGES in attributes }; - } - - const isPublic = (call: TypeSafeCallInfo): boolean => - (call.request as { metadata: { privacy: string } }).metadata.privacy === 'public'; - - beforeAll(() => { - setAsyncLocalStorageAsyncContextStrategy(); - GLOBAL_OBJ.__SENTRY_ORCHESTRION__ = { runtime: ['@typesafe-ai/sdk'] }; - endedSpans = []; - useClient(); - // The channel stays subscribed for the whole file, so each test swaps the decision functions. - typesafeIntegration({ - recordInputs: call => decideInputs(call), - recordOutputs: call => decideOutputs(call), - }).setup!(getCurrentScope().getClient()!); - }); - - afterAll(() => { - setAsyncContextStrategy(undefined); - delete GLOBAL_OBJ.__SENTRY_ORCHESTRION__; - }); - - beforeEach(() => { - endedSpans = []; - decideInputs = () => true; - decideOutputs = () => true; - useClient(); - }); - - function systemOne(params: Record): unknown { - return tracingChannel(CHANNELS.TYPESAFE_SYSTEM_ONE).traceSync(() => RESULT, { arguments: [params], self: {} }); - } - - it('decides for each call from its request', () => { - const calls: TypeSafeCallInfo[] = []; - decideInputs = call => { - calls.push(call); - return isPublic(call); - }; - decideOutputs = isPublic; - - const publicRequest = request('public'); - systemOne(publicRequest); - systemOne(request('private')); - - expect(calls[0]).toEqual({ request: publicRequest }); - expect(endedSpans.map(recorded)).toEqual([ - { inputs: true, outputs: true }, - { inputs: false, outputs: false }, - ]); - }); - - it("runs the functions in the caller's async context", async () => { - decideInputs = () => getCurrentScope().getScopeData().tags.privacy === 'public'; - decideOutputs = decideInputs; - - await withScope(async scope => { - scope.setTag('privacy', 'public'); - systemOne(request('')); - }); - await withScope(async scope => { - scope.setTag('privacy', 'private'); - systemOne(request('')); - }); - - expect(endedSpans.map(recorded)).toEqual([ - { inputs: true, outputs: true }, - { inputs: false, outputs: false }, - ]); - }); - - it('decides inputs and outputs separately', () => { - decideInputs = () => false; - decideOutputs = () => true; - - systemOne(request('')); - - expect(endedSpans.map(recorded)).toEqual([{ inputs: false, outputs: true }]); - }); - - it('instrumentTypeSafeClient decides per call, once, and keeps the decision until the response', async () => { - const recordOutputs = vi.fn(isPublic); - const response = new Response(JSON.stringify(RESULT)); - const client = { - systemOne: () => ({ asResponse: () => Promise.resolve(response.clone()) }), - }; - const instrumented = instrumentTypeSafeClient(client, { recordInputs: isPublic, recordOutputs }); - - instrumented.systemOne(request('public')); - instrumented.systemOne(request('private')); - await vi.waitFor(() => expect(endedSpans).toHaveLength(2)); - - expect(recordOutputs).toHaveBeenCalledTimes(2); - expect(endedSpans.map(recorded)).toEqual([ - { inputs: true, outputs: true }, - { inputs: false, outputs: false }, - ]); - }); -}); From d7300137f3b7da1d44847ccfa1c5e92dc92fb849 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:22:53 +0000 Subject: [PATCH 3/3] test(node): Collect TypeSafe recording-function spans until the main segment ends Under Deno the mock server's http.server span container arrives first, so wait for the scenario's spans like the neighbouring TypeSafe tests do. --- .../suites/tracing/typesafe/test.ts | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts b/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts index 96eb7011f72c..80ed8ae92a50 100644 --- a/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts @@ -142,28 +142,30 @@ describe('TypeSafe integration', () => { 'instrument-with-recording-functions.mjs', (createRunner, test) => { test('decides content recording per call when the options are functions', async () => { - await createRunner() - .expect({ - span: container => { - const spans = container.items.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE); - const recordedByModel = Object.fromEntries( - spans.map(span => [ - span.attributes[GEN_AI_REQUEST_MODEL]?.value, - { - inputs: GEN_AI_INPUT_MESSAGES in span.attributes, - outputs: GEN_AI_OUTPUT_MESSAGES in span.attributes, - }, - ]), - ); - expect(recordedByModel).toEqual({ - public: { inputs: true, outputs: true }, - private: { inputs: false, outputs: false }, - 'public-late': { inputs: true, outputs: false }, - }); + const runner = createRunner(); + const spansPromise = runner.collectStreamedSpans( + spans => + spans.some(span => span.is_segment && span.name === 'main') && + spans.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE).length >= 3, + ); + + await runner.start().completed(); + + const decideSpans = (await spansPromise).filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE); + const recordedByModel = Object.fromEntries( + decideSpans.map(span => [ + span.attributes[GEN_AI_REQUEST_MODEL]?.value, + { + inputs: GEN_AI_INPUT_MESSAGES in span.attributes, + outputs: GEN_AI_OUTPUT_MESSAGES in span.attributes, }, - }) - .start() - .completed(); + ]), + ); + expect(recordedByModel).toEqual({ + public: { inputs: true, outputs: true }, + private: { inputs: false, outputs: false }, + 'public-late': { inputs: true, outputs: false }, + }); }); }, );