diff --git a/dev-packages/node-integration-tests/suites/tracing/flue/scenario-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/flue/scenario-recording-functions.mjs new file mode 100644 index 000000000000..50d18492d36f --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/flue/scenario-recording-functions.mjs @@ -0,0 +1,56 @@ +import * as Sentry from '@sentry/node'; +import { __flueBindAgentModule, init, instrument, useModel, useTool } from '@flue/runtime'; +import { start } from '@flue/runtime/node'; +import { fauxAssistantMessage, fauxProvider, fauxToolCall } from '@earendil-works/pi-ai/providers/faux'; +import * as v from 'valibot'; + +const privacy = () => Sentry.getIsolationScope().getScopeData().tags.privacy; + +// Flue runs an agent in the isolation scope of the request that dispatched it, so the tag set there +// reaches the functions. The pi-ai integration has no options of its own: Flue's decide the chat spans. +instrument( + Sentry.createFlueInstrumentation({ + recordInputs: call => + (call.operation === 'chat' ? call.context.instanceId : call.observation.instanceId) === 'public', + recordOutputs: () => privacy() === 'public', + }), +); + +const faux = fauxProvider({ provider: 'faux', models: [{ id: 'faux-model' }] }); +const toolCall = fauxAssistantMessage(fauxToolCall('get_weather', { city: 'Berlin' }, { id: 'call_1' }), { + stopReason: 'toolUse', +}); +const answer = fauxAssistantMessage('It is sunny in Berlin.'); +faux.setResponses([toolCall, answer, toolCall, answer]); + +function Hello() { + useModel('faux/faux-model'); + useTool({ + name: 'get_weather', + description: 'Get the current weather for a city.', + input: v.object({ city: v.string() }), + run: ({ data }) => { + // A tool span decides when it starts: flipping the tag here only reaches the requests after it. + Sentry.getIsolationScope().setTag('privacy', 'public'); + return `It is sunny in ${data.city}.`; + }, + }); + return 'You are a helpful assistant.'; +} +__flueBindAgentModule(Hello, { identity: 'Hello' }); + +await Sentry.startSpan({ name: 'flue-test', op: 'function' }, async () => { + const flue = await start({ agents: [Hello], providers: [faux.provider] }); + for (const id of ['public', 'private']) { + await Sentry.withIsolationScope(async isolationScope => { + isolationScope.setTag('privacy', id); + await Sentry.startSpan({ name: `run ${id}` }, async () => { + const agent = init(Hello, { id }); + await agent.read(await agent.dispatch('What is the weather in Berlin?')); + }); + }); + } + await flue[Symbol.asyncDispose]?.(); +}); + +await Sentry.flush(2000); diff --git a/dev-packages/node-integration-tests/suites/tracing/flue/test.ts b/dev-packages/node-integration-tests/suites/tracing/flue/test.ts index 68e5dc6eb8c0..76aa547be7b6 100644 --- a/dev-packages/node-integration-tests/suites/tracing/flue/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/flue/test.ts @@ -1,4 +1,4 @@ -import type { SerializedStreamedSpanContainer } from '@sentry/core'; +import type { SerializedStreamedSpan, SerializedStreamedSpanContainer } from '@sentry/core'; import { GEN_AI_AGENT_NAME, GEN_AI_CONVERSATION_ID, @@ -22,6 +22,15 @@ import { afterAll, describe, expect } from 'vitest'; import { supports } from '../../../utils'; import { cleanupChildProcesses, createEsmTests } from '../../../utils/runner'; +function recorded(span: SerializedStreamedSpan): { span: string; inputs: boolean; outputs: boolean } { + const { attributes } = span; + return { + span: span.name, + inputs: GEN_AI_INPUT_MESSAGES in attributes || GEN_AI_TOOL_CALL_ARGUMENTS in attributes, + outputs: GEN_AI_OUTPUT_MESSAGES in attributes || GEN_AI_TOOL_CALL_RESULT in attributes, + }; +} + // `@flue/runtime` declares `engines.node >= 22.19`. describe.runIf(supports({ min: 22 }))('Flue integration', () => { afterAll(() => { @@ -217,4 +226,45 @@ describe.runIf(supports({ min: 22 }))('Flue integration', () => { .completed(); }); }); + + createEsmTests(__dirname, 'scenario-recording-functions.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { + test('decides content recording per call when the options are functions', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items; + const byId = new Map(spans.map(span => [span.span_id, span])); + const runOf = (span: SerializedStreamedSpan): string | undefined => { + let current: SerializedStreamedSpan | undefined = span; + while (current && !current.name.startsWith('run ')) { + current = current.parent_span_id ? byId.get(current.parent_span_id) : undefined; + } + return current?.name; + }; + const contentSpans = spans + .filter(span => ['auto.ai.pi_ai', 'auto.ai.flue'].includes(String(span.attributes[SENTRY_ORIGIN]?.value))) + .filter(span => span.name !== 'invoke_agent Hello') + .sort((a, b) => a.start_timestamp - b.start_timestamp); + + expect({ + public: contentSpans.filter(span => runOf(span) === 'run public').map(recorded), + private: contentSpans.filter(span => runOf(span) === 'run private').map(recorded), + }).toEqual({ + public: [ + { span: 'chat faux-model', inputs: true, outputs: true }, + { span: 'execute_tool get_weather', inputs: true, outputs: true }, + { span: 'chat faux-model', inputs: true, outputs: true }, + ], + private: [ + { span: 'chat faux-model', inputs: false, outputs: false }, + { span: 'execute_tool get_weather', inputs: false, outputs: false }, + { span: 'chat faux-model', inputs: false, outputs: true }, + ], + }); + }, + }) + .start() + .completed(); + }); + }); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/pi-ai/instrument-with-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/pi-ai/instrument-with-recording-functions.mjs new file mode 100644 index 000000000000..602cf7fa9120 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/pi-ai/instrument-with-recording-functions.mjs @@ -0,0 +1,15 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + integrations: [ + Sentry.piAiIntegration({ + recordInputs: ({ context }) => context?.messages?.[0]?.content === 'public', + recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public', + }), + ], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/pi-ai/scenario-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/pi-ai/scenario-recording-functions.mjs new file mode 100644 index 000000000000..7a3e50dbd821 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/pi-ai/scenario-recording-functions.mjs @@ -0,0 +1,31 @@ +import * as Sentry from '@sentry/node'; +import { createModels } from '@earendil-works/pi-ai/models'; +import { fauxAssistantMessage, fauxProvider } from '@earendil-works/pi-ai/providers/faux'; + +const faux = fauxProvider({ provider: 'faux', models: [{ id: 'faux-model' }] }); +const models = createModels(); +models.setProvider(faux.provider); +faux.setResponses([fauxAssistantMessage('Paris.'), fauxAssistantMessage('Paris.'), fauxAssistantMessage('Paris.')]); + +const ask = content => ({ messages: [{ role: 'user', content, timestamp: Date.now() }] }); + +await Sentry.startSpan({ name: 'pi-ai-test' }, async () => { + const model = faux.getModel(); + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'public'); + await models.complete(model, ask('public')); + }); + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'private'); + await models.complete(model, ask('private')); + + // Changing the tag after the request started must not change its outputs decision. + const events = models.stream(model, ask('public')); + Sentry.getCurrentScope().setTag('privacy', 'public'); + for await (const _event of events) { + // Drained like an app consuming the stream. + } + }); +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/pi-ai/test.ts b/dev-packages/node-integration-tests/suites/tracing/pi-ai/test.ts index 995bcb7858fa..afa3f1680a2a 100644 --- a/dev-packages/node-integration-tests/suites/tracing/pi-ai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/pi-ai/test.ts @@ -158,4 +158,34 @@ describe.runIf(supports({ min: 22 }))('pi-ai integration', () => { }, ); }); + + createEsmTests( + __dirname, + 'scenario-recording-functions.mjs', + 'instrument-with-recording-functions.mjs', + (createRunner, test) => { + test('decides content recording per request when the options are functions', async () => { + await createRunner() + .expect({ + span: container => { + const recorded = container.items + .filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.pi_ai') + .sort((a, b) => a.start_timestamp - b.start_timestamp) + .map(span => ({ + streaming: span.attributes[GEN_AI_RESPONSE_STREAMING]?.value === true, + inputs: GEN_AI_INPUT_MESSAGES in span.attributes, + outputs: GEN_AI_OUTPUT_MESSAGES in span.attributes, + })); + expect(recorded).toEqual([ + { streaming: false, inputs: true, outputs: true }, + { streaming: false, inputs: false, outputs: false }, + { streaming: true, inputs: true, outputs: false }, + ]); + }, + }) + .start() + .completed(); + }); + }, + ); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/pi-durable/instrument-with-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/pi-durable/instrument-with-recording-functions.mjs new file mode 100644 index 000000000000..e5e72fe611dc --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/pi-durable/instrument-with-recording-functions.mjs @@ -0,0 +1,19 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +// Runs start from whichever context woke the scheduler, never from the request's, so these decide from +// what pi-durable hands over. The pi-ai integration has no options of its own: these decide the chat +// spans too. +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + integrations: [ + Sentry.piDurableIntegration({ + recordInputs: call => + (call.operation === 'chat' ? call.runtime : call.api).conversationId === globalThis.publicConversationId, + recordOutputs: call => call.operation === 'chat' && globalThis.recordChatOutputs, + }), + ], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-recording-functions.mjs new file mode 100644 index 000000000000..12e455c798fd --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/pi-durable/scenario-recording-functions.mjs @@ -0,0 +1,54 @@ +import * as Sentry from '@sentry/node'; +import { BACKGROUND_CONTEXT } from '@earendil-works/chord/context'; +import { Type } from '@earendil-works/pi-ai'; +import { createModels } from '@earendil-works/pi-ai/models'; +import { fauxAssistantMessage, fauxProvider, fauxToolCall } from '@earendil-works/pi-ai/providers/faux'; +import { createRegistry, defineExtension, defineTool, Harness, MemoryStorage } from '@earendil-works/pi-durable'; + +const context = BACKGROUND_CONTEXT; + +globalThis.recordChatOutputs = true; +const respond = transcript => { + if (!transcript.messages.some(message => message.role === 'toolResult')) { + return fauxAssistantMessage(fauxToolCall('get_weather', { city: 'Berlin' }, { id: 'call_1' }), { + stopReason: 'toolUse', + }); + } + // The request is already streaming here, so this only turns off the outputs of the requests after it. + globalThis.recordChatOutputs = false; + return fauxAssistantMessage('It is sunny in Berlin.'); +}; +const faux = fauxProvider({ provider: 'faux', models: [{ id: 'faux-model' }] }); +const models = createModels(); +models.setProvider(faux.provider); +faux.setResponses([respond, respond, respond, respond]); + +const registry = createRegistry(); +registry.install( + defineExtension({ + name: 'app', + tools: [ + defineTool({ + name: 'get_weather', + description: 'Get the current weather for a city.', + parameters: Type.Object({ city: Type.String() }), + execute: async ({ city }) => ({ output: [{ type: 'text', text: `It is sunny in ${city}.` }] }), + }), + ], + }), +); + +const harness = await Harness.open(new MemoryStorage(), { models, registry }, context); +const agent = { model: { provider: 'faux', modelId: 'faux-model' } }; +const publicConversation = await harness.createConversation({ ownership: { kind: 'ownerless' }, agent }, context); +const privateConversation = await harness.createConversation({ ownership: { kind: 'ownerless' }, agent }, context); +globalThis.publicConversationId = publicConversation.id; + +for (const conversation of [publicConversation, privateConversation]) { + await ( + await conversation.submit({ type: 'input', content: 'What is the weather in Berlin?' }, context) + ).wait(context); +} +await harness.close(context); + +await Sentry.flush(2000); diff --git a/dev-packages/node-integration-tests/suites/tracing/pi-durable/test.ts b/dev-packages/node-integration-tests/suites/tracing/pi-durable/test.ts index 3a07efbaa997..b5b803f32818 100644 --- a/dev-packages/node-integration-tests/suites/tracing/pi-durable/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/pi-durable/test.ts @@ -1,3 +1,4 @@ +import type { SerializedStreamedSpanContainer } from '@sentry/core'; import { GEN_AI_CONVERSATION_ID, GEN_AI_INPUT_MESSAGES, @@ -37,6 +38,20 @@ const CONTENT_ATTRIBUTES = [ GEN_AI_TOOL_CALL_RESULT, ]; +type Recorded = { span: string; inputs: boolean; outputs: boolean }; + +/** What each span of the run records, in start order. Each run is a trace of its own. */ +function recorded(container: SerializedStreamedSpanContainer): Recorded[] { + return container.items + .filter(span => !span.is_segment) + .sort((a, b) => a.start_timestamp - b.start_timestamp) + .map(({ name, attributes }) => ({ + span: name, + inputs: GEN_AI_INPUT_MESSAGES in attributes || GEN_AI_TOOL_CALL_ARGUMENTS in attributes, + outputs: GEN_AI_OUTPUT_MESSAGES in attributes || GEN_AI_TOOL_CALL_RESULT in attributes, + })); +} + // The pi packages declare `engines.node >= 22.19`. describe.runIf(supports({ min: 22 }))('pi-durable integration', () => { afterAll(() => { @@ -651,4 +666,34 @@ describe.runIf(supports({ min: 22 }))('pi-durable integration', () => { .completed(); }); }); + + createEsmTests( + __dirname, + 'scenario-recording-functions.mjs', + 'instrument-with-recording-functions.mjs', + (createRunner, test) => { + test('decides content recording per call when the options are functions', async () => { + const runs: Recorded[][] = []; + + await createRunner() + .expect({ span: container => void runs.push(recorded(container)) }) + .expect({ span: container => void runs.push(recorded(container)) }) + .start() + .completed(); + + expect(runs).toEqual([ + [ + { span: 'chat faux-model', inputs: true, outputs: true }, + { span: 'execute_tool get_weather', inputs: true, outputs: false }, + { span: 'chat faux-model', inputs: true, outputs: true }, + ], + [ + { span: 'chat faux-model', inputs: false, outputs: false }, + { span: 'execute_tool get_weather', inputs: false, outputs: false }, + { span: 'chat faux-model', inputs: false, outputs: false }, + ], + ]); + }); + }, + ); }); diff --git a/packages/server-utils/src/ai/core/utils.ts b/packages/server-utils/src/ai/core/utils.ts index 2d3a8e801ab4..8ce0d37ac9eb 100644 --- a/packages/server-utils/src/ai/core/utils.ts +++ b/packages/server-utils/src/ai/core/utils.ts @@ -120,7 +120,8 @@ export function resolveGenAiCallRecording( }; } -function resolveRecordingOption( +/** One recording option of a call, `undefined` when it leaves the decision to `dataCollection.genAI`. */ +export function resolveRecordingOption( option: GenAiRecordingOption | undefined, call: TCall, ): boolean | undefined { diff --git a/packages/server-utils/src/ai/flue/index.ts b/packages/server-utils/src/ai/flue/index.ts index 704dd0e95fc2..611639d3830d 100644 --- a/packages/server-utils/src/ai/flue/index.ts +++ b/packages/server-utils/src/ai/flue/index.ts @@ -17,8 +17,8 @@ import { GEN_AI_OPERATION_NAME, SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; -import type { GenAiOptions } from '../core/utils'; -import { getGenAiSpanOp, resolveAIRecordingOptions } from '../core/utils'; +import type { GenAiCallOptions } from '../core/utils'; +import { getGenAiSpanOp } from '../core/utils'; import { getOrchestrionInjectedModules } from '../../orchestrion/detect'; import { piAiModuleNames } from '../../orchestrion/config/pi-ai'; import { setPiAiRecordingOptions } from '../pi-ai/recording'; @@ -27,9 +27,9 @@ import { withCleanScopes } from '../../utils/withCleanScopes'; import { FLUE_INSTRUMENTATION_KEY, FLUE_OPERATION, FLUE_ORIGIN, MAX_TRACKED_FLUE_SPANS } from './constants'; import type { SpanTracker } from './utils'; import { endToolSpan, sentryTraceFromTraceparent, startToolSpan } from './utils'; -import type { FlueInstrumentation } from './types'; +import type { FlueCallInfo, FlueInstrumentation } from './types'; -export type FlueOptions = GenAiOptions; +export type FlueOptions = GenAiCallOptions; interface Admission { isolationScope: Scope; @@ -110,8 +110,8 @@ function runInAdmissionContext(admission: Admission | undefined, next: () => * as a `gen_ai.chat` span under the agent span. * * Tool arguments and results are gated on `recordInputs`/`recordOutputs`, which fall back to the - * current client's `dataCollection.genAI` settings and are read per event. The same options apply to - * the message content of the `gen_ai.chat` spans. + * current client's `dataCollection.genAI` settings and are decided when each tool span starts. The + * same options apply to the message content of the `gen_ai.chat` spans. */ export function createFlueInstrumentation(options: FlueOptions = {}): FlueInstrumentation { // Keyed by the agent operation's own id, which is what the observations carry. That keeps @@ -143,7 +143,7 @@ export function createFlueInstrumentation(options: FlueOptions = {}): FlueInstru if (conversationId) { scope.setConversationId(conversationId); } - setPiAiRecordingOptions(scope, options); + setPiAiRecordingOptions(scope, options, { operation: 'chat', execution: operation, context: ctx }); return next(); }); } @@ -199,12 +199,7 @@ export function createFlueInstrumentation(options: FlueOptions = {}): FlueInstru : openAgentSpan(); }, - observe: observation => { - // Resolved per observation, not once at construction: `resolveAIRecordingOptions` reads the - // current client's `dataCollection.genAI`, and Cloudflare replaces the client per request, so - // values captured at isolate load would be the wrong ones for every later request. - const { recordInputs, recordOutputs } = resolveAIRecordingOptions(options); - + observe: (observation, ctx) => { // Observations are emitted synchronously from inside the agent operation, so the active span // here is the agent span the interceptor opened — tool spans parent off it without any // bookkeeping. @@ -239,10 +234,10 @@ export function createFlueInstrumentation(options: FlueOptions = {}): FlueInstru } return; case 'tool_start': - startToolSpan(observation, toolSpans, recordInputs); + startToolSpan(observation, ctx, toolSpans, options); return; case 'tool': - endToolSpan(observation, toolSpans, recordOutputs); + endToolSpan(observation, toolSpans); return; default: return; diff --git a/packages/server-utils/src/ai/flue/types.ts b/packages/server-utils/src/ai/flue/types.ts index 21f88c26bb49..cbee07efb587 100644 --- a/packages/server-utils/src/ai/flue/types.ts +++ b/packages/server-utils/src/ai/flue/types.ts @@ -74,3 +74,19 @@ export interface FlueInstrumentation { interceptor: (operation: FlueExecutionOperation, ctx: FlueExecutionContext, next: () => Promise) => Promise; dispose: () => void; } + +/** The call that `recordInputs`/`recordOutputs` decide on. */ +export type FlueCallInfo = + | { + /** A model call, which pi-ai reports as a `gen_ai.chat` span. */ + operation: 'chat'; + /** The model operation and its context, as Flue hands them to the interceptor. */ + execution: FlueExecutionOperation; + context: FlueExecutionContext; + } + | { + operation: 'execute_tool'; + /** The `tool_start` observation and its context, as Flue hands them to `observe`. */ + observation: FlueObservation; + context: FlueEventContext; + }; diff --git a/packages/server-utils/src/ai/flue/utils.ts b/packages/server-utils/src/ai/flue/utils.ts index 307642e20cb3..4e106e964043 100644 --- a/packages/server-utils/src/ai/flue/utils.ts +++ b/packages/server-utils/src/ai/flue/utils.ts @@ -15,10 +15,11 @@ import { GEN_AI_TOOL_NAME, SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; -import { getGenAiSpanOp } from '../core/utils'; +import type { GenAiCallOptions } from '../core/utils'; +import { getGenAiSpanOp, resolveGenAiCallRecording } from '../core/utils'; import { piAiContentToString } from '../pi-ai/messages'; import { FLUE_ORIGIN, MAX_TRACKED_FLUE_SPANS } from './constants'; -import type { FlueErrorInfo, FlueObservation } from './types'; +import type { FlueCallInfo, FlueErrorInfo, FlueEventContext, FlueObservation } from './types'; /** * Flue persists the incoming W3C `traceparent` at admission and replays it on the agent operation. @@ -40,6 +41,8 @@ export function sentryTraceFromTraceparent(traceparent: string): string | undefi */ export type SpanTracker = LRUMap; +const TOOL_SPANS_RECORDING_OUTPUTS = new WeakSet(); + /** * Store a span under `key`. Callers only reach this with a key the tracker does not hold, so the * one span at risk of being dropped without `end()` is the oldest entry, which `LRUMap.set` silently @@ -93,32 +96,42 @@ function captureToolError(span: Span, errorInfo: FlueErrorInfo | undefined): voi * OpenTelemetry adapter projects them: siblings of `chat`, correlated to model output by tool call * id. Keyed by `toolCallId` so concurrent tool calls in one turn cannot cross-attribute. */ -export function startToolSpan(observation: FlueObservation, toolSpans: SpanTracker, recordInputs: boolean): void { +export function startToolSpan( + observation: FlueObservation, + context: FlueEventContext, + toolSpans: SpanTracker, + options: GenAiCallOptions, +): void { const { toolCallId, toolName } = observation; if (!toolCallId || toolSpans.get(toolCallId)) { return; } - trackSpan( - toolSpans, - toolCallId, - startInactiveSpan({ - name: `execute_tool ${toolName ?? 'unknown'}`, - op: getGenAiSpanOp('execute_tool'), - attributes: { - [SENTRY_ORIGIN]: FLUE_ORIGIN, - [GEN_AI_OPERATION_NAME]: 'execute_tool', - [GEN_AI_TOOL_NAME]: toolName, - [GEN_AI_CONVERSATION_ID]: observation.conversationId, - ...(recordInputs && observation.args !== undefined - ? { [GEN_AI_TOOL_CALL_ARGUMENTS]: stringify(observation.args) } - : {}), - }, - }), - ); + const { recordInputs, recordOutputs } = resolveGenAiCallRecording(options, { + operation: 'execute_tool', + observation, + context, + }); + const span = startInactiveSpan({ + name: `execute_tool ${toolName ?? 'unknown'}`, + op: getGenAiSpanOp('execute_tool'), + attributes: { + [SENTRY_ORIGIN]: FLUE_ORIGIN, + [GEN_AI_OPERATION_NAME]: 'execute_tool', + [GEN_AI_TOOL_NAME]: toolName, + [GEN_AI_CONVERSATION_ID]: observation.conversationId, + ...(recordInputs && observation.args !== undefined + ? { [GEN_AI_TOOL_CALL_ARGUMENTS]: stringify(observation.args) } + : {}), + }, + }); + if (recordOutputs) { + TOOL_SPANS_RECORDING_OUTPUTS.add(span); + } + trackSpan(toolSpans, toolCallId, span); } -export function endToolSpan(observation: FlueObservation, toolSpans: SpanTracker, recordOutputs: boolean): void { +export function endToolSpan(observation: FlueObservation, toolSpans: SpanTracker): void { const { toolCallId } = observation; const span = toolCallId ? toolSpans.get(toolCallId) : undefined; if (!span || !toolCallId) { @@ -126,7 +139,7 @@ export function endToolSpan(observation: FlueObservation, toolSpans: SpanTracker } toolSpans.remove(toolCallId); - if (recordOutputs) { + if (TOOL_SPANS_RECORDING_OUTPUTS.has(span)) { const result = toolResultForModel(observation); if (result !== undefined) { span.setAttribute(GEN_AI_TOOL_CALL_RESULT, result); diff --git a/packages/server-utils/src/ai/pi-ai/recording.ts b/packages/server-utils/src/ai/pi-ai/recording.ts index 2abe83246ca2..6ec45867b440 100644 --- a/packages/server-utils/src/ai/pi-ai/recording.ts +++ b/packages/server-utils/src/ai/pi-ai/recording.ts @@ -1,31 +1,47 @@ import type { Scope } from '@sentry/core'; import { getCurrentScope } from '@sentry/core'; -import type { GenAiOptions } from '../core/utils'; -import { resolveAIRecordingOptions } from '../core/utils'; +import type { GenAiCallOptions, GenAiOptions } from '../core/utils'; +import { resolveGenAiCallRecording, resolveRecordingOption } from '../core/utils'; +import type { PiAiCallInfo } from './types'; const RECORDING_OPTIONS_KEY = 'piAiRecordingOptions'; +type DriverRecording = () => GenAiOptions; + /** * Apply the recording options of an integration that sends its model calls through pi-ai, such as - * Flue, to the `gen_ai.chat` spans of the requests sent under `scope`. + * Flue, to the `gen_ai.chat` spans of the requests sent under `scope`. Options that are functions + * get `call` when such a span starts. */ -export function setPiAiRecordingOptions(scope: Scope, options: GenAiOptions): void { - scope.setSDKProcessingMetadata({ [RECORDING_OPTIONS_KEY]: options }); +export function setPiAiRecordingOptions(scope: Scope, options: GenAiCallOptions, call: TCall): void { + const resolve: DriverRecording = () => ({ + recordInputs: resolveRecordingOption(options.recordInputs, call), + recordOutputs: resolveRecordingOption(options.recordOutputs, call), + }); + scope.setSDKProcessingMetadata({ [RECORDING_OPTIONS_KEY]: resolve }); } /** * The recording options of a pi-ai request, read when it starts: its response can settle outside the - * scope it was sent from. An explicit `false` on either integration keeps the content out, since an - * opt-out must not depend on which integration it was set on. Otherwise the integration driving - * pi-ai comes first, then `piAiIntegration`, then `dataCollection.genAI`. + * scope it was sent from. An explicit `false` on either integration, or from either one's function, + * keeps the content out, since an opt-out must not depend on which integration it was set on. + * Otherwise the integration driving pi-ai comes first, then `piAiIntegration`, then + * `dataCollection.genAI`. */ -export function resolvePiAiRecordingOptions(options: GenAiOptions): Required { - const driver = getCurrentScope().getScopeData().sdkProcessingMetadata[RECORDING_OPTIONS_KEY] as - | GenAiOptions - | undefined; - const { recordInputs, recordOutputs } = resolveAIRecordingOptions(options); +export function resolvePiAiRecordingOptions( + options: GenAiCallOptions, + call: PiAiCallInfo, +): Required { + const driver = ( + getCurrentScope().getScopeData().sdkProcessingMetadata[RECORDING_OPTIONS_KEY] as DriverRecording | undefined + )?.(); + const own = { + recordInputs: resolveRecordingOption(options.recordInputs, call), + recordOutputs: resolveRecordingOption(options.recordOutputs, call), + }; + const { recordInputs, recordOutputs } = resolveGenAiCallRecording(own, call); return { - recordInputs: options.recordInputs !== false && (driver?.recordInputs ?? recordInputs), - recordOutputs: options.recordOutputs !== false && (driver?.recordOutputs ?? recordOutputs), + recordInputs: own.recordInputs !== false && (driver?.recordInputs ?? recordInputs), + recordOutputs: own.recordOutputs !== false && (driver?.recordOutputs ?? recordOutputs), }; } diff --git a/packages/server-utils/src/ai/pi-ai/types.ts b/packages/server-utils/src/ai/pi-ai/types.ts index f2355d1044d8..bbd076070282 100644 --- a/packages/server-utils/src/ai/pi-ai/types.ts +++ b/packages/server-utils/src/ai/pi-ai/types.ts @@ -4,6 +4,7 @@ * treated as possibly absent at runtime. */ +import type { PiAiContext } from './messages'; import type { PiAiUsage } from './usage'; export interface PiAssistantMessage { @@ -33,3 +34,19 @@ export interface PiStreamOptions { export interface PiEventStream { result(): Promise; } + +/** What `streamDeferred` and `fetchDeferred` take in place of a context: a request the provider parked. */ +export interface PiDeferredHandle { + provider?: string; + modelId?: string; + api?: string; + id?: string; +} + +/** The request that `recordInputs`/`recordOutputs` decide on: the arguments of `stream`, `complete`, etc. */ +export interface PiAiCallInfo { + model: PiModel | undefined; + /** The deferred calls pass a {@link PiDeferredHandle} here. */ + context: PiAiContext | PiDeferredHandle | undefined; + options: PiStreamOptions | undefined; +} diff --git a/packages/server-utils/src/ai/pi-durable/index.ts b/packages/server-utils/src/ai/pi-durable/index.ts index 307d149e996e..05eef58a0eb1 100644 --- a/packages/server-utils/src/ai/pi-durable/index.ts +++ b/packages/server-utils/src/ai/pi-durable/index.ts @@ -9,13 +9,14 @@ import { withActiveSpan, } from '@sentry/core'; import { GEN_AI_CONVERSATION_ID } from '@sentry/conventions/attributes'; -import type { GenAiOptions } from '../core/utils'; +import type { GenAiCallOptions } from '../core/utils'; import { setPiAiRecordingOptions } from '../pi-ai/recording'; import { PI_DURABLE_ORIGIN, PI_TASK } from './constants'; import type { PiRuns } from './runs'; import { createRuns, endRun, observeRunControl, readProviderSessionId, startRun, toSentryConversationId } from './runs'; import { instrumentTool, instrumentToolPhase } from './tools'; import type { + PiDurableCallInfo, PiHarness, PiHarnessOptions, PiPhaseHandler, @@ -28,7 +29,7 @@ import type { import { withCleanScopes } from '../../utils/withCleanScopes'; import { bound } from './utils'; -export type PiDurableOptions = GenAiOptions; +export type PiDurableOptions = GenAiCallOptions; const INSTRUMENTED = Symbol.for('sentry.pi-durable.instrumented'); const RUNS = Symbol.for('sentry.pi-durable.runs'); @@ -228,7 +229,7 @@ function runPhase( } } scope.setConversationId(sessionId ?? toSentryConversationId(runs, conversationId)); - setPiAiRecordingOptions(scope, options); + setPiAiRecordingOptions(scope, options, { operation: 'chat', runtime, context }); try { return await phase(observed); } catch (error) { diff --git a/packages/server-utils/src/ai/pi-durable/tools.ts b/packages/server-utils/src/ai/pi-durable/tools.ts index 6a6ced8f8787..654c64dbba08 100644 --- a/packages/server-utils/src/ai/pi-durable/tools.ts +++ b/packages/server-utils/src/ai/pi-durable/tools.ts @@ -21,8 +21,8 @@ import { } from '@sentry/conventions/attributes'; import { GEN_AI_EXECUTE_TOOL } from '@sentry/conventions/op'; import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../core/gen-ai-attributes'; -import type { GenAiOptions } from '../core/utils'; -import { resolveAIRecordingOptions } from '../core/utils'; +import type { GenAiCallOptions } from '../core/utils'; +import { resolveGenAiCallRecording } from '../core/utils'; import { piAiContentToString } from '../pi-ai/messages'; import { MAX_TRACKED_PI_RUNS, PI_DURABLE_ORIGIN, PI_TOOL_RESULT_ENTRY_KIND } from './constants'; import type { PiRuns, PiToolCall } from './runs'; @@ -30,6 +30,7 @@ import { toSentryConversationId } from './runs'; import type { PiAgent, PiCommitChange, + PiDurableCallInfo, PiEntryToken, PiTaskRuntime, PiTool, @@ -104,9 +105,15 @@ export function instrumentToolPhase( } /** Wrap a tool so each call gets a `gen_ai.execute_tool` span. */ -export function instrumentTool(tool: PiTool, runs: PiRuns, options: GenAiOptions): PiTool { +export function instrumentTool(tool: PiTool, runs: PiRuns, options: GenAiCallOptions): PiTool { const execute = (args: unknown, api: PiToolExecutionApi, context: unknown): Promise => { - const { recordInputs, recordOutputs } = resolveAIRecordingOptions(options); + const { recordInputs, recordOutputs } = resolveGenAiCallRecording(options, { + operation: 'execute_tool', + tool, + args, + api, + context, + }); const phaseCalls = runs.toolCalls.get(api?.taskId); // A tool that runs another tool of its agent passes its own `api` on, so the inner call shares // the call id. Only the outer call gets the committed result; the inner one ends with its own. diff --git a/packages/server-utils/src/ai/pi-durable/types.ts b/packages/server-utils/src/ai/pi-durable/types.ts index 38c4557c8707..1d3315bd0da3 100644 --- a/packages/server-utils/src/ai/pi-durable/types.ts +++ b/packages/server-utils/src/ai/pi-durable/types.ts @@ -102,3 +102,21 @@ export interface PiHarness { /** Calls `listener` when `close()` begins. */ subscribeClose?(listener: () => void): () => void; } + +/** The call that `recordInputs`/`recordOutputs` decide on. */ +export type PiDurableCallInfo = + | { + /** A model request of a task phase, which pi-ai reports as a `gen_ai.chat` span. */ + operation: 'chat'; + /** The runtime and context pi-durable hands to the phase. */ + runtime: PiTaskRuntime; + context: unknown; + } + | { + operation: 'execute_tool'; + tool: PiTool; + /** The arguments of `tool.execute()`. */ + args: unknown; + api: PiToolExecutionApi; + context: unknown; + }; diff --git a/packages/server-utils/src/integrations/pi-ai.ts b/packages/server-utils/src/integrations/pi-ai.ts index 68bc1bdca1d0..3c4f31151acb 100644 --- a/packages/server-utils/src/integrations/pi-ai.ts +++ b/packages/server-utils/src/integrations/pi-ai.ts @@ -1,20 +1,20 @@ import * as diagnosticsChannel from '../utils/diagnosticsChannel'; import type { IntegrationFn, Span } from '@sentry/core'; import { defineIntegration, isThenable, startInactiveSpan } from '@sentry/core'; -import type { GenAiOptions } from '../ai/core/utils'; +import type { GenAiCallOptions } from '../ai/core/utils'; import { getPiAiChatSpanOptions, setPiAiResponseAttributes } from '../ai/pi-ai/chat'; import { PI_AI_INTEGRATION_NAME } from '../ai/pi-ai/constants'; import type { PiAiContext } from '../ai/pi-ai/messages'; import { isInsidePiAiSpan } from '../ai/pi-ai/providers'; import { resolvePiAiRecordingOptions } from '../ai/pi-ai/recording'; -import type { PiEventStream, PiStreamOptions } from '../ai/pi-ai/types'; +import type { PiAiCallInfo, PiEventStream, PiModel, PiStreamOptions } from '../ai/pi-ai/types'; import type { OrchestrionChannelContext } from '../orchestrion/types'; import { CHANNELS } from '../orchestrion/channels'; import { piAiModuleNames } from '../orchestrion/config/pi-ai'; import { invokeOrchestrionInstrumentation } from '../orchestrion/instrumentation'; import { bindTracingChannelToSpan } from '../tracing-channel'; -export type PiAiOptions = GenAiOptions; +export type PiAiOptions = GenAiCallOptions; type PiAiChannelContext = OrchestrionChannelContext & { _sentryRecordOutputs?: boolean }; @@ -37,15 +37,23 @@ function instrumentPiAi(options: PiAiOptions): void { if (isInsidePiAiSpan()) { return undefined; } - const [model, context, callOptions] = data.arguments ?? []; - const { recordInputs, recordOutputs } = resolvePiAiRecordingOptions(options); + const [model, context, callOptions] = (data.arguments ?? []) as [ + PiModel | undefined, + PiAiCallInfo['context'], + PiStreamOptions | undefined, + ]; + const { recordInputs, recordOutputs } = resolvePiAiRecordingOptions(options, { + model, + context, + options: callOptions, + }); data._sentryRecordOutputs = recordOutputs; return startInactiveSpan( getPiAiChatSpanOptions({ model: model ?? {}, // The deferred calls pass a handle here, which carries no request content. context: context as PiAiContext | undefined, - callOptions: callOptions as PiStreamOptions | undefined, + callOptions, streaming, recordInputs, }), diff --git a/packages/server-utils/test/ai/lib/tracing/flue.test.ts b/packages/server-utils/test/ai/lib/tracing/flue.test.ts index 42fd0fb0edad..a0137bb04f0d 100644 --- a/packages/server-utils/test/ai/lib/tracing/flue.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/flue.test.ts @@ -22,6 +22,7 @@ import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; const AGENT_OP = { type: 'agent', operationId: 'op_1' }; const AGENT_CTX = { agentName: 'Hello', submissionId: 'sub_1' }; const INNER_CTX = { conversationId: 'conv_1' }; +const PI_AI_CALL = { model: undefined, context: undefined, options: undefined }; describe('createFlueInstrumentation', () => { let endedSpans: Span[]; @@ -318,11 +319,11 @@ describe('createFlueInstrumentation', () => { let recording: unknown; await createFlueInstrumentation({ recordInputs: false }).interceptor({ type: 'model' }, {}, async () => { - recording = resolvePiAiRecordingOptions({}); + recording = resolvePiAiRecordingOptions({}, PI_AI_CALL); }); expect(recording).toEqual({ recordInputs: false, recordOutputs: true }); - expect(resolvePiAiRecordingOptions({})).toEqual({ recordInputs: true, recordOutputs: true }); + expect(resolvePiAiRecordingOptions({}, PI_AI_CALL)).toEqual({ recordInputs: true, recordOutputs: true }); }); // The `tool` operation opens no span of its own; it makes the span `observe` already opened diff --git a/packages/server-utils/test/integrations/pi-ai.test.ts b/packages/server-utils/test/integrations/pi-ai.test.ts index 260ab4423de8..0923e6f6367e 100644 --- a/packages/server-utils/test/integrations/pi-ai.test.ts +++ b/packages/server-utils/test/integrations/pi-ai.test.ts @@ -442,7 +442,7 @@ describe('piAiIntegration', () => { const result = new Promise(resolve => (settle = resolve)); const events = withScope(scope => { - setPiAiRecordingOptions(scope, { recordInputs: false, recordOutputs: false }); + setPiAiRecordingOptions(scope, { recordInputs: false, recordOutputs: false }, {}); return stream(() => ({ result: () => result }), MODEL, context); }); settle(message); @@ -469,8 +469,11 @@ describe('piAiIntegration', () => { useClient({ dataCollection: { genAI: { inputs: genAI, outputs: genAI } } }); const resolved = withScope(scope => { - setPiAiRecordingOptions(scope, { recordInputs: driver, recordOutputs: driver }); - return resolvePiAiRecordingOptions({ recordInputs: piAi, recordOutputs: piAi }); + setPiAiRecordingOptions(scope, { recordInputs: driver, recordOutputs: driver }, {}); + return resolvePiAiRecordingOptions( + { recordInputs: piAi, recordOutputs: piAi }, + { model: MODEL, context: undefined, options: undefined }, + ); }); expect(resolved).toEqual({ recordInputs: expected, recordOutputs: expected });