diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-recording-functions.mjs new file mode 100644 index 000000000000..6ba3c86ebf68 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-recording-functions.mjs @@ -0,0 +1,15 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + integrations: [ + Sentry.langChainIntegration({ + recordInputs: call => call.operation === 'chat' && call.metadata?.privacy === 'public', + recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public', + }), + ], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-recording-functions.mjs new file mode 100644 index 000000000000..1d6e9a1f04f7 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-recording-functions.mjs @@ -0,0 +1,66 @@ +import { ChatAnthropic } from '@langchain/anthropic'; +import * as Sentry from '@sentry/node'; +import express from 'express'; + +// Set to a scope whose tag the mock server flips while a call is in flight. +let scopeToFlip; + +function startMockAnthropicServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/messages', (req, res) => { + const prompt = req.body.messages[0].content; + if (prompt === 'flip') { + scopeToFlip.setTag('privacy', 'public'); + } + + res.json({ + id: `msg_${prompt}`, + type: 'message', + role: 'assistant', + content: [{ type: 'text', text: 'Paris.' }], + model: req.body.model, + stop_reason: 'end_turn', + stop_sequence: null, + usage: { input_tokens: 10, output_tokens: 15 }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockAnthropicServer(); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const model = new ChatAnthropic({ + model: 'claude-3-5-sonnet-20241022', + apiKey: 'mock-api-key', + clientOptions: { baseURL: `http://localhost:${server.address().port}` }, + }); + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'public'); + await model.invoke('public', { metadata: { privacy: 'public' } }); + }); + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'private'); + await model.invoke('private', { metadata: { privacy: 'private' } }); + + // The tag flips to public between the start and the end callback of this call. + scopeToFlip = scope; + await model.invoke('flip', { metadata: { privacy: 'public' } }); + }); + }); + + await Sentry.flush(2000); + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts index 48767041abf9..4797ba6e48ae 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts @@ -4,6 +4,7 @@ import { GEN_AI_EMBEDDINGS_INPUT, GEN_AI_INPUT_MESSAGES, GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, GEN_AI_PIPELINE_NAME, GEN_AI_PROVIDER_NAME, GEN_AI_REQUEST_MAX_TOKENS, @@ -379,4 +380,38 @@ describe('LangChain integration', () => { .completed(); }); }); + + createEsmAndCjsTests( + __dirname, + 'scenario-recording-functions.mjs', + 'instrument-with-recording-functions.mjs', + (createRunner, test) => { + test('decides content recording per call when the options are functions', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain', + ); + const recordedById = Object.fromEntries( + spans.map(span => [ + span.attributes[GEN_AI_RESPONSE_ID]?.value, + { + inputs: GEN_AI_INPUT_MESSAGES in span.attributes, + outputs: GEN_AI_RESPONSE_TEXT in span.attributes || GEN_AI_OUTPUT_MESSAGES in span.attributes, + }, + ]), + ); + expect(recordedById).toEqual({ + msg_public: { inputs: true, outputs: true }, + msg_private: { inputs: false, outputs: false }, + msg_flip: { inputs: true, outputs: false }, + }); + }, + }) + .start() + .completed(); + }); + }, + ); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/instrument-with-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/langgraph/instrument-with-recording-functions.mjs new file mode 100644 index 000000000000..76fd916d1d9a --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/instrument-with-recording-functions.mjs @@ -0,0 +1,17 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + // Spans that ignored the graph's decision would fall back to this and record. + dataCollection: { genAI: { inputs: true, outputs: true } }, + transport: loggingTransport, + integrations: [ + Sentry.langGraphIntegration({ + recordInputs: ({ config }) => config?.configurable?.privacy === 'public', + recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public', + }), + ], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/scenario-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/langgraph/scenario-recording-functions.mjs new file mode 100644 index 000000000000..76975fb70558 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/scenario-recording-functions.mjs @@ -0,0 +1,76 @@ +import { tool } from '@langchain/core/tools'; +import { ChatAnthropic } from '@langchain/anthropic'; +import { createReactAgent } from '@langchain/langgraph/prebuilt'; +import { HumanMessage } from '@langchain/core/messages'; +import * as Sentry from '@sentry/node'; +import express from 'express'; +import { z } from 'zod'; + +function startMockAnthropicServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/messages', (req, res) => { + const hasToolResult = req.body.messages.some( + message => Array.isArray(message.content) && message.content.some(part => part.type === 'tool_result'), + ); + const content = hasToolResult + ? [{ type: 'text', text: 'Paris.' }] + : [{ type: 'tool_use', id: 'toolu_1', name: 'lookup', input: { country: 'France' } }]; + + res.json({ + id: hasToolResult ? 'msg_answer' : 'msg_tool', + type: 'message', + role: 'assistant', + content, + model: req.body.model, + stop_reason: hasToolResult ? 'end_turn' : 'tool_use', + usage: { input_tokens: 10, output_tokens: 10 }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => resolve(server)); + }); +} + +async function run() { + const server = await startMockAnthropicServer(); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const llm = new ChatAnthropic({ + model: 'claude-3-5-sonnet-20241022', + apiKey: 'mock-api-key', + clientOptions: { baseURL: `http://localhost:${server.address().port}` }, + }); + const lookup = tool(async ({ country }) => `The capital of ${country} is Paris.`, { + name: 'lookup', + description: 'Look up the capital of a country', + schema: z.object({ country: z.string() }), + }); + const agent = createReactAgent({ llm, tools: [lookup], name: 'capital_assistant' }); + const input = () => ({ messages: [new HumanMessage('What is the capital of France?')] }); + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'public'); + await agent.invoke(input(), { configurable: { thread_id: 'public', privacy: 'public' } }); + }); + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'private'); + await agent.invoke(input(), { configurable: { thread_id: 'private', privacy: 'private' } }); + + // Changing the tag after the stream started must not change its outputs decision. + const stream = await agent.stream(input(), { configurable: { thread_id: 'stream', privacy: 'public' } }); + scope.setTag('privacy', 'public'); + for await (const chunk of stream) { + void chunk; + } + }); + }); + + await Sentry.flush(2000); + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts b/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts index 62d9e6cef300..48981411ce9f 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts @@ -12,6 +12,8 @@ import { GEN_AI_RESPONSE_TEXT, GEN_AI_RESPONSE_TOOL_CALLS, GEN_AI_SYSTEM_INSTRUCTIONS, + GEN_AI_TOOL_CALL_ARGUMENTS, + GEN_AI_TOOL_CALL_RESULT, GEN_AI_TOOL_DEFINITIONS, GEN_AI_TOOL_NAME, GEN_AI_USAGE_INPUT_TOKENS, @@ -453,4 +455,53 @@ describe('LangGraph integration', () => { }, ); }); + + createEsmAndCjsTests( + __dirname, + 'scenario-recording-functions.mjs', + 'instrument-with-recording-functions.mjs', + (createRunner, test) => { + test( + 'decides content recording per graph call, for every span of the run, when the options are functions', + { timeout: 30000 }, + async () => { + await createRunner() + .expect({ + span: container => { + const inputAttributes = [GEN_AI_INPUT_MESSAGES, GEN_AI_TOOL_CALL_ARGUMENTS]; + const outputAttributes = [GEN_AI_RESPONSE_TEXT, GEN_AI_OUTPUT_MESSAGES, GEN_AI_TOOL_CALL_RESULT]; + const spans = container.items.filter(span => + String(span.attributes[SENTRY_OP]?.value).startsWith('gen_ai.'), + ); + const recordedByThread: Record = {}; + for (const span of spans) { + const thread = String(span.attributes[GEN_AI_CONVERSATION_ID]?.value); + const inputs = inputAttributes.some(key => key in span.attributes); + const outputs = outputAttributes.some(key => key in span.attributes); + (recordedByThread[thread] ??= []).push(`${span.name} inputs=${inputs} outputs=${outputs}`); + } + + // The chat and tool spans of a run follow the decision of its invoke()/stream() call. + const run = (inputs: boolean, outputs: boolean): string[] => + [ + 'chat claude-3-5-sonnet-20241022', + 'chat claude-3-5-sonnet-20241022', + 'execute_tool lookup', + 'invoke_agent capital_assistant', + ].map(name => `${name} inputs=${inputs} outputs=${outputs}`); + expect( + Object.fromEntries(Object.entries(recordedByThread).map(([thread, spans]) => [thread, spans.sort()])), + ).toEqual({ + public: run(true, true), + private: run(false, false), + stream: run(true, false), + }); + }, + }) + .start() + .completed(); + }, + ); + }, + ); }); diff --git a/packages/server-utils/src/ai/langchain/embeddings.ts b/packages/server-utils/src/ai/langchain/embeddings.ts index e1c4fd25fd9b..3ed99176b080 100644 --- a/packages/server-utils/src/ai/langchain/embeddings.ts +++ b/packages/server-utils/src/ai/langchain/embeddings.ts @@ -13,7 +13,7 @@ import { GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE, GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE, } from '../core/gen-ai-attributes'; -import { resolveAIRecordingOptions } from '../core/utils'; +import { resolveGenAiCallRecording } from '../core/utils'; import { LANGCHAIN_ORIGIN } from './constants'; import type { LangChainOptions } from './types'; @@ -68,7 +68,7 @@ export function _INTERNAL_getLangChainEmbeddingsSpanOptions( input: unknown, options: LangChainOptions = {}, ): { name: string; attributes: Record } { - const { recordInputs } = resolveAIRecordingOptions(options); + const { recordInputs } = resolveGenAiCallRecording(options, { operation: 'embeddings', input }); const attributes = extractEmbeddingAttributes(instance); const modelName = attributes[GEN_AI_REQUEST_MODEL] || 'unknown'; const client = getClient(); diff --git a/packages/server-utils/src/ai/langchain/index.ts b/packages/server-utils/src/ai/langchain/index.ts index e05ea6f38a81..a8c9c8add709 100644 --- a/packages/server-utils/src/ai/langchain/index.ts +++ b/packages/server-utils/src/ai/langchain/index.ts @@ -28,7 +28,8 @@ import { } from '@sentry/conventions/attributes'; import { GEN_AI_CHAT, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op'; import { DEBUG_BUILD } from '../../debug-build'; -import { resolveAIRecordingOptions } from '../core/utils'; +import { resolveGenAiCallRecording } from '../core/utils'; +import type { GenAiOptions } from '../core/utils'; import { LANGCHAIN_ORIGIN, MAX_TRACKED_LANGCHAIN_RUNS } from './constants'; import { addTypeSafeClassifierResponseAttributes, @@ -37,6 +38,7 @@ import { } from './typesafe-classifier'; import type { LangChainCallbackHandler, + LangChainCallInfo, LangChainLLMResult, LangChainMessage, LangChainOptions, @@ -50,6 +52,7 @@ import { getAgentNameFromMetadata, getConversationIdFromMetadata, getInvocationParams, + getLangGraphRunRecording, } from './utils'; interface LangChainRun { @@ -59,6 +62,7 @@ interface LangChainRun { activeSpan?: Span; conversationId?: string; decide?: boolean; + recordOutputs: boolean; } /** @@ -68,7 +72,8 @@ interface LangChainRun { * This is a stateful handler that tracks spans across multiple LangChain executions. */ export function createLangChainCallbackHandler(options: LangChainOptions = {}): LangChainCallbackHandler { - const { recordInputs, recordOutputs } = resolveAIRecordingOptions(options); + const resolveRecording = (call: Exclude): Required => + getLangGraphRunRecording(call.metadata) ?? resolveGenAiCallRecording(options, call); // Internal state - single instance tracks all spans const runs = new LRUMap(MAX_TRACKED_LANGCHAIN_RUNS); @@ -128,7 +133,12 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): return undefined; }; - const startRunSpan = (runId: string, parentRunId: string | undefined, start: () => Span): LangChainRun => { + const startRunSpan = ( + runId: string, + parentRunId: string | undefined, + recordOutputs: boolean, + start: () => Span, + ): LangChainRun => { const ancestors = getAncestors(parentRunId); const activeSpan = getActiveSpan(); const parentSpan = getParentRunSpan(ancestors, activeSpan); @@ -143,7 +153,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): span.setAttribute(GEN_AI_CONVERSATION_ID, conversationId); } - const run: LangChainRun = { span, parentRunId, activeSpan, conversationId }; + const run: LangChainRun = { span, parentRunId, activeSpan, conversationId, recordOutputs }; trackRun(runId, run); if (conversationId) { for (const ancestor of ancestors) { @@ -187,25 +197,30 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): prompts: string[], runId: string, parentRunId?: string, - _extraParams?: Record, + extraParams?: Record, tags?: string[], metadata?: Record, _runName?: string, ) { - const invocationParams = getInvocationParams(tags); - const attributes = extractLLMRequestAttributes( - llm as LangChainSerialized, + const serialized = llm as LangChainSerialized; + const { recordInputs, recordOutputs } = resolveRecording({ + operation: 'chat', + serialized, prompts, - recordInputs, - invocationParams, + runId, + parentRunId, + extraParams, + tags, metadata, - ); + }); + const invocationParams = getInvocationParams(tags); + const attributes = extractLLMRequestAttributes(serialized, prompts, recordInputs, invocationParams, metadata); const modelName = attributes[GEN_AI_REQUEST_MODEL] || 'unknown'; const operationName = typeof attributes[GEN_AI_OPERATION_NAME] === 'string' ? attributes[GEN_AI_OPERATION_NAME] : 'unknown'; const client = getClient(); - startRunSpan(runId, parentRunId, () => + startRunSpan(runId, parentRunId, recordOutputs, () => startInactiveSpan({ // With span streaming, omit the `'unknown'` model sentinel so the name stays low-cardinality. name: @@ -233,10 +248,22 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): metadata?: Record, _runName?: string, ) { + const serialized = llm as LangChainSerialized; + const chatMessages = messages as LangChainMessage[][]; + const { recordInputs, recordOutputs } = resolveRecording({ + operation: 'chat', + serialized, + messages: chatMessages, + runId, + parentRunId, + extraParams, + tags, + metadata, + }); const invocationParams = getInvocationParams(tags); const attributes = extractChatModelRequestAttributes( - llm as LangChainSerialized, - messages as LangChainMessage[][], + serialized, + chatMessages, recordInputs, invocationParams, metadata, @@ -252,7 +279,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): typeof attributes[GEN_AI_OPERATION_NAME] === 'string' ? attributes[GEN_AI_OPERATION_NAME] : 'unknown'; const client = getClient(); - startRunSpan(runId, parentRunId, () => + startRunSpan(runId, parentRunId, recordOutputs, () => startInactiveSpan({ // With span streaming, omit the `'unknown'` model sentinel so the name stays low-cardinality. name: @@ -277,9 +304,10 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): _tags?: string[], _extraParams?: Record, ) { - const span = runs.get(runId)?.span; - if (span?.isRecording()) { - const attributes = extractLlmResponseAttributes(output as LangChainLLMResult, recordOutputs); + const run = runs.get(runId); + const span = run?.span; + if (run && span?.isRecording()) { + const attributes = extractLlmResponseAttributes(output as LangChainLLMResult, run.recordOutputs); if (attributes) { span.setAttributes(attributes); } @@ -304,14 +332,23 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): inputs: Record, runId: string, parentRunId?: string, - _tags?: string[], + tags?: string[], metadata?: Record, _runType?: string, runName?: string, ) { // A Jev call is a real model call, so it is recorded inside an agent too. if (isTypeSafeClassifier(chain)) { - startRunSpan(runId, parentRunId, () => + const { recordInputs, recordOutputs } = resolveRecording({ + operation: 'decide', + serialized: chain, + inputs, + runId, + parentRunId, + tags, + metadata, + }); + startRunSpan(runId, parentRunId, recordOutputs, () => startTypeSafeClassifierSpan(chain, inputs, metadata, recordInputs), ).decide = true; return; @@ -324,6 +361,15 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): return; } + const { recordInputs, recordOutputs } = resolveRecording({ + operation: 'invoke_agent', + serialized: chain, + inputs, + runId, + parentRunId, + tags, + metadata, + }); const chainName = runName || chain.name; const attributes: Record = { ...getConversationIdFromMetadata(metadata), @@ -341,7 +387,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): const client = getClient(); - startRunSpan(runId, parentRunId, () => + startRunSpan(runId, parentRunId, recordOutputs, () => startInactiveSpan({ // With span streaming, the name leads with the operation per the agent templates. The // chain name is bounded, so it stays; the `'unknown_chain'` sentinel is dropped instead. @@ -362,10 +408,10 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): handleChainEnd(outputs: unknown, runId: string) { const run = runs.get(runId); const span = run?.span; - if (span?.isRecording()) { - if (run?.decide) { - addTypeSafeClassifierResponseAttributes(span, outputs, recordOutputs); - } else if (recordOutputs) { + if (run && span?.isRecording()) { + if (run.decide) { + addTypeSafeClassifierResponseAttributes(span, outputs, run.recordOutputs); + } else if (run.recordOutputs) { span.setAttributes({ 'langchain.chain.outputs': JSON.stringify(outputs), }); @@ -391,7 +437,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): input: string, runId: string, parentRunId?: string, - _tags?: string[], + tags?: string[], metadata?: Record, runName?: string, ) { @@ -401,6 +447,15 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): return; } + const { recordInputs, recordOutputs } = resolveRecording({ + operation: 'execute_tool', + serialized: tool, + input, + runId, + parentRunId, + tags, + metadata, + }); // runName is set to tool.name by LangChain's StructuredTool.call() const toolName = runName || tool.name || 'unknown_tool'; const attributes: Record = { @@ -415,7 +470,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): attributes[GEN_AI_TOOL_CALL_ARGUMENTS] = input; } - startRunSpan(runId, parentRunId, () => + startRunSpan(runId, parentRunId, recordOutputs, () => startInactiveSpan({ name: `execute_tool ${toolName}`, attributes: { @@ -428,9 +483,10 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): // Tool End Handler handleToolEnd(output: unknown, runId: string) { - const span = runs.get(runId)?.span; - if (span?.isRecording()) { - if (recordOutputs) { + const run = runs.get(runId); + const span = run?.span; + if (run && span?.isRecording()) { + if (run.recordOutputs) { // LangChain tools may return ToolMessage objects — extract the content const outputObj = output as Record | undefined; const content = diff --git a/packages/server-utils/src/ai/langchain/types.ts b/packages/server-utils/src/ai/langchain/types.ts index 03039782c5bf..b2162f26dd4b 100644 --- a/packages/server-utils/src/ai/langchain/types.ts +++ b/packages/server-utils/src/ai/langchain/types.ts @@ -1,7 +1,45 @@ -import type { GenAiOptions } from '../core/utils'; +import type { GenAiCallOptions } from '../core/utils'; -/** Options for the LangChain integration. */ -export type LangChainOptions = GenAiOptions; +/** What LangChain passes to every start callback of a run, besides the component and its input. */ +interface LangChainRunCallInfo { + runId: string; + parentRunId?: string; + tags?: string[]; + metadata?: Record; +} + +/** + * The call that `recordInputs`/`recordOutputs` decide on: the gen_ai operation and the arguments of the LangChain + * start callback as passed, or the argument of `embedQuery`/`embedDocuments`. `serialized` describes the + * component, such as the chat model class and its constructor kwargs. + */ +export type LangChainCallInfo = + | (LangChainRunCallInfo & { + operation: 'chat'; + serialized: LangChainSerialized; + messages: LangChainMessage[][]; + extraParams?: Record; + }) + | (LangChainRunCallInfo & { + operation: 'chat'; + serialized: LangChainSerialized; + prompts: string[]; + extraParams?: Record; + }) + | (LangChainRunCallInfo & { + operation: 'invoke_agent' | 'decide'; + serialized: LangChainSerialized; + inputs: Record; + }) + | (LangChainRunCallInfo & { operation: 'execute_tool'; serialized: LangChainSerialized; input: string }) + | { operation: 'embeddings'; input: unknown }; + +/** + * Options for the LangChain integration. The handler decides in each start callback, which LangChain awaits + * inline in the async context of the call that started the run. Runs inside a LangGraph graph use the + * decision of the graph's `invoke()`/`stream()` call instead. + */ +export type LangChainOptions = GenAiCallOptions; /** * LangChain Serialized type (compatible with @langchain/core) diff --git a/packages/server-utils/src/ai/langchain/utils.ts b/packages/server-utils/src/ai/langchain/utils.ts index 405340e71c56..aca00ad5e507 100644 --- a/packages/server-utils/src/ai/langchain/utils.ts +++ b/packages/server-utils/src/ai/langchain/utils.ts @@ -26,6 +26,7 @@ import { } from '@sentry/conventions/attributes'; import { GEN_AI_REQUEST_STREAM_ATTRIBUTE, GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE } from '../core/gen-ai-attributes'; import { extractSystemInstructions, getTokenUsageAttributes } from '../core/utils'; +import type { GenAiOptions } from '../core/utils'; import { LANGCHAIN_ORIGIN, ROLE_MAP } from './constants'; import type { LangChainLLMResult, LangChainMessage, LangChainSerialized } from './types'; @@ -530,6 +531,15 @@ function findConversationIdEntry(source?: Record): [string, str return undefined; } +/** + * The recording decision of the LangGraph graph run that `metadata` belongs to. The graph's `invoke()`/`stream()` + * call decides once, and child runs inherit the decision through their metadata. + */ +export function getLangGraphRunRecording(metadata?: Record): Required | undefined { + const recording = metadata?.__sentry_langgraph__; + return isObjectLike(recording) ? (recording as Required) : undefined; +} + /** * Derive `gen_ai.conversation.id` from run metadata, which child runs inherit from their parent. * An id from `Sentry.setConversationId()` still wins, since `conversationIdIntegration` applies it on `spanStart`. diff --git a/packages/server-utils/src/ai/langgraph/agent-span.ts b/packages/server-utils/src/ai/langgraph/agent-span.ts index acae26fda9cf..9a72690d31e8 100644 --- a/packages/server-utils/src/ai/langgraph/agent-span.ts +++ b/packages/server-utils/src/ai/langgraph/agent-span.ts @@ -12,7 +12,8 @@ import { SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; import { GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op'; -import { extractSystemInstructions } from '../core/utils'; +import { extractSystemInstructions, resolveGenAiCallRecording } from '../core/utils'; +import type { GenAiOptions } from '../core/utils'; import type { BaseChatModel, LangChainMessage } from '../langchain/types'; import { _INTERNAL_mergeLangChainCallbackHandler, @@ -64,13 +65,22 @@ export function getAgentSpanOptions( }; } +/** Decides for one `invoke()`/`stream()` call, from its arguments as passed. */ +export function resolveGraphCallRecording(options: LangGraphOptions, args: unknown[]): Required { + return resolveGenAiCallRecording(options, { + operation: 'invoke_agent', + input: args[0], + config: args[1] as Record | undefined, + }); +} + /** Sets the request attributes on an agent span and injects the callback handler into the call's config. */ export function prepareAgentSpan( span: Span, args: unknown[], graphInstance: CompiledGraph, compileOptions: Record, - options: LangGraphOptions, + recording: Required, llm: BaseChatModel | null | undefined, sentryCallbackHandler: unknown, ): LangChainMessage[] | null { @@ -101,14 +111,15 @@ export function prepareAgentSpan( config.metadata = { ...getConversationIdMetadataFromConfig(config), ...existingMetadata, - __sentry_langgraph__: true, + // Child runs inherit the metadata, so their spans follow this call's decision. + __sentry_langgraph__: recording, ...(typeof graphName === 'string' ? { lc_agent_name: graphName } : {}), }; config.callbacks = _INTERNAL_mergeLangChainCallbackHandler(config.callbacks, sentryCallbackHandler); } - const recordInputs = options.recordInputs; + const { recordInputs } = recording; const tools = recordInputs ? extractToolsFromCompiledGraph(graphInstance) : null; if (tools) { diff --git a/packages/server-utils/src/ai/langgraph/index.ts b/packages/server-utils/src/ai/langgraph/index.ts index 82af1fa78ab6..0645b14e5a4d 100644 --- a/packages/server-utils/src/ai/langgraph/index.ts +++ b/packages/server-utils/src/ai/langgraph/index.ts @@ -1,11 +1,16 @@ import { SPAN_STATUS_ERROR, startSpan } from '@sentry/core'; -import { resolveAIRecordingOptions } from '../core/utils'; import { createLangChainCallbackHandler } from '../langchain'; import type { BaseChatModel } from '../langchain/types'; -import { callInvoke, getAgentSpanOptions, prepareAgentSpan } from './agent-span'; +import { callInvoke, getAgentSpanOptions, prepareAgentSpan, resolveGraphCallRecording } from './agent-span'; import { instrumentCompiledGraphStream } from './stream'; import type { CompiledGraph, LangGraphOptions } from './types'; -import { extractAgentNameFromParams, extractLLMFromParams, setResponseAttributes, wrapToolsWithSpans } from './utils'; +import { + extractAgentNameFromParams, + extractLLMFromParams, + getBooleanRecordingOptions, + setResponseAttributes, + wrapToolsWithSpans, +} from './utils'; // Set while a manual wrapper runs the original `compile`/`createReactAgent`. The integration's channel hooks // fire synchronously inside that call, and skip the graph so the options passed explicitly win. @@ -35,16 +40,13 @@ function runManually(callback: () => T): T { */ export function instrumentStateGraphCompile( originalCompile: (...args: unknown[]) => CompiledGraph, - rawOptions: LangGraphOptions, + options: LangGraphOptions, ): (...args: unknown[]) => CompiledGraph { if (Object.prototype.hasOwnProperty.call(originalCompile, SENTRY_PATCHED)) { return originalCompile; } - // This is exported, so callers can hand us an options object with no recording flags set. Resolving - // here (rather than only in `instrumentStateGraph`) keeps that path on the `dataCollection` defaults. - const options = resolveAIRecordingOptions(rawOptions); - const sentryHandler = createLangChainCallbackHandler(options); + const sentryHandler = createLangChainCallbackHandler(getBooleanRecordingOptions(options)); const wrapped = new Proxy(originalCompile, { apply(target, thisArg, args: unknown[]): CompiledGraph { @@ -119,6 +121,7 @@ export function instrumentCompiledGraphInvoke( ): (...args: unknown[]) => Promise { return new Proxy(originalInvoke, { apply(target, thisArg, args: unknown[]): Promise { + const recording = resolveGraphCallRecording(options, args); return startSpan(getAgentSpanOptions(args), async span => { try { const inputMessages = prepareAgentSpan( @@ -126,13 +129,13 @@ export function instrumentCompiledGraphInvoke( args, graphInstance, compileOptions, - options, + recording, llm, sentryCallbackHandler, ); const result = await callInvoke(graphInstance, () => Reflect.apply(target, thisArg, args)); - if (options.recordOutputs) { + if (recording.recordOutputs) { setResponseAttributes(span, inputMessages, result); } @@ -153,14 +156,13 @@ export function instrumentCompiledGraphInvoke( */ export function instrumentCreateReactAgent( originalCreateReactAgent: (...args: unknown[]) => CompiledGraph, - options?: LangGraphOptions, + options: LangGraphOptions = {}, ): (...args: unknown[]) => CompiledGraph { if (Object.prototype.hasOwnProperty.call(originalCreateReactAgent, SENTRY_PATCHED)) { return originalCreateReactAgent; } - const resolvedOptions = resolveAIRecordingOptions(options); - const sentryHandler = createLangChainCallbackHandler(resolvedOptions); + const sentryHandler = createLangChainCallbackHandler(getBooleanRecordingOptions(options)); const wrapped = new Proxy(originalCreateReactAgent, { apply(target, thisArg, args: unknown[]): CompiledGraph { @@ -170,13 +172,13 @@ export function instrumentCreateReactAgent( // Wrap tools with execute_tool spans (direct access gives us name, type, description) const params = args[0] as Record | undefined; if (params && Array.isArray(params.tools) && params.tools.length > 0) { - wrapToolsWithSpans(params.tools, resolvedOptions, agentName ?? undefined); + wrapToolsWithSpans(params.tools, options, agentName ?? undefined); } const compiledGraph = runManually(() => Reflect.apply(target, thisArg, args)); const compileOptions: Record = agentName ? { name: agentName } : {}; - instrumentCompiledGraph(compiledGraph, compileOptions, resolvedOptions, llm, sentryHandler); + instrumentCompiledGraph(compiledGraph, compileOptions, options, llm, sentryHandler); return compiledGraph; }, @@ -212,9 +214,9 @@ export function instrumentCreateReactAgent( // eslint-disable-next-line @typescript-eslint/no-explicit-any export function instrumentStateGraph any }>( stateGraph: T, - options?: LangGraphOptions, + options: LangGraphOptions = {}, ): T { - stateGraph.compile = instrumentStateGraphCompile(stateGraph.compile, resolveAIRecordingOptions(options)); + stateGraph.compile = instrumentStateGraphCompile(stateGraph.compile, options); return stateGraph; } diff --git a/packages/server-utils/src/ai/langgraph/stream.ts b/packages/server-utils/src/ai/langgraph/stream.ts index 3370eb9fe109..60ddb15c3a76 100644 --- a/packages/server-utils/src/ai/langgraph/stream.ts +++ b/packages/server-utils/src/ai/langgraph/stream.ts @@ -2,7 +2,7 @@ import { SPAN_STATUS_ERROR, startSpanManual } from '@sentry/core'; import { GEN_AI_RESPONSE_STREAMING } from '@sentry/conventions/attributes'; import type { BaseChatModel, LangChainMessage } from '../langchain/types'; import { _INTERNAL_addLangChainCallbackHandler } from '../langchain/utils'; -import { getAgentSpanOptions, isInvokeInternalStream, prepareAgentSpan } from './agent-span'; +import { getAgentSpanOptions, isInvokeInternalStream, prepareAgentSpan, resolveGraphCallRecording } from './agent-span'; import type { CompiledGraph, LangGraphOptions } from './types'; import { setResponseAttributes } from './utils'; @@ -26,6 +26,7 @@ export function instrumentCompiledGraphStream( return Reflect.apply(target, thisArg, args); } + const recording = resolveGraphCallRecording(options, args); return startSpanManual(getAgentSpanOptions(args, { [GEN_AI_RESPONSE_STREAMING]: true }), async span => { let inputMessages: LangChainMessage[] | null = null; let stream: { signal?: AbortSignal } | undefined; @@ -42,7 +43,7 @@ export function instrumentCompiledGraphStream( if (!stream?.signal?.aborted && !(args[1] as { signal?: AbortSignal } | undefined)?.signal?.aborted) { span.setStatus({ code: SPAN_STATUS_ERROR, message: 'internal_error' }); } - } else if (outcome && options.recordOutputs) { + } else if (outcome && recording.recordOutputs) { setResponseAttributes(span, inputMessages, outcome.outputs); } } catch { @@ -81,7 +82,7 @@ export function instrumentCompiledGraphStream( args, graphInstance, compileOptions, - options, + recording, llm, sentryCallbackHandler, ); diff --git a/packages/server-utils/src/ai/langgraph/types.ts b/packages/server-utils/src/ai/langgraph/types.ts index f8ac291c5ee4..8b5fa5de446f 100644 --- a/packages/server-utils/src/ai/langgraph/types.ts +++ b/packages/server-utils/src/ai/langgraph/types.ts @@ -1,7 +1,19 @@ -import type { GenAiOptions } from '../core/utils'; +import type { GenAiCallOptions } from '../core/utils'; -/** Options for the LangGraph integration. */ -export type LangGraphOptions = GenAiOptions; +/** The call that `recordInputs`/`recordOutputs` decide on: an `invoke()` or `stream()` call of a compiled graph. */ +export interface LangGraphCallInfo { + operation: 'invoke_agent'; + /** The graph input, the first argument of `invoke()`/`stream()` as passed. */ + input: unknown; + /** The run config, the second argument of `invoke()`/`stream()` as passed. */ + config: Record | undefined; +} + +/** + * Options for the LangGraph integration. Each `invoke()`/`stream()` call decides once, and the decision covers the + * whole graph run, including its chat model and tool spans. + */ +export type LangGraphOptions = GenAiCallOptions; /** * LangGraph Tool definition from lc_kwargs diff --git a/packages/server-utils/src/ai/langgraph/utils.ts b/packages/server-utils/src/ai/langgraph/utils.ts index adb3dfb1850a..59ac9818eebe 100644 --- a/packages/server-utils/src/ai/langgraph/utils.ts +++ b/packages/server-utils/src/ai/langgraph/utils.ts @@ -22,11 +22,13 @@ import { } from '@sentry/conventions/attributes'; import { GEN_AI_EXECUTE_TOOL } from '@sentry/conventions/op'; import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../core/gen-ai-attributes'; -import { setOutputMessagesAttribute } from '../core/utils'; +import { resolveAIRecordingOptions, setOutputMessagesAttribute } from '../core/utils'; +import type { GenAiOptions } from '../core/utils'; import type { BaseChatModel, LangChainMessage } from '../langchain/types'; import { extractMessageTokenUsageAttributes, getConversationIdFromMetadata, + getLangGraphRunRecording, normalizeLangChainMessages, } from '../langchain/utils'; import { LANGGRAPH_ORIGIN } from './constants'; @@ -58,6 +60,15 @@ export function extractAgentNameFromParams(args: unknown[]): string | null { return null; } +/** The options that hold outside a graph run, where no `invoke()`/`stream()` call decided: a function needs the graph input. */ +export function getBooleanRecordingOptions(options: LangGraphOptions): GenAiOptions { + const { recordInputs, recordOutputs } = options; + return { + recordInputs: typeof recordInputs === 'boolean' ? recordInputs : undefined, + recordOutputs: typeof recordOutputs === 'boolean' ? recordOutputs : undefined, + }; +} + /** * Wraps an array of LangChain tools so each invocation creates a gen_ai.execute_tool span. * @@ -92,11 +103,14 @@ export function wrapToolsWithSpans(tools: unknown[], options: LangGraphOptions, // Read agent name from LangChain's propagated config metadata at call time, // so shared tools get the correct agent name for each invocation const callConfig = args[1] as Record | undefined; - const callAgentName = (callConfig?.metadata as Record)?.lc_agent_name ?? agentName; + const metadata = callConfig?.metadata as Record | undefined; + const { recordInputs, recordOutputs } = + getLangGraphRunRecording(metadata) ?? resolveAIRecordingOptions(getBooleanRecordingOptions(options)); + const callAgentName = metadata?.lc_agent_name ?? agentName; if (typeof callAgentName === 'string') { spanAttributes[GEN_AI_AGENT_NAME] = callAgentName; } - Object.assign(spanAttributes, getConversationIdFromMetadata(callConfig?.metadata as Record)); + Object.assign(spanAttributes, getConversationIdFromMetadata(metadata)); if (toolDescription) { spanAttributes[GEN_AI_TOOL_DESCRIPTION] = toolDescription; @@ -109,7 +123,7 @@ export function wrapToolsWithSpans(tools: unknown[], options: LangGraphOptions, spanAttributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE] = input.id; } - if (options.recordInputs) { + if (recordInputs) { const toolArgs = 'args' in input && typeof input.args === 'object' ? input.args : input; try { spanAttributes[GEN_AI_TOOL_CALL_ARGUMENTS] = JSON.stringify(toolArgs); @@ -128,7 +142,7 @@ export function wrapToolsWithSpans(tools: unknown[], options: LangGraphOptions, try { const result = await Reflect.apply(target, thisArg, args); - if (options.recordOutputs) { + if (recordOutputs) { try { // ToolMessage objects wrap the result in .content const resultObj = result as Record | undefined; diff --git a/packages/server-utils/src/integrations/langgraph.ts b/packages/server-utils/src/integrations/langgraph.ts index ad94021c8677..821fc35e1957 100644 --- a/packages/server-utils/src/integrations/langgraph.ts +++ b/packages/server-utils/src/integrations/langgraph.ts @@ -1,12 +1,16 @@ import * as diagnosticsChannel from '../utils/diagnosticsChannel'; import type { IntegrationFn } from '@sentry/core'; import { debug, defineIntegration, isObjectLike } from '@sentry/core'; -import { resolveAIRecordingOptions } from '../ai/core/utils'; import { createLangChainCallbackHandler } from '../ai/langchain'; import { instrumentCompiledGraph, isInsideManualLangGraphInstrumentation } from '../ai/langgraph'; import { LANGGRAPH_INTEGRATION_NAME } from '../ai/langgraph/constants'; import type { LangGraphOptions } from '../ai/langgraph/types'; -import { extractAgentNameFromParams, extractLLMFromParams, wrapToolsWithSpans } from '../ai/langgraph/utils'; +import { + extractAgentNameFromParams, + extractLLMFromParams, + getBooleanRecordingOptions, + wrapToolsWithSpans, +} from '../ai/langgraph/utils'; import { DEBUG_BUILD } from '../debug-build'; import type { OrchestrionChannelContext } from '../orchestrion/types'; import { CHANNELS } from '../orchestrion/channels'; @@ -35,8 +39,7 @@ const _langGraphIntegration = ((options: LangGraphOptions = {}) => { }) satisfies IntegrationFn; function instrumentLanggraph(options: LangGraphOptions): void { - const resolvedOptions = resolveAIRecordingOptions(options); - const sentryHandler = createLangChainCallbackHandler(resolvedOptions); + const sentryHandler = createLangChainCallbackHandler(getBooleanRecordingOptions(options)); // StateGraph.compile returns synchronously; wrap the returned graph's `invoke` and `stream` at `end`. diagnosticsChannel @@ -46,7 +49,7 @@ function instrumentLanggraph(options: LangGraphOptions): void { return; } const { arguments: args, result } = message as OrchestrionChannelContext; - instrumentCompiledGraph(result, getFirstArgObject(args) ?? {}, resolvedOptions, null, sentryHandler); + instrumentCompiledGraph(result, getFirstArgObject(args) ?? {}, options, null, sentryHandler); }); // createReactAgent only wraps tools and the returned graph's `invoke` and `stream`. Tools are wrapped at @@ -64,7 +67,7 @@ function instrumentLanggraph(options: LangGraphOptions): void { const { arguments: args } = message as OrchestrionChannelContext; const params = getFirstArgObject(args); if (params && Array.isArray(params.tools) && params.tools.length > 0) { - wrapToolsWithSpans(params.tools, resolvedOptions, extractAgentNameFromParams(args) ?? undefined); + wrapToolsWithSpans(params.tools, options, extractAgentNameFromParams(args) ?? undefined); } } catch (error) { DEBUG_BUILD && debug.error('[instrumentation:langgraph] failed to wrap createReactAgent tools', error); @@ -78,7 +81,7 @@ function instrumentLanggraph(options: LangGraphOptions): void { const { arguments: args, result } = message as OrchestrionChannelContext; const agentName = extractAgentNameFromParams(args) ?? undefined; const compileOptions = agentName ? { name: agentName } : {}; - instrumentCompiledGraph(result, compileOptions, resolvedOptions, extractLLMFromParams(args), sentryHandler); + instrumentCompiledGraph(result, compileOptions, options, extractLLMFromParams(args), sentryHandler); }); // Make sure a thrown `createReactAgent` doesn't leave the suppression flag stuck on. reactAgentChannel.error.subscribe(() => { diff --git a/packages/server-utils/test/ai/lib/tracing/langgraph-invoke-agent.test.ts b/packages/server-utils/test/ai/lib/tracing/langgraph-invoke-agent.test.ts index 3f894efae3ea..8a516b80bf69 100644 --- a/packages/server-utils/test/ai/lib/tracing/langgraph-invoke-agent.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/langgraph-invoke-agent.test.ts @@ -127,7 +127,11 @@ describe('LangGraph conversation id', () => { await invoke({}, { configurable: { sessionId: 'session-1' }, metadata: { user: 'u' } }); - expect(receivedConfig?.metadata).toEqual({ sessionId: 'session-1', user: 'u', __sentry_langgraph__: true }); + expect(receivedConfig?.metadata).toEqual({ + sessionId: 'session-1', + user: 'u', + __sentry_langgraph__: { recordInputs: true, recordOutputs: true }, + }); }); it('sets the conversation id on wrapped tool spans from the call metadata', async () => {