From 7d6da130f005bdff7d3f55b7b34706167bdedb19 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2026 11:19:07 +0000 Subject: [PATCH 1/2] feat(server-utils): Decide gen_ai content recording per call for LangChain and LangGraph recordInputs and recordOutputs of langChainIntegration, createLangChainCallbackHandler, instrumentLangChainEmbeddings, langGraphIntegration and the LangGraph wrappers can be functions. LangChain decides in each start callback, from the callback arguments. LangGraph decides once per invoke()/stream() call, from the graph input and config, and the decision covers the whole graph run through the run metadata. Co-Authored-By: Francesco Gringl-Novy <2411343+mydea@users.noreply.github.com> --- .../src/ai/langchain/embeddings.ts | 4 +- .../server-utils/src/ai/langchain/index.ts | 116 +++++++--- .../server-utils/src/ai/langchain/types.ts | 44 +++- .../server-utils/src/ai/langchain/utils.ts | 10 + .../src/ai/langgraph/agent-span.ts | 19 +- .../server-utils/src/ai/langgraph/index.ts | 36 ++-- .../server-utils/src/ai/langgraph/stream.ts | 7 +- .../server-utils/src/ai/langgraph/types.ts | 18 +- .../server-utils/src/ai/langgraph/utils.ts | 24 ++- .../src/integrations/langgraph.ts | 17 +- .../lib/tracing/langchain-recording.test.ts | 188 +++++++++++++++++ .../tracing/langgraph-invoke-agent.test.ts | 6 +- .../lib/tracing/langgraph-recording.test.ts | 199 ++++++++++++++++++ 13 files changed, 613 insertions(+), 75 deletions(-) create mode 100644 packages/server-utils/test/ai/lib/tracing/langchain-recording.test.ts create mode 100644 packages/server-utils/test/ai/lib/tracing/langgraph-recording.test.ts diff --git a/packages/server-utils/src/ai/langchain/embeddings.ts b/packages/server-utils/src/ai/langchain/embeddings.ts index e1c4fd25fd9b..3ed99176b080 100644 --- a/packages/server-utils/src/ai/langchain/embeddings.ts +++ b/packages/server-utils/src/ai/langchain/embeddings.ts @@ -13,7 +13,7 @@ import { GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE, GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE, } from '../core/gen-ai-attributes'; -import { resolveAIRecordingOptions } from '../core/utils'; +import { resolveGenAiCallRecording } from '../core/utils'; import { LANGCHAIN_ORIGIN } from './constants'; import type { LangChainOptions } from './types'; @@ -68,7 +68,7 @@ export function _INTERNAL_getLangChainEmbeddingsSpanOptions( input: unknown, options: LangChainOptions = {}, ): { name: string; attributes: Record } { - const { recordInputs } = resolveAIRecordingOptions(options); + const { recordInputs } = resolveGenAiCallRecording(options, { operation: 'embeddings', input }); const attributes = extractEmbeddingAttributes(instance); const modelName = attributes[GEN_AI_REQUEST_MODEL] || 'unknown'; const client = getClient(); diff --git a/packages/server-utils/src/ai/langchain/index.ts b/packages/server-utils/src/ai/langchain/index.ts index e05ea6f38a81..a8c9c8add709 100644 --- a/packages/server-utils/src/ai/langchain/index.ts +++ b/packages/server-utils/src/ai/langchain/index.ts @@ -28,7 +28,8 @@ import { } from '@sentry/conventions/attributes'; import { GEN_AI_CHAT, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op'; import { DEBUG_BUILD } from '../../debug-build'; -import { resolveAIRecordingOptions } from '../core/utils'; +import { resolveGenAiCallRecording } from '../core/utils'; +import type { GenAiOptions } from '../core/utils'; import { LANGCHAIN_ORIGIN, MAX_TRACKED_LANGCHAIN_RUNS } from './constants'; import { addTypeSafeClassifierResponseAttributes, @@ -37,6 +38,7 @@ import { } from './typesafe-classifier'; import type { LangChainCallbackHandler, + LangChainCallInfo, LangChainLLMResult, LangChainMessage, LangChainOptions, @@ -50,6 +52,7 @@ import { getAgentNameFromMetadata, getConversationIdFromMetadata, getInvocationParams, + getLangGraphRunRecording, } from './utils'; interface LangChainRun { @@ -59,6 +62,7 @@ interface LangChainRun { activeSpan?: Span; conversationId?: string; decide?: boolean; + recordOutputs: boolean; } /** @@ -68,7 +72,8 @@ interface LangChainRun { * This is a stateful handler that tracks spans across multiple LangChain executions. */ export function createLangChainCallbackHandler(options: LangChainOptions = {}): LangChainCallbackHandler { - const { recordInputs, recordOutputs } = resolveAIRecordingOptions(options); + const resolveRecording = (call: Exclude): Required => + getLangGraphRunRecording(call.metadata) ?? resolveGenAiCallRecording(options, call); // Internal state - single instance tracks all spans const runs = new LRUMap(MAX_TRACKED_LANGCHAIN_RUNS); @@ -128,7 +133,12 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): return undefined; }; - const startRunSpan = (runId: string, parentRunId: string | undefined, start: () => Span): LangChainRun => { + const startRunSpan = ( + runId: string, + parentRunId: string | undefined, + recordOutputs: boolean, + start: () => Span, + ): LangChainRun => { const ancestors = getAncestors(parentRunId); const activeSpan = getActiveSpan(); const parentSpan = getParentRunSpan(ancestors, activeSpan); @@ -143,7 +153,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): span.setAttribute(GEN_AI_CONVERSATION_ID, conversationId); } - const run: LangChainRun = { span, parentRunId, activeSpan, conversationId }; + const run: LangChainRun = { span, parentRunId, activeSpan, conversationId, recordOutputs }; trackRun(runId, run); if (conversationId) { for (const ancestor of ancestors) { @@ -187,25 +197,30 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): prompts: string[], runId: string, parentRunId?: string, - _extraParams?: Record, + extraParams?: Record, tags?: string[], metadata?: Record, _runName?: string, ) { - const invocationParams = getInvocationParams(tags); - const attributes = extractLLMRequestAttributes( - llm as LangChainSerialized, + const serialized = llm as LangChainSerialized; + const { recordInputs, recordOutputs } = resolveRecording({ + operation: 'chat', + serialized, prompts, - recordInputs, - invocationParams, + runId, + parentRunId, + extraParams, + tags, metadata, - ); + }); + const invocationParams = getInvocationParams(tags); + const attributes = extractLLMRequestAttributes(serialized, prompts, recordInputs, invocationParams, metadata); const modelName = attributes[GEN_AI_REQUEST_MODEL] || 'unknown'; const operationName = typeof attributes[GEN_AI_OPERATION_NAME] === 'string' ? attributes[GEN_AI_OPERATION_NAME] : 'unknown'; const client = getClient(); - startRunSpan(runId, parentRunId, () => + startRunSpan(runId, parentRunId, recordOutputs, () => startInactiveSpan({ // With span streaming, omit the `'unknown'` model sentinel so the name stays low-cardinality. name: @@ -233,10 +248,22 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): metadata?: Record, _runName?: string, ) { + const serialized = llm as LangChainSerialized; + const chatMessages = messages as LangChainMessage[][]; + const { recordInputs, recordOutputs } = resolveRecording({ + operation: 'chat', + serialized, + messages: chatMessages, + runId, + parentRunId, + extraParams, + tags, + metadata, + }); const invocationParams = getInvocationParams(tags); const attributes = extractChatModelRequestAttributes( - llm as LangChainSerialized, - messages as LangChainMessage[][], + serialized, + chatMessages, recordInputs, invocationParams, metadata, @@ -252,7 +279,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): typeof attributes[GEN_AI_OPERATION_NAME] === 'string' ? attributes[GEN_AI_OPERATION_NAME] : 'unknown'; const client = getClient(); - startRunSpan(runId, parentRunId, () => + startRunSpan(runId, parentRunId, recordOutputs, () => startInactiveSpan({ // With span streaming, omit the `'unknown'` model sentinel so the name stays low-cardinality. name: @@ -277,9 +304,10 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): _tags?: string[], _extraParams?: Record, ) { - const span = runs.get(runId)?.span; - if (span?.isRecording()) { - const attributes = extractLlmResponseAttributes(output as LangChainLLMResult, recordOutputs); + const run = runs.get(runId); + const span = run?.span; + if (run && span?.isRecording()) { + const attributes = extractLlmResponseAttributes(output as LangChainLLMResult, run.recordOutputs); if (attributes) { span.setAttributes(attributes); } @@ -304,14 +332,23 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): inputs: Record, runId: string, parentRunId?: string, - _tags?: string[], + tags?: string[], metadata?: Record, _runType?: string, runName?: string, ) { // A Jev call is a real model call, so it is recorded inside an agent too. if (isTypeSafeClassifier(chain)) { - startRunSpan(runId, parentRunId, () => + const { recordInputs, recordOutputs } = resolveRecording({ + operation: 'decide', + serialized: chain, + inputs, + runId, + parentRunId, + tags, + metadata, + }); + startRunSpan(runId, parentRunId, recordOutputs, () => startTypeSafeClassifierSpan(chain, inputs, metadata, recordInputs), ).decide = true; return; @@ -324,6 +361,15 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): return; } + const { recordInputs, recordOutputs } = resolveRecording({ + operation: 'invoke_agent', + serialized: chain, + inputs, + runId, + parentRunId, + tags, + metadata, + }); const chainName = runName || chain.name; const attributes: Record = { ...getConversationIdFromMetadata(metadata), @@ -341,7 +387,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): const client = getClient(); - startRunSpan(runId, parentRunId, () => + startRunSpan(runId, parentRunId, recordOutputs, () => startInactiveSpan({ // With span streaming, the name leads with the operation per the agent templates. The // chain name is bounded, so it stays; the `'unknown_chain'` sentinel is dropped instead. @@ -362,10 +408,10 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): handleChainEnd(outputs: unknown, runId: string) { const run = runs.get(runId); const span = run?.span; - if (span?.isRecording()) { - if (run?.decide) { - addTypeSafeClassifierResponseAttributes(span, outputs, recordOutputs); - } else if (recordOutputs) { + if (run && span?.isRecording()) { + if (run.decide) { + addTypeSafeClassifierResponseAttributes(span, outputs, run.recordOutputs); + } else if (run.recordOutputs) { span.setAttributes({ 'langchain.chain.outputs': JSON.stringify(outputs), }); @@ -391,7 +437,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): input: string, runId: string, parentRunId?: string, - _tags?: string[], + tags?: string[], metadata?: Record, runName?: string, ) { @@ -401,6 +447,15 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): return; } + const { recordInputs, recordOutputs } = resolveRecording({ + operation: 'execute_tool', + serialized: tool, + input, + runId, + parentRunId, + tags, + metadata, + }); // runName is set to tool.name by LangChain's StructuredTool.call() const toolName = runName || tool.name || 'unknown_tool'; const attributes: Record = { @@ -415,7 +470,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): attributes[GEN_AI_TOOL_CALL_ARGUMENTS] = input; } - startRunSpan(runId, parentRunId, () => + startRunSpan(runId, parentRunId, recordOutputs, () => startInactiveSpan({ name: `execute_tool ${toolName}`, attributes: { @@ -428,9 +483,10 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): // Tool End Handler handleToolEnd(output: unknown, runId: string) { - const span = runs.get(runId)?.span; - if (span?.isRecording()) { - if (recordOutputs) { + const run = runs.get(runId); + const span = run?.span; + if (run && span?.isRecording()) { + if (run.recordOutputs) { // LangChain tools may return ToolMessage objects — extract the content const outputObj = output as Record | undefined; const content = diff --git a/packages/server-utils/src/ai/langchain/types.ts b/packages/server-utils/src/ai/langchain/types.ts index 03039782c5bf..b2162f26dd4b 100644 --- a/packages/server-utils/src/ai/langchain/types.ts +++ b/packages/server-utils/src/ai/langchain/types.ts @@ -1,7 +1,45 @@ -import type { GenAiOptions } from '../core/utils'; +import type { GenAiCallOptions } from '../core/utils'; -/** Options for the LangChain integration. */ -export type LangChainOptions = GenAiOptions; +/** What LangChain passes to every start callback of a run, besides the component and its input. */ +interface LangChainRunCallInfo { + runId: string; + parentRunId?: string; + tags?: string[]; + metadata?: Record; +} + +/** + * The call that `recordInputs`/`recordOutputs` decide on: the gen_ai operation and the arguments of the LangChain + * start callback as passed, or the argument of `embedQuery`/`embedDocuments`. `serialized` describes the + * component, such as the chat model class and its constructor kwargs. + */ +export type LangChainCallInfo = + | (LangChainRunCallInfo & { + operation: 'chat'; + serialized: LangChainSerialized; + messages: LangChainMessage[][]; + extraParams?: Record; + }) + | (LangChainRunCallInfo & { + operation: 'chat'; + serialized: LangChainSerialized; + prompts: string[]; + extraParams?: Record; + }) + | (LangChainRunCallInfo & { + operation: 'invoke_agent' | 'decide'; + serialized: LangChainSerialized; + inputs: Record; + }) + | (LangChainRunCallInfo & { operation: 'execute_tool'; serialized: LangChainSerialized; input: string }) + | { operation: 'embeddings'; input: unknown }; + +/** + * Options for the LangChain integration. The handler decides in each start callback, which LangChain awaits + * inline in the async context of the call that started the run. Runs inside a LangGraph graph use the + * decision of the graph's `invoke()`/`stream()` call instead. + */ +export type LangChainOptions = GenAiCallOptions; /** * LangChain Serialized type (compatible with @langchain/core) diff --git a/packages/server-utils/src/ai/langchain/utils.ts b/packages/server-utils/src/ai/langchain/utils.ts index 405340e71c56..aca00ad5e507 100644 --- a/packages/server-utils/src/ai/langchain/utils.ts +++ b/packages/server-utils/src/ai/langchain/utils.ts @@ -26,6 +26,7 @@ import { } from '@sentry/conventions/attributes'; import { GEN_AI_REQUEST_STREAM_ATTRIBUTE, GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE } from '../core/gen-ai-attributes'; import { extractSystemInstructions, getTokenUsageAttributes } from '../core/utils'; +import type { GenAiOptions } from '../core/utils'; import { LANGCHAIN_ORIGIN, ROLE_MAP } from './constants'; import type { LangChainLLMResult, LangChainMessage, LangChainSerialized } from './types'; @@ -530,6 +531,15 @@ function findConversationIdEntry(source?: Record): [string, str return undefined; } +/** + * The recording decision of the LangGraph graph run that `metadata` belongs to. The graph's `invoke()`/`stream()` + * call decides once, and child runs inherit the decision through their metadata. + */ +export function getLangGraphRunRecording(metadata?: Record): Required | undefined { + const recording = metadata?.__sentry_langgraph__; + return isObjectLike(recording) ? (recording as Required) : undefined; +} + /** * Derive `gen_ai.conversation.id` from run metadata, which child runs inherit from their parent. * An id from `Sentry.setConversationId()` still wins, since `conversationIdIntegration` applies it on `spanStart`. diff --git a/packages/server-utils/src/ai/langgraph/agent-span.ts b/packages/server-utils/src/ai/langgraph/agent-span.ts index acae26fda9cf..9a72690d31e8 100644 --- a/packages/server-utils/src/ai/langgraph/agent-span.ts +++ b/packages/server-utils/src/ai/langgraph/agent-span.ts @@ -12,7 +12,8 @@ import { SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; import { GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op'; -import { extractSystemInstructions } from '../core/utils'; +import { extractSystemInstructions, resolveGenAiCallRecording } from '../core/utils'; +import type { GenAiOptions } from '../core/utils'; import type { BaseChatModel, LangChainMessage } from '../langchain/types'; import { _INTERNAL_mergeLangChainCallbackHandler, @@ -64,13 +65,22 @@ export function getAgentSpanOptions( }; } +/** Decides for one `invoke()`/`stream()` call, from its arguments as passed. */ +export function resolveGraphCallRecording(options: LangGraphOptions, args: unknown[]): Required { + return resolveGenAiCallRecording(options, { + operation: 'invoke_agent', + input: args[0], + config: args[1] as Record | undefined, + }); +} + /** Sets the request attributes on an agent span and injects the callback handler into the call's config. */ export function prepareAgentSpan( span: Span, args: unknown[], graphInstance: CompiledGraph, compileOptions: Record, - options: LangGraphOptions, + recording: Required, llm: BaseChatModel | null | undefined, sentryCallbackHandler: unknown, ): LangChainMessage[] | null { @@ -101,14 +111,15 @@ export function prepareAgentSpan( config.metadata = { ...getConversationIdMetadataFromConfig(config), ...existingMetadata, - __sentry_langgraph__: true, + // Child runs inherit the metadata, so their spans follow this call's decision. + __sentry_langgraph__: recording, ...(typeof graphName === 'string' ? { lc_agent_name: graphName } : {}), }; config.callbacks = _INTERNAL_mergeLangChainCallbackHandler(config.callbacks, sentryCallbackHandler); } - const recordInputs = options.recordInputs; + const { recordInputs } = recording; const tools = recordInputs ? extractToolsFromCompiledGraph(graphInstance) : null; if (tools) { diff --git a/packages/server-utils/src/ai/langgraph/index.ts b/packages/server-utils/src/ai/langgraph/index.ts index 82af1fa78ab6..0645b14e5a4d 100644 --- a/packages/server-utils/src/ai/langgraph/index.ts +++ b/packages/server-utils/src/ai/langgraph/index.ts @@ -1,11 +1,16 @@ import { SPAN_STATUS_ERROR, startSpan } from '@sentry/core'; -import { resolveAIRecordingOptions } from '../core/utils'; import { createLangChainCallbackHandler } from '../langchain'; import type { BaseChatModel } from '../langchain/types'; -import { callInvoke, getAgentSpanOptions, prepareAgentSpan } from './agent-span'; +import { callInvoke, getAgentSpanOptions, prepareAgentSpan, resolveGraphCallRecording } from './agent-span'; import { instrumentCompiledGraphStream } from './stream'; import type { CompiledGraph, LangGraphOptions } from './types'; -import { extractAgentNameFromParams, extractLLMFromParams, setResponseAttributes, wrapToolsWithSpans } from './utils'; +import { + extractAgentNameFromParams, + extractLLMFromParams, + getBooleanRecordingOptions, + setResponseAttributes, + wrapToolsWithSpans, +} from './utils'; // Set while a manual wrapper runs the original `compile`/`createReactAgent`. The integration's channel hooks // fire synchronously inside that call, and skip the graph so the options passed explicitly win. @@ -35,16 +40,13 @@ function runManually(callback: () => T): T { */ export function instrumentStateGraphCompile( originalCompile: (...args: unknown[]) => CompiledGraph, - rawOptions: LangGraphOptions, + options: LangGraphOptions, ): (...args: unknown[]) => CompiledGraph { if (Object.prototype.hasOwnProperty.call(originalCompile, SENTRY_PATCHED)) { return originalCompile; } - // This is exported, so callers can hand us an options object with no recording flags set. Resolving - // here (rather than only in `instrumentStateGraph`) keeps that path on the `dataCollection` defaults. - const options = resolveAIRecordingOptions(rawOptions); - const sentryHandler = createLangChainCallbackHandler(options); + const sentryHandler = createLangChainCallbackHandler(getBooleanRecordingOptions(options)); const wrapped = new Proxy(originalCompile, { apply(target, thisArg, args: unknown[]): CompiledGraph { @@ -119,6 +121,7 @@ export function instrumentCompiledGraphInvoke( ): (...args: unknown[]) => Promise { return new Proxy(originalInvoke, { apply(target, thisArg, args: unknown[]): Promise { + const recording = resolveGraphCallRecording(options, args); return startSpan(getAgentSpanOptions(args), async span => { try { const inputMessages = prepareAgentSpan( @@ -126,13 +129,13 @@ export function instrumentCompiledGraphInvoke( args, graphInstance, compileOptions, - options, + recording, llm, sentryCallbackHandler, ); const result = await callInvoke(graphInstance, () => Reflect.apply(target, thisArg, args)); - if (options.recordOutputs) { + if (recording.recordOutputs) { setResponseAttributes(span, inputMessages, result); } @@ -153,14 +156,13 @@ export function instrumentCompiledGraphInvoke( */ export function instrumentCreateReactAgent( originalCreateReactAgent: (...args: unknown[]) => CompiledGraph, - options?: LangGraphOptions, + options: LangGraphOptions = {}, ): (...args: unknown[]) => CompiledGraph { if (Object.prototype.hasOwnProperty.call(originalCreateReactAgent, SENTRY_PATCHED)) { return originalCreateReactAgent; } - const resolvedOptions = resolveAIRecordingOptions(options); - const sentryHandler = createLangChainCallbackHandler(resolvedOptions); + const sentryHandler = createLangChainCallbackHandler(getBooleanRecordingOptions(options)); const wrapped = new Proxy(originalCreateReactAgent, { apply(target, thisArg, args: unknown[]): CompiledGraph { @@ -170,13 +172,13 @@ export function instrumentCreateReactAgent( // Wrap tools with execute_tool spans (direct access gives us name, type, description) const params = args[0] as Record | undefined; if (params && Array.isArray(params.tools) && params.tools.length > 0) { - wrapToolsWithSpans(params.tools, resolvedOptions, agentName ?? undefined); + wrapToolsWithSpans(params.tools, options, agentName ?? undefined); } const compiledGraph = runManually(() => Reflect.apply(target, thisArg, args)); const compileOptions: Record = agentName ? { name: agentName } : {}; - instrumentCompiledGraph(compiledGraph, compileOptions, resolvedOptions, llm, sentryHandler); + instrumentCompiledGraph(compiledGraph, compileOptions, options, llm, sentryHandler); return compiledGraph; }, @@ -212,9 +214,9 @@ export function instrumentCreateReactAgent( // eslint-disable-next-line @typescript-eslint/no-explicit-any export function instrumentStateGraph any }>( stateGraph: T, - options?: LangGraphOptions, + options: LangGraphOptions = {}, ): T { - stateGraph.compile = instrumentStateGraphCompile(stateGraph.compile, resolveAIRecordingOptions(options)); + stateGraph.compile = instrumentStateGraphCompile(stateGraph.compile, options); return stateGraph; } diff --git a/packages/server-utils/src/ai/langgraph/stream.ts b/packages/server-utils/src/ai/langgraph/stream.ts index 3370eb9fe109..60ddb15c3a76 100644 --- a/packages/server-utils/src/ai/langgraph/stream.ts +++ b/packages/server-utils/src/ai/langgraph/stream.ts @@ -2,7 +2,7 @@ import { SPAN_STATUS_ERROR, startSpanManual } from '@sentry/core'; import { GEN_AI_RESPONSE_STREAMING } from '@sentry/conventions/attributes'; import type { BaseChatModel, LangChainMessage } from '../langchain/types'; import { _INTERNAL_addLangChainCallbackHandler } from '../langchain/utils'; -import { getAgentSpanOptions, isInvokeInternalStream, prepareAgentSpan } from './agent-span'; +import { getAgentSpanOptions, isInvokeInternalStream, prepareAgentSpan, resolveGraphCallRecording } from './agent-span'; import type { CompiledGraph, LangGraphOptions } from './types'; import { setResponseAttributes } from './utils'; @@ -26,6 +26,7 @@ export function instrumentCompiledGraphStream( return Reflect.apply(target, thisArg, args); } + const recording = resolveGraphCallRecording(options, args); return startSpanManual(getAgentSpanOptions(args, { [GEN_AI_RESPONSE_STREAMING]: true }), async span => { let inputMessages: LangChainMessage[] | null = null; let stream: { signal?: AbortSignal } | undefined; @@ -42,7 +43,7 @@ export function instrumentCompiledGraphStream( if (!stream?.signal?.aborted && !(args[1] as { signal?: AbortSignal } | undefined)?.signal?.aborted) { span.setStatus({ code: SPAN_STATUS_ERROR, message: 'internal_error' }); } - } else if (outcome && options.recordOutputs) { + } else if (outcome && recording.recordOutputs) { setResponseAttributes(span, inputMessages, outcome.outputs); } } catch { @@ -81,7 +82,7 @@ export function instrumentCompiledGraphStream( args, graphInstance, compileOptions, - options, + recording, llm, sentryCallbackHandler, ); diff --git a/packages/server-utils/src/ai/langgraph/types.ts b/packages/server-utils/src/ai/langgraph/types.ts index f8ac291c5ee4..8b5fa5de446f 100644 --- a/packages/server-utils/src/ai/langgraph/types.ts +++ b/packages/server-utils/src/ai/langgraph/types.ts @@ -1,7 +1,19 @@ -import type { GenAiOptions } from '../core/utils'; +import type { GenAiCallOptions } from '../core/utils'; -/** Options for the LangGraph integration. */ -export type LangGraphOptions = GenAiOptions; +/** The call that `recordInputs`/`recordOutputs` decide on: an `invoke()` or `stream()` call of a compiled graph. */ +export interface LangGraphCallInfo { + operation: 'invoke_agent'; + /** The graph input, the first argument of `invoke()`/`stream()` as passed. */ + input: unknown; + /** The run config, the second argument of `invoke()`/`stream()` as passed. */ + config: Record | undefined; +} + +/** + * Options for the LangGraph integration. Each `invoke()`/`stream()` call decides once, and the decision covers the + * whole graph run, including its chat model and tool spans. + */ +export type LangGraphOptions = GenAiCallOptions; /** * LangGraph Tool definition from lc_kwargs diff --git a/packages/server-utils/src/ai/langgraph/utils.ts b/packages/server-utils/src/ai/langgraph/utils.ts index adb3dfb1850a..59ac9818eebe 100644 --- a/packages/server-utils/src/ai/langgraph/utils.ts +++ b/packages/server-utils/src/ai/langgraph/utils.ts @@ -22,11 +22,13 @@ import { } from '@sentry/conventions/attributes'; import { GEN_AI_EXECUTE_TOOL } from '@sentry/conventions/op'; import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../core/gen-ai-attributes'; -import { setOutputMessagesAttribute } from '../core/utils'; +import { resolveAIRecordingOptions, setOutputMessagesAttribute } from '../core/utils'; +import type { GenAiOptions } from '../core/utils'; import type { BaseChatModel, LangChainMessage } from '../langchain/types'; import { extractMessageTokenUsageAttributes, getConversationIdFromMetadata, + getLangGraphRunRecording, normalizeLangChainMessages, } from '../langchain/utils'; import { LANGGRAPH_ORIGIN } from './constants'; @@ -58,6 +60,15 @@ export function extractAgentNameFromParams(args: unknown[]): string | null { return null; } +/** The options that hold outside a graph run, where no `invoke()`/`stream()` call decided: a function needs the graph input. */ +export function getBooleanRecordingOptions(options: LangGraphOptions): GenAiOptions { + const { recordInputs, recordOutputs } = options; + return { + recordInputs: typeof recordInputs === 'boolean' ? recordInputs : undefined, + recordOutputs: typeof recordOutputs === 'boolean' ? recordOutputs : undefined, + }; +} + /** * Wraps an array of LangChain tools so each invocation creates a gen_ai.execute_tool span. * @@ -92,11 +103,14 @@ export function wrapToolsWithSpans(tools: unknown[], options: LangGraphOptions, // Read agent name from LangChain's propagated config metadata at call time, // so shared tools get the correct agent name for each invocation const callConfig = args[1] as Record | undefined; - const callAgentName = (callConfig?.metadata as Record)?.lc_agent_name ?? agentName; + const metadata = callConfig?.metadata as Record | undefined; + const { recordInputs, recordOutputs } = + getLangGraphRunRecording(metadata) ?? resolveAIRecordingOptions(getBooleanRecordingOptions(options)); + const callAgentName = metadata?.lc_agent_name ?? agentName; if (typeof callAgentName === 'string') { spanAttributes[GEN_AI_AGENT_NAME] = callAgentName; } - Object.assign(spanAttributes, getConversationIdFromMetadata(callConfig?.metadata as Record)); + Object.assign(spanAttributes, getConversationIdFromMetadata(metadata)); if (toolDescription) { spanAttributes[GEN_AI_TOOL_DESCRIPTION] = toolDescription; @@ -109,7 +123,7 @@ export function wrapToolsWithSpans(tools: unknown[], options: LangGraphOptions, spanAttributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE] = input.id; } - if (options.recordInputs) { + if (recordInputs) { const toolArgs = 'args' in input && typeof input.args === 'object' ? input.args : input; try { spanAttributes[GEN_AI_TOOL_CALL_ARGUMENTS] = JSON.stringify(toolArgs); @@ -128,7 +142,7 @@ export function wrapToolsWithSpans(tools: unknown[], options: LangGraphOptions, try { const result = await Reflect.apply(target, thisArg, args); - if (options.recordOutputs) { + if (recordOutputs) { try { // ToolMessage objects wrap the result in .content const resultObj = result as Record | undefined; diff --git a/packages/server-utils/src/integrations/langgraph.ts b/packages/server-utils/src/integrations/langgraph.ts index ad94021c8677..821fc35e1957 100644 --- a/packages/server-utils/src/integrations/langgraph.ts +++ b/packages/server-utils/src/integrations/langgraph.ts @@ -1,12 +1,16 @@ import * as diagnosticsChannel from '../utils/diagnosticsChannel'; import type { IntegrationFn } from '@sentry/core'; import { debug, defineIntegration, isObjectLike } from '@sentry/core'; -import { resolveAIRecordingOptions } from '../ai/core/utils'; import { createLangChainCallbackHandler } from '../ai/langchain'; import { instrumentCompiledGraph, isInsideManualLangGraphInstrumentation } from '../ai/langgraph'; import { LANGGRAPH_INTEGRATION_NAME } from '../ai/langgraph/constants'; import type { LangGraphOptions } from '../ai/langgraph/types'; -import { extractAgentNameFromParams, extractLLMFromParams, wrapToolsWithSpans } from '../ai/langgraph/utils'; +import { + extractAgentNameFromParams, + extractLLMFromParams, + getBooleanRecordingOptions, + wrapToolsWithSpans, +} from '../ai/langgraph/utils'; import { DEBUG_BUILD } from '../debug-build'; import type { OrchestrionChannelContext } from '../orchestrion/types'; import { CHANNELS } from '../orchestrion/channels'; @@ -35,8 +39,7 @@ const _langGraphIntegration = ((options: LangGraphOptions = {}) => { }) satisfies IntegrationFn; function instrumentLanggraph(options: LangGraphOptions): void { - const resolvedOptions = resolveAIRecordingOptions(options); - const sentryHandler = createLangChainCallbackHandler(resolvedOptions); + const sentryHandler = createLangChainCallbackHandler(getBooleanRecordingOptions(options)); // StateGraph.compile returns synchronously; wrap the returned graph's `invoke` and `stream` at `end`. diagnosticsChannel @@ -46,7 +49,7 @@ function instrumentLanggraph(options: LangGraphOptions): void { return; } const { arguments: args, result } = message as OrchestrionChannelContext; - instrumentCompiledGraph(result, getFirstArgObject(args) ?? {}, resolvedOptions, null, sentryHandler); + instrumentCompiledGraph(result, getFirstArgObject(args) ?? {}, options, null, sentryHandler); }); // createReactAgent only wraps tools and the returned graph's `invoke` and `stream`. Tools are wrapped at @@ -64,7 +67,7 @@ function instrumentLanggraph(options: LangGraphOptions): void { const { arguments: args } = message as OrchestrionChannelContext; const params = getFirstArgObject(args); if (params && Array.isArray(params.tools) && params.tools.length > 0) { - wrapToolsWithSpans(params.tools, resolvedOptions, extractAgentNameFromParams(args) ?? undefined); + wrapToolsWithSpans(params.tools, options, extractAgentNameFromParams(args) ?? undefined); } } catch (error) { DEBUG_BUILD && debug.error('[instrumentation:langgraph] failed to wrap createReactAgent tools', error); @@ -78,7 +81,7 @@ function instrumentLanggraph(options: LangGraphOptions): void { const { arguments: args, result } = message as OrchestrionChannelContext; const agentName = extractAgentNameFromParams(args) ?? undefined; const compileOptions = agentName ? { name: agentName } : {}; - instrumentCompiledGraph(result, compileOptions, resolvedOptions, extractLLMFromParams(args), sentryHandler); + instrumentCompiledGraph(result, compileOptions, options, extractLLMFromParams(args), sentryHandler); }); // Make sure a thrown `createReactAgent` doesn't leave the suppression flag stuck on. reactAgentChannel.error.subscribe(() => { diff --git a/packages/server-utils/test/ai/lib/tracing/langchain-recording.test.ts b/packages/server-utils/test/ai/lib/tracing/langchain-recording.test.ts new file mode 100644 index 000000000000..23ad5e8f7a73 --- /dev/null +++ b/packages/server-utils/test/ai/lib/tracing/langchain-recording.test.ts @@ -0,0 +1,188 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { + getCurrentScope, + getMainCarrier, + setAsyncContextStrategy, + setCurrentClient, + spanToJSON, + withScope, +} from '@sentry/core'; +import type { Span } from '@sentry/core'; +import { + GEN_AI_EMBEDDINGS_INPUT, + GEN_AI_INPUT_MESSAGES, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_RESPONSE_TEXT, + GEN_AI_TOOL_CALL_ARGUMENTS, + GEN_AI_TOOL_CALL_RESULT, +} from '@sentry/conventions/attributes'; +import { setAsyncLocalStorageAsyncContextStrategy } from '../../../../src/async-context'; +import { createLangChainCallbackHandler, instrumentLangChainEmbeddings } from '../../../../src/ai/langchain'; +import type { LangChainCallInfo, LangChainOptions } from '../../../../src/ai/langchain/types'; +import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; + +const CHAT_MODEL = { id: ['langchain', 'chat_models', 'ChatOpenAI'], kwargs: { model: 'gpt-4o' } }; +const MESSAGES = [[{ content: 'Capital of France?', _getType: () => 'human' }]]; +const OUTPUT = { generations: [[{ text: 'Paris.', message: { content: 'Paris.', _getType: () => 'ai' } }]] }; + +describe('LangChain per-call recording options', () => { + let endedSpans: Span[]; + + beforeAll(() => { + setAsyncLocalStorageAsyncContextStrategy(); + }); + + afterAll(() => { + setAsyncContextStrategy(undefined); + }); + + beforeEach(() => { + getMainCarrier().__SENTRY__ = undefined; + setAsyncLocalStorageAsyncContextStrategy(); + const client = new TestClient( + getDefaultTestClientOptions({ dsn: 'https://public@dsn.ingest.sentry.io/1337', tracesSampleRate: 1 }), + ); + setCurrentClient(client); + client.init(); + endedSpans = []; + client.on('spanEnd', span => endedSpans.push(span)); + }); + + function recorded(span: Span): { inputs: boolean; outputs: boolean } { + const { attributes } = spanToJSON(span); + return { + inputs: GEN_AI_INPUT_MESSAGES in attributes || GEN_AI_TOOL_CALL_ARGUMENTS in attributes, + outputs: + GEN_AI_RESPONSE_TEXT in attributes || + GEN_AI_OUTPUT_MESSAGES in attributes || + GEN_AI_TOOL_CALL_RESULT in attributes, + }; + } + + function chat(options: LangChainOptions, runId: string, metadata?: Record): void { + const handler = createLangChainCallbackHandler(options); + handler.handleChatModelStart?.(CHAT_MODEL, MESSAGES, runId, undefined, { tools: [] }, undefined, metadata); + handler.handleLLMEnd?.(OUTPUT, runId); + } + + it('passes the chat callback arguments to the functions', () => { + const calls: LangChainCallInfo[] = []; + const record = (call: LangChainCallInfo): boolean => { + calls.push(call); + return true; + }; + + chat({ recordInputs: record, recordOutputs: record }, 'run-1', { tenant: 'acme' }); + + const call = { + operation: 'chat', + serialized: CHAT_MODEL, + messages: MESSAGES, + runId: 'run-1', + parentRunId: undefined, + extraParams: { tools: [] }, + tags: undefined, + metadata: { tenant: 'acme' }, + }; + expect(calls).toEqual([call, call]); + }); + + it('decides for each run of the same handler', () => { + const record = (call: LangChainCallInfo): boolean => 'metadata' in call && call.metadata?.privacy === 'public'; + const handler = createLangChainCallbackHandler({ recordInputs: record, recordOutputs: record }); + + handler.handleChatModelStart?.(CHAT_MODEL, MESSAGES, 'public', undefined, undefined, undefined, { + privacy: 'public', + }); + handler.handleChatModelStart?.(CHAT_MODEL, MESSAGES, 'private', undefined, undefined, undefined, { + privacy: 'private', + }); + handler.handleLLMEnd?.(OUTPUT, 'public'); + handler.handleLLMEnd?.(OUTPUT, 'private'); + + expect(endedSpans.map(recorded)).toEqual([ + { inputs: true, outputs: true }, + { inputs: false, outputs: false }, + ]); + }); + + it("runs the functions in the async context of the start callback's caller", async () => { + const record = (): boolean => getCurrentScope().getScopeData().tags.privacy === 'public'; + const options = { recordInputs: record, recordOutputs: record }; + + await withScope(async scope => { + scope.setTag('privacy', 'public'); + chat(options, 'run-1'); + }); + await withScope(async scope => { + scope.setTag('privacy', 'private'); + chat(options, 'run-2'); + }); + + expect(endedSpans.map(recorded)).toEqual([ + { inputs: true, outputs: true }, + { inputs: false, outputs: false }, + ]); + }); + + it('decides inputs and outputs separately', () => { + chat({ recordInputs: () => false, recordOutputs: () => true }, 'run-1'); + + expect(endedSpans.map(recorded)).toEqual([{ inputs: false, outputs: true }]); + }); + + it('records outputs in the end callback as decided at start', () => { + let recordOutputs = true; + let outputCalls = 0; + const handler = createLangChainCallbackHandler({ + recordOutputs: () => { + outputCalls++; + return recordOutputs; + }, + }); + + handler.handleToolStart?.({ name: 'search' }, '{"q":"paris"}', 'tool-run'); + recordOutputs = false; + handler.handleToolEnd?.('Paris', 'tool-run'); + + expect(outputCalls).toBe(1); + expect(endedSpans.map(recorded)).toEqual([{ inputs: true, outputs: true }]); + }); + + it('follows the decision of the LangGraph run the callback belongs to', () => { + let called = false; + const record = (): boolean => { + called = true; + return true; + }; + + chat({ recordInputs: record, recordOutputs: record }, 'run-1', { + __sentry_langgraph__: { recordInputs: false, recordOutputs: true }, + }); + + expect(called).toBe(false); + expect(endedSpans.map(recorded)).toEqual([{ inputs: false, outputs: true }]); + }); + + it('passes the embed call argument to the functions of instrumentLangChainEmbeddings', async () => { + const calls: LangChainCallInfo[] = []; + const embeddings = instrumentLangChainEmbeddings( + { model: 'text-embedding-3-small', embedQuery: async () => [0.1], embedDocuments: async () => [[0.1]] }, + { + recordInputs: call => { + calls.push(call); + return call.operation === 'embeddings' && call.input === 'public'; + }, + }, + ); + + await embeddings.embedQuery('public'); + await embeddings.embedDocuments(['private']); + + expect(calls).toEqual([ + { operation: 'embeddings', input: 'public' }, + { operation: 'embeddings', input: ['private'] }, + ]); + expect(endedSpans.map(span => GEN_AI_EMBEDDINGS_INPUT in spanToJSON(span).attributes)).toEqual([true, false]); + }); +}); diff --git a/packages/server-utils/test/ai/lib/tracing/langgraph-invoke-agent.test.ts b/packages/server-utils/test/ai/lib/tracing/langgraph-invoke-agent.test.ts index 3f894efae3ea..8a516b80bf69 100644 --- a/packages/server-utils/test/ai/lib/tracing/langgraph-invoke-agent.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/langgraph-invoke-agent.test.ts @@ -127,7 +127,11 @@ describe('LangGraph conversation id', () => { await invoke({}, { configurable: { sessionId: 'session-1' }, metadata: { user: 'u' } }); - expect(receivedConfig?.metadata).toEqual({ sessionId: 'session-1', user: 'u', __sentry_langgraph__: true }); + expect(receivedConfig?.metadata).toEqual({ + sessionId: 'session-1', + user: 'u', + __sentry_langgraph__: { recordInputs: true, recordOutputs: true }, + }); }); it('sets the conversation id on wrapped tool spans from the call metadata', async () => { diff --git a/packages/server-utils/test/ai/lib/tracing/langgraph-recording.test.ts b/packages/server-utils/test/ai/lib/tracing/langgraph-recording.test.ts new file mode 100644 index 000000000000..d076a8ae943d --- /dev/null +++ b/packages/server-utils/test/ai/lib/tracing/langgraph-recording.test.ts @@ -0,0 +1,199 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { + getCurrentScope, + getMainCarrier, + setAsyncContextStrategy, + setCurrentClient, + spanToJSON, + withScope, +} from '@sentry/core'; +import type { Span } from '@sentry/core'; +import { + GEN_AI_INPUT_MESSAGES, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_RESPONSE_TEXT, + GEN_AI_TOOL_CALL_ARGUMENTS, + GEN_AI_TOOL_CALL_RESULT, +} from '@sentry/conventions/attributes'; +import { setAsyncLocalStorageAsyncContextStrategy } from '../../../../src/async-context'; +import { instrumentCreateReactAgent, instrumentStateGraph } from '../../../../src/ai/langgraph'; +import type { LangChainCallbackHandler } from '../../../../src/ai/langchain/types'; +import type { CompiledGraph, LangGraphCallInfo, LangGraphOptions } from '../../../../src/ai/langgraph/types'; +import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; + +type Config = { callbacks: LangChainCallbackHandler[]; metadata: Record }; + +const QUESTION = { role: 'user', content: 'Capital of France?' }; +const ANSWER = { role: 'assistant', content: 'Paris.' }; +const CHAT_MODEL = { id: ['langchain', 'chat_models', 'ChatOpenAI'] }; +const CHAT_MESSAGES = [[{ content: 'Capital of France?', _getType: () => 'human' }]]; +const CHAT_OUTPUT = { generations: [[{ text: 'Paris.', message: { content: 'Paris.', _getType: () => 'ai' } }]] }; + +// Runs one chat model call through the callbacks LangGraph hands its child runs, like a graph node would. +function runChatModel(config: Config, runId: string): void { + for (const handler of config.callbacks) { + handler.handleChatModelStart?.(CHAT_MODEL, CHAT_MESSAGES, runId, undefined, undefined, undefined, config.metadata); + handler.handleLLMEnd?.(CHAT_OUTPUT, runId); + } +} + +describe('LangGraph per-call recording options', () => { + let endedSpans: Span[]; + + beforeAll(() => { + setAsyncLocalStorageAsyncContextStrategy(); + }); + + afterAll(() => { + setAsyncContextStrategy(undefined); + }); + + beforeEach(() => { + getMainCarrier().__SENTRY__ = undefined; + setAsyncLocalStorageAsyncContextStrategy(); + const client = new TestClient( + getDefaultTestClientOptions({ dsn: 'https://public@dsn.ingest.sentry.io/1337', tracesSampleRate: 1 }), + ); + setCurrentClient(client); + client.init(); + endedSpans = []; + client.on('spanEnd', span => endedSpans.push(span)); + }); + + function recorded(span: Span): { name: string; inputs: boolean; outputs: boolean } { + const { name, attributes } = spanToJSON(span); + return { + name, + inputs: GEN_AI_INPUT_MESSAGES in attributes || GEN_AI_TOOL_CALL_ARGUMENTS in attributes, + outputs: + GEN_AI_RESPONSE_TEXT in attributes || + GEN_AI_OUTPUT_MESSAGES in attributes || + GEN_AI_TOOL_CALL_RESULT in attributes, + }; + } + + function compileGraph(options: LangGraphOptions): CompiledGraph { + const stateGraph = instrumentStateGraph( + { + compile: (): CompiledGraph => ({ + invoke: async (input: unknown, config: unknown) => { + runChatModel(config as Config, 'chat-run'); + return { messages: [...(input as { messages: unknown[] }).messages, ANSWER] }; + }, + }), + }, + options, + ); + return stateGraph.compile({ name: 'agent' }); + } + + it('passes the invoke arguments to the functions', async () => { + const calls: LangGraphCallInfo[] = []; + const record = (call: LangGraphCallInfo): boolean => { + calls.push(call); + return true; + }; + const graph = compileGraph({ recordInputs: record, recordOutputs: record }); + + const input = { messages: [QUESTION] }; + const config = { configurable: { thread_id: 'thread-1' } }; + await graph.invoke!(input, config); + + const call = { operation: 'invoke_agent', input, config }; + expect(calls).toEqual([call, call]); + expect(calls[0]!.config).toBe(config); + }); + + it('decides for each invoke call of the same graph, for the whole run', async () => { + const record = (call: LangGraphCallInfo): boolean => call.config?.configurable === 'public'; + const graph = compileGraph({ recordInputs: record, recordOutputs: record }); + + await graph.invoke!({ messages: [QUESTION] }, { configurable: 'public' }); + await graph.invoke!({ messages: [QUESTION] }, { configurable: 'private' }); + + expect(endedSpans.map(recorded)).toEqual([ + { name: 'chat unknown', inputs: true, outputs: true }, + { name: 'invoke_agent agent', inputs: true, outputs: true }, + { name: 'chat unknown', inputs: false, outputs: false }, + { name: 'invoke_agent agent', inputs: false, outputs: false }, + ]); + }); + + it("runs the functions in the caller's async context", async () => { + const record = (): boolean => getCurrentScope().getScopeData().tags.privacy === 'public'; + const graph = compileGraph({ recordInputs: record, recordOutputs: record }); + + await withScope(async scope => { + scope.setTag('privacy', 'public'); + await graph.invoke!({ messages: [QUESTION] }); + }); + await withScope(async scope => { + scope.setTag('privacy', 'private'); + await graph.invoke!({ messages: [QUESTION] }); + }); + + expect(endedSpans.filter(span => spanToJSON(span).name === 'invoke_agent agent').map(recorded)).toEqual([ + { name: 'invoke_agent agent', inputs: true, outputs: true }, + { name: 'invoke_agent agent', inputs: false, outputs: false }, + ]); + }); + + it('decides inputs and outputs separately', async () => { + const graph = compileGraph({ recordInputs: () => false, recordOutputs: () => true }); + + await graph.invoke!({ messages: [QUESTION] }); + + expect(endedSpans.map(recorded).at(-1)).toEqual({ name: 'invoke_agent agent', inputs: false, outputs: true }); + }); + + it('applies the decision of the invoke call to the tools of createReactAgent', async () => { + const tool = { name: 'search', invoke: async () => 'Paris' }; + const createReactAgent = instrumentCreateReactAgent( + (): CompiledGraph => ({ + invoke: async (_input: unknown, config: unknown) => + tool.invoke({ args: { q: 'paris' } }, { metadata: (config as Config).metadata }), + }), + { recordInputs: call => call.config?.configurable === 'public', recordOutputs: () => true }, + ); + const agent = createReactAgent({ name: 'agent', tools: [tool] }); + + await agent.invoke!({ messages: [QUESTION] }, { configurable: 'public' }); + await agent.invoke!({ messages: [QUESTION] }, { configurable: 'private' }); + + expect(endedSpans.filter(span => spanToJSON(span).name === 'execute_tool search').map(recorded)).toEqual([ + { name: 'execute_tool search', inputs: true, outputs: true }, + { name: 'execute_tool search', inputs: false, outputs: true }, + ]); + }); + + it('records the outputs of a stream as decided when it started', async () => { + let recordOutputs = true; + let outputCalls = 0; + let listener: LangChainCallbackHandler | undefined; + const stateGraph = instrumentStateGraph( + { + compile: (): CompiledGraph => ({ + stream: async (_input: unknown, config: unknown) => { + listener = (config as Config).callbacks.find(handler => handler.name === 'SentryLangGraphStreamListener'); + listener?.handleChainStart?.({}, {}, 'root'); + return {}; + }, + }), + }, + { + recordOutputs: () => { + outputCalls++; + return recordOutputs; + }, + }, + ); + const graph = stateGraph.compile({ name: 'agent' }); + + await graph.stream!({ messages: [QUESTION] }); + recordOutputs = false; + listener?.handleChainEnd?.({ messages: [QUESTION, ANSWER] }, 'root'); + + expect(outputCalls).toBe(1); + expect(endedSpans.map(recorded)).toEqual([{ name: 'invoke_agent agent', inputs: true, outputs: true }]); + }); +}); From 0c48b7c26c25936ea1345d2cde38ccd85b29d913 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2026 12:52:18 +0000 Subject: [PATCH 2/2] test(node): Cover per-call gen_ai recording for LangChain and LangGraph with node integration tests Replaces the per-call recording unit tests with node integration tests that use real @langchain/core and @langchain/langgraph against a mock Anthropic server. Co-Authored-By: Francesco Gringl-Novy <2411343+mydea@users.noreply.github.com> --- .../instrument-with-recording-functions.mjs | 15 ++ .../scenario-recording-functions.mjs | 66 ++++++ .../suites/tracing/langchain/test.ts | 35 +++ .../instrument-with-recording-functions.mjs | 17 ++ .../scenario-recording-functions.mjs | 76 +++++++ .../suites/tracing/langgraph/test.ts | 51 +++++ .../lib/tracing/langchain-recording.test.ts | 188 ----------------- .../lib/tracing/langgraph-recording.test.ts | 199 ------------------ 8 files changed, 260 insertions(+), 387 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-recording-functions.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/langchain/scenario-recording-functions.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/langgraph/instrument-with-recording-functions.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/langgraph/scenario-recording-functions.mjs delete mode 100644 packages/server-utils/test/ai/lib/tracing/langchain-recording.test.ts delete mode 100644 packages/server-utils/test/ai/lib/tracing/langgraph-recording.test.ts diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-recording-functions.mjs new file mode 100644 index 000000000000..6ba3c86ebf68 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-recording-functions.mjs @@ -0,0 +1,15 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + integrations: [ + Sentry.langChainIntegration({ + recordInputs: call => call.operation === 'chat' && call.metadata?.privacy === 'public', + recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public', + }), + ], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-recording-functions.mjs new file mode 100644 index 000000000000..1d6e9a1f04f7 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-recording-functions.mjs @@ -0,0 +1,66 @@ +import { ChatAnthropic } from '@langchain/anthropic'; +import * as Sentry from '@sentry/node'; +import express from 'express'; + +// Set to a scope whose tag the mock server flips while a call is in flight. +let scopeToFlip; + +function startMockAnthropicServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/messages', (req, res) => { + const prompt = req.body.messages[0].content; + if (prompt === 'flip') { + scopeToFlip.setTag('privacy', 'public'); + } + + res.json({ + id: `msg_${prompt}`, + type: 'message', + role: 'assistant', + content: [{ type: 'text', text: 'Paris.' }], + model: req.body.model, + stop_reason: 'end_turn', + stop_sequence: null, + usage: { input_tokens: 10, output_tokens: 15 }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockAnthropicServer(); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const model = new ChatAnthropic({ + model: 'claude-3-5-sonnet-20241022', + apiKey: 'mock-api-key', + clientOptions: { baseURL: `http://localhost:${server.address().port}` }, + }); + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'public'); + await model.invoke('public', { metadata: { privacy: 'public' } }); + }); + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'private'); + await model.invoke('private', { metadata: { privacy: 'private' } }); + + // The tag flips to public between the start and the end callback of this call. + scopeToFlip = scope; + await model.invoke('flip', { metadata: { privacy: 'public' } }); + }); + }); + + await Sentry.flush(2000); + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts index 48767041abf9..4797ba6e48ae 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts @@ -4,6 +4,7 @@ import { GEN_AI_EMBEDDINGS_INPUT, GEN_AI_INPUT_MESSAGES, GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, GEN_AI_PIPELINE_NAME, GEN_AI_PROVIDER_NAME, GEN_AI_REQUEST_MAX_TOKENS, @@ -379,4 +380,38 @@ describe('LangChain integration', () => { .completed(); }); }); + + createEsmAndCjsTests( + __dirname, + 'scenario-recording-functions.mjs', + 'instrument-with-recording-functions.mjs', + (createRunner, test) => { + test('decides content recording per call when the options are functions', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain', + ); + const recordedById = Object.fromEntries( + spans.map(span => [ + span.attributes[GEN_AI_RESPONSE_ID]?.value, + { + inputs: GEN_AI_INPUT_MESSAGES in span.attributes, + outputs: GEN_AI_RESPONSE_TEXT in span.attributes || GEN_AI_OUTPUT_MESSAGES in span.attributes, + }, + ]), + ); + expect(recordedById).toEqual({ + msg_public: { inputs: true, outputs: true }, + msg_private: { inputs: false, outputs: false }, + msg_flip: { inputs: true, outputs: false }, + }); + }, + }) + .start() + .completed(); + }); + }, + ); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/instrument-with-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/langgraph/instrument-with-recording-functions.mjs new file mode 100644 index 000000000000..76fd916d1d9a --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/instrument-with-recording-functions.mjs @@ -0,0 +1,17 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + // Spans that ignored the graph's decision would fall back to this and record. + dataCollection: { genAI: { inputs: true, outputs: true } }, + transport: loggingTransport, + integrations: [ + Sentry.langGraphIntegration({ + recordInputs: ({ config }) => config?.configurable?.privacy === 'public', + recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public', + }), + ], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/scenario-recording-functions.mjs b/dev-packages/node-integration-tests/suites/tracing/langgraph/scenario-recording-functions.mjs new file mode 100644 index 000000000000..76975fb70558 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/scenario-recording-functions.mjs @@ -0,0 +1,76 @@ +import { tool } from '@langchain/core/tools'; +import { ChatAnthropic } from '@langchain/anthropic'; +import { createReactAgent } from '@langchain/langgraph/prebuilt'; +import { HumanMessage } from '@langchain/core/messages'; +import * as Sentry from '@sentry/node'; +import express from 'express'; +import { z } from 'zod'; + +function startMockAnthropicServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/messages', (req, res) => { + const hasToolResult = req.body.messages.some( + message => Array.isArray(message.content) && message.content.some(part => part.type === 'tool_result'), + ); + const content = hasToolResult + ? [{ type: 'text', text: 'Paris.' }] + : [{ type: 'tool_use', id: 'toolu_1', name: 'lookup', input: { country: 'France' } }]; + + res.json({ + id: hasToolResult ? 'msg_answer' : 'msg_tool', + type: 'message', + role: 'assistant', + content, + model: req.body.model, + stop_reason: hasToolResult ? 'end_turn' : 'tool_use', + usage: { input_tokens: 10, output_tokens: 10 }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => resolve(server)); + }); +} + +async function run() { + const server = await startMockAnthropicServer(); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const llm = new ChatAnthropic({ + model: 'claude-3-5-sonnet-20241022', + apiKey: 'mock-api-key', + clientOptions: { baseURL: `http://localhost:${server.address().port}` }, + }); + const lookup = tool(async ({ country }) => `The capital of ${country} is Paris.`, { + name: 'lookup', + description: 'Look up the capital of a country', + schema: z.object({ country: z.string() }), + }); + const agent = createReactAgent({ llm, tools: [lookup], name: 'capital_assistant' }); + const input = () => ({ messages: [new HumanMessage('What is the capital of France?')] }); + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'public'); + await agent.invoke(input(), { configurable: { thread_id: 'public', privacy: 'public' } }); + }); + + await Sentry.withScope(async scope => { + scope.setTag('privacy', 'private'); + await agent.invoke(input(), { configurable: { thread_id: 'private', privacy: 'private' } }); + + // Changing the tag after the stream started must not change its outputs decision. + const stream = await agent.stream(input(), { configurable: { thread_id: 'stream', privacy: 'public' } }); + scope.setTag('privacy', 'public'); + for await (const chunk of stream) { + void chunk; + } + }); + }); + + await Sentry.flush(2000); + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts b/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts index 62d9e6cef300..48981411ce9f 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/test.ts @@ -12,6 +12,8 @@ import { GEN_AI_RESPONSE_TEXT, GEN_AI_RESPONSE_TOOL_CALLS, GEN_AI_SYSTEM_INSTRUCTIONS, + GEN_AI_TOOL_CALL_ARGUMENTS, + GEN_AI_TOOL_CALL_RESULT, GEN_AI_TOOL_DEFINITIONS, GEN_AI_TOOL_NAME, GEN_AI_USAGE_INPUT_TOKENS, @@ -453,4 +455,53 @@ describe('LangGraph integration', () => { }, ); }); + + createEsmAndCjsTests( + __dirname, + 'scenario-recording-functions.mjs', + 'instrument-with-recording-functions.mjs', + (createRunner, test) => { + test( + 'decides content recording per graph call, for every span of the run, when the options are functions', + { timeout: 30000 }, + async () => { + await createRunner() + .expect({ + span: container => { + const inputAttributes = [GEN_AI_INPUT_MESSAGES, GEN_AI_TOOL_CALL_ARGUMENTS]; + const outputAttributes = [GEN_AI_RESPONSE_TEXT, GEN_AI_OUTPUT_MESSAGES, GEN_AI_TOOL_CALL_RESULT]; + const spans = container.items.filter(span => + String(span.attributes[SENTRY_OP]?.value).startsWith('gen_ai.'), + ); + const recordedByThread: Record = {}; + for (const span of spans) { + const thread = String(span.attributes[GEN_AI_CONVERSATION_ID]?.value); + const inputs = inputAttributes.some(key => key in span.attributes); + const outputs = outputAttributes.some(key => key in span.attributes); + (recordedByThread[thread] ??= []).push(`${span.name} inputs=${inputs} outputs=${outputs}`); + } + + // The chat and tool spans of a run follow the decision of its invoke()/stream() call. + const run = (inputs: boolean, outputs: boolean): string[] => + [ + 'chat claude-3-5-sonnet-20241022', + 'chat claude-3-5-sonnet-20241022', + 'execute_tool lookup', + 'invoke_agent capital_assistant', + ].map(name => `${name} inputs=${inputs} outputs=${outputs}`); + expect( + Object.fromEntries(Object.entries(recordedByThread).map(([thread, spans]) => [thread, spans.sort()])), + ).toEqual({ + public: run(true, true), + private: run(false, false), + stream: run(true, false), + }); + }, + }) + .start() + .completed(); + }, + ); + }, + ); }); diff --git a/packages/server-utils/test/ai/lib/tracing/langchain-recording.test.ts b/packages/server-utils/test/ai/lib/tracing/langchain-recording.test.ts deleted file mode 100644 index 23ad5e8f7a73..000000000000 --- a/packages/server-utils/test/ai/lib/tracing/langchain-recording.test.ts +++ /dev/null @@ -1,188 +0,0 @@ -import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; -import { - getCurrentScope, - getMainCarrier, - setAsyncContextStrategy, - setCurrentClient, - spanToJSON, - withScope, -} from '@sentry/core'; -import type { Span } from '@sentry/core'; -import { - GEN_AI_EMBEDDINGS_INPUT, - GEN_AI_INPUT_MESSAGES, - GEN_AI_OUTPUT_MESSAGES, - GEN_AI_RESPONSE_TEXT, - GEN_AI_TOOL_CALL_ARGUMENTS, - GEN_AI_TOOL_CALL_RESULT, -} from '@sentry/conventions/attributes'; -import { setAsyncLocalStorageAsyncContextStrategy } from '../../../../src/async-context'; -import { createLangChainCallbackHandler, instrumentLangChainEmbeddings } from '../../../../src/ai/langchain'; -import type { LangChainCallInfo, LangChainOptions } from '../../../../src/ai/langchain/types'; -import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; - -const CHAT_MODEL = { id: ['langchain', 'chat_models', 'ChatOpenAI'], kwargs: { model: 'gpt-4o' } }; -const MESSAGES = [[{ content: 'Capital of France?', _getType: () => 'human' }]]; -const OUTPUT = { generations: [[{ text: 'Paris.', message: { content: 'Paris.', _getType: () => 'ai' } }]] }; - -describe('LangChain per-call recording options', () => { - let endedSpans: Span[]; - - beforeAll(() => { - setAsyncLocalStorageAsyncContextStrategy(); - }); - - afterAll(() => { - setAsyncContextStrategy(undefined); - }); - - beforeEach(() => { - getMainCarrier().__SENTRY__ = undefined; - setAsyncLocalStorageAsyncContextStrategy(); - const client = new TestClient( - getDefaultTestClientOptions({ dsn: 'https://public@dsn.ingest.sentry.io/1337', tracesSampleRate: 1 }), - ); - setCurrentClient(client); - client.init(); - endedSpans = []; - client.on('spanEnd', span => endedSpans.push(span)); - }); - - function recorded(span: Span): { inputs: boolean; outputs: boolean } { - const { attributes } = spanToJSON(span); - return { - inputs: GEN_AI_INPUT_MESSAGES in attributes || GEN_AI_TOOL_CALL_ARGUMENTS in attributes, - outputs: - GEN_AI_RESPONSE_TEXT in attributes || - GEN_AI_OUTPUT_MESSAGES in attributes || - GEN_AI_TOOL_CALL_RESULT in attributes, - }; - } - - function chat(options: LangChainOptions, runId: string, metadata?: Record): void { - const handler = createLangChainCallbackHandler(options); - handler.handleChatModelStart?.(CHAT_MODEL, MESSAGES, runId, undefined, { tools: [] }, undefined, metadata); - handler.handleLLMEnd?.(OUTPUT, runId); - } - - it('passes the chat callback arguments to the functions', () => { - const calls: LangChainCallInfo[] = []; - const record = (call: LangChainCallInfo): boolean => { - calls.push(call); - return true; - }; - - chat({ recordInputs: record, recordOutputs: record }, 'run-1', { tenant: 'acme' }); - - const call = { - operation: 'chat', - serialized: CHAT_MODEL, - messages: MESSAGES, - runId: 'run-1', - parentRunId: undefined, - extraParams: { tools: [] }, - tags: undefined, - metadata: { tenant: 'acme' }, - }; - expect(calls).toEqual([call, call]); - }); - - it('decides for each run of the same handler', () => { - const record = (call: LangChainCallInfo): boolean => 'metadata' in call && call.metadata?.privacy === 'public'; - const handler = createLangChainCallbackHandler({ recordInputs: record, recordOutputs: record }); - - handler.handleChatModelStart?.(CHAT_MODEL, MESSAGES, 'public', undefined, undefined, undefined, { - privacy: 'public', - }); - handler.handleChatModelStart?.(CHAT_MODEL, MESSAGES, 'private', undefined, undefined, undefined, { - privacy: 'private', - }); - handler.handleLLMEnd?.(OUTPUT, 'public'); - handler.handleLLMEnd?.(OUTPUT, 'private'); - - expect(endedSpans.map(recorded)).toEqual([ - { inputs: true, outputs: true }, - { inputs: false, outputs: false }, - ]); - }); - - it("runs the functions in the async context of the start callback's caller", async () => { - const record = (): boolean => getCurrentScope().getScopeData().tags.privacy === 'public'; - const options = { recordInputs: record, recordOutputs: record }; - - await withScope(async scope => { - scope.setTag('privacy', 'public'); - chat(options, 'run-1'); - }); - await withScope(async scope => { - scope.setTag('privacy', 'private'); - chat(options, 'run-2'); - }); - - expect(endedSpans.map(recorded)).toEqual([ - { inputs: true, outputs: true }, - { inputs: false, outputs: false }, - ]); - }); - - it('decides inputs and outputs separately', () => { - chat({ recordInputs: () => false, recordOutputs: () => true }, 'run-1'); - - expect(endedSpans.map(recorded)).toEqual([{ inputs: false, outputs: true }]); - }); - - it('records outputs in the end callback as decided at start', () => { - let recordOutputs = true; - let outputCalls = 0; - const handler = createLangChainCallbackHandler({ - recordOutputs: () => { - outputCalls++; - return recordOutputs; - }, - }); - - handler.handleToolStart?.({ name: 'search' }, '{"q":"paris"}', 'tool-run'); - recordOutputs = false; - handler.handleToolEnd?.('Paris', 'tool-run'); - - expect(outputCalls).toBe(1); - expect(endedSpans.map(recorded)).toEqual([{ inputs: true, outputs: true }]); - }); - - it('follows the decision of the LangGraph run the callback belongs to', () => { - let called = false; - const record = (): boolean => { - called = true; - return true; - }; - - chat({ recordInputs: record, recordOutputs: record }, 'run-1', { - __sentry_langgraph__: { recordInputs: false, recordOutputs: true }, - }); - - expect(called).toBe(false); - expect(endedSpans.map(recorded)).toEqual([{ inputs: false, outputs: true }]); - }); - - it('passes the embed call argument to the functions of instrumentLangChainEmbeddings', async () => { - const calls: LangChainCallInfo[] = []; - const embeddings = instrumentLangChainEmbeddings( - { model: 'text-embedding-3-small', embedQuery: async () => [0.1], embedDocuments: async () => [[0.1]] }, - { - recordInputs: call => { - calls.push(call); - return call.operation === 'embeddings' && call.input === 'public'; - }, - }, - ); - - await embeddings.embedQuery('public'); - await embeddings.embedDocuments(['private']); - - expect(calls).toEqual([ - { operation: 'embeddings', input: 'public' }, - { operation: 'embeddings', input: ['private'] }, - ]); - expect(endedSpans.map(span => GEN_AI_EMBEDDINGS_INPUT in spanToJSON(span).attributes)).toEqual([true, false]); - }); -}); diff --git a/packages/server-utils/test/ai/lib/tracing/langgraph-recording.test.ts b/packages/server-utils/test/ai/lib/tracing/langgraph-recording.test.ts deleted file mode 100644 index d076a8ae943d..000000000000 --- a/packages/server-utils/test/ai/lib/tracing/langgraph-recording.test.ts +++ /dev/null @@ -1,199 +0,0 @@ -import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; -import { - getCurrentScope, - getMainCarrier, - setAsyncContextStrategy, - setCurrentClient, - spanToJSON, - withScope, -} from '@sentry/core'; -import type { Span } from '@sentry/core'; -import { - GEN_AI_INPUT_MESSAGES, - GEN_AI_OUTPUT_MESSAGES, - GEN_AI_RESPONSE_TEXT, - GEN_AI_TOOL_CALL_ARGUMENTS, - GEN_AI_TOOL_CALL_RESULT, -} from '@sentry/conventions/attributes'; -import { setAsyncLocalStorageAsyncContextStrategy } from '../../../../src/async-context'; -import { instrumentCreateReactAgent, instrumentStateGraph } from '../../../../src/ai/langgraph'; -import type { LangChainCallbackHandler } from '../../../../src/ai/langchain/types'; -import type { CompiledGraph, LangGraphCallInfo, LangGraphOptions } from '../../../../src/ai/langgraph/types'; -import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; - -type Config = { callbacks: LangChainCallbackHandler[]; metadata: Record }; - -const QUESTION = { role: 'user', content: 'Capital of France?' }; -const ANSWER = { role: 'assistant', content: 'Paris.' }; -const CHAT_MODEL = { id: ['langchain', 'chat_models', 'ChatOpenAI'] }; -const CHAT_MESSAGES = [[{ content: 'Capital of France?', _getType: () => 'human' }]]; -const CHAT_OUTPUT = { generations: [[{ text: 'Paris.', message: { content: 'Paris.', _getType: () => 'ai' } }]] }; - -// Runs one chat model call through the callbacks LangGraph hands its child runs, like a graph node would. -function runChatModel(config: Config, runId: string): void { - for (const handler of config.callbacks) { - handler.handleChatModelStart?.(CHAT_MODEL, CHAT_MESSAGES, runId, undefined, undefined, undefined, config.metadata); - handler.handleLLMEnd?.(CHAT_OUTPUT, runId); - } -} - -describe('LangGraph per-call recording options', () => { - let endedSpans: Span[]; - - beforeAll(() => { - setAsyncLocalStorageAsyncContextStrategy(); - }); - - afterAll(() => { - setAsyncContextStrategy(undefined); - }); - - beforeEach(() => { - getMainCarrier().__SENTRY__ = undefined; - setAsyncLocalStorageAsyncContextStrategy(); - const client = new TestClient( - getDefaultTestClientOptions({ dsn: 'https://public@dsn.ingest.sentry.io/1337', tracesSampleRate: 1 }), - ); - setCurrentClient(client); - client.init(); - endedSpans = []; - client.on('spanEnd', span => endedSpans.push(span)); - }); - - function recorded(span: Span): { name: string; inputs: boolean; outputs: boolean } { - const { name, attributes } = spanToJSON(span); - return { - name, - inputs: GEN_AI_INPUT_MESSAGES in attributes || GEN_AI_TOOL_CALL_ARGUMENTS in attributes, - outputs: - GEN_AI_RESPONSE_TEXT in attributes || - GEN_AI_OUTPUT_MESSAGES in attributes || - GEN_AI_TOOL_CALL_RESULT in attributes, - }; - } - - function compileGraph(options: LangGraphOptions): CompiledGraph { - const stateGraph = instrumentStateGraph( - { - compile: (): CompiledGraph => ({ - invoke: async (input: unknown, config: unknown) => { - runChatModel(config as Config, 'chat-run'); - return { messages: [...(input as { messages: unknown[] }).messages, ANSWER] }; - }, - }), - }, - options, - ); - return stateGraph.compile({ name: 'agent' }); - } - - it('passes the invoke arguments to the functions', async () => { - const calls: LangGraphCallInfo[] = []; - const record = (call: LangGraphCallInfo): boolean => { - calls.push(call); - return true; - }; - const graph = compileGraph({ recordInputs: record, recordOutputs: record }); - - const input = { messages: [QUESTION] }; - const config = { configurable: { thread_id: 'thread-1' } }; - await graph.invoke!(input, config); - - const call = { operation: 'invoke_agent', input, config }; - expect(calls).toEqual([call, call]); - expect(calls[0]!.config).toBe(config); - }); - - it('decides for each invoke call of the same graph, for the whole run', async () => { - const record = (call: LangGraphCallInfo): boolean => call.config?.configurable === 'public'; - const graph = compileGraph({ recordInputs: record, recordOutputs: record }); - - await graph.invoke!({ messages: [QUESTION] }, { configurable: 'public' }); - await graph.invoke!({ messages: [QUESTION] }, { configurable: 'private' }); - - expect(endedSpans.map(recorded)).toEqual([ - { name: 'chat unknown', inputs: true, outputs: true }, - { name: 'invoke_agent agent', inputs: true, outputs: true }, - { name: 'chat unknown', inputs: false, outputs: false }, - { name: 'invoke_agent agent', inputs: false, outputs: false }, - ]); - }); - - it("runs the functions in the caller's async context", async () => { - const record = (): boolean => getCurrentScope().getScopeData().tags.privacy === 'public'; - const graph = compileGraph({ recordInputs: record, recordOutputs: record }); - - await withScope(async scope => { - scope.setTag('privacy', 'public'); - await graph.invoke!({ messages: [QUESTION] }); - }); - await withScope(async scope => { - scope.setTag('privacy', 'private'); - await graph.invoke!({ messages: [QUESTION] }); - }); - - expect(endedSpans.filter(span => spanToJSON(span).name === 'invoke_agent agent').map(recorded)).toEqual([ - { name: 'invoke_agent agent', inputs: true, outputs: true }, - { name: 'invoke_agent agent', inputs: false, outputs: false }, - ]); - }); - - it('decides inputs and outputs separately', async () => { - const graph = compileGraph({ recordInputs: () => false, recordOutputs: () => true }); - - await graph.invoke!({ messages: [QUESTION] }); - - expect(endedSpans.map(recorded).at(-1)).toEqual({ name: 'invoke_agent agent', inputs: false, outputs: true }); - }); - - it('applies the decision of the invoke call to the tools of createReactAgent', async () => { - const tool = { name: 'search', invoke: async () => 'Paris' }; - const createReactAgent = instrumentCreateReactAgent( - (): CompiledGraph => ({ - invoke: async (_input: unknown, config: unknown) => - tool.invoke({ args: { q: 'paris' } }, { metadata: (config as Config).metadata }), - }), - { recordInputs: call => call.config?.configurable === 'public', recordOutputs: () => true }, - ); - const agent = createReactAgent({ name: 'agent', tools: [tool] }); - - await agent.invoke!({ messages: [QUESTION] }, { configurable: 'public' }); - await agent.invoke!({ messages: [QUESTION] }, { configurable: 'private' }); - - expect(endedSpans.filter(span => spanToJSON(span).name === 'execute_tool search').map(recorded)).toEqual([ - { name: 'execute_tool search', inputs: true, outputs: true }, - { name: 'execute_tool search', inputs: false, outputs: true }, - ]); - }); - - it('records the outputs of a stream as decided when it started', async () => { - let recordOutputs = true; - let outputCalls = 0; - let listener: LangChainCallbackHandler | undefined; - const stateGraph = instrumentStateGraph( - { - compile: (): CompiledGraph => ({ - stream: async (_input: unknown, config: unknown) => { - listener = (config as Config).callbacks.find(handler => handler.name === 'SentryLangGraphStreamListener'); - listener?.handleChainStart?.({}, {}, 'root'); - return {}; - }, - }), - }, - { - recordOutputs: () => { - outputCalls++; - return recordOutputs; - }, - }, - ); - const graph = stateGraph.compile({ name: 'agent' }); - - await graph.stream!({ messages: [QUESTION] }); - recordOutputs = false; - listener?.handleChainEnd?.({ messages: [QUESTION, ANSWER] }, 'root'); - - expect(outputCalls).toBe(1); - expect(endedSpans.map(recorded)).toEqual([{ name: 'invoke_agent agent', inputs: true, outputs: true }]); - }); -});