Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import * as Sentry from '@sentry/node';
import { loggingTransport } from '@sentry-internal/node-integration-tests';

Sentry.init({
dsn: 'https://public@dsn.ingest.sentry.io/1337',
release: '1.0',
tracesSampleRate: 1.0,
transport: loggingTransport,
integrations: [
Sentry.langChainIntegration({
recordInputs: call => call.operation === 'chat' && call.metadata?.privacy === 'public',
recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public',
}),
],
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
import { ChatAnthropic } from '@langchain/anthropic';
import * as Sentry from '@sentry/node';
import express from 'express';

// Set to a scope whose tag the mock server flips while a call is in flight.
let scopeToFlip;

function startMockAnthropicServer() {
const app = express();
app.use(express.json());

app.post('/v1/messages', (req, res) => {
const prompt = req.body.messages[0].content;
if (prompt === 'flip') {
scopeToFlip.setTag('privacy', 'public');
}

res.json({
id: `msg_${prompt}`,
type: 'message',
role: 'assistant',
content: [{ type: 'text', text: 'Paris.' }],
model: req.body.model,
stop_reason: 'end_turn',
stop_sequence: null,
usage: { input_tokens: 10, output_tokens: 15 },
});
});

return new Promise(resolve => {
const server = app.listen(0, () => {
resolve(server);
});
});
}

async function run() {
const server = await startMockAnthropicServer();

await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
const model = new ChatAnthropic({
model: 'claude-3-5-sonnet-20241022',
apiKey: 'mock-api-key',
clientOptions: { baseURL: `http://localhost:${server.address().port}` },
});

await Sentry.withScope(async scope => {
scope.setTag('privacy', 'public');
await model.invoke('public', { metadata: { privacy: 'public' } });
});

await Sentry.withScope(async scope => {
scope.setTag('privacy', 'private');
await model.invoke('private', { metadata: { privacy: 'private' } });

// The tag flips to public between the start and the end callback of this call.
scopeToFlip = scope;
await model.invoke('flip', { metadata: { privacy: 'public' } });
});
});

await Sentry.flush(2000);
server.close();
}

run();
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import {
GEN_AI_EMBEDDINGS_INPUT,
GEN_AI_INPUT_MESSAGES,
GEN_AI_OPERATION_NAME,
GEN_AI_OUTPUT_MESSAGES,
GEN_AI_PIPELINE_NAME,
GEN_AI_PROVIDER_NAME,
GEN_AI_REQUEST_MAX_TOKENS,
Expand Down Expand Up @@ -379,4 +380,38 @@ describe('LangChain integration', () => {
.completed();
});
});

createEsmAndCjsTests(
__dirname,
'scenario-recording-functions.mjs',
'instrument-with-recording-functions.mjs',
(createRunner, test) => {
test('decides content recording per call when the options are functions', async () => {
await createRunner()
.expect({
span: container => {
const spans = container.items.filter(
span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain',
);
const recordedById = Object.fromEntries(
spans.map(span => [
span.attributes[GEN_AI_RESPONSE_ID]?.value,
{
inputs: GEN_AI_INPUT_MESSAGES in span.attributes,
outputs: GEN_AI_RESPONSE_TEXT in span.attributes || GEN_AI_OUTPUT_MESSAGES in span.attributes,
},
]),
);
expect(recordedById).toEqual({
msg_public: { inputs: true, outputs: true },
msg_private: { inputs: false, outputs: false },
msg_flip: { inputs: true, outputs: false },
});
},
})
.start()
.completed();
});
},
);
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
import * as Sentry from '@sentry/node';
import { loggingTransport } from '@sentry-internal/node-integration-tests';

Sentry.init({
dsn: 'https://public@dsn.ingest.sentry.io/1337',
release: '1.0',
tracesSampleRate: 1.0,
// Spans that ignored the graph's decision would fall back to this and record.
dataCollection: { genAI: { inputs: true, outputs: true } },
transport: loggingTransport,
integrations: [
Sentry.langGraphIntegration({
recordInputs: ({ config }) => config?.configurable?.privacy === 'public',
recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public',
}),
],
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
import { tool } from '@langchain/core/tools';
import { ChatAnthropic } from '@langchain/anthropic';
import { createReactAgent } from '@langchain/langgraph/prebuilt';
import { HumanMessage } from '@langchain/core/messages';
import * as Sentry from '@sentry/node';
import express from 'express';
import { z } from 'zod';

function startMockAnthropicServer() {
const app = express();
app.use(express.json());

app.post('/v1/messages', (req, res) => {
const hasToolResult = req.body.messages.some(
message => Array.isArray(message.content) && message.content.some(part => part.type === 'tool_result'),
);
const content = hasToolResult
? [{ type: 'text', text: 'Paris.' }]
: [{ type: 'tool_use', id: 'toolu_1', name: 'lookup', input: { country: 'France' } }];

res.json({
id: hasToolResult ? 'msg_answer' : 'msg_tool',
type: 'message',
role: 'assistant',
content,
model: req.body.model,
stop_reason: hasToolResult ? 'end_turn' : 'tool_use',
usage: { input_tokens: 10, output_tokens: 10 },
});
});

return new Promise(resolve => {
const server = app.listen(0, () => resolve(server));
});
}

async function run() {
const server = await startMockAnthropicServer();

await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
const llm = new ChatAnthropic({
model: 'claude-3-5-sonnet-20241022',
apiKey: 'mock-api-key',
clientOptions: { baseURL: `http://localhost:${server.address().port}` },
});
const lookup = tool(async ({ country }) => `The capital of ${country} is Paris.`, {
name: 'lookup',
description: 'Look up the capital of a country',
schema: z.object({ country: z.string() }),
});
const agent = createReactAgent({ llm, tools: [lookup], name: 'capital_assistant' });
const input = () => ({ messages: [new HumanMessage('What is the capital of France?')] });

await Sentry.withScope(async scope => {
scope.setTag('privacy', 'public');
await agent.invoke(input(), { configurable: { thread_id: 'public', privacy: 'public' } });
});

await Sentry.withScope(async scope => {
scope.setTag('privacy', 'private');
await agent.invoke(input(), { configurable: { thread_id: 'private', privacy: 'private' } });

// Changing the tag after the stream started must not change its outputs decision.
const stream = await agent.stream(input(), { configurable: { thread_id: 'stream', privacy: 'public' } });
scope.setTag('privacy', 'public');
for await (const chunk of stream) {
void chunk;
}
});
});

await Sentry.flush(2000);
server.close();
}

run();
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import {
GEN_AI_RESPONSE_TEXT,
GEN_AI_RESPONSE_TOOL_CALLS,
GEN_AI_SYSTEM_INSTRUCTIONS,
GEN_AI_TOOL_CALL_ARGUMENTS,
GEN_AI_TOOL_CALL_RESULT,
GEN_AI_TOOL_DEFINITIONS,
GEN_AI_TOOL_NAME,
GEN_AI_USAGE_INPUT_TOKENS,
Expand Down Expand Up @@ -453,4 +455,53 @@ describe('LangGraph integration', () => {
},
);
});

createEsmAndCjsTests(
__dirname,
'scenario-recording-functions.mjs',
'instrument-with-recording-functions.mjs',
(createRunner, test) => {
test(
'decides content recording per graph call, for every span of the run, when the options are functions',
{ timeout: 30000 },
async () => {
await createRunner()
.expect({
span: container => {
const inputAttributes = [GEN_AI_INPUT_MESSAGES, GEN_AI_TOOL_CALL_ARGUMENTS];
const outputAttributes = [GEN_AI_RESPONSE_TEXT, GEN_AI_OUTPUT_MESSAGES, GEN_AI_TOOL_CALL_RESULT];
const spans = container.items.filter(span =>
String(span.attributes[SENTRY_OP]?.value).startsWith('gen_ai.'),
);
const recordedByThread: Record<string, string[]> = {};
for (const span of spans) {
const thread = String(span.attributes[GEN_AI_CONVERSATION_ID]?.value);
const inputs = inputAttributes.some(key => key in span.attributes);
const outputs = outputAttributes.some(key => key in span.attributes);
(recordedByThread[thread] ??= []).push(`${span.name} inputs=${inputs} outputs=${outputs}`);
}

// The chat and tool spans of a run follow the decision of its invoke()/stream() call.
const run = (inputs: boolean, outputs: boolean): string[] =>
[
'chat claude-3-5-sonnet-20241022',
'chat claude-3-5-sonnet-20241022',
'execute_tool lookup',
'invoke_agent capital_assistant',
].map(name => `${name} inputs=${inputs} outputs=${outputs}`);
expect(
Object.fromEntries(Object.entries(recordedByThread).map(([thread, spans]) => [thread, spans.sort()])),
).toEqual({
public: run(true, true),
private: run(false, false),
stream: run(true, false),
});
},
})
.start()
.completed();
},
);
},
);
});
4 changes: 2 additions & 2 deletions packages/server-utils/src/ai/langchain/embeddings.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import {
GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE,
GEN_AI_REQUEST_ENCODING_FORMAT_ATTRIBUTE,
} from '../core/gen-ai-attributes';
import { resolveAIRecordingOptions } from '../core/utils';
import { resolveGenAiCallRecording } from '../core/utils';
import { LANGCHAIN_ORIGIN } from './constants';
import type { LangChainOptions } from './types';

Expand Down Expand Up @@ -68,7 +68,7 @@ export function _INTERNAL_getLangChainEmbeddingsSpanOptions(
input: unknown,
options: LangChainOptions = {},
): { name: string; attributes: Record<string, SpanAttributeValue> } {
const { recordInputs } = resolveAIRecordingOptions(options);
const { recordInputs } = resolveGenAiCallRecording(options, { operation: 'embeddings', input });
const attributes = extractEmbeddingAttributes(instance);
const modelName = attributes[GEN_AI_REQUEST_MODEL] || 'unknown';
const client = getClient();
Expand Down
Loading
Loading