Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import * as Sentry from '@sentry/node';
import { loggingTransport } from '@sentry-internal/node-integration-tests';

Sentry.init({
dsn: 'https://public@dsn.ingest.sentry.io/1337',
release: '1.0',
tracesSampleRate: 1.0,
transport: loggingTransport,
integrations: [
Sentry.anthropicAIIntegration({
recordInputs: call =>
call.operation === 'execute_tool'
? call.input?.city === 'Vienna'
: !!call.request?.metadata?.user_id?.startsWith('public'),
recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public',
}),
],
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
import Anthropic from '@anthropic-ai/sdk';
import * as Sentry from '@sentry/node';
import express from 'express';

const MODEL = 'claude-3-haiku-20240307';

function message(id, content, stopReason = 'end_turn') {
return {
id,
type: 'message',
role: 'assistant',
model: MODEL,
content,
stop_reason: stopReason,
usage: { input_tokens: 10, output_tokens: 5 },
};
}

function writeStream(res, id) {
res.writeHead(200, { 'Content-Type': 'text/event-stream' });
const { content, ...start } = message(id, []);
const events = [
{ type: 'message_start', message: { ...start, content } },
{ type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } },
{ type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'Paris.' } },
{ type: 'content_block_stop', index: 0 },
{ type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 5 } },
{ type: 'message_stop' },
];
for (const event of events) {
res.write(`event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`);
}
res.end();
}

function startMockServer() {
const app = express();
app.use(express.json());

app.post('/anthropic/v1/messages', (req, res) => {
const { metadata, stream, messages } = req.body;

if (metadata.user_id === 'runner') {
const answeredTools = messages.some(m => Array.isArray(m.content) && m.content[0]?.type === 'tool_result');
res.send(
answeredTools
? message('msg_tool_final', [{ type: 'text', text: 'Sunny in both.' }])
: message(
'msg_tool_use',
[
{ type: 'tool_use', id: 'toolu_vienna', name: 'weather', input: { city: 'Vienna' } },
{ type: 'tool_use', id: 'toolu_paris', name: 'weather', input: { city: 'Paris' } },
],
'tool_use',
),
);
return;
}

const id = `msg_${metadata.user_id}${stream ? '-stream' : ''}`;
if (stream) {
writeStream(res, id);
return;
}
res.send(message(id, [{ type: 'text', text: 'Paris.' }]));
});

return new Promise(resolve => {
const server = app.listen(0, () => {
resolve(server);
});
});
}

async function run() {
const server = await startMockServer();

await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
const client = new Anthropic({
apiKey: 'mock-api-key',
baseURL: `http://localhost:${server.address().port}/anthropic`,
});
const request = userId => ({
model: MODEL,
max_tokens: 100,
messages: [{ role: 'user', content: 'What is the capital of France?' }],
metadata: { user_id: userId },
});

await Sentry.withScope(async scope => {
scope.setTag('privacy', 'public');
await client.messages.create(request('public'));
});

await Sentry.withScope(async scope => {
scope.setTag('privacy', 'private');
await client.messages.create(request('private'));

// Changing the tag after a call started must not change its outputs decision.
const stream = await client.messages.create({ ...request('public'), stream: true });
Sentry.getCurrentScope().setTag('privacy', 'public');
for await (const event of stream) {
void event;
}

Sentry.getCurrentScope().setTag('privacy', 'private');
const helper = client.messages.stream(request('public-helper'));
Sentry.getCurrentScope().setTag('privacy', 'public');
await helper.finalMessage();
});

await Sentry.withScope(async scope => {
scope.setTag('privacy', 'public');
const runner = client.beta.messages.toolRunner({
...request('runner'),
tools: [
{
name: 'weather',
description: 'Get the weather by city',
input_schema: { type: 'object', properties: { city: { type: 'string' } } },
run: () => 'sunny',
},
],
});
for await (const msg of runner) {
void msg;
}
});
});

await Sentry.flush(2000);
server.close();
}

run();
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,15 @@ import {
GEN_AI_USAGE_INPUT_TOKENS,
GEN_AI_USAGE_OUTPUT_TOKENS,
GEN_AI_USAGE_TOTAL_TOKENS,
GEN_AI_TOOL_CALL_ARGUMENTS,
GEN_AI_TOOL_CALL_RESULT,
SENTRY_OP,
SENTRY_ORIGIN,
} from '@sentry/conventions/attributes';
import { GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes';
import {
GEN_AI_REQUEST_STREAM_ATTRIBUTE,
GEN_AI_TOOL_CALL_ID_ATTRIBUTE,
} from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes';
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner';

describe('Anthropic integration', () => {
Expand Down Expand Up @@ -613,4 +618,43 @@ describe('Anthropic integration', () => {
.completed();
});
});

createEsmAndCjsTests(
__dirname,
'scenario-recording-functions.mjs',
'instrument-with-recording-functions.mjs',
(createRunner, test) => {
test('decides content recording per call when the options are functions', async () => {
await createRunner()
.expect({
span: container => {
const spans = container.items.filter(
span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic',
);
const recordedById = Object.fromEntries(
spans.map(span => [
span.attributes[GEN_AI_RESPONSE_ID]?.value ?? span.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]?.value,
{
inputs: GEN_AI_INPUT_MESSAGES in span.attributes || GEN_AI_TOOL_CALL_ARGUMENTS in span.attributes,
outputs: GEN_AI_RESPONSE_TEXT in span.attributes || GEN_AI_TOOL_CALL_RESULT in span.attributes,
},
]),
);
expect(recordedById).toEqual({
msg_public: { inputs: true, outputs: true },
msg_private: { inputs: false, outputs: false },
'msg_public-stream': { inputs: true, outputs: false },
'msg_public-helper-stream': { inputs: true, outputs: false },
msg_tool_use: { inputs: false, outputs: true },
msg_tool_final: { inputs: false, outputs: true },
toolu_vienna: { inputs: true, outputs: true },
toolu_paris: { inputs: false, outputs: true },
});
},
})
.start()
.completed();
});
},
);
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
import * as Sentry from '@sentry/node';
import { loggingTransport } from '@sentry-internal/node-integration-tests';

Sentry.init({
dsn: 'https://public@dsn.ingest.sentry.io/1337',
release: '1.0',
tracesSampleRate: 1.0,
transport: loggingTransport,
integrations: [
Sentry.googleGenAIIntegration({
recordInputs: ({ operation, request }) =>
operation === 'generate_content' && !!request?.contents?.startsWith('Public'),
recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public',
}),
],
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
import { GoogleGenAI } from '@google/genai';
import * as Sentry from '@sentry/node';
import express from 'express';

const RESPONSE = {
candidates: [{ content: { parts: [{ text: 'Paris.' }], role: 'model' }, finishReason: 'STOP', index: 0 }],
usageMetadata: { promptTokenCount: 8, candidatesTokenCount: 2, totalTokenCount: 10 },
};

function startMockServer() {
const app = express();
app.use(express.json());

app.post('/v1beta/models/:model\\:generateContent', (_req, res) => {
res.send(RESPONSE);
});

app.post('/v1beta/models/:model\\:streamGenerateContent', (_req, res) => {
res.setHeader('Content-Type', 'application/json');
res.write(`data: ${JSON.stringify(RESPONSE)}\n\n`);
res.end();
});

return new Promise(resolve => {
const server = app.listen(0, () => {
resolve(server);
});
});
}

async function run() {
const server = await startMockServer();

await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
const client = new GoogleGenAI({
apiKey: 'mock-api-key',
httpOptions: { baseUrl: `http://localhost:${server.address().port}` },
});

await Sentry.withScope(async scope => {
scope.setTag('privacy', 'public');
await client.models.generateContent({ model: 'gemini-public', contents: 'Public: capital of France?' });
});

await Sentry.withScope(async scope => {
scope.setTag('privacy', 'private');
await client.models.generateContent({ model: 'gemini-private', contents: 'Private: capital of France?' });

// Changing the tag after a call started must not change its outputs decision.
const stream = await client.models.generateContentStream({
model: 'gemini-public-stream',
contents: 'Public: capital of France?',
});
Sentry.getCurrentScope().setTag('privacy', 'public');
for await (const chunk of stream) {
void chunk;
}
});
});

await Sentry.flush(2000);
server.close();
}

run();
Original file line number Diff line number Diff line change
Expand Up @@ -624,4 +624,36 @@ describe('Google GenAI integration', () => {
});
},
);

createEsmAndCjsTests(
__dirname,
'scenario-recording-functions.mjs',
'instrument-with-recording-functions.mjs',
(createRunner, test) => {
test('decides content recording per call when the options are functions', async () => {
await createRunner()
.expect({
span: container => {
const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === EXPECTED_ORIGIN);
const recordedByModel = Object.fromEntries(
spans.map(span => [
span.attributes[GEN_AI_REQUEST_MODEL]?.value,
{
inputs: GEN_AI_INPUT_MESSAGES in span.attributes,
outputs: GEN_AI_RESPONSE_TEXT in span.attributes,
},
]),
);
expect(recordedByModel).toEqual({
'gemini-public': { inputs: true, outputs: true },
'gemini-private': { inputs: false, outputs: false },
'gemini-public-stream': { inputs: true, outputs: false },
});
},
})
.start()
.completed();
});
},
);
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
import * as Sentry from '@sentry/node';
import { loggingTransport } from '@sentry-internal/node-integration-tests';

Sentry.init({
dsn: 'https://public@dsn.ingest.sentry.io/1337',
release: '1.0',
tracesSampleRate: 1.0,
transport: loggingTransport,
integrations: [
Sentry.mistralAIIntegration({
recordInputs: ({ operation, request }) =>
operation === 'chat' && !!request?.messages?.[0]?.content?.startsWith('Public'),
recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public',
}),
],
});
Loading
Loading