Skip to content

Commit defe339

Browse files
test(node): Cover per-call gen_ai recording for Mastra, TypeSafe and Bedrock with node integration tests
Replaces the per-call recording unit tests for Mastra, TypeSafe and Bedrock with one node integration test each. Workers AI has no node suite, so it keeps one focused unit test. Co-Authored-By: Francesco Gringl-Novy <2411343+mydea@users.noreply.github.com>
1 parent c570517 commit defe339

13 files changed

Lines changed: 477 additions & 378 deletions

File tree

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import * as Sentry from '@sentry/aws-serverless';
2+
import { loggingTransport } from '@sentry-internal/node-integration-tests';
3+
4+
Sentry.init({
5+
dsn: 'https://public@dsn.ingest.sentry.io/1337',
6+
release: '1.0',
7+
tracesSampleRate: 1.0,
8+
integrations: [
9+
Sentry.awsIntegration({
10+
recordInputs: ({ input }) => input.requestMetadata?.privacy === 'public',
11+
recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public',
12+
}),
13+
],
14+
transport: loggingTransport,
15+
});
Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
import { GEN_AI_INPUT_MESSAGES, GEN_AI_OUTPUT_MESSAGES, GEN_AI_REQUEST_MODEL } from '@sentry/conventions/attributes';
2+
import { afterAll, describe, expect } from 'vitest';
3+
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner';
4+
import { join } from 'node:path';
5+
6+
describe('awsIntegration - Bedrock', () => {
7+
afterAll(() => {
8+
cleanupChildProcesses();
9+
});
10+
11+
createEsmAndCjsTests(
12+
join(__dirname, '..'),
13+
'scenario-recording-functions.mjs',
14+
'instrument-with-recording-functions.mjs',
15+
(createTestRunner, test) => {
16+
test('decides content recording per call when the options are functions', { timeout: 90_000 }, async () => {
17+
await createTestRunner()
18+
.ignore('event')
19+
.expect({
20+
span: container => {
21+
const spans = container.items.filter(span => span.attributes[GEN_AI_REQUEST_MODEL]);
22+
const recordedByModel = Object.fromEntries(
23+
spans.map(span => [
24+
span.attributes[GEN_AI_REQUEST_MODEL]?.value,
25+
{
26+
inputs: GEN_AI_INPUT_MESSAGES in span.attributes,
27+
outputs: GEN_AI_OUTPUT_MESSAGES in span.attributes,
28+
},
29+
]),
30+
);
31+
expect(recordedByModel).toEqual({
32+
'amazon.nova-pro-v1:0': { inputs: true, outputs: true },
33+
'amazon.nova-lite-v1:0': { inputs: false, outputs: false },
34+
'amazon.nova-micro-v1:0': { inputs: true, outputs: false },
35+
});
36+
},
37+
})
38+
.start()
39+
.completed();
40+
});
41+
},
42+
);
43+
});
Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
import * as Sentry from '@sentry/aws-serverless';
2+
import { BedrockRuntimeClient, ConverseCommand, ConverseStreamCommand } from '@aws-sdk/client-bedrock-runtime';
3+
// Force the HTTP/1 handler so `nock` can intercept the request instead of hitting real AWS.
4+
import { NodeHttpHandler } from '@smithy/node-http-handler';
5+
import { EventStreamCodec } from '@smithy/core/event-streams';
6+
import nock from 'nock';
7+
import { Buffer } from 'node:buffer';
8+
9+
nock.disableNetConnect();
10+
11+
const region = 'us-east-1';
12+
const credentials = { accessKeyId: 'aws-test-key', secretAccessKey: 'aws-test-secret' };
13+
const host = `https://bedrock-runtime.${region}.amazonaws.com`;
14+
15+
// Bedrock streams responses in the binary AWS event stream format, not JSON. The SDK decodes and
16+
// checks each message (headers, payload, CRC32), so the mocked response must use the same format.
17+
// The SDK's own codec encodes the messages. It needs UTF-8 helpers to encode the header strings.
18+
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTSelectObjectAppendix.html
19+
const codec = new EventStreamCodec(
20+
bytes => Buffer.from(bytes).toString('utf8'),
21+
text => Buffer.from(text),
22+
);
23+
24+
// The codec needs a type for each header value. All Bedrock headers are strings.
25+
function encodeMessage(headers, payload) {
26+
return codec.encode({
27+
headers: Object.fromEntries(Object.entries(headers).map(([name, value]) => [name, { type: 'string', value }])),
28+
body: Buffer.from(JSON.stringify(payload)),
29+
});
30+
}
31+
32+
// The SDK reads `:event-type` to find the stream member (for example `messageStart` or `chunk`).
33+
function event(eventType, payload) {
34+
return encodeMessage(
35+
{ ':message-type': 'event', ':event-type': eventType, ':content-type': 'application/json' },
36+
payload,
37+
);
38+
}
39+
40+
function mockStream(path, messages) {
41+
nock(host).post(path).reply(200, Buffer.concat(messages), { 'content-type': 'application/vnd.amazon.eventstream' });
42+
}
43+
44+
function createClient() {
45+
return new BedrockRuntimeClient({ region, credentials, maxAttempts: 1, requestHandler: new NodeHttpHandler() });
46+
}
47+
48+
function converse(modelId, privacy) {
49+
nock(host)
50+
.post(/\/model\/.*\/converse$/)
51+
.reply(
52+
200,
53+
JSON.stringify({
54+
output: { message: { role: 'assistant', content: [{ text: 'Hello from Bedrock' }] } },
55+
stopReason: 'end_turn',
56+
usage: { inputTokens: 12, outputTokens: 8, totalTokens: 20 },
57+
}),
58+
{ 'content-type': 'application/json' },
59+
);
60+
61+
return createClient().send(
62+
new ConverseCommand({
63+
modelId,
64+
messages: [{ role: 'user', content: [{ text: 'Hello' }] }],
65+
requestMetadata: { privacy },
66+
}),
67+
);
68+
}
69+
70+
async function run() {
71+
await Sentry.startSpan({ name: 'Test Transaction' }, async () => {
72+
await Sentry.withScope(async scope => {
73+
scope.setTag('privacy', 'public');
74+
await converse('amazon.nova-pro-v1:0', 'public');
75+
});
76+
77+
await Sentry.withScope(async scope => {
78+
scope.setTag('privacy', 'private');
79+
await converse('amazon.nova-lite-v1:0', 'private');
80+
81+
mockStream(/\/model\/.*\/converse-stream$/, [
82+
event('messageStart', { role: 'assistant' }),
83+
event('contentBlockDelta', { contentBlockIndex: 0, delta: { text: 'Hello from Bedrock' } }),
84+
event('contentBlockStop', { contentBlockIndex: 0 }),
85+
event('messageStop', { stopReason: 'end_turn' }),
86+
]);
87+
const pending = createClient().send(
88+
new ConverseStreamCommand({
89+
modelId: 'amazon.nova-micro-v1:0',
90+
messages: [{ role: 'user', content: [{ text: 'Hello' }] }],
91+
requestMetadata: { privacy: 'public' },
92+
}),
93+
);
94+
// Changing the tag after the call started must not change its outputs decision.
95+
Sentry.getCurrentScope().setTag('privacy', 'public');
96+
const response = await pending;
97+
for await (const _ of response.stream) {
98+
// drain
99+
}
100+
});
101+
});
102+
}
103+
104+
run();
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import * as Sentry from '@sentry/node';
2+
import { loggingTransport } from '@sentry-internal/node-integration-tests';
3+
4+
Sentry.init({
5+
dsn: 'https://public@dsn.ingest.sentry.io/1337',
6+
release: '1.0',
7+
tracesSampleRate: 1.0,
8+
transport: loggingTransport,
9+
integrations: [
10+
Sentry.mastraIntegration({
11+
recordInputs: ({ span }) => span.entityName === 'public_agent',
12+
// Mastra sets `output` only when the span ends, so this records outputs only if decided at start.
13+
recordOutputs: ({ span }) => span.output === undefined,
14+
}),
15+
],
16+
});
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
import {
2+
GEN_AI_AGENT_NAME,
3+
GEN_AI_INPUT_MESSAGES,
4+
GEN_AI_OUTPUT_MESSAGES,
5+
SENTRY_OP,
6+
SENTRY_ORIGIN,
7+
} from '@sentry/conventions/attributes';
8+
import { GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op';
9+
import { afterAll, describe, expect } from 'vitest';
10+
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner';
11+
import { supports } from '../../../../utils';
12+
import { join } from 'node:path';
13+
import { MASTRA_DEPENDENCIES } from '../dependencies';
14+
15+
describe.runIf(supports({ min: 22 }))('Mastra integration', () => {
16+
afterAll(() => {
17+
cleanupChildProcesses();
18+
});
19+
20+
createEsmAndCjsTests(
21+
join(__dirname, '..'),
22+
'scenario-recording-functions.mjs',
23+
'instrument-with-recording-functions.mjs',
24+
(createRunner, test) => {
25+
test('decides content recording per span when the options are functions', async () => {
26+
await createRunner()
27+
.expect({
28+
span: container => {
29+
const agentSpans = container.items.filter(
30+
span =>
31+
span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra' &&
32+
span.attributes[SENTRY_OP]?.value === GEN_AI_INVOKE_AGENT,
33+
);
34+
const recordedByAgent = Object.fromEntries(
35+
agentSpans.map(span => [
36+
span.attributes[GEN_AI_AGENT_NAME]?.value,
37+
{
38+
inputs: GEN_AI_INPUT_MESSAGES in span.attributes,
39+
outputs: GEN_AI_OUTPUT_MESSAGES in span.attributes,
40+
},
41+
]),
42+
);
43+
expect(recordedByAgent).toEqual({
44+
public_agent: { inputs: true, outputs: true },
45+
private_agent: { inputs: false, outputs: true },
46+
});
47+
},
48+
})
49+
.start()
50+
.completed();
51+
});
52+
},
53+
MASTRA_DEPENDENCIES,
54+
);
55+
});
Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
import express from 'express';
2+
import * as Sentry from '@sentry/node';
3+
import { Mastra } from '@mastra/core';
4+
import { Agent } from '@mastra/core/agent';
5+
6+
// Inlined OpenAI-compatible mock: the ESM/CJS runner copies only this file into its temp dir.
7+
function startMockProvider(responses) {
8+
const app = express();
9+
app.use(express.json());
10+
11+
let call = 0;
12+
app.post('/v1/chat/completions', (req, res) => {
13+
const response = responses[Math.min(call, responses.length - 1)];
14+
call++;
15+
res.json({
16+
id: response.id,
17+
object: 'chat.completion',
18+
created: 1,
19+
model: req.body.model,
20+
choices: [
21+
{
22+
index: 0,
23+
finish_reason: response.toolCalls ? 'tool_calls' : 'stop',
24+
message: {
25+
role: 'assistant',
26+
content: response.content ?? null,
27+
...(response.toolCalls ? { tool_calls: response.toolCalls } : {}),
28+
},
29+
},
30+
],
31+
usage: response.usage,
32+
});
33+
});
34+
35+
const server = app.listen(0);
36+
return {
37+
url: `http://localhost:${server.address().port}/v1`,
38+
close: () => server.close(),
39+
};
40+
}
41+
42+
const provider = startMockProvider([
43+
{
44+
id: 'chatcmpl-1',
45+
content: 'It is 22C in Berlin.',
46+
usage: { prompt_tokens: 12, completion_tokens: 7, total_tokens: 19 },
47+
},
48+
]);
49+
50+
async function run() {
51+
const model = { id: 'openai/gpt-4o-mini', url: provider.url, apiKey: 'test' };
52+
const agents = {
53+
public_agent: new Agent({
54+
id: 'public_agent',
55+
name: 'public_agent',
56+
instructions: 'You report the weather.',
57+
model,
58+
}),
59+
private_agent: new Agent({
60+
id: 'private_agent',
61+
name: 'private_agent',
62+
instructions: 'You report the weather.',
63+
model,
64+
}),
65+
};
66+
const mastra = new Mastra({ agents, logger: false });
67+
68+
await Sentry.startSpan({ op: 'function', name: 'mastra-test' }, async () => {
69+
await mastra.getAgent('public_agent').generate('What is the weather in Berlin?');
70+
await mastra.getAgent('private_agent').generate('What is the weather in Berlin?');
71+
});
72+
73+
await mastra.observability.shutdown();
74+
provider.close();
75+
}
76+
77+
run();
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import * as Sentry from '@sentry/node';
2+
import { loggingTransport } from '@sentry-internal/node-integration-tests';
3+
4+
Sentry.init({
5+
dsn: 'https://public@dsn.ingest.sentry.io/1337',
6+
release: '1.0',
7+
tracesSampleRate: 1.0,
8+
transport: loggingTransport,
9+
integrations: [
10+
Sentry.typesafeIntegration({
11+
recordInputs: ({ request }) => request?.model?.startsWith('public'),
12+
recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public',
13+
}),
14+
],
15+
});
Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
import {
2+
GEN_AI_INPUT_MESSAGES,
3+
GEN_AI_OUTPUT_MESSAGES,
4+
GEN_AI_REQUEST_MODEL,
5+
SENTRY_OP,
6+
} from '@sentry/conventions/attributes';
7+
import { GEN_AI_DECIDE } from '@sentry/conventions/op';
8+
import { afterAll, describe, expect } from 'vitest';
9+
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner';
10+
import { join } from 'node:path';
11+
12+
describe('TypeSafe integration', () => {
13+
afterAll(() => {
14+
cleanupChildProcesses();
15+
});
16+
17+
createEsmAndCjsTests(
18+
join(__dirname, '..'),
19+
'scenario-recording-functions.mjs',
20+
'instrument-with-recording-functions.mjs',
21+
(createRunner, test) => {
22+
test('decides content recording per call when the options are functions', async () => {
23+
await createRunner()
24+
.expect({
25+
span: container => {
26+
const spans = container.items.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE);
27+
const recordedByModel = Object.fromEntries(
28+
spans.map(span => [
29+
span.attributes[GEN_AI_REQUEST_MODEL]?.value,
30+
{
31+
inputs: GEN_AI_INPUT_MESSAGES in span.attributes,
32+
outputs: GEN_AI_OUTPUT_MESSAGES in span.attributes,
33+
},
34+
]),
35+
);
36+
expect(recordedByModel).toEqual({
37+
public: { inputs: true, outputs: true },
38+
private: { inputs: false, outputs: false },
39+
'public-late': { inputs: true, outputs: false },
40+
});
41+
},
42+
})
43+
.start()
44+
.completed();
45+
});
46+
},
47+
);
48+
});

0 commit comments

Comments
 (0)