Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import * as Sentry from '@sentry/aws-serverless';
import { loggingTransport } from '@sentry-internal/node-integration-tests';

Sentry.init({
dsn: 'https://public@dsn.ingest.sentry.io/1337',
release: '1.0',
tracesSampleRate: 1.0,
integrations: [
Sentry.awsIntegration({
recordInputs: ({ input }) => input.requestMetadata?.privacy === 'public',
recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public',
}),
],
transport: loggingTransport,
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
import * as Sentry from '@sentry/aws-serverless';
import { BedrockRuntimeClient, ConverseCommand, ConverseStreamCommand } from '@aws-sdk/client-bedrock-runtime';
// Force the HTTP/1 handler so `nock` can intercept the request instead of hitting real AWS.
import { NodeHttpHandler } from '@smithy/node-http-handler';
import { EventStreamCodec } from '@smithy/core/event-streams';
import nock from 'nock';
import { Buffer } from 'node:buffer';

nock.disableNetConnect();

const region = 'us-east-1';
const credentials = { accessKeyId: 'aws-test-key', secretAccessKey: 'aws-test-secret' };
const host = `https://bedrock-runtime.${region}.amazonaws.com`;

// Bedrock streams responses in the binary AWS event stream format, not JSON. The SDK decodes and
// checks each message (headers, payload, CRC32), so the mocked response must use the same format.
// The SDK's own codec encodes the messages. It needs UTF-8 helpers to encode the header strings.
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTSelectObjectAppendix.html
const codec = new EventStreamCodec(
bytes => Buffer.from(bytes).toString('utf8'),
text => Buffer.from(text),
);

// The codec needs a type for each header value. All Bedrock headers are strings.
function encodeMessage(headers, payload) {
return codec.encode({
headers: Object.fromEntries(Object.entries(headers).map(([name, value]) => [name, { type: 'string', value }])),
body: Buffer.from(JSON.stringify(payload)),
});
}

// The SDK reads `:event-type` to find the stream member (for example `messageStart` or `chunk`).
function event(eventType, payload) {
return encodeMessage(
{ ':message-type': 'event', ':event-type': eventType, ':content-type': 'application/json' },
payload,
);
}

function mockStream(path, messages) {
nock(host).post(path).reply(200, Buffer.concat(messages), { 'content-type': 'application/vnd.amazon.eventstream' });
}

function createClient() {
return new BedrockRuntimeClient({ region, credentials, maxAttempts: 1, requestHandler: new NodeHttpHandler() });
}

function converse(modelId, privacy) {
nock(host)
.post(/\/model\/.*\/converse$/)
.reply(
200,
JSON.stringify({
output: { message: { role: 'assistant', content: [{ text: 'Hello from Bedrock' }] } },
stopReason: 'end_turn',
usage: { inputTokens: 12, outputTokens: 8, totalTokens: 20 },
}),
{ 'content-type': 'application/json' },
);

return createClient().send(
new ConverseCommand({
modelId,
messages: [{ role: 'user', content: [{ text: 'Hello' }] }],
requestMetadata: { privacy },
}),
);
}

async function run() {
await Sentry.startSpan({ name: 'Test Transaction' }, async () => {
await Sentry.withScope(async scope => {
scope.setTag('privacy', 'public');
await converse('amazon.nova-pro-v1:0', 'public');
});

await Sentry.withScope(async scope => {
scope.setTag('privacy', 'private');
await converse('amazon.nova-lite-v1:0', 'private');

mockStream(/\/model\/.*\/converse-stream$/, [
event('messageStart', { role: 'assistant' }),
event('contentBlockDelta', { contentBlockIndex: 0, delta: { text: 'Hello from Bedrock' } }),
event('contentBlockStop', { contentBlockIndex: 0 }),
event('messageStop', { stopReason: 'end_turn' }),
]);
const pending = createClient().send(
new ConverseStreamCommand({
modelId: 'amazon.nova-micro-v1:0',
messages: [{ role: 'user', content: [{ text: 'Hello' }] }],
requestMetadata: { privacy: 'public' },
}),
);
// Changing the tag after the call started must not change its outputs decision.
Sentry.getCurrentScope().setTag('privacy', 'public');
const response = await pending;
for await (const _ of response.stream) {
// drain
}
});
});
}

run();
Original file line number Diff line number Diff line change
Expand Up @@ -140,4 +140,37 @@ describe('awsIntegration - Bedrock', () => {
await createTestRunner().ignore('event').expect({ span: assertStreamSpans }).start().completed();
});
});

createEsmAndCjsTests(
__dirname,
'scenario-recording-functions.mjs',
'instrument-with-recording-functions.mjs',
(createTestRunner, test) => {
test('decides content recording per call when the options are functions', { timeout: 90_000 }, async () => {
await createTestRunner()
.ignore('event')
.expect({
span: container => {
const spans = container.items.filter(span => span.attributes[GEN_AI_REQUEST_MODEL]);
const recordedByModel = Object.fromEntries(
spans.map(span => [
span.attributes[GEN_AI_REQUEST_MODEL]?.value,
{
inputs: GEN_AI_INPUT_MESSAGES in span.attributes,
outputs: GEN_AI_OUTPUT_MESSAGES in span.attributes,
},
]),
);
expect(recordedByModel).toEqual({
'amazon.nova-pro-v1:0': { inputs: true, outputs: true },
'amazon.nova-lite-v1:0': { inputs: false, outputs: false },
'amazon.nova-micro-v1:0': { inputs: true, outputs: false },
});
},
})
.start()
.completed();
});
},
);
});
Original file line number Diff line number Diff line change
@@ -1,4 +1,11 @@
import { SENTRY_OP, SENTRY_ORIGIN } from '@sentry/conventions/attributes';
import {
GEN_AI_AGENT_NAME,
GEN_AI_INPUT_MESSAGES,
GEN_AI_OUTPUT_MESSAGES,
SENTRY_OP,
SENTRY_ORIGIN,
} from '@sentry/conventions/attributes';
import { GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op';
import { afterAll, expect, describe } from 'vitest';
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../../utils/runner';
import { supports } from '../../../../../utils';
Expand Down Expand Up @@ -74,4 +81,40 @@ describe.runIf(supports({ min: 22 }))('Mastra integration', () => {
},
MASTRA_DEPENDENCIES,
);

createEsmAndCjsTests(
join(__dirname, '../..'),
'scenario-recording-functions.mjs',
'instrument-with-recording-functions.mjs',
(createRunner, test) => {
test('decides content recording per span when the options are functions', async () => {
await createRunner()
.expect({
span: container => {
const agentSpans = container.items.filter(
span =>
span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.mastra' &&
span.attributes[SENTRY_OP]?.value === GEN_AI_INVOKE_AGENT,
);
const recordedByAgent = Object.fromEntries(
agentSpans.map(span => [
span.attributes[GEN_AI_AGENT_NAME]?.value,
{
inputs: GEN_AI_INPUT_MESSAGES in span.attributes,
outputs: GEN_AI_OUTPUT_MESSAGES in span.attributes,
},
]),
);
expect(recordedByAgent).toEqual({
public_agent: { inputs: true, outputs: true },
private_agent: { inputs: false, outputs: true },
});
},
})
.start()
.completed();
});
},
MASTRA_DEPENDENCIES,
);
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
import * as Sentry from '@sentry/node';
import { loggingTransport } from '@sentry-internal/node-integration-tests';

Sentry.init({
dsn: 'https://public@dsn.ingest.sentry.io/1337',
release: '1.0',
tracesSampleRate: 1.0,
transport: loggingTransport,
integrations: [
Sentry.mastraIntegration({
recordInputs: ({ span }) => span.entityName === 'public_agent',
// Mastra sets `output` only when the span ends, so this records outputs only if decided at start.
recordOutputs: ({ span }) => span.output === undefined,
}),
],
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
import express from 'express';
import * as Sentry from '@sentry/node';
import { Mastra } from '@mastra/core';
import { Agent } from '@mastra/core/agent';

// Inlined OpenAI-compatible mock: the ESM/CJS runner copies only this file into its temp dir.
function startMockProvider(responses) {
const app = express();
app.use(express.json());

let call = 0;
app.post('/v1/chat/completions', (req, res) => {
const response = responses[Math.min(call, responses.length - 1)];
call++;
res.json({
id: response.id,
object: 'chat.completion',
created: 1,
model: req.body.model,
choices: [
{
index: 0,
finish_reason: response.toolCalls ? 'tool_calls' : 'stop',
message: {
role: 'assistant',
content: response.content ?? null,
...(response.toolCalls ? { tool_calls: response.toolCalls } : {}),
},
},
],
usage: response.usage,
});
});

const server = app.listen(0);
return {
url: `http://localhost:${server.address().port}/v1`,
close: () => server.close(),
};
}

const provider = startMockProvider([
{
id: 'chatcmpl-1',
content: 'It is 22C in Berlin.',
usage: { prompt_tokens: 12, completion_tokens: 7, total_tokens: 19 },
},
]);

async function run() {
const model = { id: 'openai/gpt-4o-mini', url: provider.url, apiKey: 'test' };
const agents = {
public_agent: new Agent({
id: 'public_agent',
name: 'public_agent',
instructions: 'You report the weather.',
model,
}),
private_agent: new Agent({
id: 'private_agent',
name: 'private_agent',
instructions: 'You report the weather.',
model,
}),
};
const mastra = new Mastra({ agents, logger: false });

await Sentry.startSpan({ op: 'function', name: 'mastra-test' }, async () => {
await mastra.getAgent('public_agent').generate('What is the weather in Berlin?');
await mastra.getAgent('private_agent').generate('What is the weather in Berlin?');
});

await mastra.observability.shutdown();
provider.close();
}

run();
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import * as Sentry from '@sentry/node';
import { loggingTransport } from '@sentry-internal/node-integration-tests';

Sentry.init({
dsn: 'https://public@dsn.ingest.sentry.io/1337',
release: '1.0',
tracesSampleRate: 1.0,
transport: loggingTransport,
integrations: [
Sentry.typesafeIntegration({
recordInputs: ({ request }) => request?.model?.startsWith('public'),
recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public',
}),
],
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
import * as Sentry from '@sentry/node';
import { noul, TypeSafeClient } from '@typesafe-ai/sdk';
import express from 'express';

function startMockServer() {
const app = express();
app.use(express.json());

app.post('/v1/systemone', (req, res) => {
res.json({
model: req.body.model,
answers: { authIssue: { type: 'noul', noul: 0.98 } },
usage: { input_tokens: 275, output_tokens: 20 },
});
});

return new Promise(resolve => {
const server = app.listen(0, () => {
resolve(server);
});
});
}

async function run() {
const server = await startMockServer();
const client = new TypeSafeClient({
apiKey: 'mock-api-key',
baseURL: `http://localhost:${server.address().port}`,
retry: { maxRetries: 0 },
});
const request = { state: 'I cannot log in.', questions: { authIssue: noul('Is there a login problem?') } };

await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
await Sentry.withScope(async scope => {
scope.setTag('privacy', 'public');
await client.systemOne({ ...request, model: 'public' });
});

await Sentry.withScope(async scope => {
scope.setTag('privacy', 'private');
await client.systemOne({ ...request, model: 'private' });

// Changing the tag after the call started must not change its outputs decision.
const pending = client.systemOne({ ...request, model: 'public-late' });
Sentry.getCurrentScope().setTag('privacy', 'public');
await pending;
});
});

server.close();
}

run();
Loading
Loading