Skip to content

Commit ee79e4b

Browse files
test(node): Cover per-call gen_ai recording for pi-ai, Flue and pi-durable with node integration tests
Replaces the per-call recording unit tests with one node integration test per integration that runs the real library against its faux provider. Co-Authored-By: Francesco Gringl-Novy <2411343+mydea@users.noreply.github.com>
1 parent 2cf6bb6 commit ee79e4b

13 files changed

Lines changed: 366 additions & 362 deletions

File tree

‎dev-packages/bun-integration-tests/node-suites/excludes.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ const NODE_ONLY = [
1212
'suites/breadcrumbs/**',
1313
'suites/child-process/test.ts',
1414
'suites/thread-blocked-native/test.ts',
15-
'suites/tracing/flue/test.ts',
15+
'suites/tracing/flue/**',
1616
'suites/vercel/keep-alive/test.ts',
1717
];
1818

@@ -121,8 +121,8 @@ export const NO_AUTO_INSTRUMENTATION = [
121121
'suites/tracing/openai/v6/test.ts',
122122
'suites/tracing/openai/v7/test.ts',
123123
'suites/tracing/orchestrion-lazy-registration/test.ts',
124-
'suites/tracing/pi-ai/test.ts',
125-
'suites/tracing/pi-durable/test.ts',
124+
'suites/tracing/pi-ai/**',
125+
'suites/tracing/pi-durable/**',
126126
'suites/tracing/postgres/test.ts',
127127
'suites/tracing/postgresjs/test.ts',
128128
'suites/tracing/prisma-orm-v5/test.ts',

‎dev-packages/deno-integration-tests/node-suites/excludes.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ const NOT_TRIAGED = [
5959
'suites/tracing/anthropic/v0.129/test.ts',
6060
'suites/tracing/apollo-graphql/**',
6161
'suites/tracing/fastify/test.ts',
62-
'suites/tracing/flue/test.ts',
62+
'suites/tracing/flue/**',
6363
'suites/tracing/google-genai/test.ts',
6464
'suites/tracing/groq/test.ts',
6565
'suites/tracing/http-client-spans/http-strip-query/test.ts',
Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
import { join } from 'node:path';
2+
import type { SerializedStreamedSpan } from '@sentry/core';
3+
import {
4+
GEN_AI_INPUT_MESSAGES,
5+
GEN_AI_OUTPUT_MESSAGES,
6+
GEN_AI_TOOL_CALL_ARGUMENTS,
7+
GEN_AI_TOOL_CALL_RESULT,
8+
SENTRY_ORIGIN,
9+
} from '@sentry/conventions/attributes';
10+
import { afterAll, describe, expect } from 'vitest';
11+
import { supports } from '../../../../utils';
12+
import { cleanupChildProcesses, createEsmTests } from '../../../../utils/runner';
13+
14+
function recorded(span: SerializedStreamedSpan): { span: string; inputs: boolean; outputs: boolean } {
15+
const { attributes } = span;
16+
return {
17+
span: span.name,
18+
inputs: GEN_AI_INPUT_MESSAGES in attributes || GEN_AI_TOOL_CALL_ARGUMENTS in attributes,
19+
outputs: GEN_AI_OUTPUT_MESSAGES in attributes || GEN_AI_TOOL_CALL_RESULT in attributes,
20+
};
21+
}
22+
23+
// `@flue/runtime` declares `engines.node >= 22.19`.
24+
describe.runIf(supports({ min: 22 }))('Flue integration', () => {
25+
afterAll(() => {
26+
cleanupChildProcesses();
27+
});
28+
29+
createEsmTests(
30+
join(__dirname, '..'),
31+
'scenario-recording-functions.mjs',
32+
'instrument-with-pii.mjs',
33+
(createRunner, test) => {
34+
test('decides content recording per call when the options are functions', async () => {
35+
await createRunner()
36+
.expect({
37+
span: container => {
38+
const spans = container.items;
39+
const byId = new Map(spans.map(span => [span.span_id, span]));
40+
const runOf = (span: SerializedStreamedSpan): string | undefined => {
41+
let current: SerializedStreamedSpan | undefined = span;
42+
while (current && !current.name.startsWith('run ')) {
43+
current = current.parent_span_id ? byId.get(current.parent_span_id) : undefined;
44+
}
45+
return current?.name;
46+
};
47+
const contentSpans = spans
48+
.filter(span =>
49+
['auto.ai.pi_ai', 'auto.ai.flue'].includes(String(span.attributes[SENTRY_ORIGIN]?.value)),
50+
)
51+
.filter(span => span.name !== 'invoke_agent Hello')
52+
.sort((a, b) => a.start_timestamp - b.start_timestamp);
53+
54+
expect({
55+
public: contentSpans.filter(span => runOf(span) === 'run public').map(recorded),
56+
private: contentSpans.filter(span => runOf(span) === 'run private').map(recorded),
57+
}).toEqual({
58+
public: [
59+
{ span: 'chat faux-model', inputs: true, outputs: true },
60+
{ span: 'execute_tool get_weather', inputs: true, outputs: true },
61+
{ span: 'chat faux-model', inputs: true, outputs: true },
62+
],
63+
private: [
64+
{ span: 'chat faux-model', inputs: false, outputs: false },
65+
{ span: 'execute_tool get_weather', inputs: false, outputs: false },
66+
{ span: 'chat faux-model', inputs: false, outputs: true },
67+
],
68+
});
69+
},
70+
})
71+
.start()
72+
.completed();
73+
});
74+
},
75+
);
76+
});
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
import * as Sentry from '@sentry/node';
2+
import { __flueBindAgentModule, init, instrument, useModel, useTool } from '@flue/runtime';
3+
import { start } from '@flue/runtime/node';
4+
import { fauxAssistantMessage, fauxProvider, fauxToolCall } from '@earendil-works/pi-ai/providers/faux';
5+
import * as v from 'valibot';
6+
7+
const privacy = () => Sentry.getIsolationScope().getScopeData().tags.privacy;
8+
9+
// Flue runs an agent in the isolation scope of the request that dispatched it, so the tag set there
10+
// reaches the functions. The pi-ai integration has no options of its own: Flue's decide the chat spans.
11+
instrument(
12+
Sentry.createFlueInstrumentation({
13+
recordInputs: call =>
14+
(call.operation === 'chat' ? call.context.instanceId : call.observation.instanceId) === 'public',
15+
recordOutputs: () => privacy() === 'public',
16+
}),
17+
);
18+
19+
const faux = fauxProvider({ provider: 'faux', models: [{ id: 'faux-model' }] });
20+
const toolCall = fauxAssistantMessage(fauxToolCall('get_weather', { city: 'Berlin' }, { id: 'call_1' }), {
21+
stopReason: 'toolUse',
22+
});
23+
const answer = fauxAssistantMessage('It is sunny in Berlin.');
24+
faux.setResponses([toolCall, answer, toolCall, answer]);
25+
26+
function Hello() {
27+
useModel('faux/faux-model');
28+
useTool({
29+
name: 'get_weather',
30+
description: 'Get the current weather for a city.',
31+
input: v.object({ city: v.string() }),
32+
run: ({ data }) => {
33+
// A tool span decides when it starts: flipping the tag here only reaches the requests after it.
34+
Sentry.getIsolationScope().setTag('privacy', 'public');
35+
return `It is sunny in ${data.city}.`;
36+
},
37+
});
38+
return 'You are a helpful assistant.';
39+
}
40+
__flueBindAgentModule(Hello, { identity: 'Hello' });
41+
42+
await Sentry.startSpan({ name: 'flue-test', op: 'function' }, async () => {
43+
const flue = await start({ agents: [Hello], providers: [faux.provider] });
44+
for (const id of ['public', 'private']) {
45+
await Sentry.withIsolationScope(async isolationScope => {
46+
isolationScope.setTag('privacy', id);
47+
await Sentry.startSpan({ name: `run ${id}` }, async () => {
48+
const agent = init(Hello, { id });
49+
await agent.read(await agent.dispatch('What is the weather in Berlin?'));
50+
});
51+
});
52+
}
53+
await flue[Symbol.asyncDispose]?.();
54+
});
55+
56+
await Sentry.flush(2000);
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import * as Sentry from '@sentry/node';
2+
import { loggingTransport } from '@sentry-internal/node-integration-tests';
3+
4+
Sentry.init({
5+
dsn: 'https://public@dsn.ingest.sentry.io/1337',
6+
release: '1.0',
7+
tracesSampleRate: 1.0,
8+
transport: loggingTransport,
9+
integrations: [
10+
Sentry.piAiIntegration({
11+
recordInputs: ({ context }) => context?.messages?.[0]?.content === 'public',
12+
recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public',
13+
}),
14+
],
15+
});
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
import { join } from 'node:path';
2+
import {
3+
GEN_AI_INPUT_MESSAGES,
4+
GEN_AI_OUTPUT_MESSAGES,
5+
GEN_AI_RESPONSE_STREAMING,
6+
SENTRY_ORIGIN,
7+
} from '@sentry/conventions/attributes';
8+
import { afterAll, describe, expect } from 'vitest';
9+
import { supports } from '../../../../utils';
10+
import { cleanupChildProcesses, createEsmTests } from '../../../../utils/runner';
11+
12+
// `@earendil-works/pi-ai` declares `engines.node >= 22.19`.
13+
describe.runIf(supports({ min: 22 }))('pi-ai integration', () => {
14+
afterAll(() => {
15+
cleanupChildProcesses();
16+
});
17+
18+
createEsmTests(
19+
join(__dirname, '..'),
20+
'scenario-recording-functions.mjs',
21+
'instrument-with-recording-functions.mjs',
22+
(createRunner, test) => {
23+
test('decides content recording per request when the options are functions', async () => {
24+
await createRunner()
25+
.expect({
26+
span: container => {
27+
const recorded = container.items
28+
.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.pi_ai')
29+
.sort((a, b) => a.start_timestamp - b.start_timestamp)
30+
.map(span => ({
31+
streaming: span.attributes[GEN_AI_RESPONSE_STREAMING]?.value === true,
32+
inputs: GEN_AI_INPUT_MESSAGES in span.attributes,
33+
outputs: GEN_AI_OUTPUT_MESSAGES in span.attributes,
34+
}));
35+
expect(recorded).toEqual([
36+
{ streaming: false, inputs: true, outputs: true },
37+
{ streaming: false, inputs: false, outputs: false },
38+
{ streaming: true, inputs: true, outputs: false },
39+
]);
40+
},
41+
})
42+
.start()
43+
.completed();
44+
});
45+
},
46+
);
47+
});
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
import * as Sentry from '@sentry/node';
2+
import { createModels } from '@earendil-works/pi-ai/models';
3+
import { fauxAssistantMessage, fauxProvider } from '@earendil-works/pi-ai/providers/faux';
4+
5+
const faux = fauxProvider({ provider: 'faux', models: [{ id: 'faux-model' }] });
6+
const models = createModels();
7+
models.setProvider(faux.provider);
8+
faux.setResponses([fauxAssistantMessage('Paris.'), fauxAssistantMessage('Paris.'), fauxAssistantMessage('Paris.')]);
9+
10+
const ask = content => ({ messages: [{ role: 'user', content, timestamp: Date.now() }] });
11+
12+
await Sentry.startSpan({ name: 'pi-ai-test' }, async () => {
13+
const model = faux.getModel();
14+
15+
await Sentry.withScope(async scope => {
16+
scope.setTag('privacy', 'public');
17+
await models.complete(model, ask('public'));
18+
});
19+
20+
await Sentry.withScope(async scope => {
21+
scope.setTag('privacy', 'private');
22+
await models.complete(model, ask('private'));
23+
24+
// Changing the tag after the request started must not change its outputs decision.
25+
const events = models.stream(model, ask('public'));
26+
Sentry.getCurrentScope().setTag('privacy', 'public');
27+
for await (const _event of events) {
28+
// Drained like an app consuming the stream.
29+
}
30+
});
31+
});
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import * as Sentry from '@sentry/node';
2+
import { loggingTransport } from '@sentry-internal/node-integration-tests';
3+
4+
// Runs start from whichever context woke the scheduler, never from the request's, so these decide from
5+
// what pi-durable hands over. The pi-ai integration has no options of its own: these decide the chat
6+
// spans too.
7+
Sentry.init({
8+
dsn: 'https://public@dsn.ingest.sentry.io/1337',
9+
release: '1.0',
10+
tracesSampleRate: 1.0,
11+
transport: loggingTransport,
12+
integrations: [
13+
Sentry.piDurableIntegration({
14+
recordInputs: call =>
15+
(call.operation === 'chat' ? call.runtime : call.api).conversationId === globalThis.publicConversationId,
16+
recordOutputs: call => call.operation === 'chat' && globalThis.recordChatOutputs,
17+
}),
18+
],
19+
});
Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
import { join } from 'node:path';
2+
import type { SerializedStreamedSpanContainer } from '@sentry/core';
3+
import {
4+
GEN_AI_INPUT_MESSAGES,
5+
GEN_AI_OUTPUT_MESSAGES,
6+
GEN_AI_TOOL_CALL_ARGUMENTS,
7+
GEN_AI_TOOL_CALL_RESULT,
8+
} from '@sentry/conventions/attributes';
9+
import { afterAll, describe, expect } from 'vitest';
10+
import { supports } from '../../../../utils';
11+
import { cleanupChildProcesses, createEsmTests } from '../../../../utils/runner';
12+
13+
type Recorded = { span: string; inputs: boolean; outputs: boolean };
14+
15+
/** What each span of the run records, in start order. Each run is a trace of its own. */
16+
function recorded(container: SerializedStreamedSpanContainer): Recorded[] {
17+
return container.items
18+
.filter(span => !span.is_segment)
19+
.sort((a, b) => a.start_timestamp - b.start_timestamp)
20+
.map(({ name, attributes }) => ({
21+
span: name,
22+
inputs: GEN_AI_INPUT_MESSAGES in attributes || GEN_AI_TOOL_CALL_ARGUMENTS in attributes,
23+
outputs: GEN_AI_OUTPUT_MESSAGES in attributes || GEN_AI_TOOL_CALL_RESULT in attributes,
24+
}));
25+
}
26+
27+
// `@earendil-works/pi-durable` declares `engines.node >= 22.19`.
28+
describe.runIf(supports({ min: 22 }))('pi-durable integration', () => {
29+
afterAll(() => {
30+
cleanupChildProcesses();
31+
});
32+
33+
createEsmTests(
34+
join(__dirname, '..'),
35+
'scenario-recording-functions.mjs',
36+
'instrument-with-recording-functions.mjs',
37+
(createRunner, test) => {
38+
test('decides content recording per call when the options are functions', async () => {
39+
const runs: Recorded[][] = [];
40+
41+
await createRunner()
42+
.expect({ span: container => void runs.push(recorded(container)) })
43+
.expect({ span: container => void runs.push(recorded(container)) })
44+
.start()
45+
.completed();
46+
47+
expect(runs).toEqual([
48+
[
49+
{ span: 'chat faux-model', inputs: true, outputs: true },
50+
{ span: 'execute_tool get_weather', inputs: true, outputs: false },
51+
{ span: 'chat faux-model', inputs: true, outputs: true },
52+
],
53+
[
54+
{ span: 'chat faux-model', inputs: false, outputs: false },
55+
{ span: 'execute_tool get_weather', inputs: false, outputs: false },
56+
{ span: 'chat faux-model', inputs: false, outputs: false },
57+
],
58+
]);
59+
});
60+
},
61+
);
62+
});
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
import * as Sentry from '@sentry/node';
2+
import { BACKGROUND_CONTEXT } from '@earendil-works/chord/context';
3+
import { Type } from '@earendil-works/pi-ai';
4+
import { createModels } from '@earendil-works/pi-ai/models';
5+
import { fauxAssistantMessage, fauxProvider, fauxToolCall } from '@earendil-works/pi-ai/providers/faux';
6+
import { createRegistry, defineExtension, defineTool, Harness, MemoryStorage } from '@earendil-works/pi-durable';
7+
8+
const context = BACKGROUND_CONTEXT;
9+
10+
globalThis.recordChatOutputs = true;
11+
const respond = transcript => {
12+
if (!transcript.messages.some(message => message.role === 'toolResult')) {
13+
return fauxAssistantMessage(fauxToolCall('get_weather', { city: 'Berlin' }, { id: 'call_1' }), {
14+
stopReason: 'toolUse',
15+
});
16+
}
17+
// The request is already streaming here, so this only turns off the outputs of the requests after it.
18+
globalThis.recordChatOutputs = false;
19+
return fauxAssistantMessage('It is sunny in Berlin.');
20+
};
21+
const faux = fauxProvider({ provider: 'faux', models: [{ id: 'faux-model' }] });
22+
const models = createModels();
23+
models.setProvider(faux.provider);
24+
faux.setResponses([respond, respond, respond, respond]);
25+
26+
const registry = createRegistry();
27+
registry.install(
28+
defineExtension({
29+
name: 'app',
30+
tools: [
31+
defineTool({
32+
name: 'get_weather',
33+
description: 'Get the current weather for a city.',
34+
parameters: Type.Object({ city: Type.String() }),
35+
execute: async ({ city }) => ({ output: [{ type: 'text', text: `It is sunny in ${city}.` }] }),
36+
}),
37+
],
38+
}),
39+
);
40+
41+
const harness = await Harness.open(new MemoryStorage(), { models, registry }, context);
42+
const agent = { model: { provider: 'faux', modelId: 'faux-model' } };
43+
const publicConversation = await harness.createConversation({ ownership: { kind: 'ownerless' }, agent }, context);
44+
const privateConversation = await harness.createConversation({ ownership: { kind: 'ownerless' }, agent }, context);
45+
globalThis.publicConversationId = publicConversation.id;
46+
47+
for (const conversation of [publicConversation, privateConversation]) {
48+
await (
49+
await conversation.submit({ type: 'input', content: 'What is the weather in Berlin?' }, context)
50+
).wait(context);
51+
}
52+
await harness.close(context);
53+
54+
await Sentry.flush(2000);

0 commit comments

Comments
 (0)