Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
import * as Sentry from '@sentry/node';
import { __flueBindAgentModule, init, instrument, useModel, useTool } from '@flue/runtime';
import { start } from '@flue/runtime/node';
import { fauxAssistantMessage, fauxProvider, fauxToolCall } from '@earendil-works/pi-ai/providers/faux';
import * as v from 'valibot';

const privacy = () => Sentry.getIsolationScope().getScopeData().tags.privacy;

// Flue runs an agent in the isolation scope of the request that dispatched it, so the tag set there
// reaches the functions. The pi-ai integration has no options of its own: Flue's decide the chat spans.
instrument(
Sentry.createFlueInstrumentation({
recordInputs: call =>
(call.operation === 'chat' ? call.context.instanceId : call.observation.instanceId) === 'public',
recordOutputs: () => privacy() === 'public',
}),
);

const faux = fauxProvider({ provider: 'faux', models: [{ id: 'faux-model' }] });
const toolCall = fauxAssistantMessage(fauxToolCall('get_weather', { city: 'Berlin' }, { id: 'call_1' }), {
stopReason: 'toolUse',
});
const answer = fauxAssistantMessage('It is sunny in Berlin.');
faux.setResponses([toolCall, answer, toolCall, answer]);

function Hello() {
useModel('faux/faux-model');
useTool({
name: 'get_weather',
description: 'Get the current weather for a city.',
input: v.object({ city: v.string() }),
run: ({ data }) => {
// A tool span decides when it starts: flipping the tag here only reaches the requests after it.
Sentry.getIsolationScope().setTag('privacy', 'public');
return `It is sunny in ${data.city}.`;
},
});
return 'You are a helpful assistant.';
}
__flueBindAgentModule(Hello, { identity: 'Hello' });

await Sentry.startSpan({ name: 'flue-test', op: 'function' }, async () => {
const flue = await start({ agents: [Hello], providers: [faux.provider] });
for (const id of ['public', 'private']) {
await Sentry.withIsolationScope(async isolationScope => {
isolationScope.setTag('privacy', id);
await Sentry.startSpan({ name: `run ${id}` }, async () => {
const agent = init(Hello, { id });
await agent.read(await agent.dispatch('What is the weather in Berlin?'));
});
});
}
await flue[Symbol.asyncDispose]?.();
});

await Sentry.flush(2000);
52 changes: 51 additions & 1 deletion dev-packages/node-integration-tests/suites/tracing/flue/test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { SerializedStreamedSpanContainer } from '@sentry/core';
import type { SerializedStreamedSpan, SerializedStreamedSpanContainer } from '@sentry/core';
import {
GEN_AI_AGENT_NAME,
GEN_AI_CONVERSATION_ID,
Expand All @@ -22,6 +22,15 @@ import { afterAll, describe, expect } from 'vitest';
import { supports } from '../../../utils';
import { cleanupChildProcesses, createEsmTests } from '../../../utils/runner';

function recorded(span: SerializedStreamedSpan): { span: string; inputs: boolean; outputs: boolean } {
const { attributes } = span;
return {
span: span.name,
inputs: GEN_AI_INPUT_MESSAGES in attributes || GEN_AI_TOOL_CALL_ARGUMENTS in attributes,
outputs: GEN_AI_OUTPUT_MESSAGES in attributes || GEN_AI_TOOL_CALL_RESULT in attributes,
};
}

// `@flue/runtime` declares `engines.node >= 22.19`.
describe.runIf(supports({ min: 22 }))('Flue integration', () => {
afterAll(() => {
Expand Down Expand Up @@ -217,4 +226,45 @@ describe.runIf(supports({ min: 22 }))('Flue integration', () => {
.completed();
});
});

createEsmTests(__dirname, 'scenario-recording-functions.mjs', 'instrument-with-pii.mjs', (createRunner, test) => {
test('decides content recording per call when the options are functions', async () => {
await createRunner()
.expect({
span: container => {
const spans = container.items;
const byId = new Map(spans.map(span => [span.span_id, span]));
const runOf = (span: SerializedStreamedSpan): string | undefined => {
let current: SerializedStreamedSpan | undefined = span;
while (current && !current.name.startsWith('run ')) {
current = current.parent_span_id ? byId.get(current.parent_span_id) : undefined;
}
return current?.name;
};
const contentSpans = spans
.filter(span => ['auto.ai.pi_ai', 'auto.ai.flue'].includes(String(span.attributes[SENTRY_ORIGIN]?.value)))
.filter(span => span.name !== 'invoke_agent Hello')
.sort((a, b) => a.start_timestamp - b.start_timestamp);

expect({
public: contentSpans.filter(span => runOf(span) === 'run public').map(recorded),
private: contentSpans.filter(span => runOf(span) === 'run private').map(recorded),
}).toEqual({
public: [
{ span: 'chat faux-model', inputs: true, outputs: true },
{ span: 'execute_tool get_weather', inputs: true, outputs: true },
{ span: 'chat faux-model', inputs: true, outputs: true },
],
private: [
{ span: 'chat faux-model', inputs: false, outputs: false },
{ span: 'execute_tool get_weather', inputs: false, outputs: false },
{ span: 'chat faux-model', inputs: false, outputs: true },
],
});
},
})
.start()
.completed();
});
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import * as Sentry from '@sentry/node';
import { loggingTransport } from '@sentry-internal/node-integration-tests';

Sentry.init({
dsn: 'https://public@dsn.ingest.sentry.io/1337',
release: '1.0',
tracesSampleRate: 1.0,
transport: loggingTransport,
integrations: [
Sentry.piAiIntegration({
recordInputs: ({ context }) => context?.messages?.[0]?.content === 'public',
recordOutputs: () => Sentry.getCurrentScope().getScopeData().tags.privacy === 'public',
}),
],
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
import * as Sentry from '@sentry/node';
import { createModels } from '@earendil-works/pi-ai/models';
import { fauxAssistantMessage, fauxProvider } from '@earendil-works/pi-ai/providers/faux';

const faux = fauxProvider({ provider: 'faux', models: [{ id: 'faux-model' }] });
const models = createModels();
models.setProvider(faux.provider);
faux.setResponses([fauxAssistantMessage('Paris.'), fauxAssistantMessage('Paris.'), fauxAssistantMessage('Paris.')]);

const ask = content => ({ messages: [{ role: 'user', content, timestamp: Date.now() }] });

await Sentry.startSpan({ name: 'pi-ai-test' }, async () => {
const model = faux.getModel();

await Sentry.withScope(async scope => {
scope.setTag('privacy', 'public');
await models.complete(model, ask('public'));
});

await Sentry.withScope(async scope => {
scope.setTag('privacy', 'private');
await models.complete(model, ask('private'));

// Changing the tag after the request started must not change its outputs decision.
const events = models.stream(model, ask('public'));
Sentry.getCurrentScope().setTag('privacy', 'public');
for await (const _event of events) {
// Drained like an app consuming the stream.
}
});
});
30 changes: 30 additions & 0 deletions dev-packages/node-integration-tests/suites/tracing/pi-ai/test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -158,4 +158,34 @@ describe.runIf(supports({ min: 22 }))('pi-ai integration', () => {
},
);
});

createEsmTests(
__dirname,
'scenario-recording-functions.mjs',
'instrument-with-recording-functions.mjs',
(createRunner, test) => {
test('decides content recording per request when the options are functions', async () => {
await createRunner()
.expect({
span: container => {
const recorded = container.items
.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.pi_ai')
.sort((a, b) => a.start_timestamp - b.start_timestamp)
.map(span => ({
streaming: span.attributes[GEN_AI_RESPONSE_STREAMING]?.value === true,
inputs: GEN_AI_INPUT_MESSAGES in span.attributes,
outputs: GEN_AI_OUTPUT_MESSAGES in span.attributes,
}));
expect(recorded).toEqual([
{ streaming: false, inputs: true, outputs: true },
{ streaming: false, inputs: false, outputs: false },
{ streaming: true, inputs: true, outputs: false },
]);
},
})
.start()
.completed();
});
},
);
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import * as Sentry from '@sentry/node';
import { loggingTransport } from '@sentry-internal/node-integration-tests';

// Runs start from whichever context woke the scheduler, never from the request's, so these decide from
// what pi-durable hands over. The pi-ai integration has no options of its own: these decide the chat
// spans too.
Sentry.init({
dsn: 'https://public@dsn.ingest.sentry.io/1337',
release: '1.0',
tracesSampleRate: 1.0,
transport: loggingTransport,
integrations: [
Sentry.piDurableIntegration({
recordInputs: call =>
(call.operation === 'chat' ? call.runtime : call.api).conversationId === globalThis.publicConversationId,
recordOutputs: call => call.operation === 'chat' && globalThis.recordChatOutputs,
}),
],
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
import * as Sentry from '@sentry/node';
import { BACKGROUND_CONTEXT } from '@earendil-works/chord/context';
import { Type } from '@earendil-works/pi-ai';
import { createModels } from '@earendil-works/pi-ai/models';
import { fauxAssistantMessage, fauxProvider, fauxToolCall } from '@earendil-works/pi-ai/providers/faux';
import { createRegistry, defineExtension, defineTool, Harness, MemoryStorage } from '@earendil-works/pi-durable';

const context = BACKGROUND_CONTEXT;

globalThis.recordChatOutputs = true;
const respond = transcript => {
if (!transcript.messages.some(message => message.role === 'toolResult')) {
return fauxAssistantMessage(fauxToolCall('get_weather', { city: 'Berlin' }, { id: 'call_1' }), {
stopReason: 'toolUse',
});
}
// The request is already streaming here, so this only turns off the outputs of the requests after it.
globalThis.recordChatOutputs = false;
return fauxAssistantMessage('It is sunny in Berlin.');
};
const faux = fauxProvider({ provider: 'faux', models: [{ id: 'faux-model' }] });
const models = createModels();
models.setProvider(faux.provider);
faux.setResponses([respond, respond, respond, respond]);

const registry = createRegistry();
registry.install(
defineExtension({
name: 'app',
tools: [
defineTool({
name: 'get_weather',
description: 'Get the current weather for a city.',
parameters: Type.Object({ city: Type.String() }),
execute: async ({ city }) => ({ output: [{ type: 'text', text: `It is sunny in ${city}.` }] }),
}),
],
}),
);

const harness = await Harness.open(new MemoryStorage(), { models, registry }, context);
const agent = { model: { provider: 'faux', modelId: 'faux-model' } };
const publicConversation = await harness.createConversation({ ownership: { kind: 'ownerless' }, agent }, context);
const privateConversation = await harness.createConversation({ ownership: { kind: 'ownerless' }, agent }, context);
globalThis.publicConversationId = publicConversation.id;

for (const conversation of [publicConversation, privateConversation]) {
await (
await conversation.submit({ type: 'input', content: 'What is the weather in Berlin?' }, context)
).wait(context);
}
await harness.close(context);

await Sentry.flush(2000);
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import type { SerializedStreamedSpanContainer } from '@sentry/core';
import {
GEN_AI_CONVERSATION_ID,
GEN_AI_INPUT_MESSAGES,
Expand Down Expand Up @@ -37,6 +38,20 @@ const CONTENT_ATTRIBUTES = [
GEN_AI_TOOL_CALL_RESULT,
];

type Recorded = { span: string; inputs: boolean; outputs: boolean };

/** What each span of the run records, in start order. Each run is a trace of its own. */
function recorded(container: SerializedStreamedSpanContainer): Recorded[] {
return container.items
.filter(span => !span.is_segment)
.sort((a, b) => a.start_timestamp - b.start_timestamp)
.map(({ name, attributes }) => ({
span: name,
inputs: GEN_AI_INPUT_MESSAGES in attributes || GEN_AI_TOOL_CALL_ARGUMENTS in attributes,
outputs: GEN_AI_OUTPUT_MESSAGES in attributes || GEN_AI_TOOL_CALL_RESULT in attributes,
}));
}

// The pi packages declare `engines.node >= 22.19`.
describe.runIf(supports({ min: 22 }))('pi-durable integration', () => {
afterAll(() => {
Expand Down Expand Up @@ -651,4 +666,34 @@ describe.runIf(supports({ min: 22 }))('pi-durable integration', () => {
.completed();
});
});

createEsmTests(
__dirname,
'scenario-recording-functions.mjs',
'instrument-with-recording-functions.mjs',
(createRunner, test) => {
test('decides content recording per call when the options are functions', async () => {
const runs: Recorded[][] = [];

await createRunner()
.expect({ span: container => void runs.push(recorded(container)) })
.expect({ span: container => void runs.push(recorded(container)) })
.start()
.completed();

expect(runs).toEqual([
[
{ span: 'chat faux-model', inputs: true, outputs: true },
{ span: 'execute_tool get_weather', inputs: true, outputs: false },
{ span: 'chat faux-model', inputs: true, outputs: true },
],
[
{ span: 'chat faux-model', inputs: false, outputs: false },
{ span: 'execute_tool get_weather', inputs: false, outputs: false },
{ span: 'chat faux-model', inputs: false, outputs: false },
],
]);
});
},
);
});
3 changes: 2 additions & 1 deletion packages/server-utils/src/ai/core/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,8 @@ export function resolveGenAiCallRecording<TCall>(
};
}

function resolveRecordingOption<TCall>(
/** One recording option of a call, `undefined` when it leaves the decision to `dataCollection.genAI`. */
export function resolveRecordingOption<TCall>(
option: GenAiRecordingOption<TCall> | undefined,
call: TCall,
): boolean | undefined {
Expand Down
Loading
Loading