diff --git a/dev-packages/e2e-tests/test-applications/node-eve/agent/hooks/sentry.ts b/dev-packages/e2e-tests/test-applications/node-eve/agent/hooks/sentry.ts new file mode 100644 index 000000000000..3e7b0ad0a5d5 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-eve/agent/hooks/sentry.ts @@ -0,0 +1,7 @@ +import * as Sentry from '@sentry/node'; +import { defineHook } from 'eve/hooks'; + +// Tags every turn of an eve session with the session id as the Sentry conversation id, so a +// session's AI spans — which land in separate traces (each turn is its own durable workflow) — +// group into one conversation in Sentry. +export default defineHook(Sentry.eveConversationHook()); diff --git a/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts b/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts index 3e7adc612c8a..32141b9f3944 100644 --- a/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-eve/tests/eve.test.ts @@ -34,7 +34,7 @@ test('captures Vercel AI agent spans (invoke_agent, generate_content, execute_to ) && spansOfTrace.some(isAgentServerSpan), ); - await runAgentTurn(baseURL!, 'What is the weather in Paris?'); + const sessionId = await runAgentTurn(baseURL!, 'What is the weather in Paris?'); const traceSpans = await traceSpansPromise; @@ -66,6 +66,14 @@ test('captures Vercel AI agent spans (invoke_agent, generate_content, execute_to // The tool returns `{ city, condition: 'Sunny', temperatureC: 22 }`. expect(executeTool?.attributes?.['gen_ai.tool.call.result']?.value).toContain('Sunny'); + // `agent/hooks/sentry.ts` sets the eve session id as the conversation id via + // `Sentry.eveConversationHook()`, so every gen_ai span in the turn is tagged with it — that is + // what links a multi-turn session (each turn is its own trace) into one Sentry conversation. + expect(sessionId).toBeTruthy(); + for (const span of [invokeAgent, generateContent, executeTool]) { + expect(span?.attributes?.['gen_ai.conversation.id']?.value).toBe(sessionId); + } + // The agent turn is captured as an http.server span on one of eve's two agent // request paths (the other http.server spans — health and the event stream — // are not in this trace). diff --git a/dev-packages/e2e-tests/test-applications/node-eve/tests/utils.ts b/dev-packages/e2e-tests/test-applications/node-eve/tests/utils.ts index 6e3f0bd7a301..50542a116320 100644 --- a/dev-packages/e2e-tests/test-applications/node-eve/tests/utils.ts +++ b/dev-packages/e2e-tests/test-applications/node-eve/tests/utils.ts @@ -5,8 +5,11 @@ import { expect } from '@playwright/test'; * settle, so the agent has finished and its spans have been flushed before we * assert. eve runs the turn in a durable workflow, so the POST only needs to be * accepted; we drain the event stream to know when the turn is done. + * + * Returns the eve session id, which the SDK also records as `gen_ai.conversation.id` on the turn's + * AI spans (see `agent/hooks/sentry.ts`), so a test can assert the two match. */ -export async function runAgentTurn(baseURL: string, message: string): Promise { +export async function runAgentTurn(baseURL: string, message: string): Promise { const createRes = await fetch(`${baseURL}/eve/v1/session`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, @@ -36,4 +39,6 @@ export async function runAgentTurn(baseURL: string, message: string): Promise void; + +interface EveConversationHookOptions { + /** + * Derive the Sentry conversation id from the eve hook context. Defaults to the durable session id + * (`ctx.session.id`), which is stable across every turn of a session and so groups them into one + * conversation. + */ + getConversationId?: (context: EveHookContext) => string | null | undefined; +} + +/** + * Builds the hook definition for an eve `agent/hooks/*.ts` file that tags a session's AI spans with + * a Sentry conversation id, linking every turn of the session in the Agents "Conversations" view. + * + * ```ts + * // agent/hooks/sentry.ts + * import * as Sentry from '@sentry/node'; + * import { defineHook } from 'eve/hooks'; + * + * export default defineHook(Sentry.eveConversationHook()); + * ``` + * + * The id is set on the isolation scope; the default (always-on) `conversationIdIntegration` then + * stamps `gen_ai.conversation.id` onto the gen_ai spans the Vercel AI instrumentation records for + * that turn. That indirection is why the id has to be set here and not on the AI call: eve's session + * id never reaches the AI SDK's telemetry channel, so it can only be attached via the scope. + * + * Subscribes to both `turn.started` and `step.started`. Each eve turn is a fresh durable-workflow + * request with its own isolation scope, and a turn that parks and resumes (approvals, compaction) + * resumes in yet another request; `turn.started` alone would miss the model calls after a resume. + * `step.started` fires before every model call, so together they cover each request that produces + * spans. Re-setting the same id is idempotent, so the overlap is harmless. + */ +export function eveConversationHook(options: EveConversationHookOptions = {}): { + events: Record<'turn.started' | 'step.started', EveHookHandler>; +} { + const { getConversationId } = options; + + const setConversationIdFromContext: EveHookHandler = (_event, context) => { + const conversationId = getConversationId ? getConversationId(context) : context.session.id; + setConversationId(conversationId); + }; + + return { + events: { + 'turn.started': setConversationIdFromContext, + 'step.started': setConversationIdFromContext, + }, + }; +} diff --git a/packages/server-utils/src/index.ts b/packages/server-utils/src/index.ts index 2f918e2d5422..9a1520c560c8 100644 --- a/packages/server-utils/src/index.ts +++ b/packages/server-utils/src/index.ts @@ -13,6 +13,7 @@ export type { InstrumentationConfig } from './orchestrion/apmTypes'; // `orchestrion/bundler/moduleInjectedTransform.ts`); it is a plain runtime // helper with no orchestrion build-time dependency. export { orchestrionModuleInjected } from './utils/moduleInjected'; +export { eveConversationHook } from './eve'; export { fastifyIntegration, // oxlint-disable-next-line typescript/no-deprecated diff --git a/packages/server-utils/test/eve.test.ts b/packages/server-utils/test/eve.test.ts new file mode 100644 index 000000000000..2212f7b8e387 --- /dev/null +++ b/packages/server-utils/test/eve.test.ts @@ -0,0 +1,55 @@ +import * as SentryCore from '@sentry/core'; +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { eveConversationHook } from '../src/eve'; + +describe('eveConversationHook', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + test('subscribes to turn.started and step.started', () => { + const { events } = eveConversationHook(); + + expect(Object.keys(events).sort()).toEqual(['step.started', 'turn.started']); + }); + + test('sets the session id as the conversation id on turn.started', () => { + const setConversationId = vi.spyOn(SentryCore, 'setConversationId').mockImplementation(() => undefined); + + eveConversationHook().events['turn.started'](undefined, { session: { id: 'sess_abc' } }); + + expect(setConversationId).toHaveBeenCalledWith('sess_abc'); + }); + + test('sets it on step.started too, so model calls after a parked-turn resume are covered', () => { + const setConversationId = vi.spyOn(SentryCore, 'setConversationId').mockImplementation(() => undefined); + + eveConversationHook().events['step.started'](undefined, { session: { id: 'sess_resumed' } }); + + expect(setConversationId).toHaveBeenCalledWith('sess_resumed'); + }); + + test('honors a custom getConversationId', () => { + const setConversationId = vi.spyOn(SentryCore, 'setConversationId').mockImplementation(() => undefined); + + eveConversationHook({ getConversationId: context => `conv-${context.session.id}` }).events['turn.started']( + undefined, + { session: { id: 'xyz' } }, + ); + + expect(setConversationId).toHaveBeenCalledWith('conv-xyz'); + }); + + test.each([ + ['undefined', undefined], + ['null', null], + ])('unsets the conversation id when the resolver returns %s', (_label, returnValue) => { + const setConversationId = vi.spyOn(SentryCore, 'setConversationId').mockImplementation(() => undefined); + + eveConversationHook({ getConversationId: () => returnValue }).events['turn.started'](undefined, { + session: { id: 'xyz' }, + }); + + expect(setConversationId).toHaveBeenCalledWith(returnValue); + }); +});