Skip to content

Commit 524bd26

Browse files
RulaKhaledclaude
andauthored
feat(server-utils): Instrument Google GenAI decisions as gen_ai.decide spans (#26457)
## What `@google/genai` `decisions.create` calls (3.2.0+) now create `decide <model>` spans with op `gen_ai.decide`, recorded the same way as Jev and OpenAI decisions. ## Why These calls had no span. Google's questions and answers already use Jev's layout, so the only mapping is recording `input` as `state`. `decisions.create` takes its abort signal in its request options rather than in a `config`, so it is read from there and aborted decisions are marked `cancelled` like other AI calls. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent bc2c361 commit 524bd26

10 files changed

Lines changed: 374 additions & 9 deletions

File tree

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
import * as Sentry from '@sentry/node';
2+
import { loggingTransport } from '@sentry-internal/node-integration-tests';
3+
4+
Sentry.init({
5+
dsn: 'https://public@dsn.ingest.sentry.io/1337',
6+
release: '1.0',
7+
tracesSampleRate: 1.0,
8+
transport: loggingTransport,
9+
});
Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
import { GoogleGenAI } from '@google/genai';
2+
import * as Sentry from '@sentry/node';
3+
import express from 'express';
4+
5+
function startMockGoogleGenAIServer() {
6+
const app = express();
7+
app.use(express.json({ limit: '1mb' }));
8+
9+
app.post('/v1/decisions', (req, res) => {
10+
if (req.body.model === 'models/error-model') {
11+
// A 400, not a 500: the decisions client retries 5xx responses with backoff by default.
12+
res.status(400).end('Bad request');
13+
return;
14+
}
15+
16+
res.send({
17+
answers: {
18+
wants_refund: { type: 'noul', noul: 0.97, confidence: 0.94 },
19+
department: {
20+
type: 'choice',
21+
choice: 'billing',
22+
probabilities: { billing: 0.82, other: 0.18 },
23+
confidence: 0.82,
24+
},
25+
urgency: {
26+
type: 'score',
27+
score: 1,
28+
probabilities: { 0: 0.05, 1: 0.95 },
29+
legend: { 0: 'low', 1: 'high' },
30+
confidence: 0.93,
31+
},
32+
},
33+
usage: { total_input_tokens: 382, total_cached_tokens: 256, total_output_tokens: 0, total_tokens: 382 },
34+
});
35+
});
36+
37+
return new Promise(resolve => {
38+
const server = app.listen(0, () => {
39+
resolve(server);
40+
});
41+
});
42+
}
43+
44+
const questions = {
45+
wants_refund: {
46+
type: 'noul',
47+
instructions: 'Is a refund requested?',
48+
criteria: { true_description: 'Asks for money back' },
49+
},
50+
department: {
51+
type: 'choice',
52+
instructions: 'Which team should handle this?',
53+
criteria: { billing: 'Charges and refunds', other: 'Anything else' },
54+
},
55+
urgency: { type: 'score', instructions: 'How urgent is this ticket?', criteria: ['low', 'high'] },
56+
};
57+
58+
async function run() {
59+
const server = await startMockGoogleGenAIServer();
60+
61+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
62+
const client = new GoogleGenAI({
63+
apiKey: 'mock-api-key',
64+
httpOptions: { baseUrl: `http://localhost:${server.address().port}` },
65+
});
66+
67+
await client.decisions.create({
68+
model: 'models/gemini-3-flash',
69+
input: 'I was charged twice and want a refund.',
70+
questions,
71+
});
72+
73+
await client.decisions.create({
74+
model: 'models/gemini-3-flash',
75+
input: [
76+
{ type: 'text', text: 'Is the package damaged?' },
77+
{ type: 'image', mime_type: 'image/png', data: 'iVBORw0KGgoAAAANSUhEUg==' },
78+
],
79+
questions: { damaged: { type: 'noul', instructions: 'Is there visible damage?' } },
80+
});
81+
82+
try {
83+
await client.decisions.create({ model: 'models/error-model', input: 'Error test', questions });
84+
} catch {}
85+
});
86+
87+
server.close();
88+
}
89+
90+
run();

‎dev-packages/node-integration-tests/suites/tracing/google-genai-v3/test.ts‎

Lines changed: 94 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,21 @@
11
import { afterAll, describe, expect } from 'vitest';
22
import {
3+
GEN_AI_INPUT_MESSAGES,
34
GEN_AI_OPERATION_NAME,
5+
GEN_AI_OUTPUT_MESSAGES,
46
GEN_AI_PROVIDER_NAME,
57
GEN_AI_REQUEST_MAX_TOKENS,
68
GEN_AI_REQUEST_MODEL,
79
GEN_AI_REQUEST_TEMPERATURE,
810
GEN_AI_REQUEST_TOP_P,
11+
GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS,
912
GEN_AI_USAGE_INPUT_TOKENS,
1013
GEN_AI_USAGE_OUTPUT_TOKENS,
1114
GEN_AI_USAGE_TOTAL_TOKENS,
1215
SENTRY_OP,
1316
SENTRY_ORIGIN,
1417
} from '@sentry/conventions/attributes';
15-
import { GEN_AI_CHAT, GEN_AI_EMBEDDINGS, GEN_AI_GENERATE_CONTENT } from '@sentry/conventions/op';
18+
import { GEN_AI_CHAT, GEN_AI_DECIDE, GEN_AI_EMBEDDINGS, GEN_AI_GENERATE_CONTENT } from '@sentry/conventions/op';
1619
import { supports } from '../../../utils';
1720
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner';
1821

@@ -118,4 +121,94 @@ describe.runIf(supports({ min: 22 }))('Google GenAI integration (v3)', () => {
118121
},
119122
{ additionalDependencies: { '@google/genai': '^3' } },
120123
);
124+
125+
createEsmAndCjsTests(
126+
__dirname,
127+
'scenario-decisions.mjs',
128+
'instrument-with-recording.mjs',
129+
(createRunner, test) => {
130+
test('instruments decisions.create as decide spans on @google/genai v3', async () => {
131+
const runner = createRunner().ignore('event');
132+
const spansPromise = runner.collectStreamedSpansUntilSegment('main');
133+
134+
await runner.start().completed();
135+
136+
const allSpans = await spansPromise;
137+
const mainSpan = allSpans.find(span => span.name === 'main');
138+
const spans = allSpans.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE);
139+
expect(spans).toHaveLength(3);
140+
for (const span of spans) {
141+
expect(span.parent_span_id).toBe(mainSpan!.span_id);
142+
}
143+
144+
// Both successful spans share a name and may arrive in any order, so match on the input.
145+
const inputOf = (span: (typeof spans)[number]): string => String(span.attributes[GEN_AI_INPUT_MESSAGES]?.value);
146+
const textSpan = spans.find(span => inputOf(span).includes('charged twice'));
147+
const imageSpan = spans.find(span => inputOf(span).includes('package damaged'));
148+
149+
expect(textSpan!.name).toBe('decide models/gemini-3-flash');
150+
expect(textSpan!.status).toBe('ok');
151+
expect(textSpan!.attributes).toMatchObject({
152+
[SENTRY_ORIGIN]: { value: EXPECTED_ORIGIN },
153+
[GEN_AI_OPERATION_NAME]: { value: 'decide' },
154+
[GEN_AI_PROVIDER_NAME]: { value: 'google_genai' },
155+
[GEN_AI_REQUEST_MODEL]: { value: 'models/gemini-3-flash' },
156+
[GEN_AI_USAGE_INPUT_TOKENS]: { value: 382 },
157+
[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]: { value: 256 },
158+
[GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 0 },
159+
[GEN_AI_USAGE_TOTAL_TOKENS]: { value: 382 },
160+
});
161+
expect(JSON.parse(textSpan!.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)).toEqual([
162+
{
163+
type: 'evaluation',
164+
state: 'I was charged twice and want a refund.',
165+
questions: {
166+
wants_refund: {
167+
type: 'noul',
168+
instructions: 'Is a refund requested?',
169+
criteria: { true_description: 'Asks for money back' },
170+
},
171+
department: {
172+
type: 'choice',
173+
instructions: 'Which team should handle this?',
174+
criteria: { billing: 'Charges and refunds', other: 'Anything else' },
175+
},
176+
urgency: { type: 'score', instructions: 'How urgent is this ticket?', criteria: ['low', 'high'] },
177+
},
178+
},
179+
]);
180+
expect(JSON.parse(textSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]!.value as string)).toEqual([
181+
{
182+
type: 'evaluation',
183+
answers: {
184+
wants_refund: { type: 'noul', noul: 0.97, confidence: 0.94 },
185+
department: {
186+
type: 'choice',
187+
choice: 'billing',
188+
probabilities: { billing: 0.82, other: 0.18 },
189+
confidence: 0.82,
190+
},
191+
urgency: {
192+
type: 'score',
193+
score: 1,
194+
probabilities: { 0: 0.05, 1: 0.95 },
195+
legend: { 0: 'low', 1: 'high' },
196+
confidence: 0.93,
197+
},
198+
},
199+
},
200+
]);
201+
202+
expect(JSON.parse(imageSpan!.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)[0].state).toEqual([
203+
{ type: 'text', text: 'Is the package damaged?' },
204+
{ type: 'image', mime_type: 'image/png', data: 'iVBORw0KGgoAAAANSUhEUg==' },
205+
]);
206+
207+
const errorSpan = spans.find(span => span.name === 'decide models/error-model');
208+
expect(errorSpan!.status).toBe('error');
209+
expect(errorSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('decide');
210+
});
211+
},
212+
{ additionalDependencies: { '@google/genai': '^3' } },
213+
);
121214
});

‎packages/server-utils/src/ai/google-genai/constants.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,15 @@ export const GOOGLE_GENAI_INTEGRATION_NAME = 'Google_GenAI' as const;
66
// https://ai.google.dev/api/rest/v1/chats/sendMessage
77
// https://googleapis.github.io/js-genai/release_docs/classes/models.Models.html#generatecontentstream
88
// https://googleapis.github.io/js-genai/release_docs/classes/chats.Chat.html#sendmessagestream
9+
// https://ai.google.dev/api/decisions
910
export const GOOGLE_GENAI_METHOD_REGISTRY = {
1011
'models.generateContent': { operation: 'generate_content' },
1112
'models.generateContentStream': { operation: 'generate_content', streaming: true },
1213
'models.embedContent': { operation: 'embeddings' },
1314
'chats.create': { proxyResultPath: 'chat' },
1415
'chat.sendMessage': { operation: 'chat' },
1516
'chat.sendMessageStream': { operation: 'chat', streaming: true },
17+
'decisions.create': { operation: 'decide' },
1618
} as const satisfies InstrumentedMethodRegistry;
1719

1820
// Constants for internal use
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// Google's questions and answers already use Jev's layout, so only the evaluated input is renamed.
2+
import { GEN_AI_OUTPUT_MESSAGES } from '@sentry/conventions/attributes';
3+
import type { Span } from '@sentry/core';
4+
import { isObjectLike } from '@sentry/core';
5+
import { getEvaluationInputMessages, getEvaluationOutputMessages, getTokenUsageAttributes } from '../core/utils';
6+
7+
/**
8+
* Serialize a decisions request. Its evidence is `input` where Jev's is `state`; it is recorded as `state`
9+
* so both render the same.
10+
*/
11+
export function getDecisionInputMessages(params: Record<string, unknown>): string | undefined {
12+
return getEvaluationInputMessages({ state: params.input, questions: params.questions });
13+
}
14+
15+
export function addDecisionResponseAttributes(span: Span, response: unknown, recordOutputs: boolean): void {
16+
if (!isObjectLike(response)) {
17+
return;
18+
}
19+
20+
if (isObjectLike(response.usage)) {
21+
const usage = response.usage as Record<string, number | undefined>;
22+
span.setAttributes(
23+
getTokenUsageAttributes(
24+
usage.total_input_tokens,
25+
usage.total_output_tokens,
26+
undefined,
27+
usage.total_cached_tokens,
28+
usage.total_tokens,
29+
),
30+
);
31+
}
32+
33+
if (recordOutputs && response.answers !== undefined) {
34+
span.setAttribute(GEN_AI_OUTPUT_MESSAGES, getEvaluationOutputMessages(response.answers));
35+
}
36+
}

‎packages/server-utils/src/ai/google-genai/index.ts‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,12 @@ import {
3434
buildMethodPath,
3535
getErrorStatus,
3636
getGenAiSpanOp,
37+
getRequestSignal,
3738
getTokenUsageAttributes,
3839
resolveAIRecordingOptions,
3940
} from '../core/utils';
4041
import { GOOGLE_GENAI_METHOD_REGISTRY, GOOGLE_GENAI_SYSTEM_NAME } from './constants';
42+
import { addDecisionResponseAttributes, getDecisionInputMessages } from './decisions';
4143
import { instrumentStream } from './streaming';
4244
import type { Candidate, ContentPart, GoogleGenAIOptions, GoogleGenAIResponse } from './types';
4345
import type { ContentListUnion, Message, PartListUnion } from './utils';
@@ -135,6 +137,11 @@ export function extractRequestAttributes(
135137
attributes[GEN_AI_TOOL_DEFINITIONS] = JSON.stringify(functionDeclarations);
136138
}
137139
}
140+
141+
const decisionInput = recordInputs && operationName === 'decide' ? getDecisionInputMessages(params) : undefined;
142+
if (decisionInput) {
143+
attributes[GEN_AI_INPUT_MESSAGES] = decisionInput;
144+
}
138145
} else {
139146
attributes[GEN_AI_REQUEST_MODEL] = extractModel({}, context);
140147
}
@@ -198,9 +205,19 @@ export function addPrivateRequestAttributes(span: Span, params: Record<string, u
198205
* Add response attributes from the Google GenAI response
199206
* @see https://github.com/googleapis/js-genai/blob/v1.19.0/src/types.ts#L2313
200207
*/
201-
export function addResponseAttributes(span: Span, response: GoogleGenAIResponse, recordOutputs?: boolean): void {
208+
export function addResponseAttributes(
209+
span: Span,
210+
response: GoogleGenAIResponse,
211+
recordOutputs?: boolean,
212+
operationName?: string,
213+
): void {
202214
if (!response || typeof response !== 'object') return;
203215

216+
if (operationName === 'decide') {
217+
addDecisionResponseAttributes(span, response, !!recordOutputs);
218+
return;
219+
}
220+
204221
if (response.modelVersion) {
205222
span.setAttribute(GEN_AI_RESPONSE_MODEL, response.modelVersion);
206223
}
@@ -299,6 +316,15 @@ export function getAbortSignal(params: Record<string, unknown> | undefined): Abo
299316
return (params?.config as { abortSignal?: AbortSignal } | undefined)?.abortSignal;
300317
}
301318

319+
/** `decisions.create` takes its abort signal in its request options, not in a `config`. */
320+
export function getCallAbortSignal(
321+
operationName: string,
322+
params: Record<string, unknown> | undefined,
323+
requestOptions: unknown,
324+
): AbortSignal | undefined {
325+
return operationName === 'decide' ? getRequestSignal(requestOptions) : getAbortSignal(params);
326+
}
327+
302328
/**
303329
* Instrument any async or synchronous genai method with Sentry spans
304330
* Handles operations like models.generateContent and chat.sendMessage and chats.create
@@ -318,7 +344,7 @@ function instrumentMethod<T extends unknown[], R>(
318344
const operationName = instrumentedMethod.operation || 'unknown';
319345
const params = args[0] as Record<string, unknown> | undefined;
320346
const attributeParams = resolveChatParams(operationName, params, context);
321-
const abortSignal = getAbortSignal(attributeParams);
347+
const abortSignal = getCallAbortSignal(operationName, attributeParams, args[1]);
322348
const requestAttributes = extractRequestAttributes(
323349
operationName,
324350
attributeParams,
@@ -377,7 +403,7 @@ function instrumentMethod<T extends unknown[], R>(
377403
result => {
378404
// Only add response attributes for content-producing methods, not for embeddings
379405
if (!isEmbeddings) {
380-
addResponseAttributes(span, result, options.recordOutputs);
406+
addResponseAttributes(span, result, options.recordOutputs, operationName);
381407
}
382408
},
383409
);

‎packages/server-utils/src/integrations/google-genai.ts‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import {
1515
addResponseAttributes,
1616
extractRequestAttributes,
1717
getAbortSignal,
18+
getCallAbortSignal,
1819
resolveChatParams,
1920
} from '../ai/google-genai';
2021
import { shouldSkipAiProviderSpan } from '../ai/pi-ai/providers';
@@ -37,6 +38,7 @@ const INSTRUMENTED_CHANNELS = [
3738
{ channel: CHANNELS.GOOGLE_GENAI_GENERATE_CONTENT, operation: 'generate_content' },
3839
{ channel: CHANNELS.GOOGLE_GENAI_EMBED_CONTENT, operation: 'embeddings' },
3940
{ channel: CHANNELS.GOOGLE_GENAI_CHAT, operation: 'chat' },
41+
{ channel: CHANNELS.GOOGLE_GENAI_DECIDE, operation: 'decide' },
4042
] as const;
4143

4244
const _googleGenAIIntegration = ((options: GoogleGenAIOptions = {}) => {
@@ -61,14 +63,15 @@ function instrumentGoogleGenai(options: GoogleGenAIOptions): void {
6163
data.arguments?.[0] as Record<string, unknown> | undefined,
6264
data.self,
6365
);
64-
markCancelledIfAborted(span, getAbortSignal(params));
66+
markCancelledIfAborted(span, getCallAbortSignal(operation, params, data.arguments?.[1]));
6567
}
6668
// Embeddings responses carry no content attributes.
6769
if (operation !== 'embeddings') {
6870
addResponseAttributes(
6971
span,
7072
data.result as GoogleGenAIResponse,
7173
resolveAIRecordingOptions(options).recordOutputs,
74+
operation,
7275
);
7376
}
7477
},
@@ -171,8 +174,8 @@ function wrapStreamResult(
171174
/**
172175
* Diagnostics-channel-based Google GenAI integration. Subscribes to the
173176
* `orchestrion:@google/genai:*` diagnostics_channels injected into the SDK's `Models`
174-
* (`generateContent`/`generateContentStream`/`embedContent`) and `Chat`
175-
* (`sendMessage`/`sendMessageStream`) methods, so it requires the Sentry runtime hook or
176-
* bundler plugin.
177+
* (`generateContent`/`generateContentStream`/`embedContent`), `Chat`
178+
* (`sendMessage`/`sendMessageStream`) and decisions (`create`) methods, so it requires the Sentry
179+
* runtime hook or bundler plugin.
177180
*/
178181
export const googleGenAIIntegration = defineIntegration(_googleGenAIIntegration);

0 commit comments

Comments
 (0)