Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import * as Sentry from '@sentry/node';
import { loggingTransport } from '@sentry-internal/node-integration-tests';

Sentry.init({
dsn: 'https://public@dsn.ingest.sentry.io/1337',
release: '1.0',
tracesSampleRate: 1.0,
transport: loggingTransport,
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
import { GoogleGenAI } from '@google/genai';
import * as Sentry from '@sentry/node';
import express from 'express';

function startMockGoogleGenAIServer() {
const app = express();
app.use(express.json({ limit: '1mb' }));

app.post('/v1/decisions', (req, res) => {
if (req.body.model === 'models/error-model') {
// A 400, not a 500: the decisions client retries 5xx responses with backoff by default.
res.status(400).end('Bad request');
return;
}

res.send({
answers: {
wants_refund: { type: 'noul', noul: 0.97, confidence: 0.94 },
department: {
type: 'choice',
choice: 'billing',
probabilities: { billing: 0.82, other: 0.18 },
confidence: 0.82,
},
urgency: {
type: 'score',
score: 1,
probabilities: { 0: 0.05, 1: 0.95 },
legend: { 0: 'low', 1: 'high' },
confidence: 0.93,
},
},
usage: { total_input_tokens: 382, total_cached_tokens: 256, total_output_tokens: 0, total_tokens: 382 },
});
});

return new Promise(resolve => {
const server = app.listen(0, () => {
resolve(server);
});
});
}

const questions = {
wants_refund: {
type: 'noul',
instructions: 'Is a refund requested?',
criteria: { true_description: 'Asks for money back' },
},
department: {
type: 'choice',
instructions: 'Which team should handle this?',
criteria: { billing: 'Charges and refunds', other: 'Anything else' },
},
urgency: { type: 'score', instructions: 'How urgent is this ticket?', criteria: ['low', 'high'] },
};

async function run() {
const server = await startMockGoogleGenAIServer();

await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
const client = new GoogleGenAI({
apiKey: 'mock-api-key',
httpOptions: { baseUrl: `http://localhost:${server.address().port}` },
});

await client.decisions.create({
model: 'models/gemini-3-flash',
input: 'I was charged twice and want a refund.',
questions,
});

await client.decisions.create({
model: 'models/gemini-3-flash',
input: [
{ type: 'text', text: 'Is the package damaged?' },
{ type: 'image', mime_type: 'image/png', data: 'iVBORw0KGgoAAAANSUhEUg==' },
],
questions: { damaged: { type: 'noul', instructions: 'Is there visible damage?' } },
});

try {
await client.decisions.create({ model: 'models/error-model', input: 'Error test', questions });
} catch {}
});

server.close();
}

run();
Original file line number Diff line number Diff line change
@@ -1,18 +1,21 @@
import { afterAll, describe, expect } from 'vitest';
import {
GEN_AI_INPUT_MESSAGES,
GEN_AI_OPERATION_NAME,
GEN_AI_OUTPUT_MESSAGES,
GEN_AI_PROVIDER_NAME,
GEN_AI_REQUEST_MAX_TOKENS,
GEN_AI_REQUEST_MODEL,
GEN_AI_REQUEST_TEMPERATURE,
GEN_AI_REQUEST_TOP_P,
GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS,
GEN_AI_USAGE_INPUT_TOKENS,
GEN_AI_USAGE_OUTPUT_TOKENS,
GEN_AI_USAGE_TOTAL_TOKENS,
SENTRY_OP,
SENTRY_ORIGIN,
} from '@sentry/conventions/attributes';
import { GEN_AI_CHAT, GEN_AI_EMBEDDINGS, GEN_AI_GENERATE_CONTENT } from '@sentry/conventions/op';
import { GEN_AI_CHAT, GEN_AI_DECIDE, GEN_AI_EMBEDDINGS, GEN_AI_GENERATE_CONTENT } from '@sentry/conventions/op';
import { supports } from '../../../utils';
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner';

Expand Down Expand Up @@ -118,4 +121,94 @@ describe.runIf(supports({ min: 22 }))('Google GenAI integration (v3)', () => {
},
{ additionalDependencies: { '@google/genai': '^3' } },
);

createEsmAndCjsTests(
__dirname,
'scenario-decisions.mjs',
'instrument-with-recording.mjs',
(createRunner, test) => {
test('instruments decisions.create as decide spans on @google/genai v3', async () => {
const runner = createRunner().ignore('event');
const spansPromise = runner.collectStreamedSpansUntilSegment('main');

await runner.start().completed();

const allSpans = await spansPromise;
const mainSpan = allSpans.find(span => span.name === 'main');
const spans = allSpans.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE);
expect(spans).toHaveLength(3);
for (const span of spans) {
expect(span.parent_span_id).toBe(mainSpan!.span_id);
}

// Both successful spans share a name and may arrive in any order, so match on the input.
const inputOf = (span: (typeof spans)[number]): string => String(span.attributes[GEN_AI_INPUT_MESSAGES]?.value);
const textSpan = spans.find(span => inputOf(span).includes('charged twice'));
const imageSpan = spans.find(span => inputOf(span).includes('package damaged'));

expect(textSpan!.name).toBe('decide models/gemini-3-flash');
expect(textSpan!.status).toBe('ok');
expect(textSpan!.attributes).toMatchObject({
[SENTRY_ORIGIN]: { value: EXPECTED_ORIGIN },
[GEN_AI_OPERATION_NAME]: { value: 'decide' },
[GEN_AI_PROVIDER_NAME]: { value: 'google_genai' },
[GEN_AI_REQUEST_MODEL]: { value: 'models/gemini-3-flash' },
[GEN_AI_USAGE_INPUT_TOKENS]: { value: 382 },
[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]: { value: 256 },
[GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 0 },
[GEN_AI_USAGE_TOTAL_TOKENS]: { value: 382 },
});
expect(JSON.parse(textSpan!.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)).toEqual([
{
type: 'evaluation',
state: 'I was charged twice and want a refund.',
questions: {
wants_refund: {
type: 'noul',
instructions: 'Is a refund requested?',
criteria: { true_description: 'Asks for money back' },
},
department: {
type: 'choice',
instructions: 'Which team should handle this?',
criteria: { billing: 'Charges and refunds', other: 'Anything else' },
},
urgency: { type: 'score', instructions: 'How urgent is this ticket?', criteria: ['low', 'high'] },
},
},
]);
expect(JSON.parse(textSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]!.value as string)).toEqual([
{
type: 'evaluation',
answers: {
wants_refund: { type: 'noul', noul: 0.97, confidence: 0.94 },
department: {
type: 'choice',
choice: 'billing',
probabilities: { billing: 0.82, other: 0.18 },
confidence: 0.82,
},
urgency: {
type: 'score',
score: 1,
probabilities: { 0: 0.05, 1: 0.95 },
legend: { 0: 'low', 1: 'high' },
confidence: 0.93,
},
},
},
]);

expect(JSON.parse(imageSpan!.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)[0].state).toEqual([
{ type: 'text', text: 'Is the package damaged?' },
{ type: 'image', mime_type: 'image/png', data: 'iVBORw0KGgoAAAANSUhEUg==' },
]);

const errorSpan = spans.find(span => span.name === 'decide models/error-model');
expect(errorSpan!.status).toBe('error');
expect(errorSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('decide');
});
},
{ additionalDependencies: { '@google/genai': '^3' } },
);
});
2 changes: 2 additions & 0 deletions packages/server-utils/src/ai/google-genai/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@ export const GOOGLE_GENAI_INTEGRATION_NAME = 'Google_GenAI' as const;
// https://ai.google.dev/api/rest/v1/chats/sendMessage
// https://googleapis.github.io/js-genai/release_docs/classes/models.Models.html#generatecontentstream
// https://googleapis.github.io/js-genai/release_docs/classes/chats.Chat.html#sendmessagestream
// https://ai.google.dev/api/decisions
export const GOOGLE_GENAI_METHOD_REGISTRY = {
'models.generateContent': { operation: 'generate_content' },
'models.generateContentStream': { operation: 'generate_content', streaming: true },
'models.embedContent': { operation: 'embeddings' },
'chats.create': { proxyResultPath: 'chat' },
'chat.sendMessage': { operation: 'chat' },
'chat.sendMessageStream': { operation: 'chat', streaming: true },
'decisions.create': { operation: 'decide' },
} as const satisfies InstrumentedMethodRegistry;

// Constants for internal use
Expand Down
36 changes: 36 additions & 0 deletions packages/server-utils/src/ai/google-genai/decisions.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
// Google's questions and answers already use Jev's layout, so only the evaluated input is renamed.
import { GEN_AI_OUTPUT_MESSAGES } from '@sentry/conventions/attributes';
import type { Span } from '@sentry/core';
import { isObjectLike } from '@sentry/core';
import { getEvaluationInputMessages, getEvaluationOutputMessages, getTokenUsageAttributes } from '../core/utils';

/**
* Serialize a decisions request. Its evidence is `input` where Jev's is `state`; it is recorded as `state`
* so both render the same.
*/
export function getDecisionInputMessages(params: Record<string, unknown>): string | undefined {
return getEvaluationInputMessages({ state: params.input, questions: params.questions });
}

export function addDecisionResponseAttributes(span: Span, response: unknown, recordOutputs: boolean): void {
if (!isObjectLike(response)) {
return;
}

if (isObjectLike(response.usage)) {
const usage = response.usage as Record<string, number | undefined>;
span.setAttributes(
getTokenUsageAttributes(
usage.total_input_tokens,
usage.total_output_tokens,
undefined,
usage.total_cached_tokens,
usage.total_tokens,
),
);
}

if (recordOutputs && response.answers !== undefined) {
span.setAttribute(GEN_AI_OUTPUT_MESSAGES, getEvaluationOutputMessages(response.answers));
}
}
32 changes: 29 additions & 3 deletions packages/server-utils/src/ai/google-genai/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,12 @@ import {
buildMethodPath,
getErrorStatus,
getGenAiSpanOp,
getRequestSignal,
getTokenUsageAttributes,
resolveAIRecordingOptions,
} from '../core/utils';
import { GOOGLE_GENAI_METHOD_REGISTRY, GOOGLE_GENAI_SYSTEM_NAME } from './constants';
import { addDecisionResponseAttributes, getDecisionInputMessages } from './decisions';
import { instrumentStream } from './streaming';
import type { Candidate, ContentPart, GoogleGenAIOptions, GoogleGenAIResponse } from './types';
import type { ContentListUnion, Message, PartListUnion } from './utils';
Expand Down Expand Up @@ -135,6 +137,11 @@ export function extractRequestAttributes(
attributes[GEN_AI_TOOL_DEFINITIONS] = JSON.stringify(functionDeclarations);
}
}

const decisionInput = recordInputs && operationName === 'decide' ? getDecisionInputMessages(params) : undefined;
if (decisionInput) {
attributes[GEN_AI_INPUT_MESSAGES] = decisionInput;
}
} else {
attributes[GEN_AI_REQUEST_MODEL] = extractModel({}, context);
}
Expand Down Expand Up @@ -198,9 +205,19 @@ export function addPrivateRequestAttributes(span: Span, params: Record<string, u
* Add response attributes from the Google GenAI response
* @see https://github.com/googleapis/js-genai/blob/v1.19.0/src/types.ts#L2313
*/
export function addResponseAttributes(span: Span, response: GoogleGenAIResponse, recordOutputs?: boolean): void {
export function addResponseAttributes(
span: Span,
response: GoogleGenAIResponse,
recordOutputs?: boolean,
operationName?: string,
): void {
if (!response || typeof response !== 'object') return;

if (operationName === 'decide') {
addDecisionResponseAttributes(span, response, !!recordOutputs);
return;
}

if (response.modelVersion) {
span.setAttribute(GEN_AI_RESPONSE_MODEL, response.modelVersion);
}
Expand Down Expand Up @@ -299,6 +316,15 @@ export function getAbortSignal(params: Record<string, unknown> | undefined): Abo
return (params?.config as { abortSignal?: AbortSignal } | undefined)?.abortSignal;
}

/** `decisions.create` takes its abort signal in its request options, not in a `config`. */
export function getCallAbortSignal(
operationName: string,
params: Record<string, unknown> | undefined,
requestOptions: unknown,
): AbortSignal | undefined {
return operationName === 'decide' ? getRequestSignal(requestOptions) : getAbortSignal(params);
}

/**
* Instrument any async or synchronous genai method with Sentry spans
* Handles operations like models.generateContent and chat.sendMessage and chats.create
Expand All @@ -318,7 +344,7 @@ function instrumentMethod<T extends unknown[], R>(
const operationName = instrumentedMethod.operation || 'unknown';
const params = args[0] as Record<string, unknown> | undefined;
const attributeParams = resolveChatParams(operationName, params, context);
const abortSignal = getAbortSignal(attributeParams);
const abortSignal = getCallAbortSignal(operationName, attributeParams, args[1]);
const requestAttributes = extractRequestAttributes(
operationName,
attributeParams,
Expand Down Expand Up @@ -377,7 +403,7 @@ function instrumentMethod<T extends unknown[], R>(
result => {
// Only add response attributes for content-producing methods, not for embeddings
if (!isEmbeddings) {
addResponseAttributes(span, result, options.recordOutputs);
addResponseAttributes(span, result, options.recordOutputs, operationName);
}
},
);
Expand Down
11 changes: 7 additions & 4 deletions packages/server-utils/src/integrations/google-genai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
addResponseAttributes,
extractRequestAttributes,
getAbortSignal,
getCallAbortSignal,
resolveChatParams,
} from '../ai/google-genai';
import { shouldSkipAiProviderSpan } from '../ai/pi-ai/providers';
Expand All @@ -37,6 +38,7 @@ const INSTRUMENTED_CHANNELS = [
{ channel: CHANNELS.GOOGLE_GENAI_GENERATE_CONTENT, operation: 'generate_content' },
{ channel: CHANNELS.GOOGLE_GENAI_EMBED_CONTENT, operation: 'embeddings' },
{ channel: CHANNELS.GOOGLE_GENAI_CHAT, operation: 'chat' },
{ channel: CHANNELS.GOOGLE_GENAI_DECIDE, operation: 'decide' },
] as const;

const _googleGenAIIntegration = ((options: GoogleGenAIOptions = {}) => {
Expand All @@ -61,14 +63,15 @@ function instrumentGoogleGenai(options: GoogleGenAIOptions): void {
data.arguments?.[0] as Record<string, unknown> | undefined,
data.self,
);
markCancelledIfAborted(span, getAbortSignal(params));
markCancelledIfAborted(span, getCallAbortSignal(operation, params, data.arguments?.[1]));
}
// Embeddings responses carry no content attributes.
if (operation !== 'embeddings') {
addResponseAttributes(
span,
data.result as GoogleGenAIResponse,
resolveAIRecordingOptions(options).recordOutputs,
operation,
);
}
},
Expand Down Expand Up @@ -171,8 +174,8 @@ function wrapStreamResult(
/**
* Diagnostics-channel-based Google GenAI integration. Subscribes to the
* `orchestrion:@google/genai:*` diagnostics_channels injected into the SDK's `Models`
* (`generateContent`/`generateContentStream`/`embedContent`) and `Chat`
* (`sendMessage`/`sendMessageStream`) methods, so it requires the Sentry runtime hook or
* bundler plugin.
* (`generateContent`/`generateContentStream`/`embedContent`), `Chat`
* (`sendMessage`/`sendMessageStream`) and decisions (`create`) methods, so it requires the Sentry
* runtime hook or bundler plugin.
*/
export const googleGenAIIntegration = defineIntegration(_googleGenAIIntegration);
Loading
Loading