diff --git a/dev-packages/bun-integration-tests/node-suites/excludes.ts b/dev-packages/bun-integration-tests/node-suites/excludes.ts index ff7e526ad3db..643ebd7cb91c 100644 --- a/dev-packages/bun-integration-tests/node-suites/excludes.ts +++ b/dev-packages/bun-integration-tests/node-suites/excludes.ts @@ -86,6 +86,7 @@ export const NO_AUTO_INSTRUMENTATION = [ 'suites/tracing/genericPool/test.ts', 'suites/tracing/is-localhost/test.ts', 'suites/tracing/google-genai-v2/test.ts', + 'suites/tracing/google-genai-v3/test.ts', 'suites/tracing/google-genai/test.ts', 'suites/tracing/groq/test.ts', 'suites/tracing/hapi/test.ts', @@ -240,6 +241,7 @@ const BUN_BUILD_NOT_TRIAGED = [ 'suites/hono-sdk/test.ts', 'suites/pino/test.ts', 'suites/tracing/google-genai-v2/test.ts', + 'suites/tracing/google-genai-v3/test.ts', 'suites/tracing/google-genai/test.ts', 'suites/tracing/langchain/v1/test.ts', 'suites/tracing/langgraph/v1/test.ts', diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/test.ts b/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/test.ts index 1025569ac5d4..ed443b8ba20d 100644 --- a/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai-v2/test.ts @@ -17,10 +17,10 @@ import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runn const EXPECTED_ORIGIN = 'auto.ai.google_genai'; // `@google/genai` v2 restructured the `Models` class so `embedContent` is a constructor-assigned arrow -// property rather than a class method (v1 shape). The orchestrion config caps at `<3`, so the code -// transformer only injects the diagnostics channels for v2 when the range includes it — this suite pins -// `^2` and re-runs the core auto-instrumentation assertions to guard that path. The v1 suite lives in -// `../google-genai`; the scenario files are byte-identical because v2 kept the public API surface. +// property rather than a class method (v1 shape). The code transformer only injects the diagnostics +// channels for v2 when the orchestrion range includes it, so this suite pins `^2` and re-runs the core +// auto-instrumentation assertions to guard that path. The v1 suite lives in `../google-genai`; the +// scenario files are byte-identical because v2 kept the public API surface. describe('Google GenAI integration (v2)', () => { afterAll(() => { cleanupChildProcesses(); diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai-v3/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/google-genai-v3/instrument.mjs new file mode 100644 index 000000000000..cc192fb89834 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai-v3/instrument.mjs @@ -0,0 +1,10 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + dataCollection: { genAI: { inputs: false, outputs: false } }, + transport: loggingTransport, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai-v3/scenario-embeddings.mjs b/dev-packages/node-integration-tests/suites/tracing/google-genai-v3/scenario-embeddings.mjs new file mode 100644 index 000000000000..cf60d5d61793 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai-v3/scenario-embeddings.mjs @@ -0,0 +1,77 @@ +import { GoogleGenAI } from '@google/genai'; +import * as Sentry from '@sentry/node'; +import express from 'express'; + +function startMockGoogleGenAIServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/models/:model\\:batchEmbedContents', (req, res) => { + const model = req.params.model; + + if (model === 'error-model') { + res.status(404).set('x-request-id', 'mock-request-123').end('Model not found'); + return; + } + + res.send({ + embeddings: [ + { + values: [0.1, 0.2, 0.3, 0.4, 0.5], + }, + ], + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockGoogleGenAIServer(); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const client = new GoogleGenAI({ + apiKey: 'mock-api-key', + httpOptions: { baseUrl: `http://localhost:${server.address().port}` }, + }); + + // Test 1: Basic embedContent with string contents + await client.models.embedContent({ + model: 'text-embedding-004', + contents: 'What is the capital of France?', + }); + + // Test 2: Error handling + try { + await client.models.embedContent({ + model: 'error-model', + contents: 'This will fail', + }); + } catch { + // Expected error + } + + // Test 3: embedContent with array contents + await client.models.embedContent({ + model: 'text-embedding-004', + contents: [ + { + role: 'user', + parts: [{ text: 'First input text' }], + }, + { + role: 'user', + parts: [{ text: 'Second input text' }], + }, + ], + }); + }); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai-v3/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/google-genai-v3/scenario.mjs new file mode 100644 index 000000000000..b65449cc85e2 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai-v3/scenario.mjs @@ -0,0 +1,114 @@ +import { GoogleGenAI } from '@google/genai'; +import * as Sentry from '@sentry/node'; +import express from 'express'; + +function startMockGoogleGenAIServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/models/:model\\:generateContent', (req, res) => { + const model = req.params.model; + + if (model === 'error-model') { + res.status(404).set('x-request-id', 'mock-request-123').end('Model not found'); + return; + } + + res.send({ + candidates: [ + { + content: { + parts: [ + { + text: 'Mock response from Google GenAI!', + }, + ], + role: 'model', + }, + finishReason: 'stop', + index: 0, + }, + ], + usageMetadata: { + promptTokenCount: 8, + candidatesTokenCount: 12, + totalTokenCount: 20, + }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockGoogleGenAIServer(); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const client = new GoogleGenAI({ + apiKey: 'mock-api-key', + httpOptions: { baseUrl: `http://localhost:${server.address().port}` }, + }); + + // Test 1: chats.create and sendMessage flow + // `chats.create()` builds a local object and emits no span; only `sendMessage` does, and it + // reports the config given here. + const chat = client.chats.create({ + model: 'gemini-1.5-pro', + config: { + temperature: 0.8, + topP: 0.9, + maxOutputTokens: 150, + systemInstruction: 'You are a friendly robot who likes to be funny.', + }, + history: [ + { + role: 'user', + parts: [{ text: 'Hello, how are you?' }], + }, + ], + }); + + await chat.sendMessage({ + message: 'Tell me a joke', + }); + + // Test 2: models.generateContent + await client.models.generateContent({ + model: 'gemini-1.5-flash', + config: { + temperature: 0.7, + topP: 0.9, + maxOutputTokens: 100, + }, + contents: [ + { + role: 'user', + parts: [{ text: 'What is the capital of France?' }], + }, + ], + }); + + // Test 3: Error handling + try { + await client.models.generateContent({ + model: 'error-model', + contents: [ + { + role: 'user', + parts: [{ text: 'This will fail' }], + }, + ], + }); + } catch { + // Expected error + } + }); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/google-genai-v3/test.ts b/dev-packages/node-integration-tests/suites/tracing/google-genai-v3/test.ts new file mode 100644 index 000000000000..b63580b79057 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/google-genai-v3/test.ts @@ -0,0 +1,121 @@ +import { afterAll, describe, expect } from 'vitest'; +import { + GEN_AI_OPERATION_NAME, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MAX_TOKENS, + GEN_AI_REQUEST_MODEL, + GEN_AI_REQUEST_TEMPERATURE, + GEN_AI_REQUEST_TOP_P, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import { GEN_AI_CHAT, GEN_AI_EMBEDDINGS, GEN_AI_GENERATE_CONTENT } from '@sentry/conventions/op'; +import { supports } from '../../../utils'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; + +const EXPECTED_ORIGIN = 'auto.ai.google_genai'; + +// `@google/genai` v3 needs Node 22 and calls the `v1` API instead of `v1beta`, so the scenarios here +// differ from `../google-genai-v2` only in the mock server routes. +describe.runIf(supports({ min: 22 }))('Google GenAI integration (v3)', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('auto-instruments chat and generateContent on @google/genai v3', async () => { + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = (await spansPromise).filter(span => span.attributes[SENTRY_ORIGIN]?.value === EXPECTED_ORIGIN); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual([ + 'chat gemini-1.5-pro', + 'generate_content error-model', + 'generate_content gemini-1.5-flash', + ]); + + const chatSpan = spans.find(span => span.name === 'chat gemini-1.5-pro'); + expect(chatSpan!.status).toBe('ok'); + expect(chatSpan!.attributes[SENTRY_OP].value).toBe(GEN_AI_CHAT); + expect(chatSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(chatSpan!.attributes[SENTRY_ORIGIN].value).toBe(EXPECTED_ORIGIN); + expect(chatSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('google_genai'); + expect(chatSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gemini-1.5-pro'); + expect(chatSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(8); + expect(chatSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); + expect(chatSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); + + const generateContentSpan = spans.find(span => span.name === 'generate_content gemini-1.5-flash'); + expect(generateContentSpan!.status).toBe('ok'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe(GEN_AI_GENERATE_CONTENT); + expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); + expect(generateContentSpan!.attributes[SENTRY_ORIGIN].value).toBe(EXPECTED_ORIGIN); + expect(generateContentSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('google_genai'); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gemini-1.5-flash'); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.9); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(8); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); + + const errorSpan = spans.find(span => span.name === 'generate_content error-model'); + expect(errorSpan!.status).toBe('error'); + expect(errorSpan!.attributes[SENTRY_OP].value).toBe(GEN_AI_GENERATE_CONTENT); + expect(errorSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); + expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); + }); + }, + { additionalDependencies: { '@google/genai': '^3' } }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-embeddings.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('auto-instruments embedContent on @google/genai v3', async () => { + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = (await spansPromise).filter(span => span.attributes[SENTRY_ORIGIN]?.value === EXPECTED_ORIGIN); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual([ + 'embeddings error-model', + 'embeddings text-embedding-004', + 'embeddings text-embedding-004', + ]); + + const successfulSpans = spans.filter( + span => span.name === 'embeddings text-embedding-004' && span.status === 'ok', + ); + expect(successfulSpans).toHaveLength(2); + for (const span of successfulSpans) { + expect(span.attributes[SENTRY_OP].value).toBe(GEN_AI_EMBEDDINGS); + expect(span.attributes[GEN_AI_OPERATION_NAME].value).toBe('embeddings'); + expect(span.attributes[SENTRY_ORIGIN].value).toBe(EXPECTED_ORIGIN); + expect(span.attributes[GEN_AI_PROVIDER_NAME].value).toBe('google_genai'); + expect(span.attributes[GEN_AI_REQUEST_MODEL].value).toBe('text-embedding-004'); + } + + const errorSpan = spans.find(span => span.name === 'embeddings error-model'); + expect(errorSpan!.status).toBe('error'); + expect(errorSpan!.attributes[SENTRY_OP].value).toBe(GEN_AI_EMBEDDINGS); + expect(errorSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('embeddings'); + }); + }, + { additionalDependencies: { '@google/genai': '^3' } }, + ); +}); diff --git a/packages/server-utils/src/orchestrion/config/google-genai.ts b/packages/server-utils/src/orchestrion/config/google-genai.ts index af8b416721bf..d643ed6e59d1 100644 --- a/packages/server-utils/src/orchestrion/config/google-genai.ts +++ b/packages/server-utils/src/orchestrion/config/google-genai.ts @@ -14,7 +14,7 @@ export const googleGenAiConfig = [ ...NODE_DIST_FILES.flatMap(filePath => (['generateContent', 'generateContentStream'] as const).map(expressionName => ({ channelName: 'generate-content', - module: { name: '@google/genai', versionRange: '>=0.10.0 <3', filePath }, + module: { name: '@google/genai', versionRange: '>=0.10.0 <4', filePath }, functionQuery: { expressionName, kind: 'Auto' as const }, })), ), @@ -22,7 +22,7 @@ export const googleGenAiConfig = [ // `className`/`methodName` selector matches both shapes. ...NODE_DIST_FILES.map(filePath => ({ channelName: 'embed-content', - module: { name: '@google/genai', versionRange: '>=0.10.0 <3', filePath }, + module: { name: '@google/genai', versionRange: '>=0.10.0 <4', filePath }, functionQuery: { className: 'Models', methodName: 'embedContent', kind: 'Auto' as const }, })), // `sendMessage`/`sendMessageStream` internally delegate to `Models.generateContent(Stream)`; the @@ -30,7 +30,7 @@ export const googleGenAiConfig = [ ...NODE_DIST_FILES.flatMap(filePath => (['sendMessage', 'sendMessageStream'] as const).map(methodName => ({ channelName: 'chat', - module: { name: '@google/genai', versionRange: '>=0.10.0 <3', filePath }, + module: { name: '@google/genai', versionRange: '>=0.10.0 <4', filePath }, functionQuery: { className: 'Chat', methodName, kind: 'Auto' as const }, })), ),