|
1 | 1 | import { afterAll, describe, expect } from 'vitest'; |
2 | 2 | import { |
| 3 | + GEN_AI_INPUT_MESSAGES, |
3 | 4 | GEN_AI_OPERATION_NAME, |
| 5 | + GEN_AI_OUTPUT_MESSAGES, |
4 | 6 | GEN_AI_PROVIDER_NAME, |
5 | 7 | GEN_AI_REQUEST_MAX_TOKENS, |
6 | 8 | GEN_AI_REQUEST_MODEL, |
7 | 9 | GEN_AI_REQUEST_TEMPERATURE, |
8 | 10 | GEN_AI_REQUEST_TOP_P, |
| 11 | + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, |
9 | 12 | GEN_AI_USAGE_INPUT_TOKENS, |
10 | 13 | GEN_AI_USAGE_OUTPUT_TOKENS, |
11 | 14 | GEN_AI_USAGE_TOTAL_TOKENS, |
12 | 15 | SENTRY_OP, |
13 | 16 | SENTRY_ORIGIN, |
14 | 17 | } from '@sentry/conventions/attributes'; |
15 | | -import { GEN_AI_CHAT, GEN_AI_EMBEDDINGS, GEN_AI_GENERATE_CONTENT } from '@sentry/conventions/op'; |
| 18 | +import { GEN_AI_CHAT, GEN_AI_DECIDE, GEN_AI_EMBEDDINGS, GEN_AI_GENERATE_CONTENT } from '@sentry/conventions/op'; |
16 | 19 | import { supports } from '../../../utils'; |
17 | 20 | import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; |
18 | 21 |
|
@@ -118,4 +121,94 @@ describe.runIf(supports({ min: 22 }))('Google GenAI integration (v3)', () => { |
118 | 121 | }, |
119 | 122 | { additionalDependencies: { '@google/genai': '^3' } }, |
120 | 123 | ); |
| 124 | + |
| 125 | + createEsmAndCjsTests( |
| 126 | + __dirname, |
| 127 | + 'scenario-decisions.mjs', |
| 128 | + 'instrument-with-recording.mjs', |
| 129 | + (createRunner, test) => { |
| 130 | + test('instruments decisions.create as decide spans on @google/genai v3', async () => { |
| 131 | + const runner = createRunner().ignore('event'); |
| 132 | + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); |
| 133 | + |
| 134 | + await runner.start().completed(); |
| 135 | + |
| 136 | + const allSpans = await spansPromise; |
| 137 | + const mainSpan = allSpans.find(span => span.name === 'main'); |
| 138 | + const spans = allSpans.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE); |
| 139 | + expect(spans).toHaveLength(3); |
| 140 | + for (const span of spans) { |
| 141 | + expect(span.parent_span_id).toBe(mainSpan!.span_id); |
| 142 | + } |
| 143 | + |
| 144 | + // Both successful spans share a name and may arrive in any order, so match on the input. |
| 145 | + const inputOf = (span: (typeof spans)[number]): string => String(span.attributes[GEN_AI_INPUT_MESSAGES]?.value); |
| 146 | + const textSpan = spans.find(span => inputOf(span).includes('charged twice')); |
| 147 | + const imageSpan = spans.find(span => inputOf(span).includes('package damaged')); |
| 148 | + |
| 149 | + expect(textSpan!.name).toBe('decide models/gemini-3-flash'); |
| 150 | + expect(textSpan!.status).toBe('ok'); |
| 151 | + expect(textSpan!.attributes).toMatchObject({ |
| 152 | + [SENTRY_ORIGIN]: { value: EXPECTED_ORIGIN }, |
| 153 | + [GEN_AI_OPERATION_NAME]: { value: 'decide' }, |
| 154 | + [GEN_AI_PROVIDER_NAME]: { value: 'google_genai' }, |
| 155 | + [GEN_AI_REQUEST_MODEL]: { value: 'models/gemini-3-flash' }, |
| 156 | + [GEN_AI_USAGE_INPUT_TOKENS]: { value: 382 }, |
| 157 | + [GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]: { value: 256 }, |
| 158 | + [GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 0 }, |
| 159 | + [GEN_AI_USAGE_TOTAL_TOKENS]: { value: 382 }, |
| 160 | + }); |
| 161 | + expect(JSON.parse(textSpan!.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)).toEqual([ |
| 162 | + { |
| 163 | + type: 'evaluation', |
| 164 | + state: 'I was charged twice and want a refund.', |
| 165 | + questions: { |
| 166 | + wants_refund: { |
| 167 | + type: 'noul', |
| 168 | + instructions: 'Is a refund requested?', |
| 169 | + criteria: { true_description: 'Asks for money back' }, |
| 170 | + }, |
| 171 | + department: { |
| 172 | + type: 'choice', |
| 173 | + instructions: 'Which team should handle this?', |
| 174 | + criteria: { billing: 'Charges and refunds', other: 'Anything else' }, |
| 175 | + }, |
| 176 | + urgency: { type: 'score', instructions: 'How urgent is this ticket?', criteria: ['low', 'high'] }, |
| 177 | + }, |
| 178 | + }, |
| 179 | + ]); |
| 180 | + expect(JSON.parse(textSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]!.value as string)).toEqual([ |
| 181 | + { |
| 182 | + type: 'evaluation', |
| 183 | + answers: { |
| 184 | + wants_refund: { type: 'noul', noul: 0.97, confidence: 0.94 }, |
| 185 | + department: { |
| 186 | + type: 'choice', |
| 187 | + choice: 'billing', |
| 188 | + probabilities: { billing: 0.82, other: 0.18 }, |
| 189 | + confidence: 0.82, |
| 190 | + }, |
| 191 | + urgency: { |
| 192 | + type: 'score', |
| 193 | + score: 1, |
| 194 | + probabilities: { 0: 0.05, 1: 0.95 }, |
| 195 | + legend: { 0: 'low', 1: 'high' }, |
| 196 | + confidence: 0.93, |
| 197 | + }, |
| 198 | + }, |
| 199 | + }, |
| 200 | + ]); |
| 201 | + |
| 202 | + expect(JSON.parse(imageSpan!.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)[0].state).toEqual([ |
| 203 | + { type: 'text', text: 'Is the package damaged?' }, |
| 204 | + { type: 'image', mime_type: 'image/png', data: 'iVBORw0KGgoAAAANSUhEUg==' }, |
| 205 | + ]); |
| 206 | + |
| 207 | + const errorSpan = spans.find(span => span.name === 'decide models/error-model'); |
| 208 | + expect(errorSpan!.status).toBe('error'); |
| 209 | + expect(errorSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('decide'); |
| 210 | + }); |
| 211 | + }, |
| 212 | + { additionalDependencies: { '@google/genai': '^3' } }, |
| 213 | + ); |
121 | 214 | }); |
0 commit comments