|
| 1 | +import { expect, test } from '@playwright/test'; |
| 2 | +import { collectStreamedSpans, getSpanOp } from '@sentry-internal/test-utils'; |
| 3 | +import { APP } from './constants'; |
| 4 | +import { attr, describeTree } from './utils'; |
| 5 | + |
| 6 | +// The span of an activity is active while the activity runs, so the provider request nests under it. |
| 7 | +// Each test waits for the segment span, which ends last, so that all spans of the trace are there. |
| 8 | + |
| 9 | +test('tanstack-ai: an embed() call emits a gen_ai.embeddings span, and no span of the OpenAI integration', async ({ |
| 10 | + baseURL, |
| 11 | +}) => { |
| 12 | + const spansPromise = collectStreamedSpans( |
| 13 | + APP, |
| 14 | + spansOfTrace => |
| 15 | + spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.embeddings') && spansOfTrace.some(span => span.is_segment), |
| 16 | + ); |
| 17 | + |
| 18 | + const response = await fetch(`${baseURL}/tanstack-ai/embed`); |
| 19 | + expect(response.status).toBe(200); |
| 20 | + |
| 21 | + const spans = await spansPromise; |
| 22 | + const tree = describeTree(spans); |
| 23 | + const workflow = spans.find(span => span.name === 'ai-workflow'); |
| 24 | + const embeddings = spans.filter(span => getSpanOp(span) === 'gen_ai.embeddings'); |
| 25 | + const providerRequests = spans.filter( |
| 26 | + span => getSpanOp(span) === 'http.client' && span.parent_span_id === embeddings[0]?.span_id, |
| 27 | + ); |
| 28 | + |
| 29 | + expect(embeddings, tree).toHaveLength(1); |
| 30 | + expect(embeddings[0]!.parent_span_id, tree).toBe(workflow?.span_id); |
| 31 | + expect(embeddings[0]!.name).toBe('embeddings openai/text-embedding-3-small'); |
| 32 | + expect(embeddings[0]!.status).toBe('ok'); |
| 33 | + expect(attr(embeddings[0]!, 'sentry.origin')).toBe('auto.ai.tanstack_ai'); |
| 34 | + expect(attr(embeddings[0]!, 'gen_ai.operation.name')).toBe('embeddings'); |
| 35 | + expect(attr(embeddings[0]!, 'gen_ai.provider.name')).toBe('openai'); |
| 36 | + expect(attr(embeddings[0]!, 'gen_ai.request.model')).toBe('openai/text-embedding-3-small'); |
| 37 | + expect(typeof attr(embeddings[0]!, 'gen_ai.response.model')).toBe('string'); |
| 38 | + expect(attr(embeddings[0]!, 'gen_ai.usage.input_tokens')).toBeGreaterThan(0); |
| 39 | + expect(attr(embeddings[0]!, 'gen_ai.embeddings.input')).toBe('["a red guitar","a blue drum kit"]'); |
| 40 | + |
| 41 | + expect(providerRequests, tree).toHaveLength(1); |
| 42 | + expect(String(attr(providerRequests[0]!, 'server.address') ?? providerRequests[0]!.name)).toContain('openrouter'); |
| 43 | +}); |
| 44 | + |
| 45 | +test('tanstack-ai: a decide() call emits a gen_ai.decide span', async ({ baseURL }) => { |
| 46 | + const spansPromise = collectStreamedSpans( |
| 47 | + APP, |
| 48 | + spansOfTrace => |
| 49 | + spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.decide') && spansOfTrace.some(span => span.is_segment), |
| 50 | + ); |
| 51 | + |
| 52 | + const response = await fetch(`${baseURL}/tanstack-ai/decide`); |
| 53 | + expect(response.status).toBe(200); |
| 54 | + |
| 55 | + const spans = await spansPromise; |
| 56 | + const tree = describeTree(spans); |
| 57 | + const workflow = spans.find(span => span.name === 'ai-workflow'); |
| 58 | + const decideSpan = spans.find(span => getSpanOp(span) === 'gen_ai.decide'); |
| 59 | + const providerRequests = spans.filter( |
| 60 | + span => getSpanOp(span) === 'http.client' && span.parent_span_id === decideSpan?.span_id, |
| 61 | + ); |
| 62 | + |
| 63 | + expect(decideSpan, tree).toBeDefined(); |
| 64 | + expect(decideSpan!.parent_span_id, tree).toBe(workflow?.span_id); |
| 65 | + expect(decideSpan!.name).toBe('decide ~typesafe/jev-latest'); |
| 66 | + expect(decideSpan!.status).toBe('ok'); |
| 67 | + expect(attr(decideSpan!, 'sentry.origin')).toBe('auto.ai.tanstack_ai'); |
| 68 | + expect(attr(decideSpan!, 'gen_ai.operation.name')).toBe('decide'); |
| 69 | + expect(attr(decideSpan!, 'gen_ai.provider.name')).toBe('openrouter'); |
| 70 | + expect(attr(decideSpan!, 'gen_ai.request.model')).toBe('~typesafe/jev-latest'); |
| 71 | + expect(typeof attr(decideSpan!, 'gen_ai.response.model')).toBe('string'); |
| 72 | + expect(typeof attr(decideSpan!, 'gen_ai.response.id')).toBe('string'); |
| 73 | + expect(attr(decideSpan!, 'gen_ai.usage.input_tokens')).toBeGreaterThan(0); |
| 74 | + expect(attr(decideSpan!, 'gen_ai.usage.output_tokens')).toBeGreaterThan(0); |
| 75 | + expect(attr(decideSpan!, 'gen_ai.cost.total_tokens')).toBeGreaterThan(0); |
| 76 | + expect(attr(decideSpan!, 'gen_ai.input.messages')).toContain('I was charged twice for my subscription.'); |
| 77 | + expect(attr(decideSpan!, 'gen_ai.output.messages')).toContain('"queue":{"type":"choice"'); |
| 78 | + expect(attr(decideSpan!, 'gen_ai.output.messages')).toContain('"refund":{"type":"boolean"'); |
| 79 | + |
| 80 | + expect(providerRequests, tree).toHaveLength(1); |
| 81 | + expect(String(attr(providerRequests[0]!, 'server.address') ?? providerRequests[0]!.name)).toContain('openrouter'); |
| 82 | +}); |
| 83 | + |
| 84 | +// The rerank models of OpenRouter come from Cohere, and the zero data retention guardrail of the account |
| 85 | +// of the test key blocks them, so OpenRouter refuses the request. Thus the test checks only what the span |
| 86 | +// records when OpenRouter answers and when it refuses. |
| 87 | +test('tanstack-ai: a rerank() call emits a gen_ai.rerank span', async ({ baseURL }) => { |
| 88 | + const spansPromise = collectStreamedSpans( |
| 89 | + APP, |
| 90 | + spansOfTrace => |
| 91 | + spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.rerank') && spansOfTrace.some(span => span.is_segment), |
| 92 | + ); |
| 93 | + |
| 94 | + await fetch(`${baseURL}/tanstack-ai/rerank`); |
| 95 | + |
| 96 | + const spans = await spansPromise; |
| 97 | + const tree = describeTree(spans); |
| 98 | + const workflow = spans.find(span => span.name === 'ai-workflow'); |
| 99 | + const rerankSpan = spans.find(span => getSpanOp(span) === 'gen_ai.rerank'); |
| 100 | + const providerRequests = spans.filter( |
| 101 | + span => getSpanOp(span) === 'http.client' && span.parent_span_id === rerankSpan?.span_id, |
| 102 | + ); |
| 103 | + |
| 104 | + expect(rerankSpan, tree).toBeDefined(); |
| 105 | + expect(rerankSpan!.parent_span_id, tree).toBe(workflow?.span_id); |
| 106 | + expect(rerankSpan!.name).toBe('rerank cohere/rerank-v3.5'); |
| 107 | + expect(attr(rerankSpan!, 'sentry.origin')).toBe('auto.ai.tanstack_ai'); |
| 108 | + expect(attr(rerankSpan!, 'gen_ai.operation.name')).toBe('rerank'); |
| 109 | + expect(attr(rerankSpan!, 'gen_ai.provider.name')).toBe('openrouter'); |
| 110 | + expect(attr(rerankSpan!, 'gen_ai.request.model')).toBe('cohere/rerank-v3.5'); |
| 111 | + |
| 112 | + expect(providerRequests, tree).toHaveLength(1); |
| 113 | + expect(String(attr(providerRequests[0]!, 'server.address') ?? providerRequests[0]!.name)).toContain('openrouter'); |
| 114 | +}); |
0 commit comments