Skip to content

Commit cd29ea7

Browse files
JPeer264claude
andcommitted
test(e2e): Add TanStack AI embed(), rerank() and decide() tests
The gen-ai-libraries app now calls embed(), rerank() and decide(), so CI checks these activities on Node (runtime injection) and on Cloudflare (build-time injection). embed() goes through the OpenAI adapter, and the openai tests that run after it check that the OpenAI integration is skipped only during the TanStack AI call. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent d827a51 commit cd29ea7

5 files changed

Lines changed: 189 additions & 2 deletions

File tree

‎dev-packages/e2e-tests/test-applications/gen-ai-libraries/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
"@sentry/cloudflare": "file:../../packed/sentry-cloudflare-packed.tgz",
2323
"@sentry/node": "file:../../packed/sentry-node-packed.tgz",
2424
"@tanstack/ai": "^1.0.0",
25+
"@tanstack/ai-openai": "^1.0.0",
2526
"@tanstack/ai-openrouter": "^1.0.0",
2627
"ai": "~7.0.97",
2728
"express": "^4.21.2",

‎dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.cloudflare.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@
22
// `@sentry/cloudflare/vite` bundler plugin (build-time channel injection) and run on workerd.
33
import * as Sentry from '@sentry/cloudflare';
44
import { libraries } from './libraries';
5+
import { tanstackAiActivities } from './tanstack-ai-activities';
56

67
const byId = new Map(libraries.map(library => [library.id, library]));
8+
const tanstackAiActivityById = new Map(tanstackAiActivities.map(activity => [activity.id, activity]));
79

810
export default Sentry.withSentry(
911
(env: Env) => ({
@@ -17,8 +19,14 @@ export default Sentry.withSentry(
1719
const url = new URL(request.url);
1820
const [, id, action] = url.pathname.split('/');
1921
const library = id ? byId.get(id) : undefined;
22+
const run =
23+
library && (action === 'chat' || action === 'tools')
24+
? library[action]
25+
: id === 'tanstack-ai' && action
26+
? tanstackAiActivityById.get(action)?.run
27+
: undefined;
2028

21-
if (!library || (action !== 'chat' && action !== 'tools')) {
29+
if (!run) {
2230
return new Response('Not found', { status: 404 });
2331
}
2432

@@ -29,7 +37,7 @@ export default Sentry.withSentry(
2937

3038
try {
3139
const spanName = action === 'tools' ? 'ai-tool-workflow' : 'ai-workflow';
32-
const result = await Sentry.startSpan({ name: spanName, op: 'function' }, () => library[action](apiKey));
40+
const result = await Sentry.startSpan({ name: spanName, op: 'function' }, () => run(apiKey));
3341
return Response.json({ result });
3442
} catch (error) {
3543
return Response.json({ message: (error as Error).message }, { status: 500 });

‎dev-packages/e2e-tests/test-applications/gen-ai-libraries/src/entry.node.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import * as Sentry from '@sentry/node';
33
import express from 'express';
44
import { libraries } from './libraries';
5+
import { tanstackAiActivities } from './tanstack-ai-activities';
56

67
const apiKey = process.env.E2E_OPENROUTER_API_KEY;
78
if (!apiKey) {
@@ -35,6 +36,17 @@ for (const library of libraries) {
3536
});
3637
}
3738

39+
for (const activity of tanstackAiActivities) {
40+
app.get(`/tanstack-ai/${activity.id}`, async (_req, res, next) => {
41+
try {
42+
const result = await Sentry.startSpan({ name: 'ai-workflow', op: 'function' }, () => activity.run(apiKey));
43+
res.send({ result });
44+
} catch (error) {
45+
next(error);
46+
}
47+
});
48+
}
49+
3850
Sentry.setupExpressErrorHandler(app);
3951

4052
app.use((error: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => {
Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
// The TanStack AI activities other than `chat()`, run against OpenRouter like the libraries in
2+
// `libraries.ts`. The embedding adapter calls the OpenAI SDK, whose Sentry integration TanStack AI skips
3+
// only during its own call, so the `openai` tests that run later still get their spans.
4+
import { boolean, choice, decide, embed, rerank } from '@tanstack/ai';
5+
import { createOpenaiEmbedding } from '@tanstack/ai-openai';
6+
import { createOpenRouterDecider, createOpenRouterRerank } from '@tanstack/ai-openrouter';
7+
8+
export interface TanstackAiActivity {
9+
id: string;
10+
run: (apiKey: string) => Promise<unknown>;
11+
}
12+
13+
export const tanstackAiActivities: TanstackAiActivity[] = [
14+
{
15+
id: 'embed',
16+
run: async apiKey => {
17+
const { embeddings } = await embed({
18+
// OpenRouter names the model with its vendor, which the model type of the adapter does not allow.
19+
adapter: createOpenaiEmbedding('openai/text-embedding-3-small' as 'text-embedding-3-small', apiKey, {
20+
baseURL: 'https://openrouter.ai/api/v1',
21+
}),
22+
input: ['a red guitar', 'a blue drum kit'],
23+
});
24+
return embeddings.length;
25+
},
26+
},
27+
{
28+
id: 'decide',
29+
run: async apiKey => {
30+
const result = await decide({
31+
adapter: createOpenRouterDecider('~typesafe/jev-latest', apiKey),
32+
state: 'I was charged twice for my subscription.',
33+
questions: {
34+
queue: choice({ instructions: 'Which team handles this ticket?', options: { billing: null, tech: null } }),
35+
refund: boolean({ instructions: 'Does the customer ask for a refund?' }),
36+
},
37+
});
38+
return { queue: result.queue.value, refund: result.refund.value };
39+
},
40+
},
41+
{
42+
id: 'rerank',
43+
run: async apiKey => {
44+
const { ranking } = await rerank({
45+
adapter: createOpenRouterRerank('cohere/rerank-v3.5', apiKey),
46+
query: 'A laptop for travel',
47+
documents: ['A heavy gaming desktop', 'A lightweight ultrabook with all-day battery'],
48+
});
49+
return ranking.map(entry => entry.index);
50+
},
51+
},
52+
];
Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
import { expect, test } from '@playwright/test';
2+
import { collectStreamedSpans, getSpanOp } from '@sentry-internal/test-utils';
3+
import { APP } from './constants';
4+
import { attr, describeTree } from './utils';
5+
6+
// The span of an activity is active while the activity runs, so the provider request nests under it.
7+
// Each test waits for the segment span, which ends last, so that all spans of the trace are there.
8+
9+
test('tanstack-ai: an embed() call emits a gen_ai.embeddings span, and no span of the OpenAI integration', async ({
10+
baseURL,
11+
}) => {
12+
const spansPromise = collectStreamedSpans(
13+
APP,
14+
spansOfTrace =>
15+
spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.embeddings') && spansOfTrace.some(span => span.is_segment),
16+
);
17+
18+
const response = await fetch(`${baseURL}/tanstack-ai/embed`);
19+
expect(response.status).toBe(200);
20+
21+
const spans = await spansPromise;
22+
const tree = describeTree(spans);
23+
const workflow = spans.find(span => span.name === 'ai-workflow');
24+
const embeddings = spans.filter(span => getSpanOp(span) === 'gen_ai.embeddings');
25+
const providerRequests = spans.filter(
26+
span => getSpanOp(span) === 'http.client' && span.parent_span_id === embeddings[0]?.span_id,
27+
);
28+
29+
expect(embeddings, tree).toHaveLength(1);
30+
expect(embeddings[0]!.parent_span_id, tree).toBe(workflow?.span_id);
31+
expect(embeddings[0]!.name).toBe('embeddings openai/text-embedding-3-small');
32+
expect(embeddings[0]!.status).toBe('ok');
33+
expect(attr(embeddings[0]!, 'sentry.origin')).toBe('auto.ai.tanstack_ai');
34+
expect(attr(embeddings[0]!, 'gen_ai.operation.name')).toBe('embeddings');
35+
expect(attr(embeddings[0]!, 'gen_ai.provider.name')).toBe('openai');
36+
expect(attr(embeddings[0]!, 'gen_ai.request.model')).toBe('openai/text-embedding-3-small');
37+
expect(typeof attr(embeddings[0]!, 'gen_ai.response.model')).toBe('string');
38+
expect(attr(embeddings[0]!, 'gen_ai.usage.input_tokens')).toBeGreaterThan(0);
39+
expect(attr(embeddings[0]!, 'gen_ai.embeddings.input')).toBe('["a red guitar","a blue drum kit"]');
40+
41+
expect(providerRequests, tree).toHaveLength(1);
42+
expect(String(attr(providerRequests[0]!, 'server.address') ?? providerRequests[0]!.name)).toContain('openrouter');
43+
});
44+
45+
test('tanstack-ai: a decide() call emits a gen_ai.decide span', async ({ baseURL }) => {
46+
const spansPromise = collectStreamedSpans(
47+
APP,
48+
spansOfTrace =>
49+
spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.decide') && spansOfTrace.some(span => span.is_segment),
50+
);
51+
52+
const response = await fetch(`${baseURL}/tanstack-ai/decide`);
53+
expect(response.status).toBe(200);
54+
55+
const spans = await spansPromise;
56+
const tree = describeTree(spans);
57+
const workflow = spans.find(span => span.name === 'ai-workflow');
58+
const decideSpan = spans.find(span => getSpanOp(span) === 'gen_ai.decide');
59+
const providerRequests = spans.filter(
60+
span => getSpanOp(span) === 'http.client' && span.parent_span_id === decideSpan?.span_id,
61+
);
62+
63+
expect(decideSpan, tree).toBeDefined();
64+
expect(decideSpan!.parent_span_id, tree).toBe(workflow?.span_id);
65+
expect(decideSpan!.name).toBe('decide ~typesafe/jev-latest');
66+
expect(decideSpan!.status).toBe('ok');
67+
expect(attr(decideSpan!, 'sentry.origin')).toBe('auto.ai.tanstack_ai');
68+
expect(attr(decideSpan!, 'gen_ai.operation.name')).toBe('decide');
69+
expect(attr(decideSpan!, 'gen_ai.provider.name')).toBe('openrouter');
70+
expect(attr(decideSpan!, 'gen_ai.request.model')).toBe('~typesafe/jev-latest');
71+
expect(typeof attr(decideSpan!, 'gen_ai.response.model')).toBe('string');
72+
expect(typeof attr(decideSpan!, 'gen_ai.response.id')).toBe('string');
73+
expect(attr(decideSpan!, 'gen_ai.usage.input_tokens')).toBeGreaterThan(0);
74+
expect(attr(decideSpan!, 'gen_ai.usage.output_tokens')).toBeGreaterThan(0);
75+
expect(attr(decideSpan!, 'gen_ai.cost.total_tokens')).toBeGreaterThan(0);
76+
expect(attr(decideSpan!, 'gen_ai.input.messages')).toContain('I was charged twice for my subscription.');
77+
expect(attr(decideSpan!, 'gen_ai.output.messages')).toContain('"queue":{"type":"choice"');
78+
expect(attr(decideSpan!, 'gen_ai.output.messages')).toContain('"refund":{"type":"boolean"');
79+
80+
expect(providerRequests, tree).toHaveLength(1);
81+
expect(String(attr(providerRequests[0]!, 'server.address') ?? providerRequests[0]!.name)).toContain('openrouter');
82+
});
83+
84+
// The rerank models of OpenRouter come from Cohere, and the zero data retention guardrail of the account
85+
// of the test key blocks them, so OpenRouter refuses the request. Thus the test checks only what the span
86+
// records when OpenRouter answers and when it refuses.
87+
test('tanstack-ai: a rerank() call emits a gen_ai.rerank span', async ({ baseURL }) => {
88+
const spansPromise = collectStreamedSpans(
89+
APP,
90+
spansOfTrace =>
91+
spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.rerank') && spansOfTrace.some(span => span.is_segment),
92+
);
93+
94+
await fetch(`${baseURL}/tanstack-ai/rerank`);
95+
96+
const spans = await spansPromise;
97+
const tree = describeTree(spans);
98+
const workflow = spans.find(span => span.name === 'ai-workflow');
99+
const rerankSpan = spans.find(span => getSpanOp(span) === 'gen_ai.rerank');
100+
const providerRequests = spans.filter(
101+
span => getSpanOp(span) === 'http.client' && span.parent_span_id === rerankSpan?.span_id,
102+
);
103+
104+
expect(rerankSpan, tree).toBeDefined();
105+
expect(rerankSpan!.parent_span_id, tree).toBe(workflow?.span_id);
106+
expect(rerankSpan!.name).toBe('rerank cohere/rerank-v3.5');
107+
expect(attr(rerankSpan!, 'sentry.origin')).toBe('auto.ai.tanstack_ai');
108+
expect(attr(rerankSpan!, 'gen_ai.operation.name')).toBe('rerank');
109+
expect(attr(rerankSpan!, 'gen_ai.provider.name')).toBe('openrouter');
110+
expect(attr(rerankSpan!, 'gen_ai.request.model')).toBe('cohere/rerank-v3.5');
111+
112+
expect(providerRequests, tree).toHaveLength(1);
113+
expect(String(attr(providerRequests[0]!, 'server.address') ?? providerRequests[0]!.name)).toContain('openrouter');
114+
});

0 commit comments

Comments
 (0)