Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "gen-ai-libraries",
"description": "Real gen_ai spans for every instrumented AI library (OpenAI, Anthropic, Mistral, Together, Vercel AI), each driven through OpenRouter with a chat query and a tool call, on Node and on Cloudflare",
"description": "Real gen_ai spans for every instrumented AI library (OpenAI, Anthropic, Mistral, Together, Vercel AI, TanStack AI), each driven through OpenRouter with a chat query and a tool call, on Node and on Cloudflare",
"version": "1.0.0",
"private": true,
"type": "module",
Expand All @@ -21,6 +21,9 @@
"@openrouter/ai-sdk-provider": "~3.0.0",
"@sentry/cloudflare": "file:../../packed/sentry-cloudflare-packed.tgz",
"@sentry/node": "file:../../packed/sentry-node-packed.tgz",
"@tanstack/ai": "^1.0.0",
"@tanstack/ai-openai": "^1.0.0",
"@tanstack/ai-openrouter": "^1.0.0",
"ai": "~7.0.97",
"express": "^4.21.2",
"openai": "5.18.1",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
allowBuilds:
'@openrouter/sdk': false
esbuild: true
workerd: true
minimumReleaseAge: 0
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@
// `@sentry/cloudflare/vite` bundler plugin (build-time channel injection) and run on workerd.
import * as Sentry from '@sentry/cloudflare';
import { libraries } from './libraries';
import { tanstackAiActivities } from './tanstack-ai-activities';

const byId = new Map(libraries.map(library => [library.id, library]));
const tanstackAiActivityById = new Map(tanstackAiActivities.map(activity => [activity.id, activity]));

export default Sentry.withSentry(
(env: Env) => ({
Expand All @@ -17,8 +19,14 @@ export default Sentry.withSentry(
const url = new URL(request.url);
const [, id, action] = url.pathname.split('/');
const library = id ? byId.get(id) : undefined;
const run =
library && (action === 'chat' || action === 'tools')
? library[action]
: id === 'tanstack-ai' && action
? tanstackAiActivityById.get(action)?.run
: undefined;

if (!library || (action !== 'chat' && action !== 'tools')) {
if (!run) {
return new Response('Not found', { status: 404 });
}

Expand All @@ -29,7 +37,7 @@ export default Sentry.withSentry(

try {
const spanName = action === 'tools' ? 'ai-tool-workflow' : 'ai-workflow';
const result = await Sentry.startSpan({ name: spanName, op: 'function' }, () => library[action](apiKey));
const result = await Sentry.startSpan({ name: spanName, op: 'function' }, () => run(apiKey));
return Response.json({ result });
} catch (error) {
return Response.json({ message: (error as Error).message }, { status: 500 });
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import * as Sentry from '@sentry/node';
import express from 'express';
import { libraries } from './libraries';
import { tanstackAiActivities } from './tanstack-ai-activities';

const apiKey = process.env.E2E_OPENROUTER_API_KEY;
if (!apiKey) {
Expand Down Expand Up @@ -35,6 +36,17 @@ for (const library of libraries) {
});
}

for (const activity of tanstackAiActivities) {
app.get(`/tanstack-ai/${activity.id}`, async (_req, res, next) => {
try {
const result = await Sentry.startSpan({ name: 'ai-workflow', op: 'function' }, () => activity.run(apiKey));
res.send({ result });
} catch (error) {
next(error);
}
});
}

Sentry.setupExpressErrorHandler(app);

app.use((error: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
import Anthropic from '@anthropic-ai/sdk';
import { createOpenRouter } from '@openrouter/ai-sdk-provider';
import { Mistral } from '@mistralai/mistralai';
import { chat, toolDefinition } from '@tanstack/ai';
import { createOpenRouterText } from '@tanstack/ai-openrouter';
import OpenAI from 'openai';
import Together from 'together-ai';
import { generateText, tool } from 'ai';
Expand Down Expand Up @@ -54,9 +56,9 @@ export interface Library {
id: string;
/** The op of the model-call span; asserted by the tests. */
op: 'gen_ai.chat' | 'gen_ai.generate_content';
/** `gen_ai.provider.name` for the direct-SDK libraries; unset for the Vercel AI SDK. */
/** `gen_ai.provider.name` of the model-call span; unset for the Vercel AI SDK. */
provider?: string;
/** `sentry.origin` for the direct-SDK libraries; unset for the Vercel AI SDK. */
/** `sentry.origin` of the model-call span; unset for the Vercel AI SDK. */
origin?: string;
chat: (apiKey: string) => Promise<unknown>;
tools: (apiKey: string) => Promise<unknown>;
Expand Down Expand Up @@ -216,4 +218,49 @@ export const libraries: Library[] = [
return result.toolCalls ?? [];
},
},

{
id: 'tanstack-ai',
// TanStack AI runs an agent loop like the Vercel AI SDK: an `invoke_agent` span with one `chat` span
// per model call, and an `execute_tool` span for each tool it runs.
op: 'gen_ai.chat',
provider: 'openrouter',
origin: 'auto.ai.tanstack_ai',
chat: async apiKey => {
const { text } = await chat({
adapter: createOpenRouterText(MODEL, apiKey),
threadId: crypto.randomUUID(),
systemPrompts: [SYSTEM],
messages: [{ role: 'user', content: CHAT_PROMPT }],
modelOptions: { temperature: 0 },
stream: false,
});
return text;
},
tools: async apiKey => {
const getWeather = toolDefinition({
name: 'get_weather',
description: 'Get the current weather for a city.',
inputSchema: z.object({ city: z.string().describe('The city name') }),
}).server(async ({ city }) => `It is sunny in ${city}.`);

const { text } = await chat({
adapter: createOpenRouterText(MODEL, apiKey),
threadId: crypto.randomUUID(),
messages: [{ role: 'user', content: WEATHER_PROMPT }],
tools: [getWeather],
// A `toolChoice` option would apply to every model call of the loop, so the model could never
// answer. Forcing only the first call keeps the turn at one tool call and one answer.
middleware: [
{
name: 'force-first-tool-call',
onConfig: ctx =>
ctx.phase === 'beforeModel' && ctx.iteration === 0 ? { toolChoice: 'required' } : undefined,
},
],
stream: false,
});
return text;
},
},
];
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
// The TanStack AI activities other than `chat()`, run against OpenRouter like the libraries in
// `libraries.ts`. The embedding adapter calls the OpenAI SDK, whose Sentry integration TanStack AI skips
// only during its own call, so the `openai` tests that run later still get their spans.
import { boolean, choice, decide, embed, rerank } from '@tanstack/ai';
import { createOpenaiEmbedding } from '@tanstack/ai-openai';
import { createOpenRouterDecider, createOpenRouterRerank } from '@tanstack/ai-openrouter';

export interface TanstackAiActivity {
id: string;
run: (apiKey: string) => Promise<unknown>;
}

export const tanstackAiActivities: TanstackAiActivity[] = [
{
id: 'embed',
run: async apiKey => {
const { embeddings } = await embed({
// OpenRouter names the model with its vendor, which the model type of the adapter does not allow.
adapter: createOpenaiEmbedding('openai/text-embedding-3-small' as 'text-embedding-3-small', apiKey, {
baseURL: 'https://openrouter.ai/api/v1',
}),
input: ['a red guitar', 'a blue drum kit'],
});
return embeddings.length;
},
},
{
id: 'decide',
run: async apiKey => {
const result = await decide({
adapter: createOpenRouterDecider('~typesafe/jev-latest', apiKey),
state: 'I was charged twice for my subscription.',
questions: {
queue: choice({ instructions: 'Which team handles this ticket?', options: { billing: null, tech: null } }),
refund: boolean({ instructions: 'Does the customer ask for a refund?' }),
},
});
return { queue: result.queue.value, refund: result.refund.value };
},
},
{
id: 'rerank',
run: async apiKey => {
const { ranking } = await rerank({
adapter: createOpenRouterRerank('cohere/rerank-v3.5', apiKey),
query: 'A laptop for travel',
documents: ['A heavy gaming desktop', 'A lightweight ultrabook with all-day battery'],
});
return ranking.map(entry => entry.index);
},
},
];
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
import { expect, test } from '@playwright/test';
import { collectStreamedSpans, getSpanOp } from '@sentry-internal/test-utils';
import { APP } from './constants';
import { attr, describeTree } from './utils';

// The span of an activity is active while the activity runs, so the provider request nests under it.
// Each test waits for the segment span, which ends last, so that all spans of the trace are there.

test('tanstack-ai: an embed() call emits a gen_ai.embeddings span, and no span of the OpenAI integration', async ({
baseURL,
}) => {
const spansPromise = collectStreamedSpans(
APP,
spansOfTrace =>
spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.embeddings') && spansOfTrace.some(span => span.is_segment),
);

const response = await fetch(`${baseURL}/tanstack-ai/embed`);
expect(response.status).toBe(200);

const spans = await spansPromise;
const tree = describeTree(spans);
const workflow = spans.find(span => span.name === 'ai-workflow');
const embeddings = spans.filter(span => getSpanOp(span) === 'gen_ai.embeddings');
const providerRequests = spans.filter(
span => getSpanOp(span) === 'http.client' && span.parent_span_id === embeddings[0]?.span_id,
);

expect(embeddings, tree).toHaveLength(1);
expect(embeddings[0]!.parent_span_id, tree).toBe(workflow?.span_id);
expect(embeddings[0]!.name).toBe('embeddings openai/text-embedding-3-small');
expect(embeddings[0]!.status).toBe('ok');
expect(attr(embeddings[0]!, 'sentry.origin')).toBe('auto.ai.tanstack_ai');
expect(attr(embeddings[0]!, 'gen_ai.operation.name')).toBe('embeddings');
expect(attr(embeddings[0]!, 'gen_ai.provider.name')).toBe('openai');
expect(attr(embeddings[0]!, 'gen_ai.request.model')).toBe('openai/text-embedding-3-small');
expect(typeof attr(embeddings[0]!, 'gen_ai.response.model')).toBe('string');
expect(attr(embeddings[0]!, 'gen_ai.usage.input_tokens')).toBeGreaterThan(0);
expect(attr(embeddings[0]!, 'gen_ai.embeddings.input')).toBe('["a red guitar","a blue drum kit"]');

expect(providerRequests, tree).toHaveLength(1);
expect(String(attr(providerRequests[0]!, 'server.address') ?? providerRequests[0]!.name)).toContain('openrouter');
});

test('tanstack-ai: a decide() call emits a gen_ai.decide span', async ({ baseURL }) => {
const spansPromise = collectStreamedSpans(
APP,
spansOfTrace =>
spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.decide') && spansOfTrace.some(span => span.is_segment),
);

const response = await fetch(`${baseURL}/tanstack-ai/decide`);
expect(response.status).toBe(200);

const spans = await spansPromise;
const tree = describeTree(spans);
const workflow = spans.find(span => span.name === 'ai-workflow');
const decideSpan = spans.find(span => getSpanOp(span) === 'gen_ai.decide');
const providerRequests = spans.filter(
span => getSpanOp(span) === 'http.client' && span.parent_span_id === decideSpan?.span_id,
);

expect(decideSpan, tree).toBeDefined();
expect(decideSpan!.parent_span_id, tree).toBe(workflow?.span_id);
expect(decideSpan!.name).toBe('decide ~typesafe/jev-latest');
expect(decideSpan!.status).toBe('ok');
expect(attr(decideSpan!, 'sentry.origin')).toBe('auto.ai.tanstack_ai');
expect(attr(decideSpan!, 'gen_ai.operation.name')).toBe('decide');
expect(attr(decideSpan!, 'gen_ai.provider.name')).toBe('openrouter');
expect(attr(decideSpan!, 'gen_ai.request.model')).toBe('~typesafe/jev-latest');
expect(typeof attr(decideSpan!, 'gen_ai.response.model')).toBe('string');
expect(typeof attr(decideSpan!, 'gen_ai.response.id')).toBe('string');
expect(attr(decideSpan!, 'gen_ai.usage.input_tokens')).toBeGreaterThan(0);
expect(attr(decideSpan!, 'gen_ai.usage.output_tokens')).toBeGreaterThan(0);
expect(attr(decideSpan!, 'gen_ai.cost.total_tokens')).toBeGreaterThan(0);
expect(attr(decideSpan!, 'gen_ai.input.messages')).toContain('I was charged twice for my subscription.');
expect(attr(decideSpan!, 'gen_ai.output.messages')).toContain('"queue":{"type":"choice"');
expect(attr(decideSpan!, 'gen_ai.output.messages')).toContain('"refund":{"type":"boolean"');

expect(providerRequests, tree).toHaveLength(1);
expect(String(attr(providerRequests[0]!, 'server.address') ?? providerRequests[0]!.name)).toContain('openrouter');
});

// The rerank models of OpenRouter come from Cohere, and the zero data retention guardrail of the account
// of the test key blocks them, so OpenRouter refuses the request. Thus the test checks only what the span
// records when OpenRouter answers and when it refuses.
test('tanstack-ai: a rerank() call emits a gen_ai.rerank span', async ({ baseURL }) => {
const spansPromise = collectStreamedSpans(
APP,
spansOfTrace =>
spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.rerank') && spansOfTrace.some(span => span.is_segment),
);

await fetch(`${baseURL}/tanstack-ai/rerank`);

const spans = await spansPromise;
const tree = describeTree(spans);
const workflow = spans.find(span => span.name === 'ai-workflow');
const rerankSpan = spans.find(span => getSpanOp(span) === 'gen_ai.rerank');
const providerRequests = spans.filter(
span => getSpanOp(span) === 'http.client' && span.parent_span_id === rerankSpan?.span_id,
);

expect(rerankSpan, tree).toBeDefined();
expect(rerankSpan!.parent_span_id, tree).toBe(workflow?.span_id);
expect(rerankSpan!.name).toBe('rerank cohere/rerank-v3.5');
expect(attr(rerankSpan!, 'sentry.origin')).toBe('auto.ai.tanstack_ai');
expect(attr(rerankSpan!, 'gen_ai.operation.name')).toBe('rerank');
expect(attr(rerankSpan!, 'gen_ai.provider.name')).toBe('openrouter');
expect(attr(rerankSpan!, 'gen_ai.request.model')).toBe('cohere/rerank-v3.5');

expect(providerRequests, tree).toHaveLength(1);
expect(String(attr(providerRequests[0]!, 'server.address') ?? providerRequests[0]!.name)).toContain('openrouter');
});
Loading