Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions agentic/agentic-server/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,9 @@ app.listen(3001, () => {
| POST | `/v1/chat/completions` | OpenAI-compatible chat completion (multi-provider) |
| POST | `/v1/embeddings` | OpenAI-compatible embeddings (multi-provider) |
| POST | `/v1/usage` | Submit an external usage report to the metering sink |
| GET | `/v1/providers` | List configured providers |
| GET | `/healthz` | Health check |
| GET | `/v1/models` | Models a caller can run: `{ object: 'list', data: [{ id, name }] }`. Requires `X-Database-Id`. `id` is what to send as `model`; nothing about the providers is returned |
| GET | `/v1/providers` | List configured providers (operator) |
| GET | `/healthz` | Liveness: `{ status: 'ok' }` |

## Streaming

Expand Down
5 changes: 2 additions & 3 deletions agentic/agentic-server/__tests__/gateway.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -492,12 +492,11 @@ describe('POST /v1/usage', () => {
// ─── Health & Providers ───────────────────────────────────────────────────

describe('GET /healthz', () => {
it('returns provider status', async () => {
it('reports liveness', async () => {
const res = await fetch(`http://localhost:${agenticPort}/healthz`);
expect(res.status).toBe(200);
const data = (await res.json()) as any;
expect(data.status).toBe('ok');
expect(data.provider).toBe('ollama');
expect(data).toEqual({ status: 'ok' });
});
});

Expand Down
166 changes: 166 additions & 0 deletions agentic/agentic-server/__tests__/models.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
/**
* GET /v1/models — the models a caller can run, and nothing about who serves
* them — plus what the gateway tells a caller that named a model nobody serves.
*/

import express from 'express';
import type { Server } from 'http';
import type { AddressInfo } from 'net';

import { createAgenticServer } from '../src';

const listen = (app: express.Express): Promise<Server> =>
new Promise((resolve) => {
const server = app.listen(0, () => resolve(server));
});
const port = (server: Server): number => (server.address() as AddressInfo).port;
const close = (server: Server): Promise<void> =>
new Promise((resolve) => server.close(() => resolve()));

let ollama: Server;
let openai: Server;
let anthropic: Server;
let down: Server;
let gateway: Server;
let base: string;
let upstreamHeaders: Record<string, string | string[] | undefined>[];

beforeAll(async () => {
const ollamaApp = express();
ollamaApp.use(express.json());
ollamaApp.get('/api/tags', (_req, res) => {
res.json({ models: [{ name: 'llama3.2:latest', model: 'llama3.2:latest', size: 2019393189 }] });
});
ollamaApp.post('/api/chat', (req, res) => {
res.status(404).json({ error: `model '${req.body.model}' not found` });
});
ollama = await listen(ollamaApp);

const openaiApp = express();
openaiApp.get('/v1/models', (req, res) => {
upstreamHeaders.push(req.headers);
res.json({
object: 'list',
data: [
{ id: 'gpt-4o', object: 'model', owned_by: 'openai' },
// A family the router would otherwise send to the Ollama entry.
{ id: 'llama-hosted', object: 'model', owned_by: 'someone' },
// An upstream id whose own first segment is the dialect the router strips.
{ id: 'openai/gpt-oss-120b', object: 'model', owned_by: 'someone' }
]
});
});
openai = await listen(openaiApp);

const anthropicApp = express();
anthropicApp.get('/v1/models', (req, res) => {
if (req.query.after_id === 'claude-sonnet-4-5') {
res.json({
data: [{ id: 'claude-haiku-4-5', display_name: 'Claude Haiku 4.5', type: 'model' }],
has_more: false,
last_id: 'claude-haiku-4-5'
});
return;
}
res.json({
data: [{ id: 'claude-sonnet-4-5', display_name: 'Claude Sonnet 4.5', type: 'model' }],
has_more: true,
last_id: 'claude-sonnet-4-5'
});
});
anthropic = await listen(anthropicApp);

const downApp = express();
downApp.get('/v1/models', (_req, res) => {
res.status(500).json({ error: 'secret-internal-detail' });
});
down = await listen(downApp);

gateway = await listen(createAgenticServer({
providers: [
{ type: 'ollama', baseUrl: `http://localhost:${port(ollama)}` },
{ name: 'hosted', type: 'openai', baseUrl: `http://localhost:${port(openai)}`, apiKey: 'sk-upstream' },
{ type: 'anthropic', baseUrl: `http://localhost:${port(anthropic)}`, apiKey: 'sk-ant' }
]
}));
base = `http://localhost:${port(gateway)}`;
});

afterAll(async () => {
await Promise.all([ollama, openai, anthropic, down, gateway].map(close));
});

beforeEach(() => {
upstreamHeaders = [];
});

describe('GET /v1/models', () => {
it('is refused without the identity a run carries', async () => {
const res = await fetch(`${base}/v1/models`);
expect(res.status).toBe(400);
expect(await res.json()).toEqual({ error: { message: 'X-Database-Id is required' } });
});

it('lists each served model by the id a request addresses it with, and a display name', async () => {
const res = await fetch(`${base}/v1/models`, { headers: { 'X-Database-Id': 'db-1' } });
expect(res.status).toBe(200);
expect(await res.json()).toEqual({
object: 'list',
data: [
{ id: 'llama3.2:latest', name: 'llama3.2:latest' },
{ id: 'gpt-4o', name: 'gpt-4o' },
// Bare, this id would route to another entry; the prefix is what reaches it.
{ id: 'hosted/llama-hosted', name: 'llama-hosted' },
// Bare, the gateway would strip `openai/` before the upstream saw it.
{ id: 'hosted/openai/gpt-oss-120b', name: 'openai/gpt-oss-120b' },
// Both pages of a paged listing.
{ id: 'claude-sonnet-4-5', name: 'Claude Sonnet 4.5' },
{ id: 'claude-haiku-4-5', name: 'Claude Haiku 4.5' }
]
});
});

it('says nothing about the provider setup', async () => {
const res = await fetch(`${base}/v1/models`, { headers: { 'X-Database-Id': 'db-1' } });
const text = await res.text();
for (const internal of ['localhost', 'sk-upstream', 'sk-ant', 'owned_by', 'baseUrl', 'apiKey', 'size', '"type"', '"provider"']) {
expect(text).not.toContain(internal);
}
// The upstream is asked with the gateway's key, never the caller's headers.
expect(upstreamHeaders[0].authorization).toBe('Bearer sk-upstream');
expect(upstreamHeaders[0]['x-database-id']).toBeUndefined();
});

it('answers 502 without the upstream reason when a provider cannot be listed', async () => {
const broken = await listen(createAgenticServer({
providers: [{ type: 'openai', baseUrl: `http://localhost:${port(down)}`, apiKey: 'sk-upstream' }]
}));
const res = await fetch(`http://localhost:${port(broken)}/v1/models`, { headers: { 'X-Database-Id': 'db-1' } });
await close(broken);
expect(res.status).toBe(502);
const text = await res.text();
expect(JSON.parse(text)).toEqual({ error: { message: 'the model list is unavailable' } });
expect(text).not.toContain('secret-internal-detail');
});
});

describe('a model nobody serves', () => {
it('is told how to list the ones that are', async () => {
const res = await fetch(`${base}/v1/chat/completions`, {
method: 'POST',
headers: { 'Content-Type': 'application/json', 'X-Database-Id': 'db-1' },
body: JSON.stringify({ model: 'llama-typo', messages: [{ role: 'user', content: 'hi' }] })
});
expect(res.status).toBe(404);
const { error } = await res.json() as { error: { message: string } };
expect(error.message).toContain("model 'llama-typo' not found");
expect(error.message).toContain('GET /v1/models lists the models this gateway serves');
});
});

describe('GET /healthz', () => {
it('reports liveness and nothing about the providers', async () => {
const res = await fetch(`${base}/healthz`);
expect(await res.json()).toEqual({ status: 'ok' });
});
});
2 changes: 2 additions & 0 deletions agentic/agentic-server/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
*/

export { BadRequestError } from './errors';
export type { ServedModel } from './models';
export { listServedModels, MODEL_LIST_HINT } from './models';
export { buildProviderHeaders, resolveProvider, resolveUpstreamUrl } from './providers';
export { createRouter } from './router';
export type { AgenticServerStartOptions } from './server';
Expand Down
88 changes: 88 additions & 0 deletions agentic/agentic-server/src/models.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
import { buildProviderHeaders, resolveProvider } from './providers';
import { providerName, upstreamModel } from './transforms';
import type { AgenticServerOptions, ResolvedProvider } from './types';

/** A model a caller can run: what to send as `model`, and what to show for it. */
export interface ServedModel {
id: string;
name: string;
}

/** Appended to an upstream "no such model" answer, so the caller knows where to look. */
export const MODEL_LIST_HINT = 'GET /v1/models lists the models this gateway serves';

const configuredProviders = (options: AgenticServerOptions): ResolvedProvider[] =>
options.providers && options.providers.length > 0
? options.providers
: [resolveProvider(options)];

const text = (value: unknown): string | undefined =>
typeof value === 'string' && value ? value : undefined;

const served = (row: Record<string, unknown>, idField: string): ServedModel[] => {
const id = text(row[idField]);
return id ? [{ id, name: text(row.display_name) ?? id }] : [];
};

/** The models one provider serves, as its own listing names them. */
async function upstreamModels(provider: ResolvedProvider): Promise<ServedModel[]> {
const listing = async (url: string): Promise<Record<string, unknown>> => {
const res = await fetch(url, { headers: buildProviderHeaders(provider) });
if (!res.ok) {
throw new Error(`${providerName(provider)} answered ${res.status} to a model listing`);
}
return await res.json() as Record<string, unknown>;
};
const rows = (list: unknown): Record<string, unknown>[] => {
if (!Array.isArray(list)) {
throw new Error(`${providerName(provider)} answered a model listing with no list`);
}
return list;
};

if (provider.type === 'ollama') {
const body = await listing(`${provider.baseUrl}/api/tags`);
return rows(body.models).flatMap((row) => served(row, 'name'));
}

// A listing may be paged (Anthropic's is): follow it to the end, or the
// models on a later page read as not served.
const models: ServedModel[] = [];
let after: string | undefined;
do {
const page = after ? `?after_id=${encodeURIComponent(after)}` : '';
const body = await listing(`${provider.baseUrl}/v1/models${page}`);
models.push(...rows(body.data).flatMap((row) => served(row, 'id')));
const next = body.has_more === true ? text(body.last_id) : undefined;
if (next === after) break;
after = next;
} while (after);
return models;
}

/**
* Every model the gateway serves, under the id that reaches it: the bare
* upstream id when the router would send that, unchanged, to the provider serving it, the
* `name/` prefixed one otherwise. Only the id and a display name leave here —
* which provider serves a model, and where, is the deployment's business.
*/
export async function listServedModels(options: AgenticServerOptions): Promise<ServedModel[]> {
const providers = configuredProviders(options);
const lists = await Promise.all(providers.map(upstreamModels));
const seen = new Set<string>();
return lists.flatMap((models, index) => {
const provider = providers[index];
return models.flatMap(({ id, name }) => {
const routed = resolveProvider(options, id);
// Bare only when the id both routes here and survives the prefix strip:
// an upstream id that itself begins `openai/` would lose that segment.
const bare =
(providers.length === 1 || providerName(routed) === providerName(provider)) &&
upstreamModel(provider, id) === id;
const served = bare ? id : `${providerName(provider)}/${id}`;
if (seen.has(served)) return [];
seen.add(served);
return [{ id: served, name }];
});
Comment on lines +55 to +61

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 bug · medium

Slash ids matching a provider name are advertised bare but routed split

A model id that itself contains / and whose first segment matches a provider name is advertised bare, yet a request with that id is split and relayed without its first segment. In listServedModels, resolveProvider(options, 'hosted/thing') resolves back to the same hosted provider, so bare is true and the listing shows hosted/thing (agentic/agentic-server/src/models.ts:55-56). But upstreamModel strips the prefix equal to providerName and relays only thing (agentic/agentic-server/src/transforms.ts:31-35), so the advertised id selects a nonexistent model on the upstream. Callers picking models from /v1/models fail on their first call with every run configured with such an id.

📋 Prompt for AI Agents

In agentic/agentic-server/src/models.ts around lines 54-61, fix the bare/prefix decision in listServedModels: if the upstream id contains '/' and its first segment matches a registered provider name or type (the same test resolveProvider's split at src/providers.ts:46-52 uses), advertise the prefixed form ${providerName(provider)}/${id} instead of the bare id, because upstreamModel (src/transforms.ts:29-36) will strip the matching prefix on the request path and relay the wrong model. Add a test where a provider named 'hosted' serves id 'hosted/thing' asserting the listing shows 'hosted/hosted/thing' and a chat request with that id reaches the upstream with model 'hosted/thing'.

});
}
37 changes: 25 additions & 12 deletions agentic/agentic-server/src/router.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { Logger } from '@pgpmjs/logger';
import { Router } from 'express';

import { BadRequestError } from './errors';
import { listServedModels, MODEL_LIST_HINT } from './models';
import { buildProviderHeaders, resolveProvider, resolveUpstreamUrl } from './providers';
import { relayEventStream, withUsageStreamOptions } from './streaming';
import {
Expand Down Expand Up @@ -79,9 +80,13 @@ function upstreamErrorMessage(
? `${parsed.slice(0, UPSTREAM_REASON_LIMIT)}…`
: parsed;

return reason
const message = reason
? `${provider.type} provider error ${status}: ${reason}`
: `${provider.type} provider error ${status}`;

// 404 is how every dialect answers a model it does not serve, and a run
// configured with one fails on its first call: say where the served ones are.
return status === 404 ? `${message} — ${MODEL_LIST_HINT}` : message;
}

/**
Expand Down Expand Up @@ -433,6 +438,22 @@ export const createRouter = (options: AgenticServerOptions): Router => {
res.status(202).json({ accepted: true });
});

// GET /v1/models — the models a caller can run: id and display name, nothing
// of the providers behind them. Asked for with the identity a run carries.
router.get('/v1/models', async (req: any, res: any) => {
if (!req.get('X-Database-Id')) {
res.status(400).json({ error: { message: 'X-Database-Id is required' } });
return;
}
try {
res.json({ object: 'list', data: await listServedModels(options) });
} catch (err: any) {
// The reason names a provider; it is the operator's to read, in the log.
log.error('models error', err);
res.status(502).json({ error: { message: 'the model list is unavailable' } });
}
});

// GET /v1/providers — list configured providers
router.get('/v1/providers', (_req: any, res: any) => {
const providers = options.providers || [{
Expand All @@ -448,18 +469,10 @@ export const createRouter = (options: AgenticServerOptions): Router => {
});
});

// GET /healthz — health check
// GET /healthz — liveness only. It is unauthenticated, and whatever forwards
// to this gateway forwards here too, so it describes no provider.
router.get('/healthz', (_req: any, res: any) => {
const providers = options.providers || [{
type: options.providerType || 'ollama',
baseUrl: options.providerBaseUrl || 'http://localhost:11434'
}];
res.json({
status: 'ok',
provider: providers[0]?.type || 'ollama',
providerUrl: providers[0]?.baseUrl || options.providerBaseUrl,
providers: providers.map((p) => p.type)
});
res.json({ status: 'ok' });
});

return router;
Expand Down
Loading