diff --git a/agentic/agentic-server/README.md b/agentic/agentic-server/README.md index f404b09dac..2bd0a1f377 100644 --- a/agentic/agentic-server/README.md +++ b/agentic/agentic-server/README.md @@ -43,8 +43,9 @@ app.listen(3001, () => { | POST | `/v1/chat/completions` | OpenAI-compatible chat completion (multi-provider) | | POST | `/v1/embeddings` | OpenAI-compatible embeddings (multi-provider) | | POST | `/v1/usage` | Submit an external usage report to the metering sink | -| GET | `/v1/providers` | List configured providers | -| GET | `/healthz` | Health check | +| GET | `/v1/models` | Models a caller can run: `{ object: 'list', data: [{ id, name }] }`. Requires `X-Database-Id`. `id` is what to send as `model`; nothing about the providers is returned | +| GET | `/v1/providers` | List configured providers (operator) | +| GET | `/healthz` | Liveness: `{ status: 'ok' }` | ## Streaming diff --git a/agentic/agentic-server/__tests__/gateway.test.ts b/agentic/agentic-server/__tests__/gateway.test.ts index 524745972a..f0bac80e45 100644 --- a/agentic/agentic-server/__tests__/gateway.test.ts +++ b/agentic/agentic-server/__tests__/gateway.test.ts @@ -492,12 +492,11 @@ describe('POST /v1/usage', () => { // ─── Health & Providers ─────────────────────────────────────────────────── describe('GET /healthz', () => { - it('returns provider status', async () => { + it('reports liveness', async () => { const res = await fetch(`http://localhost:${agenticPort}/healthz`); expect(res.status).toBe(200); const data = (await res.json()) as any; - expect(data.status).toBe('ok'); - expect(data.provider).toBe('ollama'); + expect(data).toEqual({ status: 'ok' }); }); }); diff --git a/agentic/agentic-server/__tests__/models.test.ts b/agentic/agentic-server/__tests__/models.test.ts new file mode 100644 index 0000000000..33632d996e --- /dev/null +++ b/agentic/agentic-server/__tests__/models.test.ts @@ -0,0 +1,166 @@ +/** + * GET /v1/models — the models a caller can run, and nothing about who serves + * them — plus what the gateway tells a caller that named a model nobody serves. + */ + +import express from 'express'; +import type { Server } from 'http'; +import type { AddressInfo } from 'net'; + +import { createAgenticServer } from '../src'; + +const listen = (app: express.Express): Promise => + new Promise((resolve) => { + const server = app.listen(0, () => resolve(server)); + }); +const port = (server: Server): number => (server.address() as AddressInfo).port; +const close = (server: Server): Promise => + new Promise((resolve) => server.close(() => resolve())); + +let ollama: Server; +let openai: Server; +let anthropic: Server; +let down: Server; +let gateway: Server; +let base: string; +let upstreamHeaders: Record[]; + +beforeAll(async () => { + const ollamaApp = express(); + ollamaApp.use(express.json()); + ollamaApp.get('/api/tags', (_req, res) => { + res.json({ models: [{ name: 'llama3.2:latest', model: 'llama3.2:latest', size: 2019393189 }] }); + }); + ollamaApp.post('/api/chat', (req, res) => { + res.status(404).json({ error: `model '${req.body.model}' not found` }); + }); + ollama = await listen(ollamaApp); + + const openaiApp = express(); + openaiApp.get('/v1/models', (req, res) => { + upstreamHeaders.push(req.headers); + res.json({ + object: 'list', + data: [ + { id: 'gpt-4o', object: 'model', owned_by: 'openai' }, + // A family the router would otherwise send to the Ollama entry. + { id: 'llama-hosted', object: 'model', owned_by: 'someone' }, + // An upstream id whose own first segment is the dialect the router strips. + { id: 'openai/gpt-oss-120b', object: 'model', owned_by: 'someone' } + ] + }); + }); + openai = await listen(openaiApp); + + const anthropicApp = express(); + anthropicApp.get('/v1/models', (req, res) => { + if (req.query.after_id === 'claude-sonnet-4-5') { + res.json({ + data: [{ id: 'claude-haiku-4-5', display_name: 'Claude Haiku 4.5', type: 'model' }], + has_more: false, + last_id: 'claude-haiku-4-5' + }); + return; + } + res.json({ + data: [{ id: 'claude-sonnet-4-5', display_name: 'Claude Sonnet 4.5', type: 'model' }], + has_more: true, + last_id: 'claude-sonnet-4-5' + }); + }); + anthropic = await listen(anthropicApp); + + const downApp = express(); + downApp.get('/v1/models', (_req, res) => { + res.status(500).json({ error: 'secret-internal-detail' }); + }); + down = await listen(downApp); + + gateway = await listen(createAgenticServer({ + providers: [ + { type: 'ollama', baseUrl: `http://localhost:${port(ollama)}` }, + { name: 'hosted', type: 'openai', baseUrl: `http://localhost:${port(openai)}`, apiKey: 'sk-upstream' }, + { type: 'anthropic', baseUrl: `http://localhost:${port(anthropic)}`, apiKey: 'sk-ant' } + ] + })); + base = `http://localhost:${port(gateway)}`; +}); + +afterAll(async () => { + await Promise.all([ollama, openai, anthropic, down, gateway].map(close)); +}); + +beforeEach(() => { + upstreamHeaders = []; +}); + +describe('GET /v1/models', () => { + it('is refused without the identity a run carries', async () => { + const res = await fetch(`${base}/v1/models`); + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ error: { message: 'X-Database-Id is required' } }); + }); + + it('lists each served model by the id a request addresses it with, and a display name', async () => { + const res = await fetch(`${base}/v1/models`, { headers: { 'X-Database-Id': 'db-1' } }); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ + object: 'list', + data: [ + { id: 'llama3.2:latest', name: 'llama3.2:latest' }, + { id: 'gpt-4o', name: 'gpt-4o' }, + // Bare, this id would route to another entry; the prefix is what reaches it. + { id: 'hosted/llama-hosted', name: 'llama-hosted' }, + // Bare, the gateway would strip `openai/` before the upstream saw it. + { id: 'hosted/openai/gpt-oss-120b', name: 'openai/gpt-oss-120b' }, + // Both pages of a paged listing. + { id: 'claude-sonnet-4-5', name: 'Claude Sonnet 4.5' }, + { id: 'claude-haiku-4-5', name: 'Claude Haiku 4.5' } + ] + }); + }); + + it('says nothing about the provider setup', async () => { + const res = await fetch(`${base}/v1/models`, { headers: { 'X-Database-Id': 'db-1' } }); + const text = await res.text(); + for (const internal of ['localhost', 'sk-upstream', 'sk-ant', 'owned_by', 'baseUrl', 'apiKey', 'size', '"type"', '"provider"']) { + expect(text).not.toContain(internal); + } + // The upstream is asked with the gateway's key, never the caller's headers. + expect(upstreamHeaders[0].authorization).toBe('Bearer sk-upstream'); + expect(upstreamHeaders[0]['x-database-id']).toBeUndefined(); + }); + + it('answers 502 without the upstream reason when a provider cannot be listed', async () => { + const broken = await listen(createAgenticServer({ + providers: [{ type: 'openai', baseUrl: `http://localhost:${port(down)}`, apiKey: 'sk-upstream' }] + })); + const res = await fetch(`http://localhost:${port(broken)}/v1/models`, { headers: { 'X-Database-Id': 'db-1' } }); + await close(broken); + expect(res.status).toBe(502); + const text = await res.text(); + expect(JSON.parse(text)).toEqual({ error: { message: 'the model list is unavailable' } }); + expect(text).not.toContain('secret-internal-detail'); + }); +}); + +describe('a model nobody serves', () => { + it('is told how to list the ones that are', async () => { + const res = await fetch(`${base}/v1/chat/completions`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-Database-Id': 'db-1' }, + body: JSON.stringify({ model: 'llama-typo', messages: [{ role: 'user', content: 'hi' }] }) + }); + expect(res.status).toBe(404); + const { error } = await res.json() as { error: { message: string } }; + expect(error.message).toContain("model 'llama-typo' not found"); + expect(error.message).toContain('GET /v1/models lists the models this gateway serves'); + }); +}); + +describe('GET /healthz', () => { + it('reports liveness and nothing about the providers', async () => { + const res = await fetch(`${base}/healthz`); + expect(await res.json()).toEqual({ status: 'ok' }); + }); +}); diff --git a/agentic/agentic-server/src/index.ts b/agentic/agentic-server/src/index.ts index 5200bc23d0..b2059ad98c 100644 --- a/agentic/agentic-server/src/index.ts +++ b/agentic/agentic-server/src/index.ts @@ -23,6 +23,8 @@ */ export { BadRequestError } from './errors'; +export type { ServedModel } from './models'; +export { listServedModels, MODEL_LIST_HINT } from './models'; export { buildProviderHeaders, resolveProvider, resolveUpstreamUrl } from './providers'; export { createRouter } from './router'; export type { AgenticServerStartOptions } from './server'; diff --git a/agentic/agentic-server/src/models.ts b/agentic/agentic-server/src/models.ts new file mode 100644 index 0000000000..597500a410 --- /dev/null +++ b/agentic/agentic-server/src/models.ts @@ -0,0 +1,88 @@ +import { buildProviderHeaders, resolveProvider } from './providers'; +import { providerName, upstreamModel } from './transforms'; +import type { AgenticServerOptions, ResolvedProvider } from './types'; + +/** A model a caller can run: what to send as `model`, and what to show for it. */ +export interface ServedModel { + id: string; + name: string; +} + +/** Appended to an upstream "no such model" answer, so the caller knows where to look. */ +export const MODEL_LIST_HINT = 'GET /v1/models lists the models this gateway serves'; + +const configuredProviders = (options: AgenticServerOptions): ResolvedProvider[] => + options.providers && options.providers.length > 0 + ? options.providers + : [resolveProvider(options)]; + +const text = (value: unknown): string | undefined => + typeof value === 'string' && value ? value : undefined; + +const served = (row: Record, idField: string): ServedModel[] => { + const id = text(row[idField]); + return id ? [{ id, name: text(row.display_name) ?? id }] : []; +}; + +/** The models one provider serves, as its own listing names them. */ +async function upstreamModels(provider: ResolvedProvider): Promise { + const listing = async (url: string): Promise> => { + const res = await fetch(url, { headers: buildProviderHeaders(provider) }); + if (!res.ok) { + throw new Error(`${providerName(provider)} answered ${res.status} to a model listing`); + } + return await res.json() as Record; + }; + const rows = (list: unknown): Record[] => { + if (!Array.isArray(list)) { + throw new Error(`${providerName(provider)} answered a model listing with no list`); + } + return list; + }; + + if (provider.type === 'ollama') { + const body = await listing(`${provider.baseUrl}/api/tags`); + return rows(body.models).flatMap((row) => served(row, 'name')); + } + + // A listing may be paged (Anthropic's is): follow it to the end, or the + // models on a later page read as not served. + const models: ServedModel[] = []; + let after: string | undefined; + do { + const page = after ? `?after_id=${encodeURIComponent(after)}` : ''; + const body = await listing(`${provider.baseUrl}/v1/models${page}`); + models.push(...rows(body.data).flatMap((row) => served(row, 'id'))); + const next = body.has_more === true ? text(body.last_id) : undefined; + if (next === after) break; + after = next; + } while (after); + return models; +} + +/** + * Every model the gateway serves, under the id that reaches it: the bare + * upstream id when the router would send that, unchanged, to the provider serving it, the + * `name/` prefixed one otherwise. Only the id and a display name leave here — + * which provider serves a model, and where, is the deployment's business. + */ +export async function listServedModels(options: AgenticServerOptions): Promise { + const providers = configuredProviders(options); + const lists = await Promise.all(providers.map(upstreamModels)); + const seen = new Set(); + return lists.flatMap((models, index) => { + const provider = providers[index]; + return models.flatMap(({ id, name }) => { + const routed = resolveProvider(options, id); + // Bare only when the id both routes here and survives the prefix strip: + // an upstream id that itself begins `openai/` would lose that segment. + const bare = + (providers.length === 1 || providerName(routed) === providerName(provider)) && + upstreamModel(provider, id) === id; + const served = bare ? id : `${providerName(provider)}/${id}`; + if (seen.has(served)) return []; + seen.add(served); + return [{ id: served, name }]; + }); + }); +} diff --git a/agentic/agentic-server/src/router.ts b/agentic/agentic-server/src/router.ts index e323fbbcd4..69a45a260a 100644 --- a/agentic/agentic-server/src/router.ts +++ b/agentic/agentic-server/src/router.ts @@ -14,6 +14,7 @@ import { Logger } from '@pgpmjs/logger'; import { Router } from 'express'; import { BadRequestError } from './errors'; +import { listServedModels, MODEL_LIST_HINT } from './models'; import { buildProviderHeaders, resolveProvider, resolveUpstreamUrl } from './providers'; import { relayEventStream, withUsageStreamOptions } from './streaming'; import { @@ -79,9 +80,13 @@ function upstreamErrorMessage( ? `${parsed.slice(0, UPSTREAM_REASON_LIMIT)}…` : parsed; - return reason + const message = reason ? `${provider.type} provider error ${status}: ${reason}` : `${provider.type} provider error ${status}`; + + // 404 is how every dialect answers a model it does not serve, and a run + // configured with one fails on its first call: say where the served ones are. + return status === 404 ? `${message} — ${MODEL_LIST_HINT}` : message; } /** @@ -433,6 +438,22 @@ export const createRouter = (options: AgenticServerOptions): Router => { res.status(202).json({ accepted: true }); }); + // GET /v1/models — the models a caller can run: id and display name, nothing + // of the providers behind them. Asked for with the identity a run carries. + router.get('/v1/models', async (req: any, res: any) => { + if (!req.get('X-Database-Id')) { + res.status(400).json({ error: { message: 'X-Database-Id is required' } }); + return; + } + try { + res.json({ object: 'list', data: await listServedModels(options) }); + } catch (err: any) { + // The reason names a provider; it is the operator's to read, in the log. + log.error('models error', err); + res.status(502).json({ error: { message: 'the model list is unavailable' } }); + } + }); + // GET /v1/providers — list configured providers router.get('/v1/providers', (_req: any, res: any) => { const providers = options.providers || [{ @@ -448,18 +469,10 @@ export const createRouter = (options: AgenticServerOptions): Router => { }); }); - // GET /healthz — health check + // GET /healthz — liveness only. It is unauthenticated, and whatever forwards + // to this gateway forwards here too, so it describes no provider. router.get('/healthz', (_req: any, res: any) => { - const providers = options.providers || [{ - type: options.providerType || 'ollama', - baseUrl: options.providerBaseUrl || 'http://localhost:11434' - }]; - res.json({ - status: 'ok', - provider: providers[0]?.type || 'ollama', - providerUrl: providers[0]?.baseUrl || options.providerBaseUrl, - providers: providers.map((p) => p.type) - }); + res.json({ status: 'ok' }); }); return router;