diff --git a/agentic/agentic-server/README.md b/agentic/agentic-server/README.md index 95eea42af1..f404b09dac 100644 --- a/agentic/agentic-server/README.md +++ b/agentic/agentic-server/README.md @@ -50,6 +50,26 @@ app.listen(3001, () => { `POST /v1/chat/completions` with `stream: true` is relayed as a `text/event-stream`: the gateway forwards the provider's frames verbatim, so a client receives tokens as they are produced. It adds `stream_options: { include_usage: true }` unless the caller set `stream_options` itself, and meters the usage the final frame carries. Streaming is available for OpenAI-compatible providers; a provider whose stream would need translation (`ollama`, `anthropic`) is rejected `501` rather than answered with a non-streaming body. +## Providers and routing + +`providers` is a registry; each entry has a `type` (the dialect: `openai`, `anthropic`, `ollama`) and an optional `name` it is addressed by, which defaults to the type. Two entries of one dialect — an in-cluster Ollama and one on a GPU host — are both reachable by name: + +```typescript +createAgenticServer({ + providers: [ + { name: 'ollama', type: 'ollama', baseUrl: 'http://ollama:11434' }, + { name: 'host-ollama', type: 'ollama', baseUrl: 'http://host.docker.internal:11434' }, + { type: 'openai', baseUrl: 'https://api.openai.com', apiKey: process.env.OPENAI_API_KEY } + ] +}); +``` + +A request picks its provider, in this order: the `X-LLM-Provider` header (a name, or a type for an entry with no name); a `name/model` prefix such as `host-ollama/qwen3:8b` (stripped before the call goes upstream); a known model family (`claude*` → anthropic; `gpt-oss*`, `llama*`, `mistral*`, `gemma*`, `nomic*`, `qwen*`, `glm*` → ollama; `gpt*`, `o1*`, `o3*` → openai); otherwise the first entry. + +### The Ollama dialect + +A request to an `ollama` provider is rewritten for `/api/chat`: content parts are flattened to the string Ollama takes; `max_tokens` becomes `options.num_predict` and `num_ctx` becomes `options.num_ctx` (a client `options` object is merged under them); `tools`, `think` and `keep_alive` are forwarded as-is; `response_format` becomes `format` (`json_object` → `'json'`, `json_schema` → the schema; a native `format` is passed through). The reply comes back in the OpenAI shape, including `tool_calls` (arguments as a JSON string, `finish_reason: 'tool_calls'`) and `reasoning_content` for a `thinking` message, so a tool-calling harness can run on an Ollama-served model. + ## Identity & tenancy Requests carry tenant identity via headers: `X-Database-Id` (**required** — requests without it are rejected `400`), `X-Entity-Id`, and `X-Actor-Id`. Routing to a specific provider can be forced with `X-LLM-Provider`. diff --git a/agentic/agentic-server/__tests__/gateway.test.ts b/agentic/agentic-server/__tests__/gateway.test.ts index d26f49905a..524745972a 100644 --- a/agentic/agentic-server/__tests__/gateway.test.ts +++ b/agentic/agentic-server/__tests__/gateway.test.ts @@ -42,6 +42,20 @@ beforeAll(async () => { }); return; } + if (req.body?.model === 'tool-user') { + res.json({ + message: { + role: 'assistant', + content: '', + tool_calls: [{ function: { name: 'get_weather', arguments: { city: 'Paris' } } }] + }, + done: true, + done_reason: 'stop', + prompt_eval_count: 12, + eval_count: 8 + }); + return; + } res.json({ message: { role: 'assistant', content: 'Mock Ollama response' }, prompt_eval_count: 12, @@ -223,6 +237,81 @@ describe('POST /v1/chat/completions', () => { ]); }); + it('carries a tool-calling turn to ollama and back in the OpenAI shape', async () => { + const tool = { + type: 'function', + function: { name: 'get_weather', parameters: { type: 'object', properties: { city: { type: 'string' } } } } + }; + const res = await fetch(`http://localhost:${agenticPort}/v1/chat/completions`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-Database-Id': 'db-tools-1' }, + body: JSON.stringify({ + model: 'tool-user', + messages: [{ role: 'user', content: [{ type: 'text', text: 'weather in Paris?' }] }], + tools: [tool], + response_format: { type: 'json_object' }, + think: false, + keep_alive: '10m', + num_ctx: 16384 + }) + }); + + expect(res.status).toBe(200); + expect(llmRequests[0].body).toMatchObject({ + tools: [tool], + format: 'json', + think: false, + keep_alive: '10m', + options: { num_ctx: 16384 } + }); + const data = (await res.json()) as any; + expect(data.choices[0].finish_reason).toBe('tool_calls'); + expect(data.choices[0].message.tool_calls[0]).toMatchObject({ + type: 'function', + function: { name: 'get_weather', arguments: '{"city":"Paris"}' } + }); + }); + + it('rejects (400) an X-LLM-Provider that is not registered rather than routing elsewhere', async () => { + const res = await fetch(`http://localhost:${agenticPort}/v1/chat/completions`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-Database-Id': 'db-pin-1', 'X-LLM-Provider': 'gpu-ollama' }, + body: JSON.stringify({ model: 'llama3', messages: [{ role: 'user', content: 'hi' }] }) + }); + + expect(res.status).toBe(400); + const data = (await res.json()) as any; + expect(data.error.message).toBe("unknown provider 'gpu-ollama'; registered: ollama"); + await new Promise((r) => setTimeout(r, 50)); + expect(llmRequests).toHaveLength(0); + expect(sinkEntries).toHaveLength(0); + }); + + it('rejects (400) a request the ollama dialect cannot express, without calling upstream', async () => { + const res = await fetch(`http://localhost:${agenticPort}/v1/chat/completions`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-Database-Id': 'db-bad-1' }, + body: JSON.stringify({ + model: 'llama3', + messages: [ + { role: 'user', content: 'weather?' }, + { + role: 'assistant', + content: '', + tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'get_weather', arguments: '{not json' } }] + } + ] + }) + }); + + expect(res.status).toBe(400); + const data = (await res.json()) as any; + expect(data.error.message).toMatch(/^ollama: tool call 'get_weather' carries arguments that are not JSON/); + await new Promise((r) => setTimeout(r, 50)); + expect(llmRequests).toHaveLength(0); + expect(sinkEntries).toHaveLength(0); + }); + it('names the upstream reason in the error message the harness reads', async () => { const res = await fetch(`http://localhost:${agenticPort}/v1/chat/completions`, { method: 'POST', @@ -419,5 +508,6 @@ describe('GET /v1/providers', () => { const data = (await res.json()) as any; expect(data.providers).toHaveLength(1); expect(data.providers[0].type).toBe('ollama'); + expect(data.providers[0].name).toBe('ollama'); }); }); diff --git a/agentic/agentic-server/__tests__/providers.test.ts b/agentic/agentic-server/__tests__/providers.test.ts new file mode 100644 index 0000000000..a704676c1f --- /dev/null +++ b/agentic/agentic-server/__tests__/providers.test.ts @@ -0,0 +1,79 @@ +/** + * Provider selection over a registry. + * + * A deployment may run several providers of one dialect (two Ollama hosts: one + * in-cluster, one on a GPU box), so an entry is addressed by its registry + * `name`, not by its `type`. Model-name routing must also not mistake an + * open-weights model for a vendor's hosted one. + */ + +import { BadRequestError } from '../src/errors'; +import { resolveProvider } from '../src/providers'; +import type { AgenticServerOptions } from '../src/types'; + +const twoOllamas: AgenticServerOptions = { + providers: [ + { name: 'openai', type: 'openai', baseUrl: 'https://api.openai.test', apiKey: 'sk-oai' }, + { name: 'ollama', type: 'ollama', baseUrl: 'http://ollama.svc:11434' }, + { name: 'host-ollama', type: 'ollama', baseUrl: 'http://host.docker.internal:11434' } + ] +}; + +describe('resolveProvider by registry name', () => { + it('selects the second provider of a type when X-LLM-Provider names it', () => { + const provider = resolveProvider(twoOllamas, 'qwen3:8b', 'host-ollama'); + expect(provider.baseUrl).toBe('http://host.docker.internal:11434'); + }); + + it('selects a provider by a `name/model` prefix and strips the prefix', () => { + const provider = resolveProvider(twoOllamas, 'host-ollama/qwen3:8b'); + expect(provider.baseUrl).toBe('http://host.docker.internal:11434'); + expect(provider.defaultModel).toBe('qwen3:8b'); + }); + + it('still answers to the type when an entry carries no name', () => { + const options: AgenticServerOptions = { + providers: [ + { type: 'openai', baseUrl: 'https://api.openai.test', apiKey: 'sk-oai' }, + { type: 'ollama', baseUrl: 'http://ollama.svc:11434' } + ] + }; + expect(resolveProvider(options, 'anything', 'ollama').baseUrl).toBe('http://ollama.svc:11434'); + expect(resolveProvider(options, 'ollama/qwen3').baseUrl).toBe('http://ollama.svc:11434'); + }); + + it('rejects an X-LLM-Provider that names no registered provider instead of routing elsewhere', () => { + expect(() => resolveProvider(twoOllamas, 'qwen3:8b', 'host-olama')).toThrow(BadRequestError); + expect(() => resolveProvider(twoOllamas, 'qwen3:8b', 'host-olama')).toThrow( + "unknown provider 'host-olama'; registered: openai, ollama, host-ollama" + ); + }); +}); + +describe('resolveProvider by model name', () => { + const openaiAndOllama: AgenticServerOptions = { + providers: [ + { type: 'openai', baseUrl: 'https://api.openai.test', apiKey: 'sk-oai' }, + { type: 'ollama', baseUrl: 'http://ollama.svc:11434' } + ] + }; + + it('routes gpt-oss to ollama, not to openai', () => { + expect(resolveProvider(openaiAndOllama, 'gpt-oss:20b').type).toBe('ollama'); + }); + + it('still routes gpt-4o to openai', () => { + expect(resolveProvider(openaiAndOllama, 'gpt-4o').type).toBe('openai'); + }); + + it('routes qwen and glm to ollama', () => { + const ollamaLast: AgenticServerOptions = { + providers: [ + { type: 'anthropic', baseUrl: 'https://api.anthropic.test', apiKey: 'sk-ant' }, + { type: 'ollama', baseUrl: 'http://ollama.svc:11434' } + ] + }; + expect(resolveProvider(ollamaLast, 'qwen3:8b').type).toBe('ollama'); + expect(resolveProvider(ollamaLast, 'glm-4.6').type).toBe('ollama'); + }); +}); diff --git a/agentic/agentic-server/__tests__/transforms.test.ts b/agentic/agentic-server/__tests__/transforms.test.ts new file mode 100644 index 0000000000..10de5a64e8 --- /dev/null +++ b/agentic/agentic-server/__tests__/transforms.test.ts @@ -0,0 +1,170 @@ +/** + * The Ollama dialect of a chat request and response. + * + * A tool-calling harness (pi) sends `tools`, a `response_format`, and reads + * `tool_calls` back in the OpenAI shape; a model served by Ollama needs `think`, + * `keep_alive` and `num_ctx` to be reachable at all. Each must cross the + * gateway, or `ollama/…` can serve chat but no agent. + */ + +import { transformChatRequest, transformChatResponse } from '../src/transforms'; +import type { ResolvedProvider } from '../src/types'; + +const ollama: ResolvedProvider = { type: 'ollama', baseUrl: 'http://ollama.test:11434' }; + +const weatherTool = { + type: 'function', + function: { + name: 'get_weather', + description: 'Weather for a city', + parameters: { type: 'object', properties: { city: { type: 'string' } }, required: ['city'] } + } +}; + +describe('transformChatRequest for ollama', () => { + it('forwards tools and tool_choice', () => { + const body = transformChatRequest(ollama, { + model: 'qwen3:8b', + messages: [{ role: 'user', content: 'weather in Paris?' }], + tools: [weatherTool], + tool_choice: 'auto' + }); + expect(body.tools).toEqual([weatherTool]); + }); + + it('turns response_format json_object into format json', () => { + const body = transformChatRequest(ollama, { + model: 'qwen3:8b', + messages: [], + response_format: { type: 'json_object' } + }); + expect(body.format).toBe('json'); + }); + + it('turns response_format json_schema into the schema as format', () => { + const schema = { type: 'object', properties: { intent: { type: 'string' } } }; + const body = transformChatRequest(ollama, { + model: 'qwen3:8b', + messages: [], + response_format: { type: 'json_schema', json_schema: { name: 'route', schema } } + }); + expect(body.format).toEqual(schema); + }); + + it('passes an ollama-native format through', () => { + const body = transformChatRequest(ollama, { model: 'qwen3:8b', messages: [], format: 'json' }); + expect(body.format).toBe('json'); + }); + + it('forwards think and keep_alive, and num_ctx under options', () => { + const body = transformChatRequest(ollama, { + model: 'qwen3:8b', + messages: [], + think: true, + keep_alive: '30m', + num_ctx: 32768, + temperature: 0.1, + max_tokens: 512 + }); + expect(body.think).toBe(true); + expect(body.keep_alive).toBe('30m'); + expect(body.options).toEqual({ temperature: 0.1, num_predict: 512, num_ctx: 32768 }); + }); + + it('omits what the caller did not send', () => { + const body = transformChatRequest(ollama, { model: 'qwen3:8b', messages: [] }); + expect(body).toEqual({ model: 'qwen3:8b', messages: [], stream: false }); + }); + + it('parses the arguments string of an earlier assistant tool call into the object ollama takes', () => { + const body = transformChatRequest(ollama, { + model: 'qwen3:8b', + messages: [ + { role: 'user', content: 'weather in Paris?' }, + { + role: 'assistant', + content: '', + tool_calls: [ + { id: 'call_1', type: 'function', function: { name: 'get_weather', arguments: '{"city":"Paris"}' } } + ] + }, + { role: 'tool', tool_call_id: 'call_1', content: '{"temp_c":18}' } + ] + }); + const messages = body.messages as Array>; + expect(messages[1].tool_calls[0].function.arguments).toEqual({ city: 'Paris' }); + expect(messages[2]).toMatchObject({ role: 'tool', content: '{"temp_c":18}' }); + }); + + it('strips a `name/` prefix that names the provider, not only a `type/` one', () => { + const named: ResolvedProvider = { name: 'host-ollama', type: 'ollama', baseUrl: 'http://host:11434' }; + expect(transformChatRequest(named, { model: 'host-ollama/qwen3:8b', messages: [] }).model).toBe('qwen3:8b'); + expect(transformChatRequest(named, { model: 'ollama/qwen3:8b', messages: [] }).model).toBe('qwen3:8b'); + }); +}); + +describe('transformChatResponse for ollama', () => { + it('returns tool calls in the OpenAI shape with finish_reason tool_calls', () => { + const { body } = transformChatResponse( + { + message: { + role: 'assistant', + content: '', + tool_calls: [{ function: { name: 'get_weather', arguments: { city: 'Paris' } } }] + }, + done: true, + done_reason: 'stop', + prompt_eval_count: 20, + eval_count: 10 + }, + ollama + ); + const choice = (body.choices as any[])[0]; + expect(choice.finish_reason).toBe('tool_calls'); + expect(choice.message.tool_calls).toHaveLength(1); + expect(choice.message.tool_calls[0]).toMatchObject({ + type: 'function', + function: { name: 'get_weather', arguments: '{"city":"Paris"}' } + }); + expect(typeof choice.message.tool_calls[0].id).toBe('string'); + expect(choice.message.tool_calls[0].id).not.toBe(''); + }); + + it('reports a tool call cut off by the token budget as length, not as a completed tool_calls turn', () => { + const { body } = transformChatResponse( + { + message: { + role: 'assistant', + content: '', + tool_calls: [{ function: { name: 'get_weather', arguments: { city: 'Par' } } }] + }, + done: true, + done_reason: 'length' + }, + ollama + ); + const choice = (body.choices as any[])[0]; + expect(choice.finish_reason).toBe('length'); + expect(choice.message.tool_calls).toHaveLength(1); + }); + + it('reports a truncated answer as finish_reason length', () => { + const { body } = transformChatResponse( + { message: { role: 'assistant', content: 'partial' }, done: true, done_reason: 'length' }, + ollama + ); + expect((body.choices as any[])[0].finish_reason).toBe('length'); + }); + + it('surfaces thinking as reasoning_content beside the answer', () => { + const { body } = transformChatResponse( + { message: { role: 'assistant', content: 'yes', thinking: 'the user asked…' }, done: true, done_reason: 'stop' }, + ollama + ); + expect((body.choices as any[])[0].message).toEqual({ + role: 'assistant', + content: 'yes', + reasoning_content: 'the user asked…' + }); + }); +}); diff --git a/agentic/agentic-server/src/errors.ts b/agentic/agentic-server/src/errors.ts new file mode 100644 index 0000000000..217eb14d41 --- /dev/null +++ b/agentic/agentic-server/src/errors.ts @@ -0,0 +1,14 @@ +/** + * A request the gateway rejects before contacting any upstream: an unknown + * provider pinned by `X-LLM-Provider`, or a body the target dialect cannot + * express. The router answers it with `400` so a client mistake never reads + * as a provider outage. + */ +export class BadRequestError extends Error { + readonly status = 400; + + constructor(message: string) { + super(message); + this.name = 'BadRequestError'; + } +} diff --git a/agentic/agentic-server/src/index.ts b/agentic/agentic-server/src/index.ts index 6b4097a5bc..5200bc23d0 100644 --- a/agentic/agentic-server/src/index.ts +++ b/agentic/agentic-server/src/index.ts @@ -22,10 +22,19 @@ * ``` */ +export { BadRequestError } from './errors'; export { buildProviderHeaders, resolveProvider, resolveUpstreamUrl } from './providers'; export { createRouter } from './router'; export type { AgenticServerStartOptions } from './server'; export { createAgenticServer } from './server'; +export { + providerName, + transformChatRequest, + transformChatResponse, + transformEmbedRequest, + transformEmbedResponse, + upstreamModel +} from './transforms'; export type { AgenticServerOptions, InferenceAttribution, diff --git a/agentic/agentic-server/src/providers.ts b/agentic/agentic-server/src/providers.ts index ed57a5f8f7..c854d8f2d8 100644 --- a/agentic/agentic-server/src/providers.ts +++ b/agentic/agentic-server/src/providers.ts @@ -1,8 +1,28 @@ -import type { AgenticServerOptions, ResolvedProvider } from './types'; +import { BadRequestError } from './errors'; +import { providerName } from './transforms'; +import type { AgenticServerOptions, ProviderConfig, ResolvedProvider } from './types'; + +/** + * Open-weights families Ollama serves. `gpt-oss` is listed because the + * `gpt` prefix would otherwise read as OpenAI's hosted models. + */ +const OLLAMA_MODEL_PREFIXES = ['gpt-oss', 'llama', 'mistral', 'gemma', 'nomic', 'qwen', 'glm']; + +const startsWithAny = (model: string, prefixes: readonly string[]): boolean => + prefixes.some((prefix) => model.startsWith(prefix)); + +/** + * The entry a request addresses: by registry name first, so two providers of + * one dialect are both reachable, then by dialect for an entry with no name of + * its own. + */ +const findNamed = (providers: ProviderConfig[], name: string): ProviderConfig | undefined => + providers.find((p) => providerName(p) === name) ?? providers.find((p) => p.type === name); /** * Resolve which provider to use for a given request. - * Priority: model-based routing → explicit provider header → default. + * Priority: explicit provider header → `name/model` prefix → known model + * families → the first configured provider. */ export function resolveProvider( options: AgenticServerOptions, @@ -13,36 +33,36 @@ export function resolveProvider( if (providers.length > 0) { if (requestProvider) { - const match = providers.find((p) => p.type === requestProvider); + const match = findNamed(providers, requestProvider); if (match) return match; + // A request pinned to a provider that is not registered must not drift + // to whichever entry a heuristic picks. + throw new BadRequestError( + `unknown provider '${requestProvider}'; registered: ${providers.map(providerName).join(', ')}` + ); } - // Route by model prefix: "anthropic/claude-3" → anthropic provider + // Route by model prefix: "anthropic/claude-3" → the provider named anthropic if (requestModel && requestModel.includes('/')) { - const prefix = requestModel.split('/')[0]; - const match = providers.find((p) => p.type === prefix); + const [prefix, ...rest] = requestModel.split('/'); + const match = findNamed(providers, prefix); if (match) { - return { ...match, defaultModel: requestModel.split('/').slice(1).join('/') }; + return { ...match, defaultModel: rest.join('/') }; } } - // Route by known model patterns + // Route by known model families if (requestModel) { if (requestModel.startsWith('claude')) { const match = providers.find((p) => p.type === 'anthropic'); if (match) return match; } - if (requestModel.startsWith('gpt') || requestModel.startsWith('o1') || requestModel.startsWith('o3')) { - const match = providers.find((p) => p.type === 'openai'); + if (startsWithAny(requestModel, OLLAMA_MODEL_PREFIXES)) { + const match = providers.find((p) => p.type === 'ollama'); if (match) return match; } - if ( - requestModel.startsWith('llama') || - requestModel.startsWith('mistral') || - requestModel.startsWith('gemma') || - requestModel.startsWith('nomic') - ) { - const match = providers.find((p) => p.type === 'ollama'); + if (requestModel.startsWith('gpt') || requestModel.startsWith('o1') || requestModel.startsWith('o3')) { + const match = providers.find((p) => p.type === 'openai'); if (match) return match; } } @@ -51,8 +71,12 @@ export function resolveProvider( } // Single-provider mode (default: ollama for local dev) + const type = options.providerType || 'ollama'; + if (requestProvider && requestProvider !== type) { + throw new BadRequestError(`unknown provider '${requestProvider}'; registered: ${type}`); + } return { - type: options.providerType || 'ollama', + type, baseUrl: options.providerBaseUrl || 'http://localhost:11434', apiKey: options.providerApiKey, defaultModel: options.defaultModel diff --git a/agentic/agentic-server/src/router.ts b/agentic/agentic-server/src/router.ts index 03578994b5..e323fbbcd4 100644 --- a/agentic/agentic-server/src/router.ts +++ b/agentic/agentic-server/src/router.ts @@ -13,9 +13,11 @@ import { Logger } from '@pgpmjs/logger'; import { Router } from 'express'; +import { BadRequestError } from './errors'; import { buildProviderHeaders, resolveProvider, resolveUpstreamUrl } from './providers'; import { relayEventStream, withUsageStreamOptions } from './streaming'; import { + providerName, transformChatRequest, transformChatResponse, transformEmbedRequest, @@ -25,6 +27,26 @@ import type { AgenticServerOptions, InferenceAttribution, ResolvedProvider } fro const log = new Logger('agentic-server'); +/** + * Answer a request the gateway rejects itself — before any upstream call — + * with `400`, or report that `err` was something else. A client mistake must + * not be metered or reported as a provider failure. + */ +function rejectBadRequest(res: any, err: unknown): boolean { + if (!(err instanceof BadRequestError)) return false; + res.status(err.status).json({ error: { message: err.message } }); + return true; +} + +function selectProvider(options: AgenticServerOptions, req: any, res: any): ResolvedProvider | undefined { + try { + return resolveProvider(options, req.body?.model, req.get('X-LLM-Provider')); + } catch (err) { + if (rejectBadRequest(res, err)) return undefined; + throw err; + } +} + /** How long an upstream reason may be before it stops being a message. */ const UPSTREAM_REASON_LIMIT = 300; @@ -97,10 +119,10 @@ export const createRouter = (options: AgenticServerOptions): Router => { } const attribution = readAttribution(req); const { entityId } = attribution; - const requestProvider = req.get('X-LLM-Provider'); const startTime = process.hrtime.bigint(); - const provider = resolveProvider(options, req.body?.model, requestProvider); + const provider = selectProvider(options, req, res); + if (!provider) return; log.info('chat/completions', { databaseId, @@ -232,6 +254,7 @@ export const createRouter = (options: AgenticServerOptions): Router => { res.json(responseBody); } catch (err: any) { + if (!res.headersSent && rejectBadRequest(res, err)) return; const latencyMs = Number(process.hrtime.bigint() - startTime) / 1e6; log.error('chat/completions error', err); @@ -271,10 +294,10 @@ export const createRouter = (options: AgenticServerOptions): Router => { } const attribution = readAttribution(req); const { entityId } = attribution; - const requestProvider = req.get('X-LLM-Provider'); const startTime = process.hrtime.bigint(); - const provider = resolveProvider(options, req.body?.model, requestProvider); + const provider = selectProvider(options, req, res); + if (!provider) return; log.info('embeddings', { databaseId, entityId, provider: provider.type, model: req.body?.model }); @@ -338,6 +361,7 @@ export const createRouter = (options: AgenticServerOptions): Router => { res.json(responseBody); } catch (err: any) { + if (rejectBadRequest(res, err)) return; const latencyMs = Number(process.hrtime.bigint() - startTime) / 1e6; log.error('embeddings error', err); @@ -417,6 +441,7 @@ export const createRouter = (options: AgenticServerOptions): Router => { }]; res.json({ providers: providers.map((p) => ({ + name: providerName(p), type: p.type, defaultModel: p.defaultModel })) diff --git a/agentic/agentic-server/src/transforms.ts b/agentic/agentic-server/src/transforms.ts index 00b6efb62e..8902c520ff 100644 --- a/agentic/agentic-server/src/transforms.ts +++ b/agentic/agentic-server/src/transforms.ts @@ -1,3 +1,4 @@ +import { BadRequestError } from './errors'; import type { ResolvedProvider, UsageResult } from './types'; interface ContentPart { @@ -6,10 +7,32 @@ interface ContentPart { image_url?: { url?: string }; } +interface ToolCall { + id?: string; + type?: string; + function?: { name?: string; arguments?: unknown }; +} + interface ChatMessage { role: string; content?: string | ContentPart[]; images?: string[]; + tool_calls?: ToolCall[]; +} + +/** The name a request routes with: the registry name, else the dialect. */ +export const providerName = (provider: ResolvedProvider): string => provider.name ?? provider.type; + +/** + * Strip the routing prefix a client uses to pick a provider (`anthropic/claude-…`, + * `host-ollama/qwen3`), which the upstream itself does not know. + */ +export function upstreamModel(provider: ResolvedProvider, model: unknown): string | undefined { + const requested = typeof model === 'string' && model ? model : provider.defaultModel; + if (!requested) return undefined; + const [prefix, ...rest] = requested.split('/'); + if (rest.length === 0) return requested; + return prefix === providerName(provider) || prefix === provider.type ? rest.join('/') : requested; } /** @@ -25,7 +48,8 @@ interface ChatMessage { function flattenContentParts(messages: unknown): unknown { if (!Array.isArray(messages)) return messages; - return (messages as ChatMessage[]).map((message) => { + return (messages as ChatMessage[]).map((original) => { + const message = ollamaToolCalls(original); if (!Array.isArray(message.content)) return message; const text: string[] = []; @@ -37,16 +61,16 @@ function flattenContentParts(messages: unknown): unknown { } if (part.type === 'image_url') { const url = part.image_url?.url; - if (!url) throw new Error('ollama: an image_url content part carries no url'); + if (!url) throw new BadRequestError('ollama: an image_url content part carries no url'); // Ollama takes base64 bytes, never a fetchable url. const base64 = /^data:[^;]*;base64,(.*)$/.exec(url)?.[1]; if (!base64) { - throw new Error('ollama: an image content part must be a base64 data url'); + throw new BadRequestError('ollama: an image content part must be a base64 data url'); } images.push(base64); continue; } - throw new Error(`ollama: unsupported content part type '${part.type}'`); + throw new BadRequestError(`ollama: unsupported content part type '${part.type}'`); } return { @@ -57,28 +81,76 @@ function flattenContentParts(messages: unknown): unknown { }); } +/** + * An earlier assistant turn replays its tool calls with `arguments` as the JSON + * string OpenAI returns them in; Ollama's struct wants the object, and answers + * the string with an unmarshal error. + */ +function ollamaToolCalls(message: ChatMessage): ChatMessage { + if (!Array.isArray(message.tool_calls)) return message; + return { + ...message, + tool_calls: message.tool_calls.map((call) => { + const args = call.function?.arguments; + if (typeof args !== 'string') return call; + let parsed: unknown; + try { + parsed = JSON.parse(args); + } catch (err) { + throw new BadRequestError( + `ollama: tool call '${call.function?.name ?? ''}' carries arguments that are not JSON: ${(err as Error).message}` + ); + } + return { ...call, function: { ...call.function, arguments: parsed } }; + }) + }; +} + +/** + * Ollama's `format` from an OpenAI `response_format`: `json_object` is the + * `'json'` mode, `json_schema` is the schema itself. A native `format` wins + * when a caller already speaks the dialect. + */ +function ollamaFormat(body: Record): unknown { + if (body.format !== undefined) return body.format; + const responseFormat = body.response_format as { type?: string; json_schema?: { schema?: unknown } } | undefined; + if (!responseFormat || typeof responseFormat !== 'object') return undefined; + if (responseFormat.type === 'json_object') return 'json'; + if (responseFormat.type === 'json_schema') { + if (responseFormat.json_schema?.schema === undefined) { + throw new BadRequestError('ollama: response_format json_schema carries no schema'); + } + return responseFormat.json_schema.schema; + } + return undefined; +} + export function transformChatRequest( provider: ResolvedProvider, body: Record ): Record { - let model = body.model || provider.defaultModel; - if (typeof model === 'string' && model.includes('/')) { - const prefix = model.split('/')[0]; - if (prefix === provider.type) { - model = model.split('/').slice(1).join('/'); - } - } + const model = upstreamModel(provider, body.model); if (provider.type === 'ollama') { - // Ollama takes sampling settings under `options`; `max_tokens` is `num_predict`. + // Ollama takes sampling and runtime settings under `options`; `max_tokens` + // is `num_predict`, and `num_ctx` (the context window the model is loaded + // with) has no OpenAI spelling, so it rides at the top level and moves here. + const clientOptions = (body.options && typeof body.options === 'object' ? body.options : {}) as Record; const options = { + ...clientOptions, ...(body.temperature !== undefined && { temperature: body.temperature }), - ...(body.max_tokens !== undefined && { num_predict: body.max_tokens }) + ...(body.max_tokens !== undefined && { num_predict: body.max_tokens }), + ...(body.num_ctx !== undefined && { num_ctx: body.num_ctx }) }; + const format = ollamaFormat(body); return { model: model || 'llama3', messages: flattenContentParts(body.messages), stream: false, + ...(body.tools !== undefined && { tools: body.tools }), + ...(format !== undefined && { format }), + ...(body.think !== undefined && { think: body.think }), + ...(body.keep_alive !== undefined && { keep_alive: body.keep_alive }), ...(Object.keys(options).length > 0 && { options }) }; } @@ -104,7 +176,7 @@ export function transformEmbedRequest( provider: ResolvedProvider, body: Record ): Record { - const model = body.model || provider.defaultModel; + const model = upstreamModel(provider, body.model); if (provider.type === 'ollama') { return { @@ -120,7 +192,9 @@ export function transformChatResponse( provider: ResolvedProvider ): { body: Record; usage: UsageResult } { if (provider.type === 'ollama') { - const msg = data.message as { role: string; content: string } | undefined; + const msg = data.message as + | { role?: string; content?: string; thinking?: string; tool_calls?: ToolCall[] } + | undefined; const promptTokens = (data.prompt_eval_count as number) || 0; const completionTokens = (data.eval_count as number) || 0; const usage = { @@ -128,13 +202,23 @@ export function transformChatResponse( completion_tokens: completionTokens, total_tokens: promptTokens + completionTokens }; + const toolCalls = openAiToolCalls(msg?.tool_calls); + // A generation cut off by the token budget may hold half a tool call; + // only a completed turn is reported as `tool_calls`. + const reason = finishReason(data.done_reason); + const message: Record = { + role: msg?.role || 'assistant', + content: msg?.content ?? '', + ...(msg?.thinking ? { reasoning_content: msg.thinking } : {}), + ...(toolCalls ? { tool_calls: toolCalls } : {}) + }; return { body: { id: `chatcmpl-${Date.now()}`, object: 'chat.completion', choices: [{ - message: msg || { role: 'assistant', content: '' }, - finish_reason: 'stop', + message, + finish_reason: toolCalls && reason === 'stop' ? 'tool_calls' : reason, index: 0 }], usage @@ -172,6 +256,31 @@ export function transformChatResponse( return { body: data, usage }; } +/** + * Ollama's tool calls in the OpenAI shape a harness reads: `arguments` as a JSON + * string, a `type`, and an `id` (Ollama gives none; the harness echoes the id + * back on the tool result, so any stable per-response id works). + */ +function openAiToolCalls(calls: ToolCall[] | undefined): Record[] | undefined { + if (!Array.isArray(calls) || calls.length === 0) return undefined; + return calls.map((call, index) => { + const args = call.function?.arguments; + return { + id: call.id || `call_${index}`, + type: 'function', + function: { + name: call.function?.name ?? '', + arguments: typeof args === 'string' ? args : JSON.stringify(args ?? {}) + } + }; + }); +} + +function finishReason(doneReason: unknown): string { + if (doneReason === 'length' || doneReason === 'max_tokens') return 'length'; + return 'stop'; +} + export function transformEmbedResponse( data: Record, provider: ResolvedProvider diff --git a/agentic/agentic-server/src/types.ts b/agentic/agentic-server/src/types.ts index 1f3fb9b9c1..494fd89df4 100644 --- a/agentic/agentic-server/src/types.ts +++ b/agentic/agentic-server/src/types.ts @@ -1,4 +1,9 @@ export interface ProviderConfig { + /** Registry name a request addresses this entry by (`X-LLM-Provider`, a + * `name/model` prefix). Defaults to `type`, so a deployment with one entry + * per dialect needs no names. */ + name?: string; + /** Dialect: 'openai' | 'anthropic' | 'ollama'. Chooses URL, auth and body shape. */ type: string; baseUrl: string; apiKey?: string; @@ -32,6 +37,7 @@ export interface AgenticServerOptions { } export interface ResolvedProvider { + name?: string; type: string; baseUrl: string; apiKey?: string;