From 8aba74b37073ffcb7151d5400c0a0e2d3f88936c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Po=C5=99=C3=ADzek?= <33246262+Rhiz3K@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:52:11 +0200 Subject: [PATCH] feat: advanced OpenAI settings per provider Providers of type openai and openai_compatible get an Advanced section: endpoint (auto, Responses API, Chat Completions), reasoning effort, reasoning summary, verbosity, max output tokens and temperature. They are stored as one JSON column (migration 0013) and validated by the routes. Empty fields send nothing. The user's output limit never goes above a caller's own cap. Temperature is not sent to OpenAI with an effort other than none, and a server that refuses it is asked again without it. The summary retry now keeps a user-set effort, and an effort the model refuses is shown instead of turning summaries off. --- CHANGELOG.md | 1 + locales/en.json | 29 +- migrations/mysql/0013_provider_advanced.sql | 4 + .../postgres/0013_provider_advanced.sql | 4 + migrations/snapshot.json | 3 + migrations/sqlite/0013_provider_advanced.sql | 4 + src/backend/ops/routes.ts | 1 + src/backend/providers/openai-responses.ts | 87 ++- src/backend/providers/openai.ts | 46 +- src/backend/providers/types.ts | 14 + src/backend/repository.ts | 26 +- src/backend/routes.ts | 147 ++++- src/backend/tables.ts | 2 + src/backend/tasks/runner.ts | 1 + src/frontend/AiProviderSettings.tsx | 294 ++++++++- src/frontend/ai-api.ts | 20 + tests/backend/migrations.test.ts | 38 +- .../providers/openai-responses.test.ts | 559 ++++++++++++++++++ tests/backend/providers/openai.test.ts | 495 ++++++++++++++++ tests/backend/routes.test.ts | 228 +++++++ tests/frontend/AiProviderSettings.test.tsx | 230 +++++++ 21 files changed, 2210 insertions(+), 23 deletions(-) create mode 100644 migrations/mysql/0013_provider_advanced.sql create mode 100644 migrations/postgres/0013_provider_advanced.sql create mode 100644 migrations/sqlite/0013_provider_advanced.sql create mode 100644 tests/backend/providers/openai.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e6e041..11d1dca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Added - The assistant starts on the model you used last +- Advanced settings for OpenAI and OpenAI compatible providers: endpoint (Auto, Responses API or Chat Completions), reasoning effort, reasoning summary, verbosity, max output tokens and temperature. Left on Default, nothing extra is sent ### Changed diff --git a/locales/en.json b/locales/en.json index a0dc36d..f365598 100644 --- a/locales/en.json +++ b/locales/en.json @@ -218,7 +218,34 @@ "contextSourceDefault": "default for unknown models", "contextLoadFailed": "Could not read the context windows", "contextRefresh": "Refresh", - "contextWindowInvalid": "Enter the context window as a whole number of tokens, at least 1024." + "contextWindowInvalid": "Enter the context window as a whole number of tokens, at least 1024.", + "advanced": "Advanced", + "advancedHint": "Leave a setting on Default to use the model's own.", + "advancedDefault": "Default", + "advancedDefaultAuto": "Default (auto)", + "advancedAuto": "Auto", + "advancedEndpoint": "Endpoint", + "advancedEndpointResponses": "Responses API", + "advancedEndpointChat": "Chat Completions", + "advancedReasoningEffort": "Reasoning effort", + "advancedReasoningSummary": "Reasoning summary", + "advancedReasoningSummaryHint": "Responses API only.", + "advancedVerbosity": "Verbosity", + "advancedNone": "None", + "advancedMinimal": "Minimal", + "advancedLow": "Low", + "advancedMedium": "Medium", + "advancedHigh": "High", + "advancedXhigh": "Extra high", + "advancedMax": "Max", + "advancedConcise": "Concise", + "advancedDetailed": "Detailed", + "advancedOff": "Off", + "advancedMaxOutputTokens": "Max output tokens", + "advancedMaxOutputTokensInvalid": "Enter a whole number from 1 to 1,000,000.", + "advancedTemperature": "Temperature", + "advancedTemperatureHint": "OpenAI reasoning models accept this only with reasoning effort None, so with any other effort it is not sent.", + "advancedTemperatureInvalid": "Enter a number from 0 to 2." }, "nav": { "ai": "AI assistant" diff --git a/migrations/mysql/0013_provider_advanced.sql b/migrations/mysql/0013_provider_advanced.sql new file mode 100644 index 0000000..af34be6 --- /dev/null +++ b/migrations/mysql/0013_provider_advanced.sql @@ -0,0 +1,4 @@ +-- ai 0013: provider_advanced +-- Generated by termix-plugin migrations. Review before committing. + +ALTER TABLE `p_ai_providers` ADD COLUMN `advanced` longtext; diff --git a/migrations/postgres/0013_provider_advanced.sql b/migrations/postgres/0013_provider_advanced.sql new file mode 100644 index 0000000..4feabbc --- /dev/null +++ b/migrations/postgres/0013_provider_advanced.sql @@ -0,0 +1,4 @@ +-- ai 0013: provider_advanced +-- Generated by termix-plugin migrations. Review before committing. + +ALTER TABLE "p_ai_providers" ADD COLUMN "advanced" text; diff --git a/migrations/snapshot.json b/migrations/snapshot.json index 3f56246..094cdd2 100644 --- a/migrations/snapshot.json +++ b/migrations/snapshot.json @@ -163,6 +163,9 @@ "modelOverrides": { "type": "text" }, + "advanced": { + "type": "text" + }, "enabled": { "type": "boolean", "notNull": true, diff --git a/migrations/sqlite/0013_provider_advanced.sql b/migrations/sqlite/0013_provider_advanced.sql new file mode 100644 index 0000000..4feabbc --- /dev/null +++ b/migrations/sqlite/0013_provider_advanced.sql @@ -0,0 +1,4 @@ +-- ai 0013: provider_advanced +-- Generated by termix-plugin migrations. Review before committing. + +ALTER TABLE "p_ai_providers" ADD COLUMN "advanced" text; diff --git a/src/backend/ops/routes.ts b/src/backend/ops/routes.ts index 9b51991..d7907dd 100644 --- a/src/backend/ops/routes.ts +++ b/src/backend/ops/routes.ts @@ -162,6 +162,7 @@ export function registerOpsRoutes( providerType: provider.providerType as ProviderConfig["providerType"], baseUrl: provider.baseUrl, apiKey: provider.apiKey, + advanced: provider.advanced, fetch: createProviderFetch( ctx.fetch, await readPrivateAllowlist(ctx.settings), diff --git a/src/backend/providers/openai-responses.ts b/src/backend/providers/openai-responses.ts index 608a814..3607ce6 100644 --- a/src/backend/providers/openai-responses.ts +++ b/src/backend/providers/openai-responses.ts @@ -14,7 +14,24 @@ export function resetResponsesMemory(): void { reasoningDisabled.clear(); } +/** The user's output limit, never above a caller's own cap. */ +export function cappedMaxTokens( + limit: number | undefined, + request: Pick, +): number | undefined { + if (limit === undefined) return request.maxTokens; + return request.maxTokens ? Math.min(limit, request.maxTokens) : limit; +} + export function usesResponsesApi(config: ProviderConfig): boolean { + if ( + config.providerType !== "openai" && + config.providerType !== "openai_compatible" + ) { + return false; + } + if (config.advanced?.endpoint === "responses") return true; + if (config.advanced?.endpoint === "chat") return false; if (config.providerType !== "openai") return false; const base = config.baseUrl?.trim(); if (!base) return true; @@ -76,17 +93,39 @@ export async function* streamResponses( headers: Record, ): AsyncIterable { const memoryKey = `${url}|${request.model}`; + const advanced = config.advanced ?? {}; + const reasoning: Record = {}; + if (advanced.reasoningEffort !== undefined) { + reasoning.effort = advanced.reasoningEffort; + } + const summary = advanced.reasoningSummary ?? "auto"; + if (summary !== "off" && !reasoningDisabled.has(memoryKey)) { + reasoning.summary = summary; + } + const maxTokens = cappedMaxTokens(advanced.maxOutputTokens, request); + const skipTemperature = + advanced.temperature !== undefined && + config.providerType === "openai" && + advanced.reasoningEffort !== undefined && + advanced.reasoningEffort !== "none"; + if (skipTemperature) { + config.log?.warn( + `OpenAI skipped temperature for ${request.model} because reasoning effort is ${advanced.reasoningEffort}; temperature requires reasoning effort none.`, + ); + } let body: Record = { model: request.model, instructions: request.system, input: toResponsesInput(request), stream: true, store: false, - ...(!reasoningDisabled.has(memoryKey) - ? { reasoning: { summary: "auto" } } + ...(Object.keys(reasoning).length ? { reasoning } : {}), + ...(maxTokens !== undefined ? { max_output_tokens: maxTokens } : {}), + ...(advanced.verbosity !== undefined + ? { text: { verbosity: advanced.verbosity } } : {}), - ...(request.maxTokens !== undefined - ? { max_output_tokens: request.maxTokens } + ...(advanced.temperature !== undefined && !skipTemperature + ? { temperature: advanced.temperature } : {}), ...(request.tools.length ? { @@ -109,27 +148,49 @@ export async function* streamResponses( }); let response = await post(body); - if (response.status === 400) { + let temperatureDropped = false; + for (let attempt = 0; attempt < 2; attempt++) { + if (response.status !== 400 && response.status !== 422) break; const detail = await response.text().catch(() => ""); - if (/reasoning|summar|encrypted/i.test(detail) && body.reasoning) { + const reasoning = body.reasoning as Record | undefined; + const temperatureError = /temperature/i.test(detail); + if (temperatureError && body.temperature !== undefined) { + config.log?.warn( + `OpenAI refused temperature for ${request.model}, continuing without it: ${detail.slice(0, 300)}`, + ); + const { temperature: _temperature, ...rest } = body; + body = rest; + temperatureDropped = true; + } else if ( + response.status === 400 && + !(temperatureError && temperatureDropped) && + // An effort the model refuses is the user's to fix, not a summary issue. + !(advanced.reasoningEffort !== undefined && /effort/i.test(detail)) && + /reasoning|summar|encrypted/i.test(detail) && + reasoning?.summary !== undefined + ) { config.log?.warn( `OpenAI refused reasoning summaries for ${request.model}, continuing without them (no Thinking block): ${detail.slice(0, 300)}`, ); reasoningDisabled.add(memoryKey); - const { reasoning: _dropped, ...rest } = body; - body = rest; - response = await post(body); + const { summary: _summary, ...remainingReasoning } = reasoning; + const { reasoning: _reasoning, ...rest } = body; + body = Object.keys(remainingReasoning).length + ? { ...rest, reasoning: remainingReasoning } + : rest; } else { response = new Response(detail, { status: response.status, statusText: response.statusText, }); + break; } + response = await post(body); } await assertOk(response, "OpenAI"); const calls: ToolCall[] = []; - const reasoning: unknown[] = []; + const reasoningBlocks: unknown[] = []; let usage: { input_tokens?: number; output_tokens?: number } | null = null; let stopReason: string | undefined; let previousReasoningKey: string | undefined; @@ -167,7 +228,7 @@ export async function* streamResponses( break; case "response.output_item.done": { const item = frame.item; - if (item?.type === "reasoning") reasoning.push(item); + if (item?.type === "reasoning") reasoningBlocks.push(item); if (item?.type !== "function_call" || !item.name) break; let args: Record = {}; try { @@ -208,7 +269,9 @@ export async function* streamResponses( } for (const call of calls) yield { type: "tool_call", call }; - if (reasoning.length) yield { type: "reasoning_state", blocks: reasoning }; + if (reasoningBlocks.length) { + yield { type: "reasoning_state", blocks: reasoningBlocks }; + } if (usage) { yield { type: "usage", diff --git a/src/backend/providers/openai.ts b/src/backend/providers/openai.ts index 05bdeaa..c02ee19 100644 --- a/src/backend/providers/openai.ts +++ b/src/backend/providers/openai.ts @@ -1,5 +1,9 @@ import { assertOk, firstCount, joinUrl, readSseLines } from "./http.js"; -import { streamResponses, usesResponsesApi } from "./openai-responses.js"; +import { + cappedMaxTokens, + streamResponses, + usesResponsesApi, +} from "./openai-responses.js"; import { createThinkSplitter } from "./think-tags.js"; import type { ChatChunk, @@ -91,6 +95,22 @@ export const openAiAdapter: ProviderAdapter = { return; } + const advanced = config.advanced; + const maxTokens = cappedMaxTokens(advanced?.maxOutputTokens, request); + let temperature = advanced?.temperature; + // OpenAI's reasoning models take temperature only with effort none. + if ( + temperature !== undefined && + config.providerType === "openai" && + advanced?.reasoningEffort !== undefined && + advanced.reasoningEffort !== "none" + ) { + config.log?.warn( + `OpenAI skipped temperature for ${request.model} because reasoning effort is ${advanced.reasoningEffort}; temperature requires reasoning effort none.`, + ); + temperature = undefined; + } + let body: Record = { model: request.model, messages: toOpenAiMessages(request), @@ -100,11 +120,18 @@ export const openAiAdapter: ProviderAdapter = { stream_options: { include_usage: true }, // OpenAI's own API refuses max_tokens for its reasoning models; // compatible servers (vLLM, LM Studio, Groq, OpenRouter) know max_tokens. - ...(request.maxTokens + ...(maxTokens ? config.providerType === "openai" - ? { max_completion_tokens: request.maxTokens } - : { max_tokens: request.maxTokens } + ? { max_completion_tokens: maxTokens } + : { max_tokens: maxTokens } + : {}), + ...(advanced?.reasoningEffort !== undefined + ? { reasoning_effort: advanced.reasoningEffort } : {}), + ...(advanced?.verbosity !== undefined + ? { verbosity: advanced.verbosity } + : {}), + ...(temperature !== undefined ? { temperature } : {}), ...(request.tools.length ? { tools: request.tools.map((tool) => ({ @@ -130,12 +157,21 @@ export const openAiAdapter: ProviderAdapter = { // A server can refuse an option it does not know, or OpenAI's newer // models refuse tools with their default reasoning effort. Ask again // without the option, or with reasoning off. - for (let attempt = 0; attempt < 2; attempt++) { + for (let attempt = 0; attempt < 3; attempt++) { if (response.status !== 400 && response.status !== 422) break; const detail = await response.text().catch(() => ""); if (/stream_options/i.test(detail) && body.stream_options) { const { stream_options: _dropped, ...rest } = body; body = rest; + } else if ( + /temperature/i.test(detail) && + body.temperature !== undefined + ) { + config.log?.warn( + `OpenAI refused temperature for ${request.model}, continuing without it: ${detail.slice(0, 300)}`, + ); + const { temperature: _dropped, ...rest } = body; + body = rest; } else if ( /reasoning_effort/i.test(detail) && /tools/i.test(detail) && diff --git a/src/backend/providers/types.ts b/src/backend/providers/types.ts index 5d405e2..8531c19 100644 --- a/src/backend/providers/types.ts +++ b/src/backend/providers/types.ts @@ -95,10 +95,24 @@ export type ProviderFetch = ( init: RequestInit, ) => Promise; +export type ReasoningEffort = + "none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max"; + +export interface ProviderAdvanced { + endpoint?: "auto" | "responses" | "chat"; + reasoningEffort?: ReasoningEffort; + reasoningSummary?: "auto" | "concise" | "detailed" | "off"; + verbosity?: "low" | "medium" | "high"; + maxOutputTokens?: number; + temperature?: number; +} + export interface ProviderConfig { providerType: AiProviderType; baseUrl?: string | null; apiKey?: string | null; + /** The user's API settings; absent fields keep the model's defaults. */ + advanced?: ProviderAdvanced; fetch: ProviderFetch; /** Reports provider fallbacks to the server log. */ log?: { warn(message: string): void }; diff --git a/src/backend/repository.ts b/src/backend/repository.ts index 29a73c2..238f0e2 100644 --- a/src/backend/repository.ts +++ b/src/backend/repository.ts @@ -1,6 +1,7 @@ import { and, desc, eq, gte, inArray } from "drizzle-orm"; import type { PluginContext } from "@termix-ssh/plugin-sdk/backend"; import type { ModelOverrides } from "./context-window.js"; +import type { ProviderAdvanced } from "./providers/types.js"; import { cliRuns as cliRunsDef, conversations as conversationsDef, @@ -27,6 +28,7 @@ export interface AiProviderRecord { defaultModel: string | null; /** The user's per-model settings, such as a context window override. */ modelOverrides: ModelOverrides; + advanced: ProviderAdvanced; enabled: boolean; createdAt: string; updatedAt: string; @@ -127,6 +129,7 @@ export interface AiProviderInput { apiKey?: string | null; defaultModel?: string | null; modelOverrides?: ModelOverrides; + advanced?: ProviderAdvanced; enabled?: boolean; } @@ -161,12 +164,28 @@ function parseOverrides(raw: unknown): ModelOverrides { const storedOverrides = (overrides: ModelOverrides) => Object.keys(overrides).length ? JSON.stringify(overrides) : null; +/** The column's JSON as settings; anything unreadable is model defaults. */ +function parseAdvanced(raw: unknown): ProviderAdvanced { + try { + const parsed = typeof raw === "string" ? JSON.parse(raw) : null; + return parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? parsed + : {}; + } catch { + return {}; + } +} + +const storedAdvanced = (advanced: ProviderAdvanced) => + Object.keys(advanced).length ? JSON.stringify(advanced) : null; + /** A provider row as callers see it, without the retired window column. */ function toProvider(row: Record): AiProviderRecord { - const { contextWindow: _retired, modelOverrides, ...rest } = row; + const { contextWindow: _retired, modelOverrides, advanced, ...rest } = row; return { - ...(rest as Omit), + ...(rest as Omit), modelOverrides: parseOverrides(modelOverrides), + advanced: parseAdvanced(advanced), }; } @@ -283,6 +302,7 @@ export async function createAiRepository( apiKeyPrefix: apiKeyPrefix(input.apiKey), defaultModel: input.defaultModel ?? null, modelOverrides: storedOverrides(input.modelOverrides ?? {}), + advanced: storedAdvanced(input.advanced ?? {}), enabled: input.enabled ?? true, createdAt: stamp, updatedAt: stamp, @@ -313,6 +333,8 @@ export async function createAiRepository( updates.defaultModel = input.defaultModel; if (input.modelOverrides !== undefined) updates.modelOverrides = storedOverrides(input.modelOverrides); + if (input.advanced !== undefined) + updates.advanced = storedAdvanced(input.advanced); if (input.enabled !== undefined) updates.enabled = input.enabled; // An empty string clears the key; undefined leaves it untouched. diff --git a/src/backend/routes.ts b/src/backend/routes.ts index 66d187e..3636856 100644 --- a/src/backend/routes.ts +++ b/src/backend/routes.ts @@ -22,7 +22,11 @@ import { REQUIRES_API_KEY, REQUIRES_BASE_URL, } from "./providers/registry.js"; -import type { AiProviderType, ProviderConfig } from "./providers/types.js"; +import type { + AiProviderType, + ProviderAdvanced, + ProviderConfig, +} from "./providers/types.js"; import { isAiProviderType } from "./providers/types.js"; import type { AiRepository } from "./repository.js"; import { serviceAvailable } from "./services.js"; @@ -66,6 +70,81 @@ function parseModelOverrides(raw: unknown): ModelOverrides | undefined | false { return overrides; } +/** Advanced settings replace the stored ones; null clears them. */ +function parseAdvanced(raw: unknown): ProviderAdvanced | undefined | false { + if (raw === undefined) return undefined; + if (raw === null) return {}; + if (typeof raw !== "object" || Array.isArray(raw)) return false; + const advanced: ProviderAdvanced = {}; + for (const [key, value] of Object.entries(raw)) { + const empty = value === undefined || value === null || value === ""; + switch (key) { + case "endpoint": + if (empty) break; + if (value !== "auto" && value !== "responses" && value !== "chat") + return false; + advanced.endpoint = value; + break; + case "reasoningEffort": + if (empty) break; + if ( + value !== "none" && + value !== "minimal" && + value !== "low" && + value !== "medium" && + value !== "high" && + value !== "xhigh" && + value !== "max" + ) + return false; + advanced.reasoningEffort = value; + break; + case "reasoningSummary": + if (empty) break; + if ( + value !== "auto" && + value !== "concise" && + value !== "detailed" && + value !== "off" + ) + return false; + advanced.reasoningSummary = value; + break; + case "verbosity": + if (empty) break; + if (value !== "low" && value !== "medium" && value !== "high") + return false; + advanced.verbosity = value; + break; + case "maxOutputTokens": + if (empty) break; + if ( + typeof value !== "number" || + !Number.isInteger(value) || + value < 1 || + value > 1_000_000 + ) + return false; + advanced.maxOutputTokens = value; + break; + case "temperature": + if (empty) break; + if ( + typeof value !== "number" || + !Number.isFinite(value) || + value < 0 || + value > 2 + ) + return false; + advanced.temperature = value; + break; + default: + return false; + } + } + return advanced; +} + function parseId(raw: unknown): number | null { const id = typeof raw === "string" ? parseInt(raw, 10) : Number(raw); return Number.isInteger(id) && id > 0 ? id : null; @@ -112,12 +191,14 @@ export function registerAiRoutes( providerType: string; baseUrl: string | null; apiKey: string | null; + advanced?: ProviderAdvanced; }): Promise { const allowlist = await readPrivateAllowlist(ctx.settings); return { providerType: input.providerType as ProviderConfig["providerType"], baseUrl: input.baseUrl, apiKey: input.apiKey, + advanced: input.advanced, fetch: createProviderFetch(ctx.fetch, allowlist), log: ctx.log, }; @@ -270,6 +351,32 @@ export function registerAiRoutes( * properties: * contextWindow: * type: integer + * advanced: + * type: object + * nullable: true + * additionalProperties: false + * description: Optional provider request settings. Null clears them. + * properties: + * endpoint: + * type: string + * enum: [auto, responses, chat] + * reasoningEffort: + * type: string + * enum: [none, minimal, low, medium, high, xhigh, max] + * reasoningSummary: + * type: string + * enum: [auto, concise, detailed, off] + * verbosity: + * type: string + * enum: [low, medium, high] + * maxOutputTokens: + * type: integer + * minimum: 1 + * maximum: 1000000 + * temperature: + * type: number + * minimum: 0 + * maximum: 2 * responses: * 201: * description: Provider created. @@ -285,6 +392,7 @@ export function registerAiRoutes( const { providerType, label, baseUrl, apiKey, defaultModel } = req.body ?? {}; const modelOverrides = parseModelOverrides(req.body?.modelOverrides); + const advanced = parseAdvanced(req.body?.advanced); if (!isAiProviderType(providerType)) { return res.status(400).json({ error: "Unknown provider type" }); @@ -292,6 +400,9 @@ export function registerAiRoutes( if (modelOverrides === false) { return res.status(400).json({ error: "Invalid context window" }); } + if (advanced === false) { + return res.status(400).json({ error: "Invalid advanced settings" }); + } if (typeof label !== "string" || !label.trim()) { return res.status(400).json({ error: "label is required" }); } @@ -315,6 +426,7 @@ export function registerAiRoutes( defaultModel: typeof defaultModel === "string" ? defaultModel.trim() : null, modelOverrides, + advanced, }); await audit({ @@ -373,9 +485,37 @@ export function registerAiRoutes( * properties: * contextWindow: * type: integer + * advanced: + * type: object + * nullable: true + * additionalProperties: false + * description: Replaces every advanced setting. Null clears them. + * properties: + * endpoint: + * type: string + * enum: [auto, responses, chat] + * reasoningEffort: + * type: string + * enum: [none, minimal, low, medium, high, xhigh, max] + * reasoningSummary: + * type: string + * enum: [auto, concise, detailed, off] + * verbosity: + * type: string + * enum: [low, medium, high] + * maxOutputTokens: + * type: integer + * minimum: 1 + * maximum: 1000000 + * temperature: + * type: number + * minimum: 0 + * maximum: 2 * responses: * 200: * description: Provider updated. + * 400: + * description: Invalid request body. * 404: * description: Provider not found. */ @@ -396,9 +536,13 @@ export function registerAiRoutes( return res.status(400).json({ error: "Unknown provider type" }); } const modelOverrides = parseModelOverrides(body.modelOverrides); + const advanced = parseAdvanced(body.advanced); if (modelOverrides === false) { return res.status(400).json({ error: "Invalid context window" }); } + if (advanced === false) { + return res.status(400).json({ error: "Invalid advanced settings" }); + } const text = (value: unknown) => typeof value === "string" ? value.trim() : undefined; @@ -415,6 +559,7 @@ export function registerAiRoutes( ? null : (text(body.defaultModel) ?? undefined), modelOverrides, + advanced, enabled: typeof body.enabled === "boolean" ? body.enabled : undefined, }); if (!updated) diff --git a/src/backend/tables.ts b/src/backend/tables.ts index f19dee1..752d16e 100644 --- a/src/backend/tables.ts +++ b/src/backend/tables.ts @@ -45,6 +45,8 @@ export const providers = adoptLegacyTable( contextWindow: integer(), // JSON { "": { "contextWindow": tokens } }, set by the user. modelOverrides: text(), + // JSON ProviderAdvanced, set by the user; empty means the model's defaults. + advanced: text(), enabled: boolean().notNull().default(true), createdAt: timestamp().notNull().defaultNow(), updatedAt: timestamp().notNull().defaultNow(), diff --git a/src/backend/tasks/runner.ts b/src/backend/tasks/runner.ts index b8504be..a56eb24 100644 --- a/src/backend/tasks/runner.ts +++ b/src/backend/tasks/runner.ts @@ -105,6 +105,7 @@ export async function providerConfig( providerType: provider.providerType as ProviderConfig["providerType"], baseUrl: provider.baseUrl, apiKey: provider.apiKey, + advanced: provider.advanced, fetch: createProviderFetch( ctx.fetch, await readPrivateAllowlist(ctx.settings), diff --git a/src/frontend/AiProviderSettings.tsx b/src/frontend/AiProviderSettings.tsx index 8bb24b0..844471e 100644 --- a/src/frontend/AiProviderSettings.tsx +++ b/src/frontend/AiProviderSettings.tsx @@ -2,7 +2,7 @@ import { getErrorMessage } from "./errors"; import { useCallback, useEffect, useState } from "react"; import { useTranslation } from "@termix-ssh/plugin-sdk/frontend"; import { toast } from "sonner"; -import { Loader2, Pencil, RefreshCw, Trash2 } from "lucide-react"; +import { ChevronRight, Loader2, Pencil, RefreshCw, Trash2 } from "lucide-react"; import { Button, Input, @@ -29,6 +29,7 @@ import { CONTEXT_SOURCE_KEYS, type ModelContext, type ModelOverrides, + type ProviderAdvanced, } from "./ai-api"; const PROVIDER_TYPES: Array<{ @@ -74,6 +75,148 @@ const PROVIDER_TYPES: Array<{ const FORM = "flex flex-col gap-3 border-b border-border py-3"; const FIELD = "flex flex-col gap-1.5"; +/** Only these provider types read the advanced settings. */ +function hasAdvanced(type: AiProviderType): boolean { + return type === "openai" || type === "openai_compatible"; +} + +/** The advanced settings as the form edits them; empty means the default. */ +type AdvancedFields = Record; + +/** Radix Select needs a non-empty value, so the default gets one. */ +const DEFAULT_CHOICE = "__default__"; + +const ADVANCED_CHOICES: Array<{ + key: "endpoint" | "reasoningEffort" | "reasoningSummary" | "verbosity"; + labelKey: string; + hintKey?: string; + defaultKey: string; + options: Array<{ value: string; labelKey: string }>; +}> = [ + { + key: "endpoint", + labelKey: "ai.advancedEndpoint", + defaultKey: "ai.advancedAuto", + options: [ + { value: "responses", labelKey: "ai.advancedEndpointResponses" }, + { value: "chat", labelKey: "ai.advancedEndpointChat" }, + ], + }, + { + key: "reasoningEffort", + labelKey: "ai.advancedReasoningEffort", + defaultKey: "ai.advancedDefault", + options: [ + { value: "none", labelKey: "ai.advancedNone" }, + { value: "minimal", labelKey: "ai.advancedMinimal" }, + { value: "low", labelKey: "ai.advancedLow" }, + { value: "medium", labelKey: "ai.advancedMedium" }, + { value: "high", labelKey: "ai.advancedHigh" }, + { value: "xhigh", labelKey: "ai.advancedXhigh" }, + { value: "max", labelKey: "ai.advancedMax" }, + ], + }, + { + key: "reasoningSummary", + labelKey: "ai.advancedReasoningSummary", + hintKey: "ai.advancedReasoningSummaryHint", + defaultKey: "ai.advancedDefaultAuto", + options: [ + { value: "auto", labelKey: "ai.advancedAuto" }, + { value: "concise", labelKey: "ai.advancedConcise" }, + { value: "detailed", labelKey: "ai.advancedDetailed" }, + { value: "off", labelKey: "ai.advancedOff" }, + ], + }, + { + key: "verbosity", + labelKey: "ai.advancedVerbosity", + defaultKey: "ai.advancedDefault", + options: [ + { value: "low", labelKey: "ai.advancedLow" }, + { value: "medium", labelKey: "ai.advancedMedium" }, + { value: "high", labelKey: "ai.advancedHigh" }, + ], + }, +]; + +/** The ranges match what the server accepts. */ +const ADVANCED_NUMBERS: Array<{ + key: "maxOutputTokens" | "temperature"; + labelKey: string; + hintKey?: string; + errorKey: string; + min: number; + max: number; + step: number; + integer: boolean; +}> = [ + { + key: "maxOutputTokens", + labelKey: "ai.advancedMaxOutputTokens", + errorKey: "ai.advancedMaxOutputTokensInvalid", + min: 1, + max: 1_000_000, + step: 1, + integer: true, + }, + { + key: "temperature", + labelKey: "ai.advancedTemperature", + hintKey: "ai.advancedTemperatureHint", + errorKey: "ai.advancedTemperatureInvalid", + min: 0, + max: 2, + step: 0.1, + integer: false, + }, +]; + +/** The field's value as a number: null when empty, false when not valid. */ +function parseAdvancedNumber( + value: string, + range: { min: number; max: number; integer: boolean }, +): number | null | false { + if (!value.trim()) return null; + const number = Number(value); + if (!Number.isFinite(number) || number < range.min || number > range.max) { + return false; + } + return range.integer && !Number.isInteger(number) ? false : number; +} + +function fromAdvanced(advanced: ProviderAdvanced): AdvancedFields { + return { + // Auto is what the server does without one, so it shows as the default. + endpoint: advanced.endpoint === "auto" ? "" : (advanced.endpoint ?? ""), + reasoningEffort: advanced.reasoningEffort ?? "", + reasoningSummary: advanced.reasoningSummary ?? "", + verbosity: advanced.verbosity ?? "", + maxOutputTokens: advanced.maxOutputTokens?.toString() ?? "", + temperature: advanced.temperature?.toString() ?? "", + }; +} + +const EMPTY_ADVANCED = fromAdvanced({}); + +/** + * The settings the fields describe with only the ones that are set: null + * when none are, false when a number is not valid. + */ +function toAdvanced(fields: AdvancedFields): ProviderAdvanced | null | false { + const advanced: Record = {}; + for (const { key } of ADVANCED_CHOICES) { + if (fields[key]) advanced[key] = fields[key]; + } + for (const spec of ADVANCED_NUMBERS) { + const number = parseAdvancedNumber(fields[spec.key], spec); + if (number === false) return false; + if (number !== null) advanced[spec.key] = number; + } + // The choices only hold values from their option lists. + return Object.keys(advanced).length ? (advanced as ProviderAdvanced) : null; +} + interface AiProviderSettingsProps { providers: AiProvider[]; /** selectId names a provider that should become the active one. */ @@ -298,6 +441,117 @@ function ContextWindowTable({ ); } +/** Request settings for OpenAI-style providers, folded away until asked for. */ +function AdvancedSection({ + id, + fields, + onChange, +}: { + id: string; + fields: AdvancedFields; + onChange: (key: keyof AdvancedFields, value: string) => void; +}) { + const { t } = useTranslation(); + const [open, setOpen] = useState(false); + + return ( +
+ + + {open && ( +
+

+ {t("ai.advancedHint")} +

+ + {ADVANCED_CHOICES.map((choice) => ( +
+ + + {choice.hintKey && ( +

+ {t(choice.hintKey)} +

+ )} +
+ ))} + + {ADVANCED_NUMBERS.map((spec) => { + const invalid = + parseAdvancedNumber(fields[spec.key], spec) === false; + return ( +
+ + onChange(spec.key, event.target.value)} + /> + {spec.hintKey && ( +

+ {t(spec.hintKey)} +

+ )} + {invalid && ( +

+ {t(spec.errorKey)} +

+ )} +
+ ); + })} +
+ )} +
+ ); +} + function AiProviderEditForm({ provider, onSaved, @@ -318,12 +572,18 @@ function AiProviderEditForm({ ]), ), ); + const [advancedFields, setAdvancedFields] = useState(() => + fromAdvanced(provider.advanced), + ); const [models, setModels] = useState([]); const [customModel, setCustomModel] = useState(false); const [detecting, setDetecting] = useState(false); const [detectWarning, setDetectWarning] = useState(null); const [saving, setSaving] = useState(false); + const showAdvanced = hasAdvanced(provider.providerType); + const advanced = showAdvanced ? toAdvanced(advancedFields) : null; + const detectModels = useCallback(async () => { setDetecting(true); setDetectWarning(null); @@ -357,6 +617,7 @@ function AiProviderEditForm({ toast.error(t("ai.contextWindowInvalid")); return; } + if (advanced === false) return; setSaving(true); try { @@ -364,6 +625,8 @@ function AiProviderEditForm({ label: label.trim(), defaultModel: defaultModel.trim() || null, modelOverrides, + // Null when nothing is set, so earlier settings are cleared. + ...(showAdvanced && { advanced }), }); toast.success(t("ai.providerUpdated")); onSaved(); @@ -413,10 +676,21 @@ function AiProviderEditForm({ } /> + {showAdvanced && ( + + setAdvancedFields((current) => ({ ...current, [key]: value })) + } + /> + )} + void handleSave()} onCancel={onCancel} /> @@ -439,6 +713,7 @@ export function AiProviderSettings({ const [baseUrl, setBaseUrl] = useState("http://localhost:11434"); const [apiKey, setApiKey] = useState(""); const [defaultModel, setDefaultModel] = useState(""); + const [advancedFields, setAdvancedFields] = useState(EMPTY_ADVANCED); const [models, setModels] = useState([]); const [detecting, setDetecting] = useState(false); @@ -446,6 +721,9 @@ export function AiProviderSettings({ const [customModel, setCustomModel] = useState(false); const spec = PROVIDER_TYPES.find((entry) => entry.value === providerType)!; + const advanced = hasAdvanced(providerType) + ? toAdvanced(advancedFields) + : null; useEffect(() => { setBaseUrl(spec.defaultBaseUrl ?? ""); @@ -498,6 +776,7 @@ export function AiProviderSettings({ toast.error(t("ai.labelRequired")); return; } + if (advanced === false) return; setSaving(true); try { const created = await createAiProvider({ @@ -506,11 +785,13 @@ export function AiProviderSettings({ baseUrl: baseUrl.trim() || null, apiKey: apiKey.trim() || null, defaultModel: defaultModel.trim() || null, + ...(advanced && { advanced }), }); setAdding(false); setLabel(""); setApiKey(""); setDefaultModel(""); + setAdvancedFields(EMPTY_ADVANCED); onChanged(created.id); onAdded?.(); } catch (error) { @@ -687,10 +968,21 @@ export function AiProviderSettings({ onRefresh={() => void detectModels()} /> + {hasAdvanced(providerType) && ( + + setAdvancedFields((current) => ({ ...current, [key]: value })) + } + /> + )} + void handleAdd()} onCancel={() => setAdding(false)} /> diff --git a/src/frontend/ai-api.ts b/src/frontend/ai-api.ts index f2d72a5..bca3492 100644 --- a/src/frontend/ai-api.ts +++ b/src/frontend/ai-api.ts @@ -51,12 +51,29 @@ export interface AiProvider { defaultModel: string | null; /** Per model id, a context window that replaces the provider's. */ modelOverrides: ModelOverrides; + /** Request settings for OpenAI-style providers, empty when none are set. */ + advanced: ProviderAdvanced; enabled: boolean; createdAt: string; } export type ModelOverrides = Record; +export type ReasoningEffort = + "none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max"; + +/** Absent fields keep the model's defaults and are not sent. */ +export interface ProviderAdvanced { + endpoint?: "auto" | "responses" | "chat"; + reasoningEffort?: ReasoningEffort; + reasoningSummary?: "auto" | "concise" | "detailed" | "off"; + verbosity?: "low" | "medium" | "high"; + /** A whole number, 1 to 1,000,000. */ + maxOutputTokens?: number; + /** 0 to 2. */ + temperature?: number; +} + /** Where a context window came from, best first. */ export type ContextSource = "override" | "provider" | "guess" | "default"; @@ -139,6 +156,7 @@ export async function createAiProvider(input: { baseUrl?: string | null; apiKey?: string | null; defaultModel?: string | null; + advanced?: ProviderAdvanced | null; }): Promise { try { const provider = (await api().post("/providers", input)).data.provider; @@ -157,6 +175,8 @@ export async function updateAiProvider( apiKey: string | null; defaultModel: string | null; modelOverrides: ModelOverrides; + /** Replaces the whole object; null clears it. */ + advanced: ProviderAdvanced | null; enabled: boolean; }>, ): Promise { diff --git a/tests/backend/migrations.test.ts b/tests/backend/migrations.test.ts index c05fb1c..8d541b0 100644 --- a/tests/backend/migrations.test.ts +++ b/tests/backend/migrations.test.ts @@ -75,7 +75,11 @@ describe("ai adoption migration", () => { .prepare("SELECT * FROM p_ai_providers WHERE id = 3") .get() as Record; // The key stays until core's boot migration moves it into ctx.secrets. - expect(provider).toMatchObject({ label: "Local", api_key: "enc-key" }); + expect(provider).toMatchObject({ + label: "Local", + api_key: "enc-key", + advanced: null, + }); expect(indexes(db.sqlite, "p_ai_providers")).toEqual([ "idx_ai_providers_user_label", @@ -152,6 +156,38 @@ describe("ai adoption migration", () => { }); }); +describe("provider advanced migration", () => { + it("adds a nullable advanced column", async () => { + db = await createTestDb(pluginDir); + const columns = db.sqlite + .prepare("PRAGMA table_info(p_ai_providers)") + .all(); + expect(columns).toContainEqual( + expect.objectContaining({ name: "advanced", type: "TEXT", notnull: 0 }), + ); + }); + + it("writes only the plugin's own tables and uses longtext on mysql", () => { + for (const dialect of ["sqlite", "postgres", "mysql"]) { + const sql = fs.readFileSync( + path.join( + pluginDir, + "migrations", + dialect, + "0013_provider_advanced.sql", + ), + "utf8", + ); + expect(findUnownedTableWrites("ai", sql)).toEqual([]); + if (dialect === "mysql") { + expect(sql).toContain( + "ALTER TABLE `p_ai_providers` ADD COLUMN `advanced` longtext;", + ); + } + } + }); +}); + describe("asset inventory migration", () => { it("keeps one asset per key and one edge per pair and type", async () => { db = await createTestDb(pluginDir); diff --git a/tests/backend/providers/openai-responses.test.ts b/tests/backend/providers/openai-responses.test.ts index 5515f25..fba12f9 100644 --- a/tests/backend/providers/openai-responses.test.ts +++ b/tests/backend/providers/openai-responses.test.ts @@ -789,3 +789,562 @@ describe("openAiAdapter compatible chat requests", () => { }, ); }); + +const RESPONSES_BODY = { + model: REQUEST.model, + instructions: REQUEST.system, + input: REQUEST.messages, + stream: true, + store: false, +}; + +describe("OpenAI advanced Responses routing", () => { + it.each([ + { + label: "OpenAI forced to Responses at a proxy", + config: { + ...OPENAI, + baseUrl: "https://proxy.example/v1", + advanced: { endpoint: "responses" as const }, + }, + expected: true, + }, + { + label: "a compatible provider forced to Responses", + config: { + ...COMPATIBLE, + advanced: { endpoint: "responses" as const }, + }, + expected: true, + }, + { + label: "OpenAI forced to chat at the official URL", + config: { + ...OPENAI, + baseUrl: "https://api.openai.com/v1", + advanced: { endpoint: "chat" as const }, + }, + expected: false, + }, + { + label: "a compatible provider forced to chat", + config: { + ...COMPATIBLE, + advanced: { endpoint: "chat" as const }, + }, + expected: false, + }, + { + label: "OpenAI using auto at the official URL", + config: { ...OPENAI, advanced: { endpoint: "auto" as const } }, + expected: true, + }, + { + label: "OpenAI using auto at a proxy", + config: { + ...OPENAI, + baseUrl: COMPATIBLE.baseUrl, + advanced: { endpoint: "auto" as const }, + }, + expected: false, + }, + { + label: "a compatible provider using auto at the official URL", + config: { + ...COMPATIBLE, + baseUrl: "https://api.openai.com/v1", + advanced: { endpoint: "auto" as const }, + }, + expected: false, + }, + ])("routes $label", ({ config, expected }) => { + expect(usesResponsesApi(config)).toBe(expected); + expect(providerFetch).not.toHaveBeenCalled(); + }); + + it.each([OPENAI, COMPATIBLE])( + "posts to Responses when $providerType overrides its proxy endpoint", + async (config) => { + providerFetch.mockResolvedValueOnce( + streamingResponse([sse({ type: "response.completed" })]), + ); + + await collect( + openAiAdapter.streamChat( + { + ...config, + baseUrl: "https://proxy.example/v1", + advanced: { endpoint: "responses" }, + }, + REQUEST, + ), + ); + + expect(providerFetch).toHaveBeenCalledTimes(1); + expect(providerFetch.mock.calls[0][0]).toBe( + "https://proxy.example/v1/responses", + ); + expect(JSON.parse(providerFetch.mock.calls[0][1].body)).toEqual({ + ...RESPONSES_BODY, + reasoning: { summary: "auto" }, + }); + }, + ); + + it("posts to chat when OpenAI overrides the official Responses endpoint", async () => { + providerFetch.mockResolvedValueOnce( + streamingResponse([ + sse({ choices: [{ delta: {}, finish_reason: "stop" }] }), + "data: [DONE]\n", + ]), + ); + + await collect( + openAiAdapter.streamChat( + { + ...OPENAI, + baseUrl: "https://api.openai.com/v1", + advanced: { endpoint: "chat" }, + }, + REQUEST, + ), + ); + + expect(providerFetch).toHaveBeenCalledTimes(1); + expect(providerFetch.mock.calls[0][0]).toBe( + "https://api.openai.com/v1/chat/completions", + ); + expect(JSON.parse(providerFetch.mock.calls[0][1].body)).toEqual({ + model: REQUEST.model, + messages: [ + { role: "system", content: REQUEST.system }, + ...REQUEST.messages, + ], + stream: true, + stream_options: { include_usage: true }, + }); + }); +}); + +describe("OpenAI advanced Responses requests", () => { + it("maps every advanced field and lets effort none use temperature", async () => { + providerFetch.mockResolvedValueOnce( + streamingResponse([sse({ type: "response.completed" })]), + ); + + await collect( + openAiAdapter.streamChat( + { + ...OPENAI, + advanced: { + endpoint: "responses", + reasoningEffort: "none", + reasoningSummary: "detailed", + verbosity: "high", + maxOutputTokens: 1024, + temperature: 0.25, + }, + }, + { ...REQUEST, maxTokens: 64, tools: [RUN_TOOL] }, + ), + ); + + expect(providerFetch).toHaveBeenCalledTimes(1); + expect(JSON.parse(providerFetch.mock.calls[0][1].body)).toEqual({ + ...RESPONSES_BODY, + reasoning: { effort: "none", summary: "detailed" }, + text: { verbosity: "high" }, + max_output_tokens: 64, + temperature: 0.25, + tools: [{ type: "function", ...RUN_TOOL, strict: false }], + }); + }); + + it.each([undefined, {}])( + "keeps the legacy body with advanced settings %j", + async (advanced) => { + providerFetch.mockResolvedValueOnce( + streamingResponse([sse({ type: "response.completed" })]), + ); + + await collect( + openAiAdapter.streamChat( + { ...OPENAI, advanced }, + { ...REQUEST, maxTokens: 512 }, + ), + ); + + expect(JSON.parse(providerFetch.mock.calls[0][1].body)).toEqual({ + ...RESPONSES_BODY, + reasoning: { summary: "auto" }, + max_output_tokens: 512, + }); + }, + ); + + it.each([ + { effort: undefined, expected: RESPONSES_BODY }, + { + effort: "high" as const, + expected: { ...RESPONSES_BODY, reasoning: { effort: "high" } }, + }, + ])("disables summaries with effort $effort", async ({ effort, expected }) => { + providerFetch.mockResolvedValueOnce( + streamingResponse([sse({ type: "response.completed" })]), + ); + + await collect( + openAiAdapter.streamChat( + { + ...OPENAI, + advanced: { reasoningSummary: "off", reasoningEffort: effort }, + }, + REQUEST, + ), + ); + + expect(JSON.parse(providerFetch.mock.calls[0][1].body)).toEqual(expected); + }); + + it.each([undefined, "none" as const, "high" as const])( + "sends temperature only when effort %s allows it and warns once when skipped", + async (reasoningEffort) => { + const warn = vi.fn(); + providerFetch.mockResolvedValueOnce( + streamingResponse([sse({ type: "response.completed" })]), + ); + + await collect( + openAiAdapter.streamChat( + { + ...OPENAI, + log: { warn }, + advanced: { reasoningEffort, temperature: 0 }, + }, + REQUEST, + ), + ); + + expect(JSON.parse(providerFetch.mock.calls[0][1].body)).toEqual({ + ...RESPONSES_BODY, + reasoning: { + ...(reasoningEffort !== undefined ? { effort: reasoningEffort } : {}), + summary: "auto", + }, + ...(reasoningEffort === "high" ? {} : { temperature: 0 }), + }); + if (reasoningEffort === "high") { + expect(warn).toHaveBeenCalledTimes(1); + expect(warn).toHaveBeenCalledWith( + `OpenAI skipped temperature for ${REQUEST.model} because reasoning effort is high; temperature requires reasoning effort none.`, + ); + } else { + expect(warn).not.toHaveBeenCalled(); + } + }, + ); + + it.each([ + { + label: "uses a zero user limit below the request cap", + maxOutputTokens: 0, + maxTokens: 512, + expected: 0, + }, + { + label: "uses a user limit below the request cap", + maxOutputTokens: 128, + maxTokens: 512, + expected: 128, + }, + { + label: "caps a user limit above the request cap", + maxOutputTokens: 1024, + maxTokens: 512, + expected: 512, + }, + { + label: "uses a user limit when the request cap is absent", + maxOutputTokens: 1024, + maxTokens: undefined, + expected: 1024, + }, + ])("$label", async ({ maxOutputTokens, maxTokens, expected }) => { + providerFetch.mockResolvedValueOnce( + streamingResponse([sse({ type: "response.completed" })]), + ); + + await collect( + openAiAdapter.streamChat( + { ...OPENAI, advanced: { maxOutputTokens } }, + maxTokens === undefined ? REQUEST : { ...REQUEST, maxTokens }, + ), + ); + + expect(JSON.parse(providerFetch.mock.calls[0][1].body)).toEqual({ + ...RESPONSES_BODY, + reasoning: { summary: "auto" }, + max_output_tokens: expected, + }); + }); +}); + +describe("OpenAI advanced Responses retries", () => { + it("preserves effort when summaries are refused and remembered", async () => { + const warn = vi.fn(); + const detail = "reasoning.summary is unsupported"; + const config = { + ...OPENAI, + log: { warn }, + advanced: { + reasoningEffort: "high" as const, + reasoningSummary: "concise" as const, + verbosity: "low" as const, + }, + }; + providerFetch + .mockResolvedValueOnce(new Response(detail, { status: 400 })) + .mockImplementation(async () => + streamingResponse([sse({ type: "response.completed" })]), + ); + + await collect(openAiAdapter.streamChat(config, REQUEST)); + await collect(openAiAdapter.streamChat(config, REQUEST)); + + const withoutSummary = { + ...RESPONSES_BODY, + reasoning: { effort: "high" }, + text: { verbosity: "low" }, + }; + expect(providerFetch).toHaveBeenCalledTimes(3); + expect( + providerFetch.mock.calls.map(([, options]) => JSON.parse(options.body)), + ).toEqual([ + { + ...withoutSummary, + reasoning: { effort: "high", summary: "concise" }, + }, + withoutSummary, + withoutSummary, + ]); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn).toHaveBeenCalledWith( + `OpenAI refused reasoning summaries for ${REQUEST.model}, continuing without them (no Thinking block): ${detail}`, + ); + }); + + it("surfaces rejected reasoning effort when summaries are off", async () => { + const warn = vi.fn(); + providerFetch.mockResolvedValueOnce( + new Response("reasoning.effort is unsupported", { status: 400 }), + ); + + await expect( + collect( + openAiAdapter.streamChat( + { + ...OPENAI, + log: { warn }, + advanced: { reasoningEffort: "high", reasoningSummary: "off" }, + }, + REQUEST, + ), + ), + ).rejects.toMatchObject({ + status: 400, + message: "OpenAI request failed (400): reasoning.effort is unsupported", + }); + + expect(providerFetch).toHaveBeenCalledTimes(1); + expect(JSON.parse(providerFetch.mock.calls[0][1].body)).toEqual({ + ...RESPONSES_BODY, + reasoning: { effort: "high" }, + }); + expect(warn).not.toHaveBeenCalled(); + }); + + it("surfaces rejected reasoning effort without retrying or remembering a summary fallback", async () => { + const warn = vi.fn(); + const detail = "reasoning.effort 'xhigh' is not supported"; + const config = { + ...OPENAI, + log: { warn }, + advanced: { reasoningEffort: "xhigh" as const }, + }; + providerFetch + .mockResolvedValueOnce(new Response(detail, { status: 400 })) + .mockResolvedValueOnce( + streamingResponse([sse({ type: "response.completed" })]), + ); + + const result = collect(openAiAdapter.streamChat(config, REQUEST)); + + await expect(result).rejects.toBeInstanceOf(AiProviderError); + await expect(result).rejects.toMatchObject({ + status: 400, + message: `OpenAI request failed (400): ${detail}`, + }); + expect(providerFetch).toHaveBeenCalledTimes(1); + expect(warn).not.toHaveBeenCalled(); + + await collect(openAiAdapter.streamChat(config, REQUEST)); + + expect(providerFetch).toHaveBeenCalledTimes(2); + expect(providerFetch.mock.calls.map(([url]) => url)).toEqual([ + "https://api.openai.com/v1/responses", + "https://api.openai.com/v1/responses", + ]); + const expected = { + ...RESPONSES_BODY, + reasoning: { effort: "xhigh", summary: "auto" }, + }; + expect( + providerFetch.mock.calls.map(([, options]) => JSON.parse(options.body)), + ).toEqual([expected, expected]); + expect(warn).not.toHaveBeenCalled(); + }); + + it.each([400, 422])( + "retries a temperature %i once without changing other fields", + async (status) => { + const warn = vi.fn(); + const detail = "temperature is not supported"; + providerFetch + .mockResolvedValueOnce(new Response(detail, { status })) + .mockResolvedValueOnce( + streamingResponse([sse({ type: "response.completed" })]), + ); + + await collect( + openAiAdapter.streamChat( + { + ...OPENAI, + log: { warn }, + advanced: { + reasoningEffort: "none", + reasoningSummary: "detailed", + verbosity: "medium", + maxOutputTokens: 256, + temperature: 0.5, + }, + }, + REQUEST, + ), + ); + + const withoutTemperature = { + ...RESPONSES_BODY, + reasoning: { effort: "none", summary: "detailed" }, + text: { verbosity: "medium" }, + max_output_tokens: 256, + }; + expect(providerFetch).toHaveBeenCalledTimes(2); + expect( + providerFetch.mock.calls.map(([, options]) => JSON.parse(options.body)), + ).toEqual([ + { ...withoutTemperature, temperature: 0.5 }, + withoutTemperature, + ]); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn).toHaveBeenCalledWith( + `OpenAI refused temperature for ${REQUEST.model}, continuing without it: ${detail}`, + ); + }, + ); + + it.each([400, 422])( + "surfaces a second temperature %i mentioning reasoning without retrying again", + async (status) => { + const warn = vi.fn(); + const config = { + ...OPENAI, + log: { warn }, + advanced: { temperature: 0.5 }, + }; + const detail = "temperature is unsupported with reasoning models"; + providerFetch.mockImplementation( + async () => new Response(detail, { status }), + ); + + await expect( + collect(openAiAdapter.streamChat(config, REQUEST)), + ).rejects.toMatchObject({ + status, + message: `OpenAI request failed (${status}): ${detail}`, + }); + + expect(providerFetch).toHaveBeenCalledTimes(2); + expect(warn).toHaveBeenCalledTimes(1); + expect(JSON.parse(providerFetch.mock.calls[1][1].body)).toEqual({ + ...RESPONSES_BODY, + reasoning: { summary: "auto" }, + }); + + providerFetch.mockResolvedValueOnce( + streamingResponse([sse({ type: "response.completed" })]), + ); + await collect(openAiAdapter.streamChat(config, REQUEST)); + + expect(providerFetch).toHaveBeenCalledTimes(3); + expect(warn).toHaveBeenCalledTimes(1); + expect(JSON.parse(providerFetch.mock.calls[2][1].body)).toEqual({ + ...RESPONSES_BODY, + reasoning: { summary: "auto" }, + temperature: 0.5, + }); + }, + ); + + it.each(["summary", "temperature"])( + "handles both fallbacks when %s is refused first", + async (first) => { + const warn = vi.fn(); + const summaryFailure = () => + new Response("reasoning.summary is unsupported", { status: 400 }); + const temperatureFailure = () => + new Response("temperature is unsupported", { status: 422 }); + providerFetch + .mockResolvedValueOnce( + first === "summary" ? summaryFailure() : temperatureFailure(), + ) + .mockResolvedValueOnce( + first === "summary" ? temperatureFailure() : summaryFailure(), + ) + .mockResolvedValueOnce( + streamingResponse([sse({ type: "response.completed" })]), + ); + + await collect( + openAiAdapter.streamChat( + { + ...OPENAI, + log: { warn }, + advanced: { reasoningEffort: "none", temperature: 0.5 }, + }, + REQUEST, + ), + ); + + const initial = { + ...RESPONSES_BODY, + reasoning: { effort: "none", summary: "auto" }, + temperature: 0.5, + }; + expect(providerFetch).toHaveBeenCalledTimes(3); + expect( + providerFetch.mock.calls.map(([, options]) => JSON.parse(options.body)), + ).toEqual([ + initial, + first === "summary" + ? { + ...RESPONSES_BODY, + reasoning: { effort: "none" }, + temperature: 0.5, + } + : { ...RESPONSES_BODY, reasoning: initial.reasoning }, + { ...RESPONSES_BODY, reasoning: { effort: "none" } }, + ]); + expect(warn).toHaveBeenCalledTimes(2); + }, + ); +}); diff --git a/tests/backend/providers/openai.test.ts b/tests/backend/providers/openai.test.ts new file mode 100644 index 0000000..4d484fc --- /dev/null +++ b/tests/backend/providers/openai.test.ts @@ -0,0 +1,495 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { + ChatChunk, + ProviderAdvanced, +} from "../../../src/backend/providers/types.js"; + +const providerFetch = vi.fn(); + +const { openAiAdapter } = + await import("../../../src/backend/providers/openai.js"); + +function streamingResponse(lines: string[]): Response { + const encoder = new TextEncoder(); + return new Response( + new ReadableStream({ + start(controller) { + for (const line of lines) controller.enqueue(encoder.encode(line)); + controller.close(); + }, + }), + ); +} + +const sse = (payload: unknown) => `data: ${JSON.stringify(payload)}\n`; + +async function collect(iterable: AsyncIterable) { + const chunks: ChatChunk[] = []; + for await (const chunk of iterable) chunks.push(chunk); + return chunks; +} + +const REQUEST = { + model: "gpt-5", + system: "system", + messages: [{ role: "user" as const, content: "hi" }], + tools: [], +}; + +const CONFIGS = [ + { + providerType: "openai" as const, + fetch: providerFetch, + baseUrl: "https://proxy.example/v1", + tokenField: "max_completion_tokens", + }, + { + providerType: "openai_compatible" as const, + fetch: providerFetch, + baseUrl: "https://proxy.example/v1", + tokenField: "max_tokens", + }, +]; + +const BODY = { + model: REQUEST.model, + messages: [{ role: "system", content: REQUEST.system }, ...REQUEST.messages], + stream: true, + stream_options: { include_usage: true }, +}; + +const RUN_TOOL = { name: "run", description: "Run a command", parameters: {} }; +const TOOLS = [{ type: "function", function: RUN_TOOL }]; + +beforeEach(() => { + vi.resetAllMocks(); +}); + +describe.each(CONFIGS)( + "openAiAdapter $providerType Chat requests", + (config) => { + it("maps all advanced fields within the request token limit", async () => { + providerFetch.mockResolvedValueOnce( + streamingResponse(["data: [DONE]\n"]), + ); + + await collect( + openAiAdapter.streamChat( + { + ...config, + advanced: { + endpoint: "chat", + reasoningEffort: "none", + reasoningSummary: "detailed", + verbosity: "high", + maxOutputTokens: 2048, + temperature: 0.5, + }, + }, + { ...REQUEST, maxTokens: 512, tools: [RUN_TOOL] }, + ), + ); + + expect(providerFetch).toHaveBeenCalledTimes(1); + expect(providerFetch.mock.calls[0][0]).toBe( + "https://proxy.example/v1/chat/completions", + ); + expect(JSON.parse(providerFetch.mock.calls[0][1].body)).toEqual({ + ...BODY, + [config.tokenField]: 512, + reasoning_effort: "none", + verbosity: "high", + temperature: 0.5, + tools: TOOLS, + }); + }); + + it.each([undefined, {}, { endpoint: "auto" as const }])( + "preserves the default body with advanced %j", + async (advanced) => { + providerFetch.mockResolvedValueOnce( + streamingResponse(["data: [DONE]\n"]), + ); + + await collect( + openAiAdapter.streamChat( + { ...config, advanced }, + { ...REQUEST, maxTokens: 512, tools: [RUN_TOOL] }, + ), + ); + + expect(JSON.parse(providerFetch.mock.calls[0][1].body)).toEqual({ + ...BODY, + [config.tokenField]: 512, + tools: TOOLS, + }); + }, + ); + + it.each([undefined, 0])( + "preserves the default body with request maxTokens %j", + async (maxTokens) => { + providerFetch.mockResolvedValueOnce( + streamingResponse(["data: [DONE]\n"]), + ); + + await collect( + openAiAdapter.streamChat( + { ...config, advanced: {} }, + { ...REQUEST, maxTokens }, + ), + ); + + expect(JSON.parse(providerFetch.mock.calls[0][1].body)).toEqual(BODY); + }, + ); + + it.each([ + { maxOutputTokens: 256, maxTokens: 512, expected: 256 }, + { maxOutputTokens: 1024, maxTokens: 512, expected: 512 }, + { maxOutputTokens: 1024, maxTokens: undefined, expected: 1024 }, + ])( + "sends maxOutputTokens $maxOutputTokens with request maxTokens $maxTokens as $expected", + async ({ maxOutputTokens, maxTokens, expected }) => { + providerFetch.mockResolvedValueOnce( + streamingResponse(["data: [DONE]\n"]), + ); + + await collect( + openAiAdapter.streamChat( + { ...config, advanced: { maxOutputTokens } }, + maxTokens === undefined ? REQUEST : { ...REQUEST, maxTokens }, + ), + ); + + expect(JSON.parse(providerFetch.mock.calls[0][1].body)).toEqual({ + ...BODY, + [config.tokenField]: expected, + }); + }, + ); + + it("ignores the Responses summary setting when it is off", async () => { + providerFetch.mockResolvedValueOnce( + streamingResponse(["data: [DONE]\n"]), + ); + + await collect( + openAiAdapter.streamChat( + { ...config, advanced: { reasoningSummary: "off" } }, + REQUEST, + ), + ); + + expect(JSON.parse(providerFetch.mock.calls[0][1].body)).toEqual(BODY); + }); + + it.each>([ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max", + ])("sends the explicit effort %s", async (reasoningEffort) => { + providerFetch.mockResolvedValueOnce( + streamingResponse(["data: [DONE]\n"]), + ); + + await collect( + openAiAdapter.streamChat( + { ...config, advanced: { reasoningEffort } }, + REQUEST, + ), + ); + + expect(JSON.parse(providerFetch.mock.calls[0][1].body)).toEqual({ + ...BODY, + reasoning_effort: reasoningEffort, + }); + }); + + it("applies provider temperature handling with high effort across retries", async () => { + const warn = vi.fn(); + providerFetch + .mockResolvedValueOnce( + new Response("stream_options unsupported", { status: 400 }), + ) + .mockResolvedValueOnce(streamingResponse(["data: [DONE]\n"])); + + await collect( + openAiAdapter.streamChat( + { + ...config, + log: { warn }, + advanced: { reasoningEffort: "high", temperature: 0 }, + }, + REQUEST, + ), + ); + + const { stream_options, ...withoutStreamOptions } = BODY; + const temperature = + config.providerType === "openai_compatible" ? { temperature: 0 } : {}; + expect( + providerFetch.mock.calls.map(([, init]) => JSON.parse(init.body)), + ).toEqual([ + { ...BODY, reasoning_effort: "high", ...temperature }, + { ...withoutStreamOptions, reasoning_effort: "high", ...temperature }, + ]); + if (config.providerType === "openai") { + expect(warn).toHaveBeenCalledTimes(1); + expect(warn).toHaveBeenCalledWith( + `OpenAI skipped temperature for ${REQUEST.model} because reasoning effort is high; temperature requires reasoning effort none.`, + ); + } else { + expect(warn).not.toHaveBeenCalled(); + } + }); + + it.each([ + { temperature: 0 }, + { temperature: 2 }, + { reasoningEffort: "none", temperature: 0.7 }, + ])("sends allowed temperature with %j", async (advanced) => { + const warn = vi.fn(); + providerFetch.mockResolvedValueOnce( + streamingResponse(["data: [DONE]\n"]), + ); + + await collect( + openAiAdapter.streamChat( + { ...config, log: { warn }, advanced }, + REQUEST, + ), + ); + + expect(JSON.parse(providerFetch.mock.calls[0][1].body)).toEqual({ + ...BODY, + ...(advanced.reasoningEffort + ? { reasoning_effort: advanced.reasoningEffort } + : {}), + temperature: advanced.temperature, + }); + expect(warn).not.toHaveBeenCalled(); + }); + + it.each([400, 422])( + "retries a temperature rejection %i without changing other fields", + async (status) => { + const warn = vi.fn(); + const reasoningEffort = + config.providerType === "openai" ? "none" : "high"; + const detail = "Temperature is not supported with reasoning models"; + providerFetch + .mockResolvedValueOnce(new Response(detail, { status })) + .mockResolvedValueOnce(streamingResponse(["data: [DONE]\n"])); + + await collect( + openAiAdapter.streamChat( + { + ...config, + log: { warn }, + advanced: { + reasoningEffort, + verbosity: "low", + temperature: 0, + }, + }, + { ...REQUEST, tools: [RUN_TOOL] }, + ), + ); + + const body = { + ...BODY, + reasoning_effort: reasoningEffort, + verbosity: "low", + tools: TOOLS, + }; + expect( + providerFetch.mock.calls.map(([, init]) => JSON.parse(init.body)), + ).toEqual([{ ...body, temperature: 0 }, body]); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn).toHaveBeenCalledWith( + `OpenAI refused temperature for ${REQUEST.model}, continuing without it: ${detail}`, + ); + }, + ); + + it("retries a temperature rejection only once", async () => { + const warn = vi.fn(); + providerFetch.mockImplementation( + async () => new Response("temperature unsupported", { status: 400 }), + ); + + await expect( + collect( + openAiAdapter.streamChat( + { ...config, log: { warn }, advanced: { temperature: 1 } }, + REQUEST, + ), + ), + ).rejects.toMatchObject({ + status: 400, + message: "OpenAI request failed (400): temperature unsupported", + }); + + expect( + providerFetch.mock.calls.map(([, init]) => JSON.parse(init.body)), + ).toEqual([{ ...BODY, temperature: 1 }, BODY]); + expect(warn).toHaveBeenCalledTimes(1); + }); + + it.each([400, 422, 500])( + "surfaces temperature error %i when no retry applies", + async (status) => { + const warn = vi.fn(); + providerFetch.mockResolvedValueOnce( + new Response("temperature unsupported", { status }), + ); + + await expect( + collect( + openAiAdapter.streamChat({ ...config, log: { warn } }, REQUEST), + ), + ).rejects.toMatchObject({ + status, + message: `OpenAI request failed (${status}): temperature unsupported`, + }); + + expect(providerFetch).toHaveBeenCalledTimes(1); + expect(JSON.parse(providerFetch.mock.calls[0][1].body)).toEqual(BODY); + expect(warn).not.toHaveBeenCalled(); + }, + ); + + it.each(["none", "high"] as const)( + "does not replace the user's %s effort for tools", + async (reasoningEffort) => { + const detail = "tools require reasoning_effort none"; + providerFetch.mockResolvedValueOnce( + new Response(detail, { status: 400 }), + ); + + await expect( + collect( + openAiAdapter.streamChat( + { ...config, advanced: { reasoningEffort } }, + { ...REQUEST, tools: [RUN_TOOL] }, + ), + ), + ).rejects.toMatchObject({ + status: 400, + message: `OpenAI request failed (400): ${detail}`, + }); + + expect(providerFetch).toHaveBeenCalledTimes(1); + expect(JSON.parse(providerFetch.mock.calls[0][1].body)).toEqual({ + ...BODY, + reasoning_effort: reasoningEffort, + tools: TOOLS, + }); + }, + ); + + it.each([ + ["stream_options", "tools require reasoning_effort none", "temperature"], + ["temperature", "stream_options", "tools require reasoning_effort none"], + ["tools require reasoning_effort none", "temperature", "stream_options"], + ])("allows all three fallbacks in order %j", async (...details) => { + const warn = vi.fn(); + for (const detail of details) { + providerFetch.mockResolvedValueOnce( + new Response(detail, { status: 400 }), + ); + } + providerFetch.mockResolvedValueOnce( + streamingResponse([ + sse({ + choices: [{ delta: { content: "ok" }, finish_reason: "stop" }], + }), + ]), + ); + + const chunks = await collect( + openAiAdapter.streamChat( + { ...config, log: { warn }, advanced: { temperature: 1 } }, + { ...REQUEST, tools: [RUN_TOOL] }, + ), + ); + + let body: Record = { + ...BODY, + temperature: 1, + tools: TOOLS, + }; + const bodies = [body]; + for (const detail of details) { + if (detail === "stream_options") { + const { stream_options, ...rest } = body; + body = rest; + } else if (detail === "temperature") { + const { temperature, ...rest } = body; + body = rest; + } else { + body = { ...body, reasoning_effort: "none" }; + } + bodies.push(body); + } + expect( + providerFetch.mock.calls.map(([, init]) => JSON.parse(init.body)), + ).toEqual(bodies); + expect(chunks).toEqual([ + { type: "text", text: "ok" }, + { type: "done", stopReason: "stop" }, + ]); + expect(warn).toHaveBeenCalledTimes(1); + }); + }, +); + +it("sends temperature with high effort for a compatible provider without warning", async () => { + const warn = vi.fn(); + providerFetch.mockResolvedValueOnce(streamingResponse(["data: [DONE]\n"])); + + await collect( + openAiAdapter.streamChat( + { + ...CONFIGS[1], + log: { warn }, + advanced: { reasoningEffort: "high", temperature: 0.7 }, + }, + REQUEST, + ), + ); + + expect(providerFetch).toHaveBeenCalledTimes(1); + expect(JSON.parse(providerFetch.mock.calls[0][1].body)).toEqual({ + ...BODY, + reasoning_effort: "high", + temperature: 0.7, + }); + expect(warn).not.toHaveBeenCalled(); +}); + +it("forces Chat Completions at the official OpenAI base", async () => { + providerFetch.mockResolvedValueOnce(streamingResponse(["data: [DONE]\n"])); + + await collect( + openAiAdapter.streamChat( + { + fetch: providerFetch, + providerType: "openai", + advanced: { endpoint: "chat" }, + }, + REQUEST, + ), + ); + + expect(providerFetch).toHaveBeenCalledTimes(1); + expect(providerFetch.mock.calls[0][0]).toBe( + "https://api.openai.com/v1/chat/completions", + ); + expect(JSON.parse(providerFetch.mock.calls[0][1].body)).toEqual(BODY); +}); diff --git a/tests/backend/routes.test.ts b/tests/backend/routes.test.ts index a13dc54..eacc067 100644 --- a/tests/backend/routes.test.ts +++ b/tests/backend/routes.test.ts @@ -167,6 +167,234 @@ describe("ai routes", () => { }); }); +describe("per-provider advanced settings", () => { + const provider = { + providerType: "openai", + label: "Work", + apiKey: "sk-test", + }; + + it("returns persisted settings, replaces them on PATCH, and clears them", async () => { + server = await startServer(); + await server.enableFor("user-1"); + + const advanced = { + endpoint: "responses", + reasoningEffort: "high", + reasoningSummary: "detailed", + verbosity: "low", + maxOutputTokens: 8192, + temperature: 0.5, + }; + const created = await server.request("POST", "/providers", { + body: { ...provider, advanced }, + }); + expect(created.status).toBe(201); + expect(created.body.provider.advanced).toEqual(advanced); + const id = created.body.provider.id; + + const list = await server.request("GET", "/providers"); + expect(list.status).toBe(200); + expect(list.body.providers[0].advanced).toEqual(advanced); + + const replacement = { endpoint: "chat", temperature: 0 }; + const patched = await server.request("PATCH", `/providers/${id}`, { + body: { advanced: replacement }, + }); + expect(patched.status).toBe(200); + expect(patched.body.provider.advanced).toEqual(replacement); + + const unchanged = await server.request("PATCH", `/providers/${id}`, { + body: { label: "Renamed" }, + }); + expect(unchanged.status).toBe(200); + expect(unchanged.body.provider.advanced).toEqual(replacement); + + for (const advanced of [null, {}]) { + const cleared = await server.request("PATCH", `/providers/${id}`, { + body: { advanced }, + }); + expect(cleared.status).toBe(200); + expect(cleared.body.provider.advanced).toEqual({}); + const row = server.db.sqlite + .prepare("SELECT advanced FROM p_ai_providers WHERE id = ?") + .get(id) as { advanced: string | null }; + expect(row.advanced).toBeNull(); + if (advanced === null) { + await server.request("PATCH", `/providers/${id}`, { + body: { advanced: replacement }, + }); + } + } + const clearedList = await server.request("GET", "/providers"); + expect(clearedList.body.providers[0].advanced).toEqual({}); + }); + + it("defaults omitted, null and empty settings to an empty object", async () => { + server = await startServer(); + await server.enableFor("user-1"); + + for (const [index, advanced] of [undefined, null, {}].entries()) { + const created = await server.request("POST", "/providers", { + body: { ...provider, label: `Work ${index}`, advanced }, + }); + expect(created.status).toBe(201); + expect(created.body.provider.advanced).toEqual({}); + const row = server.db.sqlite + .prepare("SELECT advanced FROM p_ai_providers WHERE id = ?") + .get(created.body.provider.id) as { advanced: string | null }; + expect(row.advanced).toBeNull(); + } + }); + + it.each(["POST", "PATCH"])( + "%s omits null and empty fields while keeping explicit defaults and zero", + async (method) => { + server = await startServer(); + await server.enableFor("user-1"); + let path = "/providers"; + if (method === "PATCH") { + const created = await server.request("POST", path, { + body: { ...provider, advanced: { reasoningEffort: "high" } }, + }); + path += `/${created.body.provider.id}`; + } + + for (const empty of [null, ""]) { + const response = await server.request(method, path, { + body: { + ...provider, + label: `Work ${empty === null ? "null" : "empty"}`, + advanced: { + endpoint: empty, + reasoningEffort: empty, + reasoningSummary: empty, + verbosity: empty, + maxOutputTokens: empty, + temperature: empty, + }, + }, + }); + expect(response.status).toBe(method === "POST" ? 201 : 200); + expect(response.body.provider.advanced).toEqual({}); + } + const response = await server.request(method, path, { + body: { + ...provider, + advanced: { endpoint: "auto", temperature: 0 }, + }, + }); + expect(response.status).toBe(method === "POST" ? 201 : 200); + expect(response.body.provider.advanced).toEqual({ + endpoint: "auto", + temperature: 0, + }); + }, + ); + + it("accepts every enum value and numeric boundary", async () => { + server = await startServer(); + await server.enableFor("user-1"); + const created = await server.request("POST", "/providers", { + body: provider, + }); + const path = `/providers/${created.body.provider.id}`; + + for (const [field, values] of Object.entries({ + endpoint: ["auto", "responses", "chat"], + reasoningEffort: [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max", + ], + reasoningSummary: ["auto", "concise", "detailed", "off"], + verbosity: ["low", "medium", "high"], + maxOutputTokens: [1, 1000000], + temperature: [0, 2], + })) { + for (const value of values) { + const advanced = { [field]: value }; + const response = await server.request("PATCH", path, { + body: { advanced }, + }); + expect(response.status, JSON.stringify(advanced)).toBe(200); + expect(response.body.provider.advanced).toEqual(advanced); + } + } + }); + + it.each(["POST", "PATCH"])( + "%s rejects invalid advanced settings and unknown keys", + async (method) => { + server = await startServer(); + await server.enableFor("user-1"); + let path = "/providers"; + if (method === "PATCH") { + const created = await server.request("POST", path, { + body: provider, + }); + path += `/${created.body.provider.id}`; + } + + for (const advanced of [ + "responses", + "", + 1, + true, + [], + ["responses"], + { endpoint: "invalid" }, + { endpoint: 1 }, + { reasoningEffort: "invalid" }, + { reasoningEffort: false }, + { reasoningSummary: "invalid" }, + { reasoningSummary: [] }, + { verbosity: "invalid" }, + { verbosity: {} }, + { maxOutputTokens: 0 }, + { maxOutputTokens: -1 }, + { maxOutputTokens: 1000001 }, + { maxOutputTokens: 1.5 }, + { maxOutputTokens: "100" }, + { maxOutputTokens: true }, + { temperature: -0.1 }, + { temperature: 2.1 }, + { temperature: "1" }, + { temperature: false }, + { unknown: "value" }, + { endpoint: "auto", unknown: null }, + ]) { + const response = await server.request(method, path, { + body: { ...provider, advanced }, + }); + expect(response.status, JSON.stringify(advanced)).toBe(400); + } + }, + ); + + it("reads malformed and non-object stored JSON as empty settings", async () => { + server = await startServer(); + await server.enableFor("user-1"); + const created = await server.request("POST", "/providers", { + body: provider, + }); + const id = created.body.provider.id; + + for (const stored of ["{", "[]", '"responses"', "1", "null"]) { + server.db.sqlite + .prepare("UPDATE p_ai_providers SET advanced = ? WHERE id = ?") + .run(stored, id); + const list = await server.request("GET", "/providers"); + expect(list.status).toBe(200); + expect(list.body.providers[0].advanced).toEqual({}); + } + }); +}); + describe("chat stream", () => { it("offers the model only the tools whose plugins are running", async () => { const bodies: Array<{ tools?: Array<{ function?: { name: string } }> }> = diff --git a/tests/frontend/AiProviderSettings.test.tsx b/tests/frontend/AiProviderSettings.test.tsx index 8befcd4..f8ab44b 100644 --- a/tests/frontend/AiProviderSettings.test.tsx +++ b/tests/frontend/AiProviderSettings.test.tsx @@ -42,21 +42,52 @@ const provider: AiProvider = { apiKeyPrefix: null, defaultModel: "llama3.1", modelOverrides: {}, + advanced: {}, enabled: true, createdAt: "2026-08-21T00:00:00Z", }; +const openai: AiProvider = { + ...provider, + id: 8, + providerType: "openai", + label: "OpenAI", + baseUrl: null, + apiKeyPrefix: "sk-proj", + defaultModel: "gpt-5", +}; + beforeEach(() => { api.getAiProviderModels.mockReset(); api.getAiProviderModels.mockResolvedValue([]); api.updateAiProvider.mockReset(); api.updateAiProvider.mockResolvedValue(provider); + api.createAiProvider.mockReset(); + api.createAiProvider.mockResolvedValue({ ...openai, id: 9 }); + api.probeAiModels.mockReset(); + api.probeAiModels.mockResolvedValue({ models: [], source: "live" }); api.getProviderContext.mockReset(); api.getProviderContext.mockResolvedValue([]); + // Radix Select calls these, and jsdom has neither. + Element.prototype.scrollIntoView = vi.fn(); + Element.prototype.hasPointerCapture = vi.fn(() => false); }); afterEach(cleanup); +async function choose(field: string, option: string) { + fireEvent.keyDown(screen.getByRole("combobox", { name: field }), { + key: "Enter", + }); + fireEvent.keyDown(await screen.findByRole("option", { name: option }), { + key: "Enter", + }); +} + +function openAdvanced() { + fireEvent.click(screen.getByRole("button", { name: "ai.advanced" })); +} + describe("AiProviderSettings", () => { it("edits an existing provider name and configured model", async () => { const onChanged = vi.fn(); @@ -174,4 +205,203 @@ describe("AiProviderSettings", () => { }), ); }); + + describe("advanced settings", () => { + it("are hidden for providers other than OpenAI", () => { + render( + {}} />, + ); + + fireEvent.click(screen.getByRole("button", { name: "ai.editProvider" })); + expect(screen.queryByRole("button", { name: "ai.advanced" })).toBeNull(); + + fireEvent.click(screen.getByRole("button", { name: "ai.cancel" })); + fireEvent.click(screen.getByRole("button", { name: "ai.addProvider" })); + expect(screen.queryByRole("button", { name: "ai.advanced" })).toBeNull(); + }); + + it("start closed and open from the toggle", () => { + render( {}} />); + fireEvent.click(screen.getByRole("button", { name: "ai.editProvider" })); + + const toggle = screen.getByRole("button", { name: "ai.advanced" }); + expect(toggle.getAttribute("aria-expanded")).toBe("false"); + expect( + screen.queryByRole("combobox", { name: "ai.advancedEndpoint" }), + ).toBeNull(); + + fireEvent.click(toggle); + expect(toggle.getAttribute("aria-expanded")).toBe("true"); + expect( + screen.getByRole("combobox", { name: "ai.advancedEndpoint" }), + ).toBeTruthy(); + expect(screen.getByText("ai.advancedReasoningSummaryHint")).toBeTruthy(); + expect(screen.getByText("ai.advancedTemperatureHint")).toBeTruthy(); + + fireEvent.click(toggle); + expect( + screen.queryByRole("combobox", { name: "ai.advancedEndpoint" }), + ).toBeNull(); + }); + + it("saves only the settings that are set", async () => { + render( {}} />); + fireEvent.click(screen.getByRole("button", { name: "ai.editProvider" })); + openAdvanced(); + + await choose("ai.advancedReasoningEffort", "ai.advancedHigh"); + fireEvent.change(screen.getByLabelText("ai.advancedMaxOutputTokens"), { + target: { value: "4096" }, + }); + fireEvent.click(screen.getByRole("button", { name: "ai.save" })); + + await waitFor(() => + expect(api.updateAiProvider).toHaveBeenCalledWith(8, { + label: "OpenAI", + defaultModel: "gpt-5", + modelOverrides: {}, + advanced: { reasoningEffort: "high", maxOutputTokens: 4096 }, + }), + ); + }); + + it("clears them on edit when every field is empty", async () => { + render( + {}} + />, + ); + fireEvent.click(screen.getByRole("button", { name: "ai.editProvider" })); + openAdvanced(); + + await choose("ai.advancedVerbosity", "ai.advancedDefault"); + fireEvent.click(screen.getByRole("button", { name: "ai.save" })); + + await waitFor(() => + expect(api.updateAiProvider).toHaveBeenCalledWith(8, { + label: "OpenAI", + defaultModel: "gpt-5", + modelOverrides: {}, + advanced: null, + }), + ); + }); + + it("are left out on create when empty and sent when set", async () => { + render( {}} />); + + fireEvent.click(screen.getByRole("button", { name: "ai.addProvider" })); + await choose("ai.providerType", "ai.providerOpenai"); + fireEvent.change(screen.getByLabelText("ai.providerLabel"), { + target: { value: "Work" }, + }); + fireEvent.click(screen.getByRole("button", { name: "ai.save" })); + + await waitFor(() => + expect(api.createAiProvider).toHaveBeenCalledTimes(1), + ); + expect(api.createAiProvider.mock.calls[0][0]).toEqual({ + providerType: "openai", + label: "Work", + baseUrl: null, + apiKey: null, + defaultModel: null, + }); + expect(api.createAiProvider.mock.calls[0][0]).not.toHaveProperty( + "advanced", + ); + + fireEvent.click(screen.getByRole("button", { name: "ai.addProvider" })); + await choose("ai.providerType", "ai.providerOpenaiCompatible"); + fireEvent.change(screen.getByLabelText("ai.providerLabel"), { + target: { value: "Gateway" }, + }); + openAdvanced(); + await choose("ai.advancedEndpoint", "ai.advancedEndpointChat"); + fireEvent.change(screen.getByLabelText("ai.advancedTemperature"), { + target: { value: "0.7" }, + }); + fireEvent.click(screen.getByRole("button", { name: "ai.save" })); + + await waitFor(() => + expect(api.createAiProvider).toHaveBeenCalledTimes(2), + ); + expect(api.createAiProvider.mock.calls[1][0]).toMatchObject({ + providerType: "openai_compatible", + label: "Gateway", + advanced: { endpoint: "chat", temperature: 0.7 }, + }); + }); + + it("are filled in from the provider when editing", async () => { + const advanced = { + endpoint: "chat", + reasoningEffort: "xhigh", + reasoningSummary: "off", + verbosity: "high", + maxOutputTokens: 2048, + temperature: 0.2, + } as const; + render( + {}} + />, + ); + fireEvent.click(screen.getByRole("button", { name: "ai.editProvider" })); + openAdvanced(); + + const shown = (name: string) => + screen.getByRole("combobox", { name }).textContent; + expect(shown("ai.advancedEndpoint")).toBe("ai.advancedEndpointChat"); + expect(shown("ai.advancedReasoningEffort")).toBe("ai.advancedXhigh"); + expect(shown("ai.advancedReasoningSummary")).toBe("ai.advancedOff"); + expect(shown("ai.advancedVerbosity")).toBe("ai.advancedHigh"); + expect( + ( + screen.getByLabelText( + "ai.advancedMaxOutputTokens", + ) as HTMLInputElement + ).value, + ).toBe("2048"); + expect( + (screen.getByLabelText("ai.advancedTemperature") as HTMLInputElement) + .value, + ).toBe("0.2"); + + fireEvent.click(screen.getByRole("button", { name: "ai.save" })); + await waitFor(() => + expect(api.updateAiProvider).toHaveBeenCalledWith( + 8, + expect.objectContaining({ advanced }), + ), + ); + }); + + it("block saving while the temperature is out of range", async () => { + render( {}} />); + fireEvent.click(screen.getByRole("button", { name: "ai.editProvider" })); + openAdvanced(); + + const temperature = screen.getByLabelText("ai.advancedTemperature"); + const save = screen.getByRole("button", { name: "ai.save" }); + fireEvent.change(temperature, { target: { value: "2.5" } }); + + expect(screen.getByText("ai.advancedTemperatureInvalid")).toBeTruthy(); + expect((save as HTMLButtonElement).disabled).toBe(true); + fireEvent.click(save); + expect(api.updateAiProvider).not.toHaveBeenCalled(); + + fireEvent.change(temperature, { target: { value: "1.5" } }); + expect(screen.queryByText("ai.advancedTemperatureInvalid")).toBeNull(); + fireEvent.click(save); + await waitFor(() => + expect(api.updateAiProvider).toHaveBeenCalledWith( + 8, + expect.objectContaining({ advanced: { temperature: 1.5 } }), + ), + ); + }); + }); });