diff --git a/extensions/vision.ts b/extensions/vision.ts index 14bff73..5489ba9 100644 --- a/extensions/vision.ts +++ b/extensions/vision.ts @@ -51,6 +51,7 @@ import { } from "../lib/config.ts"; import { delegateToVisionModel, type DelegateParams } from "../lib/delegate.ts"; import { VisionCache } from "../lib/cache.ts"; +import { isUsableVisionModel } from "../lib/supported.ts"; import { setSharedState } from "../lib/state.ts"; import { createPreviewComponent, makePreviewImage, detectProtocol, formatImageMetadata } from "../lib/preview.ts"; import { matchesKey } from "@earendil-works/pi-tui"; @@ -195,9 +196,11 @@ function applyAndSave(id: string, value: string, pi: ExtensionAPI, ctx: Extensio if (id === "cachePersist" || id === "cacheMaxEntries") rebuildCache(); } -/** Vision-capable authed models from the registry (input includes "image"). */ +/** Vision models the delegate path can actually run (ADR-0001): image + * capable + a supported API type. Unsupported types (e.g. anthropic-messages) + * stay hidden until implemented — see lib/supported.ts. */ function visionCapableModels(ctx: ExtensionContext): Model[] { - return ctx.modelRegistry.getAvailable().filter((m) => m.input.includes("image")); + return ctx.modelRegistry.getAvailable().filter((m) => isUsableVisionModel(m)); } /** Open pi's native select picker over vision-capable models. Sets provider + diff --git a/lib/defaults.ts b/lib/defaults.ts index 01ac7fa..a40fb9e 100644 --- a/lib/defaults.ts +++ b/lib/defaults.ts @@ -20,6 +20,7 @@ * (auto-detect only fires when both provider + model are unset). */ import type { Api, Model } from "@earendil-works/pi-ai"; +import { isUsableVisionModel } from "./supported.ts"; export interface DetectedDefaults { provider: string | undefined; @@ -46,7 +47,10 @@ export const PREFERRED_PRIMARY_PROVIDER = "Ollama"; * Pure + deterministic (same input in any order → same output). */ export function autoDetectDefaults(models: Model[]): DetectedDefaults { - const visionModels = models.filter((m) => m.input?.includes("image")); + // ADR-0001: only models the delegate path can actually run (supported API + // types). Unsupported types (e.g. anthropic-messages) are invisible here so + // auto-detect never picks a model that would fail on every call. + const visionModels = models.filter((m) => isUsableVisionModel(m)); if (visionModels.length === 0) { return { provider: undefined, model: undefined }; } diff --git a/lib/delegate.ts b/lib/delegate.ts index 8d6d85a..fe88771 100644 --- a/lib/delegate.ts +++ b/lib/delegate.ts @@ -25,6 +25,7 @@ import { loadImage, type LoadedImage } from "./image.ts"; import { cacheKey, type VisionCache } from "./cache.ts"; import { AbortError, classifyError, withRetry } from "./resilience.ts"; import { appendAuditEntry, resolveAuditPath, truncateImagePathForLog, type AuditEntry } from "./audit.ts"; +import { SUPPORTED_VISION_APIS, isSupportedVisionApi } from "./supported.ts"; export interface DelegateParams { image_path: string; @@ -59,8 +60,8 @@ export interface DelegateFailure { export type DelegateResult = DelegateSuccess | DelegateFailure; -/** Build provider-specific reasoning params for the request body, if any. */ -function buildReasoningParams( +/** Build completions-API reasoning params (flat `reasoning_effort`), if any. */ +function buildCompletionsReasoningParams( visionModel: Model, level: ReasoningLevel, ): Record | undefined { @@ -68,13 +69,124 @@ function buildReasoningParams( return { reasoning_effort: level }; } +/** Build responses-API reasoning params (nested `reasoning: { effort }`), if any. */ +function buildResponsesReasoningParams( + visionModel: Model, + level: ReasoningLevel, +): Record | undefined { + if (!visionModel.reasoning || level === "off") return undefined; + return { reasoning: { effort: level } }; +} + +/** Error for a vision model whose declared API type we cannot delegate to. */ +function unsupportedApiError(api: string): Error { + return new Error( + `Vision model API type "${api}" is not supported. Supported: ${[...SUPPORTED_VISION_APIS].join(", ")}.`, + ); +} + +/** Build the shared headers (auth + provider headers) for a vision request. */ +function buildHeaders( + apiKey: string | undefined, + providerHeaders: Record | undefined, +): Record { + const headers: Record = { "Content-Type": "application/json" }; + if (apiKey) headers.Authorization = `Bearer ${apiKey}`; + if (providerHeaders) Object.assign(headers, providerHeaders); + return headers; +} + +/** Extract the message content (text or reasoning) from a chat/completions response. */ +function extractCompletionsText(json: unknown): string | undefined { + const j = json as { + choices?: Array<{ message?: { content?: string; reasoning_content?: string } }>; + }; + const msg = j.choices?.[0]?.message; + return msg?.content || msg?.reasoning_content; +} + +/** Extract the first `output_text` from a responses-API response. */ +function extractResponsesText(json: unknown): string | undefined { + const j = json as { + output?: Array<{ + type?: string; + content?: Array<{ type?: string; text?: string }>; + }>; + }; + for (const item of j.output ?? []) { + if (item.type !== "message") continue; + for (const part of item.content ?? []) { + if (part.type === "output_text" && typeof part.text === "string" && part.text.length > 0) { + return part.text; + } + } + } + return undefined; +} + +/** POST the image (data URL) + prompt to the model's responses-API endpoint. */ +async function callResponsesEndpoint( + visionModel: Model, + apiKey: string | undefined, + providerHeaders: Record | undefined, + image: LoadedImage, + prompt: string, + signal: AbortSignal | undefined, + reasoning: ReasoningLevel, + systemPrompt?: string, +): Promise { + const baseUrl = visionModel.baseUrl.replace(/\/+$/, ""); + const input: unknown[] = []; + if (systemPrompt && systemPrompt.length > 0) { + input.push({ role: "system", content: systemPrompt }); + } + input.push({ + role: "user", + content: [ + { + type: "input_image", + image_url: `data:${image.mimeType};base64,${image.data}`, + }, + { type: "input_text", text: prompt }, + ], + }); + const body: Record = { + model: visionModel.id, + input, + // OpenAI Responses rejects max_output_tokens below 16. + max_output_tokens: 4096, + temperature: 0, + }; + const reasoningParams = buildResponsesReasoningParams(visionModel, reasoning); + if (reasoningParams) Object.assign(body, reasoningParams); + + const headers = buildHeaders(apiKey, providerHeaders); + const response = await fetch(`${baseUrl}/responses`, { + method: "POST", + headers, + body: JSON.stringify(body), + signal, + }); + + if (!response.ok) { + const errBody = await response.text().catch(() => ""); + throw new Error( + `Vision model returned ${response.status}: ${errBody.slice(0, 500)}`, + ); + } + + const text = extractResponsesText(await response.json()); + if (!text) { + throw new Error("Vision model returned no content in the response"); + } + return text; +} + /** - * Call the vision model's OpenAI-compat chat/completions endpoint with the - * image as a data URL + the user's prompt (and an optional system prompt). - * Returns the model's text response. Exported + fetch-based so tests can mock - * `globalThis.fetch`. + * POST the image (data URL) + prompt to the model's OpenAI-compat + * chat/completions endpoint. Returns the model's text response. */ -export async function callVisionModel( +async function callCompletionsEndpoint( visionModel: Model, apiKey: string | undefined, providerHeaders: Record | undefined, @@ -105,13 +217,10 @@ export async function callVisionModel( max_tokens: 4096, temperature: 0, }; - const reasoningParams = buildReasoningParams(visionModel, reasoning); + const reasoningParams = buildCompletionsReasoningParams(visionModel, reasoning); if (reasoningParams) Object.assign(body, reasoningParams); - const headers: Record = { "Content-Type": "application/json" }; - if (apiKey) headers.Authorization = `Bearer ${apiKey}`; - if (providerHeaders) Object.assign(headers, providerHeaders); - + const headers = buildHeaders(apiKey, providerHeaders); const response = await fetch(`${baseUrl}/chat/completions`, { method: "POST", headers, @@ -126,17 +235,42 @@ export async function callVisionModel( ); } - const json = (await response.json()) as { - choices?: Array<{ message?: { content?: string; reasoning_content?: string } }>; - }; - const msg = json.choices?.[0]?.message; - const text = msg?.content || msg?.reasoning_content; + const text = extractCompletionsText(await response.json()); if (!text) { throw new Error("Vision model returned no content in the response"); } return text; } +/** + * Call the vision model's endpoint with the image as a data URL + the + * user's prompt (and an optional system prompt). Returns the model's text + * response. Exported + fetch-based so tests can mock `globalThis.fetch`. + * + * Dispatches the request/response shape on the model's declared API type + * (ADR-0001): each supported type gets a branch, unknown types throw and + * flow into the caller's fallback. Add a new API type here with an `else if` + * branch + a matching endpoint function. + */ +export async function callVisionModel( + visionModel: Model, + apiKey: string | undefined, + providerHeaders: Record | undefined, + image: LoadedImage, + prompt: string, + signal: AbortSignal | undefined, + reasoning: ReasoningLevel, + systemPrompt?: string, +): Promise { + if (visionModel.api === "openai-completions") { + return callCompletionsEndpoint(visionModel, apiKey, providerHeaders, image, prompt, signal, reasoning, systemPrompt); + } else if (visionModel.api === "openai-responses") { + return callResponsesEndpoint(visionModel, apiKey, providerHeaders, image, prompt, signal, reasoning, systemPrompt); + } else { + throw unsupportedApiError(visionModel.api); + } +} + function formatImageError(error: { code: string; path?: string; message?: string }, inputPath: string): string { switch (error.code) { case "not_found": diff --git a/lib/supported.ts b/lib/supported.ts new file mode 100644 index 0000000..88fbbe6 --- /dev/null +++ b/lib/supported.ts @@ -0,0 +1,30 @@ +/** + * Supported vision-model API types (ADR-0001). + * + * `callVisionModel` dispatches its request/response shape on the model's + * declared `api` type. This module is the single source of truth for which + * types the extension can delegate to today — pickers, auto-detect, and the + * delegate path all consult it so users never see a model that can't run. + * + * When a new API type is implemented (e.g. `anthropic-messages`), add it + * here + add a dispatch branch in `lib/delegate.ts`; the picker/auto-detect + * filters relax in one place. + */ +import type { Api, Model } from "@earendil-works/pi-ai"; + +/** API types the delegate path can speak (ADR-0001). */ +export const SUPPORTED_VISION_APIS: ReadonlySet = new Set([ + "openai-completions", + "openai-responses", +]); + +/** Whether a model's declared API type is delegatable. */ +export function isSupportedVisionApi(api: Api | string): boolean { + return SUPPORTED_VISION_APIS.has(api); +} + +/** Whether a model is usable as a vision model: image-capable + supported + * API type (the picker/auto-detect filter, ADR-0001). */ +export function isUsableVisionModel(model: Model): boolean { + return (model.input?.includes("image") ?? false) && isSupportedVisionApi(model.api); +} diff --git a/package.json b/package.json index ad5f913..55f8f4b 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,8 @@ }, "peerDependencies": { "@earendil-works/pi-coding-agent": "*", + "@earendil-works/pi-ai": "*", + "@earendil-works/pi-tui": "*", "typebox": "*" }, "devDependencies": { diff --git a/tests/defaults.test.ts b/tests/defaults.test.ts index 75a7e63..4664808 100644 --- a/tests/defaults.test.ts +++ b/tests/defaults.test.ts @@ -83,6 +83,41 @@ test("only one provider with vision → fallback undefined (no other provider)", assert.equal(result.model, "minimax-m3:cloud"); }); +test("ADR-0001: vision models with unsupported api types are filtered out", () => { + // anthropic-messages models are vision-capable but not delegatable (yet). + // auto-detect must ignore them — picking one would fail on every call. + const anthropicVision = (p: string, id: string) => + ({ ...vision(p, id), api: "anthropic-messages" as Api }); + const result = autoDetectDefaults([ + anthropicVision("Ollama", "minimax-m3:cloud"), + anthropicVision("Ollama", "qwen3.7-plus"), + ]); + assert.equal(result.provider, undefined, "unsupported api types must not be auto-picked"); + assert.equal(result.model, undefined); +}); + +test("ADR-0001: unsupported api filtered → supported model from other provider wins", () => { + const anthropicVision = (p: string, id: string) => + ({ ...vision(p, id), api: "anthropic-messages" as Api }); + const result = autoDetectDefaults([ + anthropicVision("Ollama", "minimax-m3:cloud"), // unsupported, would win sort otherwise + vision("OpenRouter", "gpt-4o"), + ]); + assert.equal(result.provider, "OpenRouter"); + assert.equal(result.model, "gpt-4o", "unsupported Ollama skipped; supported model picked"); +}); + +test("ADR-0001: openai-responses models are supported by auto-detect", () => { + const responsesVision = (p: string, id: string) => + ({ ...vision(p, id), api: "openai-responses" as Api }); + const result = autoDetectDefaults([ + responsesVision("OpenRouter", "gpt-5.6-luna"), + responsesVision("OpenRouter", "grok-4.5"), + ]); + assert.equal(result.provider, "OpenRouter"); + assert.equal(result.model, "gpt-5.6-luna", "sorted by id: gpt < grok"); +}); + test("three providers with vision → primary Ollama (v0.5.1: no auto-fallback)", () => { const result = autoDetectDefaults([ vision("Ollama", "minimax-m3:cloud"), diff --git a/tests/delegate.test.ts b/tests/delegate.test.ts index 415f25d..63180b3 100644 --- a/tests/delegate.test.ts +++ b/tests/delegate.test.ts @@ -93,12 +93,17 @@ function makeCtx(opts: { apiKey?: string; headers?: Record; cwd?: string; + /** Per-model lookup override: `(provider, id) → model`. When provided, the + * registry resolves each requested model individually (primary + fallback + * can differ). Falls back to `opts.model` when the override returns undefined. */ + findOverride?: (provider: string, id: string) => Model | undefined; }): ExtensionContext { const model = opts.model ?? makeVisionModel(); return { cwd: opts.cwd ?? "/tmp", modelRegistry: { - find: () => opts.model === null ? undefined : model, + find: (provider: string, id: string) => + opts.model === null ? undefined : opts.findOverride?.(provider, id) ?? model, getApiKeyAndHeaders: async () => opts.authOk === false ? { ok: false, error: opts.authError ?? "no api key" } @@ -261,6 +266,118 @@ test("callVisionModel: no content in response throws", async () => { } }); +test("callVisionModel: openai-responses → POST /responses with input[] body", async () => { + const m = mockFetch({ + status: 200, + body: { + output: [ + { + type: "message", + content: [{ type: "output_text", text: "a red square" }], + }, + ], + }, + }); + try { + const text = await callVisionModel( + makeVisionModel({ api: "openai-responses", baseUrl: "https://api.example.com/v1" }), + "key-456", + undefined, + { data: PNG_1x1_B64, mimeType: "image/png" }, + "describe this", + undefined, + "off", + "You are a careful analyst.", + ); + assert.equal(text, "a red square"); + assert.equal(m.calls.length, 1); + assert.equal(m.calls[0]!.url, "https://api.example.com/v1/responses"); + const init = m.calls[0]!.init; + assert.equal(init.method, "POST"); + const headers = init.headers as Record; + assert.equal(headers.Authorization, "Bearer key-456"); + assert.equal(headers["Content-Type"], "application/json"); + const body = JSON.parse(init.body as string); + assert.equal(body.model, "minimax-m3:cloud"); + assert.equal(body.input.length, 2); + assert.equal(body.input[0].role, "system"); + assert.equal(body.input[0].content, "You are a careful analyst."); + assert.equal(body.input[1].role, "user"); + assert.equal(body.input[1].content[0].type, "input_image"); + assert.ok(body.input[1].content[0].image_url.startsWith("data:image/png;base64,")); + assert.equal(body.input[1].content[1].type, "input_text"); + assert.equal(body.input[1].content[1].text, "describe this"); + assert.ok(body.max_output_tokens >= 16, "max_output_tokens must be >= 16"); + assert.equal(body.temperature, 0); + } finally { + m.restore(); + } +}); + +test("callVisionModel: openai-responses with reasoning → nested reasoning.effort", async () => { + const m = mockFetch({ + status: 200, + body: { output: [{ type: "message", content: [{ type: "output_text", text: "ok" }] }] }, + }); + try { + await callVisionModel( + makeVisionModel({ api: "openai-responses", reasoning: true }), + "k", + undefined, + { data: PNG_1x1_B64, mimeType: "image/png" }, + "p", + undefined, + "high", + ); + const body = JSON.parse(m.calls[0]!.init.body as string); + assert.deepEqual(body.reasoning, { effort: "high" }); + assert.equal(body.reasoning_effort, undefined, "flat reasoning_effort must NOT be sent for responses"); + } finally { + m.restore(); + } +}); + +test("callVisionModel: openai-responses empty output → no content error", async () => { + const m = mockFetch({ status: 200, body: { output: [] } }); + try { + await assert.rejects( + callVisionModel( + makeVisionModel({ api: "openai-responses" }), + "k", + undefined, + { data: PNG_1x1_B64, mimeType: "image/png" }, + "p", + undefined, + "off", + ), + /no content/, + ); + } finally { + m.restore(); + } +}); + +test("callVisionModel: unsupported api type throws clear error (no fetch)", async () => { + const m = mockFetch({ status: 200, body: { choices: [{ message: { content: "x" } }] } }); + try { + await assert.rejects( + callVisionModel( + makeVisionModel({ api: "anthropic-messages" }), + "k", + undefined, + { data: PNG_1x1_B64, mimeType: "image/png" }, + "p", + undefined, + "off", + ), + /anthropic-messages.*not supported/, + ); + assert.equal(m.calls.length, 0, "unsupported api type must not hit the network"); + } finally { + m.restore(); + } +}); + test("delegateToVisionModel: disabled → actionable error", async () => { const ctx = makeCtx({}); const r = await delegateToVisionModel(ctx, { ...DEFAULT_CONFIG, enabled: false }, { @@ -510,6 +627,43 @@ test("delegateToVisionModel: 4xx (client) → no retry → fallback fires", asyn } }); +test("delegateToVisionModel: unsupported primary api type → no retry → fallback fires", async () => { + const dir = tmpDir(); + // Unsupported primary must throw synchronously in the call layer (no fetch), + // so the sequence only sees the fallback request. + const m = mockFetchSeq([ + { status: 200, body: { choices: [{ message: { content: "fb-desc" } }] } }, // fallback + ]); + try { + const file = join(dir, "pixel.png"); + writeFileSync(file, PNG_BYTES); + const ctx = makeCtx({ + cwd: dir, + // Primary resolves to an unsupported api type; fallback resolves to a + // supported completions model (ADR-0001: unsupported → fallback fires). + findOverride: (provider, id) => + provider === "openrouter" && id === "qwen3.5:cloud" + ? makeVisionModel({ id: "qwen3.5:cloud", provider: "openrouter", api: "openai-completions" }) + : makeVisionModel({ api: "anthropic-messages" }), + }); + const cfg = { + ...DEFAULT_CONFIG, provider: "ollama", model: "minimax-m3:cloud", + retryAttempts: 3, retryBackoffMs: 1, + fallbackProvider: "openrouter", fallbackModel: "qwen3.5:cloud", + }; + const r = await delegateToVisionModel(ctx, cfg, { image_path: file, prompt: "p", compress: false, reasoning: "off" }, undefined); + assert.equal(r.ok, true); + if (r.ok) { + assert.equal(r.details.fallback, true); + assert.equal(r.text, "fb-desc"); + } + assert.equal(m.calls.length, 1, "unsupported primary = 0 primary calls + 1 fallback"); + } finally { + m.restore(); + rmSync(dir, { recursive: true, force: true }); + } +}); + test("delegateToVisionModel: 5xx exhausts retries → fallback fires", async () => { const dir = tmpDir(); const m = mockFetchSeq([