feat: Add openai-responses-api support and filter out unsupported models.

extensions/vision.ts

@@ -51,6 +51,7 @@ import {
 } from "../lib/config.ts";
 import { delegateToVisionModel, type DelegateParams } from "../lib/delegate.ts";
 import { VisionCache } from "../lib/cache.ts";
+import { isUsableVisionModel } from "../lib/supported.ts";
 import { setSharedState } from "../lib/state.ts";
 import { createPreviewComponent, makePreviewImage, detectProtocol, formatImageMetadata } from "../lib/preview.ts";
 import { matchesKey } from "@earendil-works/pi-tui";
@@ -195,9 +196,11 @@ function applyAndSave(id: string, value: string, pi: ExtensionAPI, ctx: Extensio
   if (id === "cachePersist" || id === "cacheMaxEntries") rebuildCache();
 }
 
-/** Vision-capable authed models from the registry (input includes "image"). */
+/** Vision models the delegate path can actually run (ADR-0001): image
+ *  capable + a supported API type. Unsupported types (e.g. anthropic-messages)
+ *  stay hidden until implemented — see lib/supported.ts. */
 function visionCapableModels(ctx: ExtensionContext): Model<Api>[] {
-  return ctx.modelRegistry.getAvailable().filter((m) => m.input.includes("image"));
+  return ctx.modelRegistry.getAvailable().filter((m) => isUsableVisionModel(m));
 }
 
 /** Open pi's native select picker over vision-capable models. Sets provider +

lib/defaults.ts

@@ -20,6 +20,7 @@
  * (auto-detect only fires when both provider + model are unset).
  */
 import type { Api, Model } from "@earendil-works/pi-ai";
+import { isUsableVisionModel } from "./supported.ts";
 
 export interface DetectedDefaults {
   provider: string | undefined;
@@ -46,7 +47,10 @@ export const PREFERRED_PRIMARY_PROVIDER = "Ollama";
  * Pure + deterministic (same input in any order → same output).
  */
 export function autoDetectDefaults(models: Model<Api>[]): DetectedDefaults {
-  const visionModels = models.filter((m) => m.input?.includes("image"));
+  // ADR-0001: only models the delegate path can actually run (supported API
+  // types). Unsupported types (e.g. anthropic-messages) are invisible here so
+  // auto-detect never picks a model that would fail on every call.
+  const visionModels = models.filter((m) => isUsableVisionModel(m));
   if (visionModels.length === 0) {
     return { provider: undefined, model: undefined };
   }

lib/delegate.ts

@@ -25,6 +25,7 @@ import { loadImage, type LoadedImage } from "./image.ts";
 import { cacheKey, type VisionCache } from "./cache.ts";
 import { AbortError, classifyError, withRetry } from "./resilience.ts";
 import { appendAuditEntry, resolveAuditPath, truncateImagePathForLog, type AuditEntry } from "./audit.ts";
+import { SUPPORTED_VISION_APIS, isSupportedVisionApi } from "./supported.ts";
 
 export interface DelegateParams {
   image_path: string;
@@ -59,22 +60,133 @@ export interface DelegateFailure {
 
 export type DelegateResult = DelegateSuccess | DelegateFailure;
 
-/** Build provider-specific reasoning params for the request body, if any. */
-function buildReasoningParams(
+/** Build completions-API reasoning params (flat `reasoning_effort`), if any. */
+function buildCompletionsReasoningParams(
   visionModel: Model<Api>,
   level: ReasoningLevel,
 ): Record<string, unknown> | undefined {
   if (!visionModel.reasoning || level === "off") return undefined;
   return { reasoning_effort: level };
 }
 
+/** Build responses-API reasoning params (nested `reasoning: { effort }`), if any. */
+function buildResponsesReasoningParams(
+  visionModel: Model<Api>,
+  level: ReasoningLevel,
+): Record<string, unknown> | undefined {
+  if (!visionModel.reasoning || level === "off") return undefined;
+  return { reasoning: { effort: level } };
+}
+
+/** Error for a vision model whose declared API type we cannot delegate to. */
+function unsupportedApiError(api: string): Error {
+  return new Error(
+    `Vision model API type "${api}" is not supported. Supported: ${[...SUPPORTED_VISION_APIS].join(", ")}.`,
+  );
+}
+
+/** Build the shared headers (auth + provider headers) for a vision request. */
+function buildHeaders(
+  apiKey: string | undefined,
+  providerHeaders: Record<string, string> | undefined,
+): Record<string, string> {
+  const headers: Record<string, string> = { "Content-Type": "application/json" };
+  if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
+  if (providerHeaders) Object.assign(headers, providerHeaders);
+  return headers;
+}
+
+/** Extract the message content (text or reasoning) from a chat/completions response. */
+function extractCompletionsText(json: unknown): string | undefined {
+  const j = json as {
+    choices?: Array<{ message?: { content?: string; reasoning_content?: string } }>;
+  };
+  const msg = j.choices?.[0]?.message;
+  return msg?.content || msg?.reasoning_content;
+}
+
+/** Extract the first `output_text` from a responses-API response. */
+function extractResponsesText(json: unknown): string | undefined {
+  const j = json as {
+    output?: Array<{
+      type?: string;
+      content?: Array<{ type?: string; text?: string }>;
+    }>;
+  };
+  for (const item of j.output ?? []) {
+    if (item.type !== "message") continue;
+    for (const part of item.content ?? []) {
+      if (part.type === "output_text" && typeof part.text === "string" && part.text.length > 0) {
+        return part.text;
+      }
+    }
+  }
+  return undefined;
+}
+
+/** POST the image (data URL) + prompt to the model's responses-API endpoint. */
+async function callResponsesEndpoint(
+  visionModel: Model<Api>,
+  apiKey: string | undefined,
+  providerHeaders: Record<string, string> | undefined,
+  image: LoadedImage,
+  prompt: string,
+  signal: AbortSignal | undefined,
+  reasoning: ReasoningLevel,
+  systemPrompt?: string,
+): Promise<string> {
+  const baseUrl = visionModel.baseUrl.replace(/\/+$/, "");
+  const input: unknown[] = [];
+  if (systemPrompt && systemPrompt.length > 0) {
+    input.push({ role: "system", content: systemPrompt });
+  }
+  input.push({
+    role: "user",
+    content: [
+      {
+        type: "input_image",
+        image_url: `data:${image.mimeType};base64,${image.data}`,
+      },
+      { type: "input_text", text: prompt },
+    ],
+  });
+  const body: Record<string, unknown> = {
+    model: visionModel.id,
+    input,
+    // OpenAI Responses rejects max_output_tokens below 16.
+    max_output_tokens: 4096,
+    temperature: 0,
+  };
+  const reasoningParams = buildResponsesReasoningParams(visionModel, reasoning);
+  if (reasoningParams) Object.assign(body, reasoningParams);
+
+  const headers = buildHeaders(apiKey, providerHeaders);
+  const response = await fetch(`${baseUrl}/responses`, {
+    method: "POST",
+    headers,
+    body: JSON.stringify(body),
+    signal,
+  });
+
+  if (!response.ok) {
+    const errBody = await response.text().catch(() => "");
+    throw new Error(
+      `Vision model returned ${response.status}: ${errBody.slice(0, 500)}`,
+    );
+  }
+
+  const text = extractResponsesText(await response.json());
+  if (!text) {
+    throw new Error("Vision model returned no content in the response");
+  }
+  return text;
+}
+
 /**
- * Call the vision model's OpenAI-compat chat/completions endpoint with the
- * image as a data URL + the user's prompt (and an optional system prompt).
- * Returns the model's text response. Exported + fetch-based so tests can mock
- * `globalThis.fetch`.
+ * POST the image (data URL) + prompt to the model's OpenAI-compat
+ * chat/completions endpoint. Returns the model's text response.
  */
-export async function callVisionModel(
+async function callCompletionsEndpoint(
   visionModel: Model<Api>,
   apiKey: string | undefined,
   providerHeaders: Record<string, string> | undefined,
@@ -105,13 +217,10 @@ export async function callVisionModel(
     max_tokens: 4096,
     temperature: 0,
   };
-  const reasoningParams = buildReasoningParams(visionModel, reasoning);
+  const reasoningParams = buildCompletionsReasoningParams(visionModel, reasoning);
   if (reasoningParams) Object.assign(body, reasoningParams);
 
-  const headers: Record<string, string> = { "Content-Type": "application/json" };
-  if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
-  if (providerHeaders) Object.assign(headers, providerHeaders);
-
+  const headers = buildHeaders(apiKey, providerHeaders);
   const response = await fetch(`${baseUrl}/chat/completions`, {
     method: "POST",
     headers,
@@ -126,17 +235,42 @@ export async function callVisionModel(
     );
   }
 
-  const json = (await response.json()) as {
-    choices?: Array<{ message?: { content?: string; reasoning_content?: string } }>;
-  };
-  const msg = json.choices?.[0]?.message;
-  const text = msg?.content || msg?.reasoning_content;
+  const text = extractCompletionsText(await response.json());
   if (!text) {
     throw new Error("Vision model returned no content in the response");
   }
   return text;
 }
 
+/**
+ * Call the vision model's endpoint with the image as a data URL + the
+ * user's prompt (and an optional system prompt). Returns the model's text
+ * response. Exported + fetch-based so tests can mock `globalThis.fetch`.
+ *
+ * Dispatches the request/response shape on the model's declared API type
+ * (ADR-0001): each supported type gets a branch, unknown types throw and
+ * flow into the caller's fallback. Add a new API type here with an `else if`
+ * branch + a matching endpoint function.
+ */
+export async function callVisionModel(
+  visionModel: Model<Api>,
+  apiKey: string | undefined,
+  providerHeaders: Record<string, string> | undefined,
+  image: LoadedImage,
+  prompt: string,
+  signal: AbortSignal | undefined,
+  reasoning: ReasoningLevel,
+  systemPrompt?: string,
+): Promise<string> {
+  if (visionModel.api === "openai-completions") {
+    return callCompletionsEndpoint(visionModel, apiKey, providerHeaders, image, prompt, signal, reasoning, systemPrompt);
+  } else if (visionModel.api === "openai-responses") {
+    return callResponsesEndpoint(visionModel, apiKey, providerHeaders, image, prompt, signal, reasoning, systemPrompt);
+  } else {
+    throw unsupportedApiError(visionModel.api);
+  }
+}
+
 function formatImageError(error: { code: string; path?: string; message?: string }, inputPath: string): string {
   switch (error.code) {
     case "not_found":

lib/supported.ts

@@ -0,0 +1,30 @@
+/**
+ * Supported vision-model API types (ADR-0001).
+ *
+ * `callVisionModel` dispatches its request/response shape on the model's
+ * declared `api` type. This module is the single source of truth for which
+ * types the extension can delegate to today — pickers, auto-detect, and the
+ * delegate path all consult it so users never see a model that can't run.
+ *
+ * When a new API type is implemented (e.g. `anthropic-messages`), add it
+ * here + add a dispatch branch in `lib/delegate.ts`; the picker/auto-detect
+ * filters relax in one place.
+ */
+import type { Api, Model } from "@earendil-works/pi-ai";
+
+/** API types the delegate path can speak (ADR-0001). */
+export const SUPPORTED_VISION_APIS: ReadonlySet<string> = new Set([
+  "openai-completions",
+  "openai-responses",
+]);
+
+/** Whether a model's declared API type is delegatable. */
+export function isSupportedVisionApi(api: Api | string): boolean {
+  return SUPPORTED_VISION_APIS.has(api);
+}
+
+/** Whether a model is usable as a vision model: image-capable + supported
+ *  API type (the picker/auto-detect filter, ADR-0001). */
+export function isUsableVisionModel(model: Model<Api>): boolean {
+  return (model.input?.includes("image") ?? false) && isSupportedVisionApi(model.api);
+}

package.json

@@ -50,6 +50,8 @@
   },
   "peerDependencies": {
     "@earendil-works/pi-coding-agent": "*",
+    "@earendil-works/pi-ai": "*",
+    "@earendil-works/pi-tui": "*",
     "typebox": "*"
   },
   "devDependencies": {

tests/defaults.test.ts

@@ -83,6 +83,41 @@ test("only one provider with vision → fallback undefined (no other provider)",
   assert.equal(result.model, "minimax-m3:cloud");
 });
 
+test("ADR-0001: vision models with unsupported api types are filtered out", () => {
+  // anthropic-messages models are vision-capable but not delegatable (yet).
+  // auto-detect must ignore them — picking one would fail on every call.
+  const anthropicVision = (p: string, id: string) =>
+    ({ ...vision(p, id), api: "anthropic-messages" as Api });
+  const result = autoDetectDefaults([
+    anthropicVision("Ollama", "minimax-m3:cloud"),
+    anthropicVision("Ollama", "qwen3.7-plus"),
+  ]);
+  assert.equal(result.provider, undefined, "unsupported api types must not be auto-picked");
+  assert.equal(result.model, undefined);
+});
+
+test("ADR-0001: unsupported api filtered → supported model from other provider wins", () => {
+  const anthropicVision = (p: string, id: string) =>
+    ({ ...vision(p, id), api: "anthropic-messages" as Api });
+  const result = autoDetectDefaults([
+    anthropicVision("Ollama", "minimax-m3:cloud"), // unsupported, would win sort otherwise
+    vision("OpenRouter", "gpt-4o"),
+  ]);
+  assert.equal(result.provider, "OpenRouter");
+  assert.equal(result.model, "gpt-4o", "unsupported Ollama skipped; supported model picked");
+});
+
+test("ADR-0001: openai-responses models are supported by auto-detect", () => {
+  const responsesVision = (p: string, id: string) =>
+    ({ ...vision(p, id), api: "openai-responses" as Api });
+  const result = autoDetectDefaults([
+    responsesVision("OpenRouter", "gpt-5.6-luna"),
+    responsesVision("OpenRouter", "grok-4.5"),
+  ]);
+  assert.equal(result.provider, "OpenRouter");
+  assert.equal(result.model, "gpt-5.6-luna", "sorted by id: gpt < grok");
+});
+
 test("three providers with vision → primary Ollama (v0.5.1: no auto-fallback)", () => {
   const result = autoDetectDefaults([
     vision("Ollama", "minimax-m3:cloud"),