diff --git a/memory-bank/architecture.md b/memory-bank/architecture.md index 72c5eee..9233c75 100644 --- a/memory-bank/architecture.md +++ b/memory-bank/architecture.md @@ -33,7 +33,11 @@ The bridge owns **wire protocols, transport quirks, credentials, and model disco not own host policy: cache placement, model-capability eligibility rules, and pricing decisions belong to consumers. Capabilities arrive as **explicit per-request chat params** (`supportsImages`, `usesExplicitPromptCaching`, …), never inferred from model IDs inside a -client. A client acts only on the params it is handed; when a param is absent the provider's +client. Exception: `PositAiClient` reads ai-config's Model APIs table +(`getModelApisModelCapabilities`) to decide how to send thinking "off" (Baseten's +`reasoning_effort: "none"`), because that table is the same source that decides whether "off" +is offered. Do not extend this to other capabilities without revisiting the charter. A client +acts only on the params it is handed; when a param is absent the provider's default behavior applies, so a client that ignores a capability param is correct by default. Clients may still veto a host instruction on transport grounds (e.g. `BedrockClient` honors the explicit prompt-caching opt-in only on its Mantle Responses route) — that is wire knowledge, not diff --git a/packages/ai-config/README.md b/packages/ai-config/README.md index c192d5b..59f61b6 100644 --- a/packages/ai-config/README.md +++ b/packages/ai-config/README.md @@ -172,6 +172,7 @@ import { getDeepSeekModelCapabilities, getGeminiModelCapabilities, getGemmaModelCapabilities, + getModelApisModelCapabilities, getOpenAIModelCapabilities, openaiMaxInputTokens, getPositAiModelCapabilities, diff --git a/packages/ai-config/src/index.ts b/packages/ai-config/src/index.ts index 8dac296..8931269 100644 --- a/packages/ai-config/src/index.ts +++ b/packages/ai-config/src/index.ts @@ -149,6 +149,7 @@ export type { PortkeyModelClassificationInput, PortkeyModelFamily, } from "./model-capabilities/portkey-helpers.js"; +export { getModelApisModelCapabilities } from "./model-capabilities/model-apis-helpers.js"; export { getPositAiModelCapabilities } from "./model-capabilities/positai-helpers.js"; export { getSnowflakeCortexModelCapabilities, diff --git a/packages/ai-config/src/model-capabilities/model-apis-helpers.ts b/packages/ai-config/src/model-capabilities/model-apis-helpers.ts index ad1c0c2..7e48026 100644 --- a/packages/ai-config/src/model-capabilities/model-apis-helpers.ts +++ b/packages/ai-config/src/model-capabilities/model-apis-helpers.ts @@ -13,7 +13,8 @@ import type { InferredModelCapabilities as ModelInfo } from "../types.js"; * Models with `requiresChatTemplateKwargs` expose a binary toggle via the * vLLM-style `chat_template_kwargs: { enable_thinking: true }` request field; * models with named effort levels take a top-level OpenAI-style - * `reasoning_effort` instead. + * `reasoning_effort` instead. Listing "off" next to named levels means Baseten + * accepts `reasoning_effort: "none"`; the bridge sends that for thinking "off". */ const MODEL_APIS_CAPABILITIES: Record> = { "zai-org/GLM-5.2": { diff --git a/packages/ai-provider-bridge/src/__tests__/utils.test.ts b/packages/ai-provider-bridge/src/__tests__/utils.test.ts index 1a50040..3b3c6fe 100644 --- a/packages/ai-provider-bridge/src/__tests__/utils.test.ts +++ b/packages/ai-provider-bridge/src/__tests__/utils.test.ts @@ -53,22 +53,25 @@ describe("thinkingRequestFields", () => { }); describe("positAiThinkingRequestFields", () => { - it("maps DeepSeek off to none and reuses named efforts", () => { - expect( - positAiThinkingRequestFields("deepseek-ai/DeepSeek-V4.1-Flash-1017", "off", false), - ).toEqual({ + it("sends off as none when the table offers off next to named effort levels", () => { + expect(positAiThinkingRequestFields("moonshotai/Kimi-K3", "off", false)).toEqual({ reasoning_effort: "none", }); - expect( - positAiThinkingRequestFields("deepseek-ai/DeepSeek-V4.1-Flash-1017", "high", false), - ).toEqual({ reasoning_effort: "high" }); }); - it("maps Kimi K3 off to none instead of omitting the field", () => { - expect(positAiThinkingRequestFields("moonshotai/Kimi-K3", "off", false)).toEqual({ - reasoning_effort: "none", + it("passes named efforts through unchanged", () => { + expect(positAiThinkingRequestFields("moonshotai/Kimi-K3", "high", false)).toEqual({ + reasoning_effort: "high", }); }); + + it("sends nothing for off on a binary-toggle model", () => { + expect(positAiThinkingRequestFields("zai-org/GLM-5.2", "off", true)).toBeUndefined(); + }); + + it("sends nothing for off on a model the table does not know", () => { + expect(positAiThinkingRequestFields("example/unknown-model", "off", false)).toBeUndefined(); + }); }); describe("normalizeProviderBaseUrl", () => { diff --git a/packages/ai-provider-bridge/src/utils.ts b/packages/ai-provider-bridge/src/utils.ts index e74152d..e2f6b8e 100644 --- a/packages/ai-provider-bridge/src/utils.ts +++ b/packages/ai-provider-bridge/src/utils.ts @@ -7,6 +7,8 @@ * Kept here to avoid depending on any consumer package. */ +import { getModelApisModelCapabilities } from "ai-config"; + // --------------------------------------------------------------------------- // Thinking effort // --------------------------------------------------------------------------- @@ -135,17 +137,26 @@ export function thinkingRequestFields( : undefined; } +/** Whether the Model APIs table offers "off" next to at least one named effort level. */ +function offIsNamedEffortLevel(modelId: string): boolean { + const levels = getModelApisModelCapabilities(modelId)?.thinkingEffortLevels ?? []; + return levels.includes("off") && levels.some((level) => level !== "off" && level !== "on"); +} + +/** + * `thinkingRequestFields` for Posit AI Pass's OpenAI-chat route, plus Baseten's + * spelling of "off". On a model whose Model APIs table entry offers "off" next to + * named effort levels, "off" goes out as `reasoning_effort: "none"`; omitting the + * field would leave Baseten's default effort in place. The table is the same + * source that decides whether the picker offers "off", so for built-in models the + * two cannot disagree (see the charter exception in `memory-bank/architecture.md`). + */ export function positAiThinkingRequestFields( modelId: string, effort: string | undefined, requiresChatTemplateKwargs: boolean, ): Record | undefined { - if ( - (modelId === "deepseek-ai/DeepSeek-V4.1-Flash-1017" || - modelId === "moonshotai/Kimi-K3" || - modelId === "moonshotai/Kimi-K3.1-Pro") && - effort === "off" - ) { + if (effort === "off" && offIsNamedEffortLevel(modelId)) { return { reasoning_effort: "none" }; } return thinkingRequestFields(effort, requiresChatTemplateKwargs);