From a5df088dde3d4a3b9941a302e0ffc6c0dc11f5ae Mon Sep 17 00:00:00 2001 From: thestreamcode Date: Wed, 14 Oct 2026 20:34:54 +0200 Subject: [PATCH 1/5] chore(deps-dev): fix high-severity brace-expansion audit failure Bumps transitive brace-expansion 5.0.9 -> 5.0.12 via npm audit fix (chain: eslint -> minimatch -> brace-expansion). Unblocks the npm audit gate failing on CI. --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9b3feee..2314349 100644 --- a/package-lock.json +++ b/package-lock.json @@ -340,9 +340,9 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.9", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", - "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", + "version": "5.0.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", + "integrity": "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==", "dev": true, "license": "MIT", "dependencies": { From 04be774c096ea7d40fb5fe4302583e3e742aec3e Mon Sep 17 00:00:00 2001 From: thestreamcode Date: Wed, 14 Oct 2026 21:00:47 +0200 Subject: [PATCH 2/5] feat: discovery-only catalog with coding-oriented frontier providers The built-in catalog ships endpoints only: model lists always come from live discovery, with the remembered list as the offline fallback. - Ranking no longer prefers any model family; vision is read from live metadata. - muse and nim providers added; stepfun, doubao, hunyuan, qianfan removed. - catalog:update is offline (no models.dev); catalog:audit checks usable models. - --no-discovery is now an offline mode; --list-providers shows key state. BREAKING CHANGE: configs referencing the removed provider ids need updating. --- AGENTS.md | 26 +- CHANGELOG.md | 28 ++ README.md | 77 +++-- docs/provider-verification.md | 50 ++-- docs/troubleshooting.md | 16 +- examples/providers.example.json | 74 +---- schemas/providers.schema.json | 2 +- scripts/audit-catalog.mjs | 62 ++-- scripts/catalog-sources.json | 205 ++----------- scripts/update-catalog.mjs | 73 +---- src/args.mjs | 2 +- src/cli.mjs | 28 +- src/config.mjs | 6 +- src/discovery.mjs | 60 +++- src/launcher.mjs | 29 +- src/model-cache.mjs | 10 +- src/model-ranking.mjs | 20 -- src/process-env.mjs | 4 + src/providers.default.json | 493 ++------------------------------ test/catalog.test.mjs | 14 +- test/config.test.mjs | 20 +- test/discovery.test.mjs | 46 ++- test/keys-command.test.mjs | 7 +- test/launcher.test.mjs | 22 +- test/model-cache.test.mjs | 6 +- test/model-ranking.test.mjs | 15 +- test/process-env.test.mjs | 5 + 27 files changed, 440 insertions(+), 960 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ce2c66c..490050c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,7 +48,7 @@ src/router.mjs The local proxy: model injection, provider routing, pa src/catalog.mjs Turns providers into the model entries Copilot expects from GET /models src/launcher.mjs Starts the router on an ephemeral port and spawns Copilot against it src/upstream.mjs Resolves the Copilot API tier (individual / business / enterprise) -src/discovery.mjs Asks each provider for its live model list; falls back to the catalog +src/discovery.mjs Asks each provider for its live model list; falls back to the remembered list src/config.mjs Provider config loading and validation; loads providers.default.json src/providers.default.json Generated catalog — never edit by hand, run npm run catalog:update src/keystore.mjs Optional on-disk key store (0600 / owner-only ACL) @@ -60,8 +60,8 @@ src/copilot-bin.mjs Resolves the Copilot executable, skipping the stale VS src/model-ranking.mjs Pure ranking of provider model catalogs (legacy mode) src/process-env.mjs Strips stale/secret variables from the child environment src/provider-env.mjs Builds the COPILOT_PROVIDER_* variables (legacy mode) -scripts/update-catalog.mjs Rebuilds the catalog from models.dev -scripts/catalog-sources.json Curated provider/model selection feeding the generator +scripts/update-catalog.mjs Rebuilds the endpoint catalog (offline, no network) +scripts/catalog-sources.json Provider endpoints feeding the generator extensions/byok/extension.mjs The /byok slash command, copied into ~/.copilot/extensions/ test/*.test.mjs One suite per src module schemas/providers.schema.json Published JSON Schema for providers.json @@ -80,7 +80,7 @@ All commands run from the repository root. | Test | `npm test` | | Test with coverage | `npm run test:coverage` | | Full quality gate | `npm run check` (lint + test) | -| Verify curated models still exist | `npm run catalog:audit` | +| Verify providers serve usable models | `npm run catalog:audit` | | Package contents check | `npm pack --dry-run` | | Dependency audit | `npm audit --omit=dev --audit-level=high` | | Local install for manual testing | `npm link` then `copilot-byok --help` | @@ -132,20 +132,18 @@ This project handles third-party provider API keys. Every change must preserve t The catalog is generated. **Never hand-edit `src/providers.default.json`** — it is overwritten. 1. Add the provider to `scripts/catalog-sources.json`, using only endpoints documented by the - provider, then run `npm run catalog:update`. List model ids under `models`; the generator fills - in context window and capabilities from models.dev and drops anything without tool-calling - support. Leave `models` empty when names are account-specific (Volcengine endpoint ids) or the - catalog is too large to curate (OpenRouter). -2. Run `npm run catalog:audit` after changing any curated list. A model that does - not exist on the provider is worse than a missing one: it is offered to the user - and fails only when picked. + provider, then run `npm run catalog:update`. List no models: the picker is filled + by live discovery from the provider's `GET /models`. +2. Run `npm run catalog:audit` after changing the catalog. A provider that answers + but yields no usable chat model fails the audit; providers without a configured + key are reported as unverifiable. 3. Verify the endpoint yourself before adding it: `POST {baseUrl}/chat/completions` with an invalid key must answer `400`/`401`/`403`, not `404`. A `404` usually means the path is wrong. -3. For authenticated providers, add every credential environment name to `DEFAULT_SECRET_SOURCE_ENV` +4. For authenticated providers, add every credential environment name to `DEFAULT_SECRET_SOURCE_ENV` in `src/process-env.mjs`. For intentionally authless providers set `authRequired: false` and do not invent a credential environment name. -4. Add the row to the provider table in `README.md`. -5. Update `docs/provider-verification.md` honestly: endpoint reachability, authenticated catalog +5. Add the row to the provider table in `README.md`. +6. Update `docs/provider-verification.md` honestly: endpoint reachability, authenticated catalog access, and end-to-end inference are three distinct evidence levels. Never claim a level that was not actually verified. diff --git a/CHANGELOG.md b/CHANGELOG.md index d4c8c6f..06a2376 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,34 @@ All notable changes to this project are documented in this file. ## [Unreleased] +### Added + +- Meta Muse (`muse`) and NVIDIA NIM (`nim`) providers, both verified with + OpenAI-compatible chat and model-list endpoints. +- Image-input support is now read from each provider's live catalog, the same + way reasoning support already was. +- `--no-discovery` is now an offline mode: providers use their pinned models or + the list each key returned last time, without any network request. + +### Changed + +- The built-in catalog ships endpoints only; model lists always come from live + discovery. There are no shipped model lists to go stale. +- Model ranking no longer prefers any model family; order is recency, version, + context window, then name. +- `npm run catalog:update` is offline and instant: no more models.dev fetch. +- `npm run catalog:audit` now checks that every provider serves at least one + usable chat model instead of checking pinned model ids. +- `--list-providers` marks providers with a configured key as usable and shows + pinned or remembered model counts. + +### Removed + +- **Breaking:** `stepfun`, `doubao`, `hunyuan` and `qianfan` providers were + removed: the catalog keeps only coding-oriented frontier providers. Configs + referencing them need updating; account-specific endpoint ids still work + through a custom provider entry. + ## [1.4.2] - 2026-08-12 ### Changed diff --git a/README.md b/README.md index 9dd0146..c9f4661 100644 --- a/README.md +++ b/README.md @@ -100,18 +100,18 @@ Every provider below is reached through its OpenAI-compatible endpoint, verified |---|---|---|---| | OpenAI | `OPENAI_API_KEY` | DeepSeek | `DEEPSEEK_API_KEY` | | Anthropic | `ANTHROPIC_API_KEY` | Alibaba Qwen | `DASHSCOPE_API_KEY` | -| Google Gemini | `GEMINI_API_KEY` | Z.AI GLM | `ZAI_API_KEY` | -| xAI Grok | `XAI_API_KEY` | Zhipu AI (CN) | `ZHIPU_CN_API_KEY` | -| Mistral | `MISTRAL_API_KEY` | Moonshot Kimi | `MOONSHOT_API_KEY` | -| Groq | `GROQ_API_KEY` | MiniMax | `MINIMAX_API_KEY` | -| Cerebras | `CEREBRAS_API_KEY` | StepFun | `STEPFUN_API_KEY` | -| Together AI | `TOGETHER_API_KEY` | ByteDance Doubao | `ARK_API_KEY` | -| DeepInfra | `DEEPINFRA_API_KEY` | Tencent Hunyuan | `HUNYUAN_API_KEY` | -| Fireworks AI | `FIREWORKS_API_KEY` | Baidu Qianfan | `QIANFAN_API_KEY` | -| OpenRouter | `OPENROUTER_API_KEY` | SiliconFlow | `SILICONFLOW_API_KEY` | -| Chutes | `CHUTES_API_KEY` | ModelScope | `MODELSCOPE_API_KEY` | -| OpenCode Zen | `OPENCODE_ZEN_API_KEY` | Alibaba Token Plan | `ALIBABA_TOKEN_PLAN_API_KEY` | -| OpenCode Go | `OPENCODE_GO_API_KEY` | Ollama / LM Studio | no key needed | +| Google Gemini | `GEMINI_API_KEY` | Alibaba Qwen (CN) | `DASHSCOPE_API_KEY_CN` | +| xAI Grok | `XAI_API_KEY` | Z.AI GLM | `ZAI_API_KEY` | +| Mistral | `MISTRAL_API_KEY` | Zhipu AI (CN) | `ZHIPU_CN_API_KEY` | +| Meta Muse | `MODEL_API_KEY` | Moonshot Kimi | `MOONSHOT_API_KEY` | +| NVIDIA NIM | `NVIDIA_API_KEY` | MiniMax | `MINIMAX_API_KEY` | +| Groq | `GROQ_API_KEY` | SiliconFlow | `SILICONFLOW_API_KEY` | +| Cerebras | `CEREBRAS_API_KEY` | ModelScope | `MODELSCOPE_API_KEY` | +| Together AI | `TOGETHER_API_KEY` | OpenRouter | `OPENROUTER_API_KEY` | +| DeepInfra | `DEEPINFRA_API_KEY` | Chutes | `CHUTES_API_KEY` | +| Fireworks AI | `FIREWORKS_API_KEY` | OpenCode Zen | `OPENCODE_ZEN_API_KEY` | +| Alibaba Token Plan | `ALIBABA_TOKEN_PLAN_API_KEY` | OpenCode Go | `OPENCODE_GO_API_KEY` | +| Ollama | no key needed | LM Studio | no key needed | Each provider also accepts a `COPILOT_BYOK__API_KEY` variant, useful when you already use the plain name for something else. @@ -126,17 +126,17 @@ The store lives at `~/.config/copilot-byok/keys.json` (`%APPDATA%\copilot-byok\k Either way the keys stay out of the Copilot process: the router uses them in its own, and strips them from the environment it hands to the agent. -Some entries ship without a preset model list, either because their model names are account-specific (Volcengine endpoint ids) or because the catalog is too large to curate (OpenRouter). Add the ones you want in your own config — see below. +No entry ships a model list: the picker is filled by live discovery. To pin exact models instead — for an offline setup or a fixed line-up — add them in your own config — see below. ### Where the model list comes from **Providers are asked directly.** At startup each configured provider is queried for the models it serves right now, so the picker shows today's line-up rather -than whatever was curated at release time — a model released this morning is -there this morning. All 30 providers were verified to expose an OpenAI-style -`GET /models`; the ones that also publish capabilities (Chutes, OpenRouter and -others) supply the real context window, output limit and tool-calling support -along with it. +than whatever was pinned at release time — a model released this morning is +there this morning. All 28 providers expose an OpenAI-style `GET /models` +(verified in `docs/provider-verification.md`); the ones that also publish +capabilities (Chutes, OpenRouter, NIM and others) supply the real context +window, output limit and tool-calling support along with it. While this happens the startup shows what it is waiting on, since the pause comes before Copilot has drawn anything: @@ -153,31 +153,28 @@ Only the providers you have a key for are queried, in parallel, with a hard timeout. Measured on three configured providers: about a second. Results are cached for ten minutes, failures for one. -Anything the provider does not report is filled in from the shipped catalog, -which is generated from [models.dev](https://models.dev) and refreshed with: - -```sh -npm run catalog:update -``` +Anything the provider does not report falls back to conservative defaults (128k +context, 32k output); image input and the reasoning selector appear only when +the provider's own metadata declares them. If a request fails — a rate limit, a blip, an expired key — the list that key returned last time is used instead, kept in `~/.cache/copilot-byok/models.json` -(`%LOCALAPPDATA%` on Windows). That matters: the shipped list is generic, while a -provider's catalog belongs to the plan behind your key, so falling back to it can -offer models your plan does not include and hide ones it does. Only when nothing -was ever remembered does the shipped list step in, so a hiccup never empties the -picker. To skip discovery entirely and always use the shipped lists: +(`%LOCALAPPDATA%` on Windows). That matters: a provider's catalog belongs to the +plan behind your key, so only what your key actually returned is ever offered — +never a generic guess. When nothing was ever remembered, the provider stays out +of the picker and the log says why. To skip discovery entirely and work from +remembered lists: ```sh copilot-byok --no-discovery ``` -Models that cannot call tools are excluded, whether that is known from the -provider's own metadata or from the catalog: Copilot's harness trusts what a -model entry declares, and an agent that cannot call tools breaks halfway through -a session instead of failing cleanly. Embedding, image, audio and moderation -models are filtered out for the same reason. Each provider contributes at most -twelve models, so a catalog like OpenRouter's 400+ cannot swamp the picker. +Models that cannot call tools are excluded based on the provider's own metadata: +Copilot's harness trusts what a model entry declares, and an agent that cannot +call tools breaks halfway through a session instead of failing cleanly. +Embedding, image, audio and moderation models are filtered out for the same +reason. Each provider contributes at most twelve models, so a catalog like +OpenRouter's 460+ cannot swamp the picker. ## The /byok command @@ -356,13 +353,13 @@ Everything has a working default; these are for when you need to change one. ### Keeping the catalog honest ```sh -npm run catalog:update # refresh models and limits from models.dev -npm run catalog:audit # check every curated model still exists on its provider +npm run catalog:update # regenerate the endpoint catalog from scripts/catalog-sources.json +npm run catalog:audit # check every provider serves a usable model list ``` -The audit exits non-zero when a curated entry is not served by its provider. -Curated lists are only a fallback, but a wrong entry is worse than a missing one: -it is offered in the picker and fails only when selected. +The audit queries each provider's live catalog and exits non-zero when one that +answers yields no usable chat model. Providers that need a key you have not +configured are reported as unverifiable rather than failing. ### Single-provider mode diff --git a/docs/provider-verification.md b/docs/provider-verification.md index f2f9b57..45f7f43 100644 --- a/docs/provider-verification.md +++ b/docs/provider-verification.md @@ -1,6 +1,6 @@ # Provider verification -Last verified: 2026-08-12 (model-list endpoints re-verified the same day) +Last verified: 2026-10-14 for endpoints and model lists; end-to-end runs below are from 2026-08-12. This document separates three different levels of evidence, because they are not interchangeable: @@ -19,6 +19,8 @@ interchangeable: - GitHub Copilot CLI 1.0.79, personal (free) plan - Endpoint probes: `POST /chat/completions` with a deliberately invalid key and a 25-second timeout. No credentials were printed. +- Endpoint and model-list tables were re-probed on 2026-10-14 (Windows, + Node.js 24.21, invalid-key probes, no credentials). ## Endpoint reachability @@ -45,15 +47,13 @@ All entries below answered on their OpenAI-compatible route. | Zhipu AI (CN) | `https://open.bigmodel.cn/api/paas/v4` | 401 | | Moonshot Kimi | `https://api.moonshot.ai/v1` | 401 | | MiniMax | `https://api.minimax.io/v1` | 401 | -| StepFun | `https://api.stepfun.com/v1` | 401 | +| Meta Muse | `https://api.meta.ai/v1` | 401 | | SiliconFlow | `https://api.siliconflow.com/v1` | 401 | | ModelScope | `https://api-inference.modelscope.cn/v1` | 400 | -| ByteDance Doubao | `https://ark.cn-beijing.volces.com/api/v3` | 401 | -| Tencent Hunyuan | `https://api.hunyuan.cloud.tencent.com/v1` | 401 | -| Baidu Qianfan | `https://qianfan.baidubce.com/v2` | 401 | | Baichuan | `https://api.baichuan-ai.com/v1` | 401 | | Chutes | `https://llm.chutes.ai/v1` | 401 | | OpenCode Zen | `https://opencode.ai/zen/v1` | 401 | +| OpenCode Go | `https://opencode.ai/zen/go/v1` | 400 | | Baseten | `https://inference.baseten.co/v1` | 403 | Two endpoints answered `404` to the probe because they validate the model name @@ -64,35 +64,47 @@ before the credential, and the probe deliberately sends a non-existent model: | Fireworks AI | `https://api.fireworks.ai/inference/v1` | 404 on an unknown model | | Nvidia NIM | `https://integrate.api.nvidia.com/v1` | 404 on an unknown model | +Perplexity, Baichuan and Baseten were probed but are not shipped: Perplexity is +search-focused rather than coding-oriented, and Baichuan and Baseten are +unverified for coding use. StepFun, Doubao, Hunyuan and Qianfan were removed +from the catalog for the same reason — the catalog keeps only coding-oriented +frontier providers. + ## Model-list endpoints Discovery reads each provider's `GET /models`, so that route was probed for every -catalog entry with an invalid key. **None returned 404** — every provider exposes -it. Four serve it without authentication, which is how the model counts below -were observed: +catalog entry. Sending no credentials returns the public catalogs below; +sending an invalid key is rejected instead, so a 401 proves the route exists +but says nothing about catalog contents. Observed 2026-10-14 via +`npm run catalog:audit` with no keys configured: | Provider | Result | |---|---| -| ModelScope | 200 — 43 models | -| OpenCode Go | 200 — 25 models | -| OpenCode Zen | 200 — 61 models | -| OpenRouter | 200 — 408 models, 340 declaring tool support | -| Gemini, xAI | 400 (route exists, request rejected) | -| Qianfan | 403 | +| Chutes | 200 — 14 models, all usable | +| DeepInfra | 200 — 179 models, 137 usable after ranking | +| ModelScope | 200 — 36 models, 34 usable | +| Nvidia NIM | 200 — 80 models, 69 usable | +| OpenCode Go | 200 — 40 models, all usable | +| OpenCode Zen | 200 — 85 models, all usable | +| OpenRouter | 200 — 462 models, 447 usable | +| Gemini | 404 without a key (400 with an invalid one) | +| xAI | 400 (route exists, request rejected) | | All others | 401 | | Ollama, LM Studio | connection refused — local servers, not running here | Some providers publish the metadata discovery needs alongside the list, which is -what lets the picker show real limits rather than guesses: +what lets the picker show real limits rather than defaults: | Provider | Fields observed | |---|---| | Chutes | `context_length`, `max_output_length`, `supported_features: ["json_mode","tools","structured_outputs","reasoning"]` | | OpenRouter | `context_length`, `top_provider.max_completion_tokens`, `supported_parameters` including `tools` | +| Nvidia NIM | ids only (`id`, `created`, `owned_by`) | +| ModelScope | ids only (`id`, `created`, `owned_by`) | -Providers that publish only ids still work: their limits come from the shipped -catalog, and a model is excluded only when a provider explicitly says it cannot -call tools. +Providers that publish only ids still work: their limits fall back to +conservative defaults (128k context, 32k output), and a model is excluded only +when a provider explicitly says it cannot call tools. ### Discovery against real keys @@ -100,7 +112,7 @@ call tools. |---|---| | Chutes | 12 models, with context and output read from the API | | Alibaba Token Plan | 7 models | -| OpenAI | 401 — the key on this machine is not valid for `api.openai.com`; the curated list was used, as designed | +| OpenAI | 401 — the key on this machine is not valid for `api.openai.com`; nothing remembered, so the provider stayed out of the picker, as designed | Measured wall-clock for the three configured providers, queried in parallel: 1.2s. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 61dceaf..bcf9538 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -85,9 +85,8 @@ The list comes from the provider itself. Three things can hide a model: chosen by the same ranking used elsewhere. Raise it with `maxDiscoveredModels` in your config, or pin the exact models you want under `models`. - **Discovery fell back.** If the provider was unreachable or rejected the key, - the list it returned last time is used; failing that, the shipped one. The log - says which and why. A shipped list is generic, so it may name models your plan - does not include — if you see one that should not be there, that is the sign. + the list it returned last time is used; failing that, the provider stays out + of the picker. The log says which and why. Results are cached for ten minutes, so a model added upstream may take that long to appear — restart to pick it up immediately. @@ -101,16 +100,15 @@ provider actually offers: copilot-byok --legacy --provider --list-models ``` -Then update `models` in your config, or run `npm run catalog:update` if you use -the built-in catalog. +Then pin the working names under `models` in your own config file. ## The agent starts but fails mid-task Almost always a capability mismatch: the model does not really support tool -calling, and the harness discovers it once it tries to use a tool. The generated -catalog only publishes models that report tool support, but a hand-written config -entry can claim anything. Verify the model supports function calling and -streaming before adding it. +calling, and the harness discovers it once it tries to use a tool. Discovery +only publishes models that do not rule out tool support, but a hand-written +config entry can claim anything. Verify the model supports function calling and +streaming before pinning it. ## Copilot CLI is not found diff --git a/examples/providers.example.json b/examples/providers.example.json index 8ec1c0e..7ca0e8c 100644 --- a/examples/providers.example.json +++ b/examples/providers.example.json @@ -1,6 +1,6 @@ { "$schema": "../schemas/providers.schema.json", - "_comment": "Custom configuration: copy to ~/.config/copilot-byok/providers.json (or %APPDATA%\\copilot-byok\\providers.json) to replace the built-in catalog. Use it for different models, a company endpoint, or a local runtime. Never write keys here: only the names of the environment variables that hold them.", + "_comment": "Custom configuration: copy to ~/.config/copilot-byok/providers.json (or %APPDATA%\\copilot-byok\\providers.json) to replace the built-in catalog. Providers need only an endpoint: model lists come from live discovery. Pin models only to fix the list or work fully offline. Never write keys here: only the names of the environment variables that hold them.", "providers": [ { "id": "openai", @@ -8,66 +8,6 @@ "baseUrl": "https://api.openai.com/v1", "apiKeyEnv": [ "OPENAI_API_KEY" - ], - "models": [ - { - "model": "gpt-5.5", - "label": "GPT-5.5 (my key)", - "contextWindow": 1050000, - "maxOutputTokens": 128000 - }, - { - "model": "gpt-5.4-mini", - "label": "GPT-5.4 mini", - "contextWindow": 1050000, - "maxOutputTokens": 128000 - } - ] - }, - { - "id": "anthropic", - "name": "Anthropic", - "baseUrl": "https://api.anthropic.com/v1", - "apiKeyEnv": [ - "ANTHROPIC_API_KEY" - ], - "models": [ - { - "model": "claude-sonnet-4-6", - "label": "Claude Sonnet 4.6", - "contextWindow": 1000000, - "maxOutputTokens": 128000 - } - ] - }, - { - "id": "deepseek", - "name": "DeepSeek", - "baseUrl": "https://api.deepseek.com/v1", - "apiKeyEnv": [ - "DEEPSEEK_API_KEY" - ], - "models": [ - { - "model": "deepseek-v4-pro", - "label": "DeepSeek V4 Pro", - "contextWindow": 1000000, - "maxOutputTokens": 384000 - } - ] - }, - { - "id": "ollama", - "name": "Local Ollama", - "baseUrl": "http://127.0.0.1:11434/v1", - "authRequired": false, - "models": [ - { - "model": "qwen3-coder:30b", - "label": "Qwen3 Coder 30B (local)", - "contextWindow": 262144, - "maxOutputTokens": 32768 - } ] }, { @@ -78,16 +18,22 @@ "COMPANY_LLM_KEY" ], "headers": { - "X-Tenant-Id": "platform-team" + "X-Tenant": "my-team" }, "models": [ { - "model": "internal-model", - "label": "Internal model", + "model": "company-coder-large", + "label": "Company Coder Large", "contextWindow": 200000, "maxOutputTokens": 32000 } ] + }, + { + "id": "ollama", + "name": "Local Ollama", + "baseUrl": "http://127.0.0.1:11434/v1", + "authRequired": false } ] } diff --git a/schemas/providers.schema.json b/schemas/providers.schema.json index 0d9e5e6..cbbfb4d 100644 --- a/schemas/providers.schema.json +++ b/schemas/providers.schema.json @@ -157,7 +157,7 @@ }, "models": { "type": "array", - "description": "Models published to the /model picker. Without this field the provider is only usable with --legacy.", + "description": "Optional pinned models published to the /model picker. When absent or empty, models come from live discovery.", "items": { "oneOf": [ { diff --git a/scripts/audit-catalog.mjs b/scripts/audit-catalog.mjs index b0a4ffb..30ca12e 100644 --- a/scripts/audit-catalog.mjs +++ b/scripts/audit-catalog.mjs @@ -1,17 +1,19 @@ #!/usr/bin/env node -// Checks that every curated model still exists on its provider. +// Checks that every catalog provider serves a usable model list. // // npm run catalog:audit // -// The curated lists are only a fallback, but a wrong entry is worse than a missing -// one: it offers a model the user's plan does not include, and the failure only -// shows up when they pick it. Providers with a public catalog are checked without -// credentials; the rest use whatever key is configured, and are reported as -// unverifiable when there is none. +// Discovery shows whatever the provider returns, so a provider whose catalog is +// reachable but yields no usable chat model is a problem worth failing on. +// Providers with a public catalog are checked without credentials; the rest use +// whatever key is configured, and are reported as unverifiable when there is +// none. Unreachable providers are reported but do not fail the audit: the cause +// is usually the network, not the catalog. import { readFile } from 'node:fs/promises'; import { loadKeys } from '../src/keystore.mjs'; +import { rankModels } from '../src/model-ranking.mjs'; const CATALOG_URL = new URL('../src/providers.default.json', import.meta.url); const TIMEOUT_MS = 20_000; @@ -21,26 +23,23 @@ const stored = await loadKeys(process.env); const rows = await Promise.all(catalog.providers.map((provider) => audit(provider, stored))); -let wrong = 0; +let bad = 0; for (const row of rows.sort((a, b) => a.id.localeCompare(b.id))) { - const flag = row.missing?.length ? '!!' : ' '; - process.stdout.write(`${flag} ${row.id.padEnd(20)} ${String(row.curated).padStart(2)} curated ${row.verdict}\n`); - if (row.missing?.length) { - wrong += 1; - process.stdout.write(` not served by the provider: ${row.missing.join(', ')}\n`); + const flag = row.problem ? '!!' : ' '; + process.stdout.write(`${flag} ${row.id.padEnd(20)} ${row.verdict}\n`); + if (row.problem) { + bad += 1; + process.stdout.write(` ${row.problem}\n`); } } -process.stdout.write(wrong === 0 - ? '\nEvery curated model that could be checked exists on its provider.\n' - : `\n${wrong} provider(s) list models that do not exist. Remove them from scripts/catalog-sources.json and re-run npm run catalog:update.\n`); +process.stdout.write(bad === 0 + ? '\nEvery reachable provider serves at least one usable chat model.\n' + : `\n${bad} provider(s) need attention (see above).\n`); -process.exitCode = wrong === 0 ? 0 : 1; +process.exitCode = bad === 0 ? 0 : 1; async function audit(provider, keys) { - const curated = (provider.models || []).map((model) => model.model); - if (curated.length === 0) return { id: provider.id, curated: 0, verdict: 'no curated models' }; - const key = (provider.apiKeyEnv || []).map((name) => process.env[name]).find(Boolean) || keys[provider.id]; try { @@ -52,24 +51,25 @@ async function audit(provider, keys) { if (!response.ok) { return { id: provider.id, - curated: curated.length, - verdict: `unverifiable (${response.status}${key ? '' : ', no key configured'})`, + verdict: key + ? `unverifiable (${response.status} with a configured key)` + : `unverifiable (${response.status}, no key configured)`, }; } const body = await response.json(); const items = body?.data || body?.models || (Array.isArray(body) ? body : []); - const live = new Set(items.map((item) => (typeof item === 'string' ? item : item?.id))); - const missing = curated.filter((model) => !live.has(model)); - - return { - id: provider.id, - curated: curated.length, - missing, - verdict: missing.length === 0 ? `all present (${live.size} served)` : `${missing.length} do not exist`, - }; + const usable = rankModels({ payload: body }); + if (usable.length === 0) { + return { + id: provider.id, + verdict: `${items.length} served, 0 usable`, + problem: 'reachable but yields no usable chat model: check the endpoint or the ranking filters', + }; + } + return { id: provider.id, verdict: `${items.length} served, ${usable.length} usable` }; } catch (error) { const reason = error.name === 'TimeoutError' ? 'timeout' : (error.cause?.code || error.message); - return { id: provider.id, curated: curated.length, verdict: `unreachable (${reason})` }; + return { id: provider.id, verdict: `unreachable (${reason})` }; } } diff --git a/scripts/catalog-sources.json b/scripts/catalog-sources.json index badddf8..4fcf749 100644 --- a/scripts/catalog-sources.json +++ b/scripts/catalog-sources.json @@ -1,404 +1,259 @@ { - "_comment": "Curated selection used by scripts/update-catalog.mjs to generate src/providers.default.json. Model metadata (context, output, capabilities) comes from models.dev, so it stays aligned with reality instead of being hand-written.", + "_comment": "Provider endpoints used by scripts/update-catalog.mjs to generate src/providers.default.json. Endpoints only: model lists always come from live discovery, so there is nothing here that can go stale. Use only base URLs documented by the provider.", "providers": [ { "id": "openai", "name": "OpenAI", - "source": "openai", "baseUrl": "https://api.openai.com/v1", "apiKeyEnv": [ "OPENAI_API_KEY", "COPILOT_BYOK_OPENAI_API_KEY" - ], - "models": [ - "gpt-5.6-sol", - "gpt-5.5", - "gpt-5.4", - "gpt-5.4-mini" ] }, { "id": "anthropic", "name": "Anthropic", - "source": "anthropic", "baseUrl": "https://api.anthropic.com/v1", "apiKeyEnv": [ "ANTHROPIC_API_KEY", "COPILOT_BYOK_ANTHROPIC_API_KEY" - ], - "models": [ - "claude-sonnet-4-6", - "claude-opus-4-6", - "claude-sonnet-4-5", - "claude-haiku-4-5" ] }, { "id": "gemini", "name": "Google Gemini", - "source": "google", "baseUrl": "https://generativelanguage.googleapis.com/v1beta/openai", "apiKeyEnv": [ "GEMINI_API_KEY", "GOOGLE_API_KEY", "COPILOT_BYOK_GEMINI_API_KEY" - ], - "models": [ - "gemini-3.5-flash", - "gemini-2.5-flash", - "gemini-3.5-flash-lite" ] }, { "id": "xai", "name": "xAI Grok", - "source": "xai", "baseUrl": "https://api.x.ai/v1", "apiKeyEnv": [ "XAI_API_KEY", "COPILOT_BYOK_XAI_API_KEY" - ], - "models": [ - "grok-4.5", - "grok-4.3" ] }, { "id": "mistral", "name": "Mistral", - "source": "mistral", "baseUrl": "https://api.mistral.ai/v1", "apiKeyEnv": [ "MISTRAL_API_KEY", "COPILOT_BYOK_MISTRAL_API_KEY" - ], - "models": [ - "mistral-medium-latest", - "devstral-medium-latest" ] }, { "id": "deepseek", "name": "DeepSeek", - "source": "deepseek", "baseUrl": "https://api.deepseek.com/v1", "apiKeyEnv": [ "DEEPSEEK_API_KEY", "COPILOT_BYOK_DEEPSEEK_API_KEY" - ], - "models": [ - "deepseek-v4-pro", - "deepseek-v4-flash", - "deepseek-reasoner", - "deepseek-chat" ] }, { "id": "qwen", "name": "Alibaba Qwen", - "source": "alibaba", "baseUrl": "https://dashscope-intl.aliyuncs.com/compatible-mode/v1", "apiKeyEnv": [ "DASHSCOPE_API_KEY", "QWEN_API_KEY", "COPILOT_BYOK_QWEN_API_KEY" - ], - "models": [ - "qwen3.7-max", - "qwen3.7-plus", - "qwen3-coder-plus" ] }, { "id": "qwen-cn", "name": "Alibaba Qwen (China)", - "source": "alibaba-cn", "baseUrl": "https://dashscope.aliyuncs.com/compatible-mode/v1", "apiKeyEnv": [ "DASHSCOPE_API_KEY_CN", "COPILOT_BYOK_QWEN_CN_API_KEY" - ], - "models": [ - "qwen3.7-max", - "qwen3-coder-plus" ] }, { "id": "zai", "name": "Z.AI GLM", - "source": "zai", "baseUrl": "https://api.z.ai/api/paas/v4", "apiKeyEnv": [ "ZAI_API_KEY", "ZHIPU_API_KEY", "COPILOT_BYOK_ZAI_API_KEY" - ], - "models": [ - "glm-5.2", - "glm-5.1", - "glm-4.7" ] }, { "id": "zhipu-cn", "name": "Zhipu AI (China)", - "source": "zhipuai", "baseUrl": "https://open.bigmodel.cn/api/paas/v4", "apiKeyEnv": [ "ZHIPU_CN_API_KEY", "COPILOT_BYOK_ZHIPU_CN_API_KEY" - ], - "models": [ - "glm-5.2", - "glm-4.7" ] }, { "id": "kimi", "name": "Moonshot Kimi", - "source": "moonshotai", "baseUrl": "https://api.moonshot.ai/v1", "apiKeyEnv": [ "MOONSHOT_API_KEY", "KIMI_API_KEY", "COPILOT_BYOK_KIMI_API_KEY" - ], - "models": [ - "kimi-k3", - "kimi-k2.6", - "kimi-k2.5" ] }, { "id": "minimax", "name": "MiniMax", - "source": "minimax", "baseUrl": "https://api.minimax.io/v1", "apiKeyEnv": [ "MINIMAX_API_KEY", "COPILOT_BYOK_MINIMAX_API_KEY" - ], - "models": [ - "MiniMax-M3", - "MiniMax-M2.7" ] }, - { - "id": "stepfun", - "name": "StepFun", - "source": "stepfun", - "baseUrl": "https://api.stepfun.com/v1", - "apiKeyEnv": [ - "STEPFUN_API_KEY", - "COPILOT_BYOK_STEPFUN_API_KEY" - ], - "models": [] - }, - { - "id": "doubao", - "name": "ByteDance Doubao", - "source": null, - "baseUrl": "https://ark.cn-beijing.volces.com/api/v3", - "apiKeyEnv": [ - "ARK_API_KEY", - "VOLCENGINE_API_KEY", - "COPILOT_BYOK_DOUBAO_API_KEY" - ], - "models": [], - "note": "Volcengine model names are account-specific endpoint ids: add yours under `models`." - }, - { - "id": "hunyuan", - "name": "Tencent Hunyuan", - "source": null, - "baseUrl": "https://api.hunyuan.cloud.tencent.com/v1", - "apiKeyEnv": [ - "HUNYUAN_API_KEY", - "COPILOT_BYOK_HUNYUAN_API_KEY" - ], - "models": [] - }, - { - "id": "qianfan", - "name": "Baidu Qianfan", - "source": null, - "baseUrl": "https://qianfan.baidubce.com/v2", - "apiKeyEnv": [ - "QIANFAN_API_KEY", - "BAIDU_API_KEY", - "COPILOT_BYOK_QIANFAN_API_KEY" - ], - "models": [] - }, { "id": "siliconflow", "name": "SiliconFlow", - "source": "siliconflow", "baseUrl": "https://api.siliconflow.com/v1", "apiKeyEnv": [ "SILICONFLOW_API_KEY", "COPILOT_BYOK_SILICONFLOW_API_KEY" - ], - "models": [] + ] }, { "id": "modelscope", "name": "ModelScope", - "source": "modelscope", "baseUrl": "https://api-inference.modelscope.cn/v1", "apiKeyEnv": [ "MODELSCOPE_API_KEY", "COPILOT_BYOK_MODELSCOPE_API_KEY" - ], - "models": [] + ] }, { "id": "openrouter", "name": "OpenRouter", - "source": "openrouter", "baseUrl": "https://openrouter.ai/api/v1", "apiKeyEnv": [ "OPENROUTER_API_KEY", "COPILOT_BYOK_OPENROUTER_API_KEY" - ], - "models": [] + ] }, { "id": "groq", "name": "Groq", - "source": "groq", "baseUrl": "https://api.groq.com/openai/v1", "apiKeyEnv": [ "GROQ_API_KEY", "COPILOT_BYOK_GROQ_API_KEY" - ], - "models": [ - "openai/gpt-oss-20b", - "llama-3.3-70b-versatile" ] }, { "id": "cerebras", "name": "Cerebras", - "source": "cerebras", "baseUrl": "https://api.cerebras.ai/v1", "apiKeyEnv": [ "CEREBRAS_API_KEY", "COPILOT_BYOK_CEREBRAS_API_KEY" - ], - "models": [] + ] }, { "id": "together", "name": "Together AI", - "source": "togetherai", "baseUrl": "https://api.together.xyz/v1", "apiKeyEnv": [ "TOGETHER_API_KEY", "COPILOT_BYOK_TOGETHER_API_KEY" - ], - "models": [] + ] }, { "id": "deepinfra", "name": "DeepInfra", - "source": "deepinfra", "baseUrl": "https://api.deepinfra.com/v1/openai", "apiKeyEnv": [ "DEEPINFRA_API_KEY", "COPILOT_BYOK_DEEPINFRA_API_KEY" - ], - "models": [] + ] }, { "id": "fireworks", "name": "Fireworks AI", - "source": "fireworks-ai", "baseUrl": "https://api.fireworks.ai/inference/v1", "apiKeyEnv": [ "FIREWORKS_API_KEY", "COPILOT_BYOK_FIREWORKS_API_KEY" - ], - "models": [] + ] }, { "id": "chutes", "name": "Chutes", - "source": "chutes", "baseUrl": "https://llm.chutes.ai/v1", "apiKeyEnv": [ "CHUTES_API_KEY", "COPILOT_BYOK_CHUTES_API_KEY" - ], - "models": [ - "zai-org/GLM-5.2-TEE", - "moonshotai/Kimi-K2.6-TEE", - "deepseek-ai/DeepSeek-V4-Flash-0731-TEE" ] }, { "id": "opencode-zen", "name": "OpenCode Zen", - "source": "opencode", "baseUrl": "https://opencode.ai/zen/v1", "apiKeyEnv": [ "OPENCODE_ZEN_API_KEY", "COPILOT_BYOK_OPENCODE_ZEN_API_KEY" - ], - "models": [ - "claude-sonnet-4-6", - "gpt-5.5-pro", - "gemini-3.5-flash" ] }, { "id": "opencode-go", "name": "OpenCode Go", - "source": "opencode-go", "baseUrl": "https://opencode.ai/zen/go/v1", "apiKeyEnv": [ "OPENCODE_GO_API_KEY", "COPILOT_BYOK_OPENCODE_GO_API_KEY" - ], - "models": [ - "glm-5.2", - "kimi-k2.6", - "minimax-m3", - "qwen3.7-max" ] }, { "id": "alibaba-token-plan", "name": "Alibaba Token Plan", - "source": "alibaba-token-plan", "baseUrl": "https://token-plan.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1", "apiKeyEnv": [ "ALIBABA_TOKEN_PLAN_API_KEY", "COPILOT_BYOK_ALIBABA_TOKEN_PLAN_API_KEY" - ], - "models": [ - "qwen3.8-max", - "deepseek-v4-pro", - "glm-5.2" + ] + }, + { + "id": "muse", + "name": "Meta Muse", + "baseUrl": "https://api.meta.ai/v1", + "apiKeyEnv": [ + "MODEL_API_KEY", + "META_API_KEY", + "COPILOT_BYOK_MUSE_API_KEY" + ] + }, + { + "id": "nim", + "name": "NVIDIA NIM", + "baseUrl": "https://integrate.api.nvidia.com/v1", + "apiKeyEnv": [ + "NVIDIA_API_KEY", + "NGC_API_KEY", + "COPILOT_BYOK_NIM_API_KEY" ] }, { "id": "ollama", "name": "Ollama (locale)", - "source": null, "baseUrl": "http://127.0.0.1:11434/v1", "authRequired": false, - "enabled": false, - "models": [] + "enabled": false }, { "id": "lmstudio", "name": "LM Studio (locale)", - "source": null, "baseUrl": "http://127.0.0.1:1234/v1", "authRequired": false, - "enabled": false, - "models": [] + "enabled": false } ] } diff --git a/scripts/update-catalog.mjs b/scripts/update-catalog.mjs index 49da1e8..7423389 100644 --- a/scripts/update-catalog.mjs +++ b/scripts/update-catalog.mjs @@ -1,62 +1,44 @@ #!/usr/bin/env node -// Rebuilds src/providers.default.json by combining the curated selection in -// scripts/catalog-sources.json with the model metadata published on models.dev. +// Rebuilds src/providers.default.json from scripts/catalog-sources.json. // // node scripts/update-catalog.mjs // -// Context limits and capabilities come from the source rather than hand-written -// values: declaring capabilities a model lacks makes the Copilot harness fail. +// The shipped catalog carries provider endpoints only: model lists always come +// from live discovery, so there is nothing here that can go stale. The only +// derivation is modelsUrl, the conventional OpenAI list endpoint under baseUrl. import { readFile, writeFile } from 'node:fs/promises'; const SOURCES_URL = new URL('./catalog-sources.json', import.meta.url); const OUTPUT_URL = new URL('../src/providers.default.json', import.meta.url); -const MODELS_DEV = 'https://models.dev/api.json'; const sources = JSON.parse(await readFile(SOURCES_URL, 'utf8')); -process.stdout.write(`Fetching ${MODELS_DEV}... -`); -const response = await fetch(MODELS_DEV, { signal: AbortSignal.timeout(120_000) }); -if (!response.ok) throw new Error(`models.dev answered ${response.status}`); -const catalog = await response.json(); - -const warnings = []; const providers = []; - for (const source of sources.providers) { - const upstream = source.source ? catalog[source.source] : null; - if (source.source && !upstream) { - warnings.push(`provider "${source.source}" is not present on models.dev`); - } - - const models = []; - for (const modelId of source.models || []) { - const meta = upstream?.models?.[modelId]; - if (!meta) { - warnings.push(`${source.id}: model "${modelId}" not found on models.dev, using conservative values`); - } - if (meta && meta.tool_call === false) { - warnings.push(`${source.id}: "${modelId}" does not support tool calling, skipped`); - continue; - } - models.push(buildModel(modelId, meta, source)); + if (!source.id || !source.name || !source.baseUrl) { + throw new Error(`Provider entry needs id, name and baseUrl: ${JSON.stringify(source)}`); } providers.push({ id: source.id, name: source.name, baseUrl: source.baseUrl, - // Standard OpenAI endpoint: used by `--list-models` and by the legacy mode. - modelsUrl: `${source.baseUrl.replace(/\/$/, '')}/models`, + // Standard OpenAI endpoint: used by discovery, --list-models and legacy mode. + modelsUrl: source.modelsUrl || `${source.baseUrl.replace(/\/$/, '')}/models`, ...(source.apiKeyEnv ? { apiKeyEnv: source.apiKeyEnv } : {}), ...(source.authRequired === false ? { authRequired: false } : {}), ...(source.enabled === false ? { enabled: false } : {}), ...(source.note ? { note: source.note } : {}), - models, }); } +const ids = providers.map((provider) => provider.id); +const duplicates = [...new Set(ids.filter((id, index) => ids.indexOf(id) !== index))]; +if (duplicates.length > 0) { + throw new Error(`Duplicate provider ids: ${duplicates.join(', ')}`); +} + const output = { $schema: '../schemas/providers.schema.json', _generated: 'npm run catalog:update — do not edit by hand', @@ -64,29 +46,4 @@ const output = { }; await writeFile(OUTPUT_URL, `${JSON.stringify(output, null, 2)}\n`, 'utf8'); - -const modelCount = providers.reduce((total, p) => total + p.models.length, 0); -process.stdout.write(` -Wrote ${providers.length} providers and ${modelCount} models. -`); -if (warnings.length) { - process.stdout.write(` -Warnings (${warnings.length}): -`); - for (const warning of warnings) process.stdout.write(` - ${warning}\n`); -} - -function buildModel(modelId, meta, source) { - const limit = meta?.limit || {}; - const context = limit.context || 128_000; - const output = limit.output || 32_000; - - return { - model: modelId, - label: `${meta?.name || modelId} (${source.name})`, - contextWindow: context, - maxOutputTokens: output, - ...(meta?.reasoning ? { reasoning: true } : {}), - ...(meta?.attachment ? { vision: true } : {}), - }; -} +process.stdout.write(`Wrote ${providers.length} providers (endpoints only, no shipped models).\n`); diff --git a/src/args.mjs b/src/args.mjs index 1c09153..054d786 100644 --- a/src/args.mjs +++ b/src/args.mjs @@ -47,7 +47,7 @@ export function parseArgs(argv) { continue; } - // Skip asking providers for their catalogs and use the shipped lists instead. + // Skip asking providers for their catalogs; use pinned or remembered lists instead. if (arg === '--no-discovery') { result.noDiscovery = true; continue; diff --git a/src/cli.mjs b/src/cli.mjs index a10230d..7f57680 100644 --- a/src/cli.mjs +++ b/src/cli.mjs @@ -9,7 +9,8 @@ import { findProvider, loadConfig } from './config.mjs'; import { rankModels } from './model-ranking.mjs'; import { buildProviderEnvironment } from './provider-env.mjs'; import { sanitizeCopilotEnvironment } from './process-env.mjs'; -import { isQueryable, resolveLiveModels, runCopilotWithRouter, selectActiveProviders, startRouter } from './launcher.mjs'; +import { isQueryable, resolveLiveModels, resolveOfflineModels, runCopilotWithRouter, selectActiveProviders, startRouter } from './launcher.mjs'; +import { readCache, recallModels } from './model-cache.mjs'; import { runKeysCommand } from './keys-command.mjs'; import { loadKeys } from './keystore.mjs'; import { runExtensionCommand } from './extension-install.mjs'; @@ -62,7 +63,7 @@ export async function main(argv = process.argv.slice(2), io = defaultIo()) { if (args.listProviders) { const config = await loadConfig({ configPath: args.configPath, env: io.env, secrets }); - return listProviders({ config, io }); + return await listProviders({ config, io }); } if (!args.legacy) { @@ -77,11 +78,11 @@ export async function main(argv = process.argv.slice(2), io = defaultIo()) { async function runRouterMode({ args, config, io }) { const sessionLog = createSessionLog(io); - // Discovery has to run before deciding whether anything is usable: a provider - // that ships no curated models — OpenRouter, Cerebras and the rest — only gains - // them here, and checking first would discard it while it holds a valid key. + // Discovery has to run before deciding whether anything is usable: providers + // ship no model lists, so a keyed provider only gains its models here, and + // checking first would discard it while it holds a valid key. const providers = args.noDiscovery - ? config.providers + ? await resolveOfflineModels(config.providers, { env: io.env }) : await discoverWithProgress({ config, io, sessionLog }); if (selectActiveProviders(providers).length === 0) { @@ -104,7 +105,7 @@ async function runRouterMode({ args, config, io }) { // A key added mid-session belongs to a provider that was not queried at // startup, so its models are discovered on the next rebuild. return args.noDiscovery - ? fresh.providers + ? resolveOfflineModels(fresh.providers, { env: io.env }) : resolveLiveModels(fresh.providers, { onEvent: sessionLog.onEvent }); }, }); @@ -161,16 +162,17 @@ async function discoverWithProgress({ config, io, sessionLog }) { } } -function listProviders({ config, io }) { - const active = new Set(selectActiveProviders(config.providers).map((provider) => provider.id)); +async function listProviders({ config, io }) { + const diskCache = await readCache(io.env).catch(() => ({})); for (const provider of config.providers) { - const usable = active.has(provider.id); + const usable = isQueryable(provider); + const known = provider.models?.length || recallModels(diskCache, provider.id, Date.now)?.length || 0; const keys = (provider.apiKeyEnv || []).join(' | ') || (provider.authRequired === false ? 'no key needed' : '-'); - io.stdout.write(`${usable ? '*' : ' '} ${provider.id.padEnd(20)} ${String(provider.models?.length || 0).padStart(2)} models ${keys}\n`); + io.stdout.write(`${usable ? '*' : ' '} ${provider.id.padEnd(20)} ${String(known).padStart(2)} models ${keys}\n`); } - io.stdout.write('\n* = usable right now. The others are waiting for the environment variable shown.\n'); + io.stdout.write('\n* = key configured; live model lists load at startup. Counts show pinned or remembered lists.\n'); return 0; } @@ -493,7 +495,7 @@ Options: -m, --model Starting model (picker id, e.g. byok-openai-gpt-5-5) -c, --config Provider configuration file --upstream Force the Copilot API tier (individual/business/enterprise) - --no-discovery Use the shipped model lists instead of asking providers + --no-discovery Skip live discovery; use pinned or remembered lists (offline mode) --dry-run Print what would start, then exit -h, --help Show this help -v, --version Print the copilot-byok version diff --git a/src/config.mjs b/src/config.mjs index d53af82..f629d2b 100644 --- a/src/config.mjs +++ b/src/config.mjs @@ -11,8 +11,8 @@ const TRANSPORTS = new Set(['http', 'websockets']); const MODELS_AUTH_MODES = new Set([true, false, 'none', 'bearer', 'x-api-key', 'api-key']); const SECRET_QUERY_NAME_PATTERN = /(^|[-_.])(api[-_.]?key|subscription[-_.]?key|auth(?:orization)?|bearer|credential|password|secret|signature|sig|token)([-_.]|$)/i; -// Default catalog: generated from models.dev with `npm run catalog:update`, so -// context windows and capabilities stay aligned with what providers actually offer. +// Default catalog: generated with `npm run catalog:update`. It ships provider +// endpoints only; model lists always come from live discovery. const DEFAULT_PROVIDERS = JSON.parse( readFileSync(new URL('./providers.default.json', import.meta.url), 'utf8') ).providers; @@ -136,7 +136,7 @@ function normalizeProvider(provider, env, index, secrets = {}) { }; } -/** Models published to the Copilot picker. Absent => the provider is legacy-only. */ +/** Pinned models published to the Copilot picker. Absent or empty: models come from live discovery. */ function normalizeModels(value, providerName) { if (value == null) return []; if (!Array.isArray(value)) { diff --git a/src/discovery.mjs b/src/discovery.mjs index f8158b3..c597d3f 100644 --- a/src/discovery.mjs +++ b/src/discovery.mjs @@ -3,16 +3,17 @@ // // Every provider in the catalog exposes an OpenAI-style GET /models (verified at // release time), and most return the metadata that matters with it: context -// window, output limit, and whether the model can call tools. What is missing is -// filled in from the shipped catalog, and a provider that cannot be reached falls -// back to it entirely — a network hiccup must never empty the picker. +// window, output limit, and whether the model can call tools. What is missing +// falls back to conservative defaults, and a provider that cannot be reached +// falls back to the list its key returned last time — or, when nothing was ever +// remembered, it stays out of the picker with the reason logged. import { rankModels } from './model-ranking.mjs'; import { recallModels, rememberModels } from './model-cache.mjs'; // Providers are queried in parallel, so this is the worst-case startup delay, not // a per-provider cost. Measured: the slower catalogs answer in about five seconds, -// and anything past this keeps its shipped list rather than holding up the session. +// and anything past this falls back to its remembered list rather than holding up the session. const DEFAULT_TIMEOUT_MS = 9000; const DEFAULT_LIMIT = 12; const MAX_CATALOG_BYTES = 5 * 1024 * 1024; @@ -42,14 +43,14 @@ export async function discoverModels(provider, { diskCache = null, env = process.env, } = {}) { - // The shipped list is generic; a provider's catalog belongs to the plan behind - // the key. Preferring what this key last returned avoids offering models the - // user does not have when a request fails. + // A provider's catalog belongs to the plan behind the key, so what this key + // last returned beats any pinned list: preferring it avoids offering models + // the user does not have when a request fails. const remembered = diskCache ? recallModels(diskCache, provider.id, now) : null; - const curated = remembered || provider.models || []; + const configured = remembered || provider.models || []; if (provider.discover === false || !provider.modelsUrl) { - return { models: curated, source: 'catalog', reason: 'discovery disabled' }; + return { models: configured, source: 'catalog', reason: 'discovery disabled' }; } // The catalog is rebuilt whenever Copilot asks for the model list, which would @@ -82,12 +83,12 @@ export async function discoverModels(provider, { // Silence is not a denial: plenty of providers publish no capability field. if (tools === false) continue; - models.push(mergeWithCurated(id, entry, curated, provider)); + models.push(mergeWithConfigured(id, entry, configured, provider)); if (models.length >= limit) break; } const result = models.length === 0 - ? { models: curated, source: remembered ? 'remembered' : 'catalog', reason: 'no usable models returned' } + ? { models: configured, source: remembered ? 'remembered' : 'catalog', reason: 'no usable models returned' } : { models, source: 'provider' }; cache?.set(cacheKey, { at: now(), result }); @@ -97,7 +98,7 @@ export async function discoverModels(provider, { return result; } catch (error) { const reason = error.name === 'TimeoutError' ? `timed out after ${timeoutMs}ms` : error.message; - const result = { models: curated, source: remembered ? 'remembered' : 'catalog', reason }; + const result = { models: configured, source: remembered ? 'remembered' : 'catalog', reason }; // Cache failures too: a provider that is down or rejecting the key would // otherwise be retried on every single model-list request. @@ -184,6 +185,36 @@ export function declaresReasoning(entry) { return Boolean(supports?.reasoning || supports?.reasoning_effort); } +/** + * Whether the model accepts images. Copilot only offers image input for models + * that say so; unlike reasoning this was previously known from pinned lists + * alone, so sparse catalogs simply omit the flag. + */ +export function declaresVision(entry) { + if (!entry || typeof entry !== 'object') return false; + + for (const field of ['modalities', 'input_modalities', 'inputModalities']) { + const value = entry[field]; + if (Array.isArray(value) && value.some((item) => /^(image|vision)$/i.test(String(item)))) { + return true; + } + } + + const architecture = entry.architecture; + if (architecture && typeof architecture === 'object') { + for (const field of ['modalities', 'input_modalities', 'inputModalities']) { + const value = architecture[field]; + if (Array.isArray(value) && value.some((item) => /^(image|vision)$/i.test(String(item)))) { + return true; + } + } + } + + if (entry.attachment === true) return true; + const supports = entry.capabilities?.supports || entry.capabilities; + return Boolean(supports?.vision); +} + export function readContextWindow(entry) { return firstPositive( entry?.context_length, @@ -205,8 +236,8 @@ export function readOutputLimit(entry) { ); } -function mergeWithCurated(id, entry, curated, provider) { - const known = curated.find((model) => model.model === id); +function mergeWithConfigured(id, entry, configured, provider) { + const known = configured.find((model) => model.model === id); const context = readContextWindow(entry) || known?.contextWindow; const output = readOutputLimit(entry) || known?.maxOutputTokens; @@ -216,6 +247,7 @@ function mergeWithCurated(id, entry, curated, provider) { ...(context ? { contextWindow: context } : {}), ...(output ? { maxOutputTokens: output } : {}), ...(declaresReasoning(entry) || known?.reasoning ? { reasoning: true } : {}), + ...(declaresVision(entry) || known?.vision ? { vision: true } : {}), }; } diff --git a/src/launcher.mjs b/src/launcher.mjs index 3fed763..a36c118 100644 --- a/src/launcher.mjs +++ b/src/launcher.mjs @@ -5,7 +5,7 @@ import spawn from 'cross-spawn'; import { buildCatalog } from './catalog.mjs'; import { discoverModels } from './discovery.mjs'; -import { readCache } from './model-cache.mjs'; +import { readCache, recallModels } from './model-cache.mjs'; import { createRouter } from './router.mjs'; import { createUpstreamResolver } from './upstream.mjs'; import { resolveCopilotBin } from './copilot-bin.mjs'; @@ -22,9 +22,9 @@ export function selectActiveProviders(providers) { } /** - * Replaces each provider's shipped model list with what it actually serves right - * now. Providers are queried in parallel; one that fails keeps its curated list, - * so a slow or unreachable provider never empties the picker. + * Replaces each provider's pinned model list with what it actually serves right + * now. Providers are queried in parallel; one that fails falls back to what its + * key returned last time, or stays out of the picker when nothing was remembered. */ export async function resolveLiveModels(providers, { onEvent = () => {}, discoverImpl = discoverModels, env = process.env } = {}) { const diskCache = await readCache(env).catch(() => ({})); @@ -42,7 +42,13 @@ export async function resolveLiveModels(providers, { onEvent = () => {}, discove message: `model discovery failed (${reason}); using the list this key returned last time`, }); } else if (source === 'catalog' && reason) { - onEvent({ type: 'error', provider: provider.id, message: `model discovery fell back to the shipped list: ${reason}` }); + onEvent({ + type: 'error', + provider: provider.id, + message: models.length === 0 + ? `model discovery failed (${reason}); no remembered models, provider unavailable` + : `model discovery failed (${reason}); using its pinned models`, + }); } else { onEvent({ type: 'discovery', provider: provider.id, count: models.length, source }); } @@ -50,6 +56,19 @@ export async function resolveLiveModels(providers, { onEvent = () => {}, discove })); } +/** + * Offline mode for --no-discovery: no provider is contacted. Each provider + * keeps its pinned models, or the list its key returned last time. + */ +export async function resolveOfflineModels(providers, { env = process.env, now = Date.now } = {}) { + const diskCache = await readCache(env).catch(() => ({})); + + return providers.map((provider) => ({ + ...provider, + models: recallModels(diskCache, provider.id, now) || provider.models || [], + })); +} + export function isQueryable(provider) { return hasCredential(provider); } diff --git a/src/model-cache.mjs b/src/model-cache.mjs index 8c18281..afe9ae3 100644 --- a/src/model-cache.mjs +++ b/src/model-cache.mjs @@ -1,11 +1,11 @@ // Remembers the last model list each provider actually returned. // // When discovery fails — a rate limit, a network blip, an expired key — falling -// back to the shipped list is misleading: that list is generic, while a provider's -// catalog is specific to the plan behind the key. It can therefore offer models the -// user does not have (a Kimi entry on an Alibaba plan that never included it) and -// omit ones they do. Yesterday's real answer is a far better guess than a generic -// list, so it is kept on disk and preferred over it. +// back to a generic list is misleading: a provider's catalog is specific to the +// plan behind the key, so a generic entry can offer models the user does not have +// (a Kimi entry on an Alibaba plan that never included it) and omit ones they do. +// Yesterday's real answer is a far better guess, so it is kept on disk and +// preferred over any pinned list. import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises'; import { homedir, platform } from 'node:os'; diff --git a/src/model-ranking.mjs b/src/model-ranking.mjs index 1ac549d..36a8fe4 100644 --- a/src/model-ranking.mjs +++ b/src/model-ranking.mjs @@ -1,13 +1,3 @@ -const FAMILY_PRIORITIES = [ - [/minimax/i, 100], - [/kimi|moonshot/i, 95], - [/deepseek/i, 90], - [/qwen/i, 80], - [/glm|zai-org/i, 75], - [/gpt-oss|gpt/i, 60], - [/mistral|mixtral|codestral/i, 50], -]; - const DENY_PATTERNS = [ /(^|[/_.-])(flux|sdxl|stable-diffusion|diffusion|dall-e|sora|image|video|moderation|realtime)([/_.-]|$)/i, /(^|[/_.-])(embedding|embed|rerank|ocr|guard|safeguard|safety|tts|whisper|audio|speech|transcription|computer-use)([/_.-]|$)/i, @@ -37,7 +27,6 @@ export function rankModels({ id: entry.id, updated: getUpdatedTicks(entry.model), created: getCreatedTicks(entry.model), - family: getFamilyPriority(entry.id), version: getVersionScore(entry.id), context: getContextLength(entry.model), index: entry.index, @@ -45,7 +34,6 @@ export function rankModels({ .sort((a, b) => ( b.updated - a.updated || b.created - a.created || - b.family - a.family || b.version - a.version || b.context - a.context || a.id.localeCompare(b.id) || @@ -200,14 +188,6 @@ function getContextLength(model) { return 0; } -function getFamilyPriority(id) { - for (const [pattern, priority] of FAMILY_PRIORITIES) { - if (pattern.test(id)) return priority; - } - - return 0; -} - function getVersionScore(id) { const matches = [...id.toLowerCase().matchAll(/(?_API_KEY: the pattern covers the diff --git a/src/providers.default.json b/src/providers.default.json index 0375249..6279a4c 100644 --- a/src/providers.default.json +++ b/src/providers.default.json @@ -10,40 +10,6 @@ "apiKeyEnv": [ "OPENAI_API_KEY", "COPILOT_BYOK_OPENAI_API_KEY" - ], - "models": [ - { - "model": "gpt-5.6-sol", - "label": "GPT-5.6 Sol (OpenAI)", - "contextWindow": 1050000, - "maxOutputTokens": 128000, - "reasoning": true, - "vision": true - }, - { - "model": "gpt-5.5", - "label": "GPT-5.5 (OpenAI)", - "contextWindow": 1050000, - "maxOutputTokens": 128000, - "reasoning": true, - "vision": true - }, - { - "model": "gpt-5.4", - "label": "GPT-5.4 (OpenAI)", - "contextWindow": 1050000, - "maxOutputTokens": 128000, - "reasoning": true, - "vision": true - }, - { - "model": "gpt-5.4-mini", - "label": "GPT-5.4 mini (OpenAI)", - "contextWindow": 400000, - "maxOutputTokens": 128000, - "reasoning": true, - "vision": true - } ] }, { @@ -54,40 +20,6 @@ "apiKeyEnv": [ "ANTHROPIC_API_KEY", "COPILOT_BYOK_ANTHROPIC_API_KEY" - ], - "models": [ - { - "model": "claude-sonnet-4-6", - "label": "Claude Sonnet 4.6 (Anthropic)", - "contextWindow": 1000000, - "maxOutputTokens": 128000, - "reasoning": true, - "vision": true - }, - { - "model": "claude-opus-4-6", - "label": "Claude Opus 4.6 (Anthropic)", - "contextWindow": 1000000, - "maxOutputTokens": 128000, - "reasoning": true, - "vision": true - }, - { - "model": "claude-sonnet-4-5", - "label": "Claude Sonnet 4.5 (latest) (Anthropic)", - "contextWindow": 1000000, - "maxOutputTokens": 64000, - "reasoning": true, - "vision": true - }, - { - "model": "claude-haiku-4-5", - "label": "Claude Haiku 4.5 (latest) (Anthropic)", - "contextWindow": 200000, - "maxOutputTokens": 64000, - "reasoning": true, - "vision": true - } ] }, { @@ -99,32 +31,6 @@ "GEMINI_API_KEY", "GOOGLE_API_KEY", "COPILOT_BYOK_GEMINI_API_KEY" - ], - "models": [ - { - "model": "gemini-3.5-flash", - "label": "Gemini 3.5 Flash (Google Gemini)", - "contextWindow": 1048576, - "maxOutputTokens": 65536, - "reasoning": true, - "vision": true - }, - { - "model": "gemini-2.5-flash", - "label": "Gemini 2.5 Flash (Google Gemini)", - "contextWindow": 1048576, - "maxOutputTokens": 65536, - "reasoning": true, - "vision": true - }, - { - "model": "gemini-3.5-flash-lite", - "label": "Gemini 3.5 Flash Lite (Google Gemini)", - "contextWindow": 1048576, - "maxOutputTokens": 65536, - "reasoning": true, - "vision": true - } ] }, { @@ -135,24 +41,6 @@ "apiKeyEnv": [ "XAI_API_KEY", "COPILOT_BYOK_XAI_API_KEY" - ], - "models": [ - { - "model": "grok-4.5", - "label": "Grok 4.5 (xAI Grok)", - "contextWindow": 500000, - "maxOutputTokens": 500000, - "reasoning": true, - "vision": true - }, - { - "model": "grok-4.3", - "label": "Grok 4.3 (xAI Grok)", - "contextWindow": 1000000, - "maxOutputTokens": 30000, - "reasoning": true, - "vision": true - } ] }, { @@ -163,22 +51,6 @@ "apiKeyEnv": [ "MISTRAL_API_KEY", "COPILOT_BYOK_MISTRAL_API_KEY" - ], - "models": [ - { - "model": "mistral-medium-latest", - "label": "Mistral Medium (latest) (Mistral)", - "contextWindow": 262144, - "maxOutputTokens": 262144, - "reasoning": true, - "vision": true - }, - { - "model": "devstral-medium-latest", - "label": "Devstral 2 (latest) (Mistral)", - "contextWindow": 262144, - "maxOutputTokens": 262144 - } ] }, { @@ -189,37 +61,6 @@ "apiKeyEnv": [ "DEEPSEEK_API_KEY", "COPILOT_BYOK_DEEPSEEK_API_KEY" - ], - "models": [ - { - "model": "deepseek-v4-pro", - "label": "DeepSeek V4 Pro (DeepSeek)", - "contextWindow": 1000000, - "maxOutputTokens": 384000, - "reasoning": true - }, - { - "model": "deepseek-v4-flash", - "label": "DeepSeek V4 Flash (DeepSeek)", - "contextWindow": 1000000, - "maxOutputTokens": 384000, - "reasoning": true - }, - { - "model": "deepseek-reasoner", - "label": "DeepSeek Reasoner (DeepSeek)", - "contextWindow": 1000000, - "maxOutputTokens": 384000, - "reasoning": true, - "vision": true - }, - { - "model": "deepseek-chat", - "label": "DeepSeek Chat (DeepSeek)", - "contextWindow": 1000000, - "maxOutputTokens": 384000, - "vision": true - } ] }, { @@ -231,28 +72,6 @@ "DASHSCOPE_API_KEY", "QWEN_API_KEY", "COPILOT_BYOK_QWEN_API_KEY" - ], - "models": [ - { - "model": "qwen3.7-max", - "label": "Qwen3.7 Max (Alibaba Qwen)", - "contextWindow": 1000000, - "maxOutputTokens": 65536, - "reasoning": true - }, - { - "model": "qwen3.7-plus", - "label": "Qwen3.7 Plus (Alibaba Qwen)", - "contextWindow": 1000000, - "maxOutputTokens": 65536, - "reasoning": true - }, - { - "model": "qwen3-coder-plus", - "label": "Qwen3 Coder Plus (Alibaba Qwen)", - "contextWindow": 1048576, - "maxOutputTokens": 65536 - } ] }, { @@ -263,21 +82,6 @@ "apiKeyEnv": [ "DASHSCOPE_API_KEY_CN", "COPILOT_BYOK_QWEN_CN_API_KEY" - ], - "models": [ - { - "model": "qwen3.7-max", - "label": "Qwen3.7 Max (Alibaba Qwen (China))", - "contextWindow": 1000000, - "maxOutputTokens": 65536, - "reasoning": true - }, - { - "model": "qwen3-coder-plus", - "label": "Qwen3 Coder Plus (Alibaba Qwen (China))", - "contextWindow": 1048576, - "maxOutputTokens": 65536 - } ] }, { @@ -289,29 +93,6 @@ "ZAI_API_KEY", "ZHIPU_API_KEY", "COPILOT_BYOK_ZAI_API_KEY" - ], - "models": [ - { - "model": "glm-5.2", - "label": "GLM-5.2 (Z.AI GLM)", - "contextWindow": 1000000, - "maxOutputTokens": 131072, - "reasoning": true - }, - { - "model": "glm-5.1", - "label": "GLM-5.1 (Z.AI GLM)", - "contextWindow": 200000, - "maxOutputTokens": 131072, - "reasoning": true - }, - { - "model": "glm-4.7", - "label": "GLM-4.7 (Z.AI GLM)", - "contextWindow": 204800, - "maxOutputTokens": 131072, - "reasoning": true - } ] }, { @@ -322,22 +103,6 @@ "apiKeyEnv": [ "ZHIPU_CN_API_KEY", "COPILOT_BYOK_ZHIPU_CN_API_KEY" - ], - "models": [ - { - "model": "glm-5.2", - "label": "GLM-5.2 (Zhipu AI (China))", - "contextWindow": 1000000, - "maxOutputTokens": 131072, - "reasoning": true - }, - { - "model": "glm-4.7", - "label": "GLM-4.7 (Zhipu AI (China))", - "contextWindow": 204800, - "maxOutputTokens": 131072, - "reasoning": true - } ] }, { @@ -349,31 +114,6 @@ "MOONSHOT_API_KEY", "KIMI_API_KEY", "COPILOT_BYOK_KIMI_API_KEY" - ], - "models": [ - { - "model": "kimi-k3", - "label": "Kimi K3 (Moonshot Kimi)", - "contextWindow": 1048576, - "maxOutputTokens": 131072, - "reasoning": true, - "vision": true - }, - { - "model": "kimi-k2.6", - "label": "Kimi K2.6 (Moonshot Kimi)", - "contextWindow": 262144, - "maxOutputTokens": 262144, - "reasoning": true, - "vision": true - }, - { - "model": "kimi-k2.5", - "label": "Kimi K2.5 (Moonshot Kimi)", - "contextWindow": 262144, - "maxOutputTokens": 262144, - "reasoning": true - } ] }, { @@ -384,72 +124,8 @@ "apiKeyEnv": [ "MINIMAX_API_KEY", "COPILOT_BYOK_MINIMAX_API_KEY" - ], - "models": [ - { - "model": "MiniMax-M3", - "label": "MiniMax-M3 (MiniMax)", - "contextWindow": 1000000, - "maxOutputTokens": 128000, - "reasoning": true, - "vision": true - }, - { - "model": "MiniMax-M2.7", - "label": "MiniMax-M2.7 (MiniMax)", - "contextWindow": 204800, - "maxOutputTokens": 131072, - "reasoning": true - } ] }, - { - "id": "stepfun", - "name": "StepFun", - "baseUrl": "https://api.stepfun.com/v1", - "modelsUrl": "https://api.stepfun.com/v1/models", - "apiKeyEnv": [ - "STEPFUN_API_KEY", - "COPILOT_BYOK_STEPFUN_API_KEY" - ], - "models": [] - }, - { - "id": "doubao", - "name": "ByteDance Doubao", - "baseUrl": "https://ark.cn-beijing.volces.com/api/v3", - "modelsUrl": "https://ark.cn-beijing.volces.com/api/v3/models", - "apiKeyEnv": [ - "ARK_API_KEY", - "VOLCENGINE_API_KEY", - "COPILOT_BYOK_DOUBAO_API_KEY" - ], - "note": "Volcengine model names are account-specific endpoint ids: add yours under `models`.", - "models": [] - }, - { - "id": "hunyuan", - "name": "Tencent Hunyuan", - "baseUrl": "https://api.hunyuan.cloud.tencent.com/v1", - "modelsUrl": "https://api.hunyuan.cloud.tencent.com/v1/models", - "apiKeyEnv": [ - "HUNYUAN_API_KEY", - "COPILOT_BYOK_HUNYUAN_API_KEY" - ], - "models": [] - }, - { - "id": "qianfan", - "name": "Baidu Qianfan", - "baseUrl": "https://qianfan.baidubce.com/v2", - "modelsUrl": "https://qianfan.baidubce.com/v2/models", - "apiKeyEnv": [ - "QIANFAN_API_KEY", - "BAIDU_API_KEY", - "COPILOT_BYOK_QIANFAN_API_KEY" - ], - "models": [] - }, { "id": "siliconflow", "name": "SiliconFlow", @@ -458,8 +134,7 @@ "apiKeyEnv": [ "SILICONFLOW_API_KEY", "COPILOT_BYOK_SILICONFLOW_API_KEY" - ], - "models": [] + ] }, { "id": "modelscope", @@ -469,8 +144,7 @@ "apiKeyEnv": [ "MODELSCOPE_API_KEY", "COPILOT_BYOK_MODELSCOPE_API_KEY" - ], - "models": [] + ] }, { "id": "openrouter", @@ -480,8 +154,7 @@ "apiKeyEnv": [ "OPENROUTER_API_KEY", "COPILOT_BYOK_OPENROUTER_API_KEY" - ], - "models": [] + ] }, { "id": "groq", @@ -491,21 +164,6 @@ "apiKeyEnv": [ "GROQ_API_KEY", "COPILOT_BYOK_GROQ_API_KEY" - ], - "models": [ - { - "model": "openai/gpt-oss-20b", - "label": "GPT OSS 20B (Groq)", - "contextWindow": 131072, - "maxOutputTokens": 65536, - "reasoning": true - }, - { - "model": "llama-3.3-70b-versatile", - "label": "Llama 3.3 70B (Groq)", - "contextWindow": 131072, - "maxOutputTokens": 32768 - } ] }, { @@ -516,8 +174,7 @@ "apiKeyEnv": [ "CEREBRAS_API_KEY", "COPILOT_BYOK_CEREBRAS_API_KEY" - ], - "models": [] + ] }, { "id": "together", @@ -527,8 +184,7 @@ "apiKeyEnv": [ "TOGETHER_API_KEY", "COPILOT_BYOK_TOGETHER_API_KEY" - ], - "models": [] + ] }, { "id": "deepinfra", @@ -538,8 +194,7 @@ "apiKeyEnv": [ "DEEPINFRA_API_KEY", "COPILOT_BYOK_DEEPINFRA_API_KEY" - ], - "models": [] + ] }, { "id": "fireworks", @@ -549,8 +204,7 @@ "apiKeyEnv": [ "FIREWORKS_API_KEY", "COPILOT_BYOK_FIREWORKS_API_KEY" - ], - "models": [] + ] }, { "id": "chutes", @@ -560,30 +214,6 @@ "apiKeyEnv": [ "CHUTES_API_KEY", "COPILOT_BYOK_CHUTES_API_KEY" - ], - "models": [ - { - "model": "zai-org/GLM-5.2-TEE", - "label": "GLM 5.2 TEE (Chutes)", - "contextWindow": 1048576, - "maxOutputTokens": 65535, - "reasoning": true - }, - { - "model": "moonshotai/Kimi-K2.6-TEE", - "label": "Kimi K2.6 TEE (Chutes)", - "contextWindow": 262144, - "maxOutputTokens": 65535, - "reasoning": true, - "vision": true - }, - { - "model": "deepseek-ai/DeepSeek-V4-Flash-0731-TEE", - "label": "DeepSeek V4 Flash 0731 TEE (Chutes)", - "contextWindow": 1048576, - "maxOutputTokens": 131072, - "reasoning": true - } ] }, { @@ -594,32 +224,6 @@ "apiKeyEnv": [ "OPENCODE_ZEN_API_KEY", "COPILOT_BYOK_OPENCODE_ZEN_API_KEY" - ], - "models": [ - { - "model": "claude-sonnet-4-6", - "label": "Claude Sonnet 4.6 (OpenCode Zen)", - "contextWindow": 1000000, - "maxOutputTokens": 64000, - "reasoning": true, - "vision": true - }, - { - "model": "gpt-5.5-pro", - "label": "GPT-5.5 Pro (OpenCode Zen)", - "contextWindow": 1050000, - "maxOutputTokens": 128000, - "reasoning": true, - "vision": true - }, - { - "model": "gemini-3.5-flash", - "label": "Gemini 3.5 Flash (OpenCode Zen)", - "contextWindow": 1048576, - "maxOutputTokens": 65536, - "reasoning": true, - "vision": true - } ] }, { @@ -630,37 +234,6 @@ "apiKeyEnv": [ "OPENCODE_GO_API_KEY", "COPILOT_BYOK_OPENCODE_GO_API_KEY" - ], - "models": [ - { - "model": "glm-5.2", - "label": "GLM-5.2 (OpenCode Go)", - "contextWindow": 1000000, - "maxOutputTokens": 131072, - "reasoning": true - }, - { - "model": "kimi-k2.6", - "label": "Kimi K2.6 (OpenCode Go)", - "contextWindow": 262144, - "maxOutputTokens": 65536, - "reasoning": true, - "vision": true - }, - { - "model": "minimax-m3", - "label": "MiniMax-M3 (OpenCode Go)", - "contextWindow": 1000000, - "maxOutputTokens": 131072, - "reasoning": true - }, - { - "model": "qwen3.7-max", - "label": "Qwen3.7 Max (OpenCode Go)", - "contextWindow": 1000000, - "maxOutputTokens": 65536, - "reasoning": true - } ] }, { @@ -671,30 +244,28 @@ "apiKeyEnv": [ "ALIBABA_TOKEN_PLAN_API_KEY", "COPILOT_BYOK_ALIBABA_TOKEN_PLAN_API_KEY" - ], - "models": [ - { - "model": "qwen3.8-max", - "label": "Qwen3.8 Max (Alibaba Token Plan)", - "contextWindow": 1000000, - "maxOutputTokens": 131072, - "reasoning": true, - "vision": true - }, - { - "model": "deepseek-v4-pro", - "label": "DeepSeek V4 Pro (Alibaba Token Plan)", - "contextWindow": 1000000, - "maxOutputTokens": 384000, - "reasoning": true - }, - { - "model": "glm-5.2", - "label": "GLM-5.2 (Alibaba Token Plan)", - "contextWindow": 1000000, - "maxOutputTokens": 131072, - "reasoning": true - } + ] + }, + { + "id": "muse", + "name": "Meta Muse", + "baseUrl": "https://api.meta.ai/v1", + "modelsUrl": "https://api.meta.ai/v1/models", + "apiKeyEnv": [ + "MODEL_API_KEY", + "META_API_KEY", + "COPILOT_BYOK_MUSE_API_KEY" + ] + }, + { + "id": "nim", + "name": "NVIDIA NIM", + "baseUrl": "https://integrate.api.nvidia.com/v1", + "modelsUrl": "https://integrate.api.nvidia.com/v1/models", + "apiKeyEnv": [ + "NVIDIA_API_KEY", + "NGC_API_KEY", + "COPILOT_BYOK_NIM_API_KEY" ] }, { @@ -703,8 +274,7 @@ "baseUrl": "http://127.0.0.1:11434/v1", "modelsUrl": "http://127.0.0.1:11434/v1/models", "authRequired": false, - "enabled": false, - "models": [] + "enabled": false }, { "id": "lmstudio", @@ -712,8 +282,7 @@ "baseUrl": "http://127.0.0.1:1234/v1", "modelsUrl": "http://127.0.0.1:1234/v1/models", "authRequired": false, - "enabled": false, - "models": [] + "enabled": false } ] } diff --git a/test/catalog.test.mjs b/test/catalog.test.mjs index 0c3becd..309824c 100644 --- a/test/catalog.test.mjs +++ b/test/catalog.test.mjs @@ -96,18 +96,16 @@ test('prompt and output budgets fit inside the context window', () => { } }); -test('the shipped catalog never advertises more than a model can hold', async () => { +test('the generated catalog ships endpoints only, with unique ids', async () => { const { readFile } = await import('node:fs/promises'); const catalog = JSON.parse(await readFile(new URL('../src/providers.default.json', import.meta.url), 'utf8')); - const providers = catalog.providers - .filter((provider) => provider.models.length) - .map((provider) => ({ ...provider, apiKey: 'k' })); + const ids = catalog.providers.map((provider) => provider.id); + assert.equal(new Set(ids).size, ids.length, 'provider ids must be unique'); - for (const entry of buildCatalog(providers).entries) { - const { max_prompt_tokens: prompt, max_output_tokens: output, max_context_window_tokens: context } = - entry.capabilities.limits; - assert.ok(prompt + output <= context, `${entry.id}: ${prompt} + ${output} > ${context}`); + for (const provider of catalog.providers) { + assert.ok(provider.modelsUrl, `${provider.id} needs a models endpoint`); + assert.ok(!provider.models || provider.models.length === 0, `${provider.id} must not ship models`); } }); diff --git a/test/config.test.mjs b/test/config.test.mjs index 5935074..7a36734 100644 --- a/test/config.test.mjs +++ b/test/config.test.mjs @@ -33,14 +33,20 @@ test('loads the generated provider catalog', async () => { const dir = await mkdtemp(join(tmpdir(), 'copilot-byok-empty-config-')); const config = await loadConfig({ env: { XDG_CONFIG_HOME: dir } }); - // The catalog is regenerated from models.dev, so the test checks the shape and - // the fixed points rather than the exact list of models. - assert.ok(config.providers.length >= 20); + // The catalog ships endpoints only; model lists come from live discovery. + // The test checks the fixed points rather than the full list. + assert.equal(config.providers.length, 28); const openai = findProvider(config, 'openai'); assert.equal(openai.baseUrl, 'https://api.openai.com/v1'); assert.ok(openai.apiKeyEnv.includes('OPENAI_API_KEY')); - assert.ok(openai.models.length > 0); + assert.deepEqual(openai.models, []); + + const muse = findProvider(config, 'muse'); + assert.equal(muse.baseUrl, 'https://api.meta.ai/v1'); + + const nim = findProvider(config, 'nim'); + assert.equal(nim.baseUrl, 'https://integrate.api.nvidia.com/v1'); const anthropic = findProvider(config, 'anthropic'); assert.equal(anthropic.baseUrl, 'https://api.anthropic.com/v1'); @@ -54,6 +60,7 @@ test('loads the generated provider catalog', async () => { for (const provider of config.providers) { assert.equal(provider.type, 'openai', `${provider.id} must be OpenAI-compatible`); + assert.match(provider.modelsUrl, /^https?:\/\//, `${provider.id} needs a models endpoint`); for (const model of provider.models) { assert.ok(model.model, `${provider.id} has a model without a name`); if (model.contextWindow != null) assert.ok(model.contextWindow > 0); @@ -83,9 +90,12 @@ test('the published example is a valid configuration', async () => { for (const provider of example.providers) { assert.ok(provider.id); assert.ok(provider.baseUrl); - assert.ok(provider.models.length > 0, `${provider.id} must declare at least one model`); } + // The example covers both styles: discovery-only and pinned models. + assert.ok(example.providers.some((provider) => provider.models.length === 0)); + assert.ok(example.providers.some((provider) => provider.models.length > 0)); + // The example also covers the credential-free case (local runtime). assert.ok(example.providers.some((provider) => provider.authRequired === false)); }); diff --git a/test/discovery.test.mjs b/test/discovery.test.mjs index c2287fa..1158afa 100644 --- a/test/discovery.test.mjs +++ b/test/discovery.test.mjs @@ -4,6 +4,7 @@ import test, { beforeEach } from 'node:test'; import { clearDiscoveryCache, declaresToolSupport, + declaresVision, discoverModels, readContextWindow, readOutputLimit, @@ -16,7 +17,7 @@ const provider = { baseUrl: 'https://api.acme.test/v1', modelsUrl: 'https://api.acme.test/v1/models', apiKey: 'secret', - models: [{ model: 'curated-one', label: 'Curated One', contextWindow: 1000, maxOutputTokens: 100 }], + models: [{ model: 'configured-one', label: 'Configured One', contextWindow: 1000, maxOutputTokens: 100 }], }; // Discovery caches per session; each test needs to start from a clean slate. @@ -92,7 +93,7 @@ test('the list is capped so one provider cannot flood the picker', async () => { assert.equal(result.models.length, 5); }); -test('an unreachable provider keeps its shipped list', async () => { +test('an unreachable provider keeps its configured list', async () => { const result = await discoverModels(provider, { fetchImpl: async () => { throw new Error('ECONNREFUSED'); }, }); @@ -102,7 +103,7 @@ test('an unreachable provider keeps its shipped list', async () => { assert.deepEqual(result.models, provider.models); }); -test('an error response keeps the shipped list rather than emptying the picker', async () => { +test('an error response keeps the configured list rather than emptying the picker', async () => { const result = await discoverModels(provider, { fetchImpl: respondWith(null, { ok: false, status: 401 }), }); @@ -201,12 +202,45 @@ test('namespaced ids get a readable label while the id itself is preserved', asy assert.equal(model.label, 'Qwen3.5-397B-A17B-TEE (Acme)'); }); -test('a curated label wins over the generated one', async () => { +test('a configured label wins over the generated one', async () => { const result = await discoverModels(provider, { - fetchImpl: respondWith({ data: [{ id: 'curated-one' }] }), + fetchImpl: respondWith({ data: [{ id: 'configured-one' }] }), }); - assert.equal(result.models[0].label, 'Curated One'); + assert.equal(result.models[0].label, 'Configured One'); +}); + +test('without configured models a failure yields an empty list with a reason', async () => { + const result = await discoverModels({ ...provider, models: [] }, { + fetchImpl: async () => { throw new Error('down'); }, + cache: new Map(), + }); + + assert.equal(result.source, 'catalog'); + assert.match(result.reason, /down/); + assert.deepEqual(result.models, []); +}); + +test('vision support is read from the provider payload', () => { + assert.equal(declaresVision({ input_modalities: ['text', 'image'] }), true); + assert.equal(declaresVision({ architecture: { input_modalities: ['image'] } }), true); + assert.equal(declaresVision({ attachment: true }), true); + assert.equal(declaresVision({ capabilities: { supports: { vision: true } } }), true); + assert.equal(declaresVision({ input_modalities: ['text'] }), false); + assert.equal(declaresVision({ id: 'plain' }), false); + assert.equal(declaresVision(null), false); +}); + +test('a discovered model carries live vision and reasoning flags', async () => { + const result = await discoverModels({ ...provider, models: [] }, { + fetchImpl: respondWith({ + data: [{ id: 'acme-omni', input_modalities: ['text', 'image'], supported_parameters: ['tools', 'reasoning'] }], + }), + cache: new Map(), + }); + + assert.equal(result.models[0].vision, true); + assert.equal(result.models[0].reasoning, true); }); test('the catalog is fetched once per provider, not on every rebuild', async () => { diff --git a/test/keys-command.test.mjs b/test/keys-command.test.mjs index ec8da6c..b63c730 100644 --- a/test/keys-command.test.mjs +++ b/test/keys-command.test.mjs @@ -7,6 +7,7 @@ import test from 'node:test'; import { main } from '../src/cli.mjs'; import { loadKeys, saveKey } from '../src/keystore.mjs'; +import { rememberModels } from '../src/model-cache.mjs'; function captureWritable() { const chunks = []; @@ -114,10 +115,12 @@ test('an unknown keys action fails with guidance', async () => { }); test('a stored key activates its provider in the router catalog', async () => { - const { io, stdout } = await scratchIo(); + const cacheDir = await mkdtemp(join(tmpdir(), 'copilot-byok-keys-cache-')); + const { io, stdout } = await scratchIo({ env: { COPILOT_BYOK_MODEL_CACHE: join(cacheDir, 'models.json') } }); await saveKey('deepseek', 'stored-key', io.env); + await rememberModels('deepseek', [{ model: 'deepseek-chat', label: 'DeepSeek Chat' }], io.env); - const exitCode = await main(['--dry-run'], io); + const exitCode = await main(['--dry-run', '--no-discovery'], io); assert.equal(exitCode, 0); const report = JSON.parse(stdout.text()); diff --git a/test/launcher.test.mjs b/test/launcher.test.mjs index 8e3b1bb..47f15f6 100644 --- a/test/launcher.test.mjs +++ b/test/launcher.test.mjs @@ -1,7 +1,11 @@ import assert from 'node:assert/strict'; +import { mkdtemp } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import test from 'node:test'; -import { startRouter } from '../src/launcher.mjs'; +import { resolveOfflineModels, startRouter } from '../src/launcher.mjs'; +import { rememberModels } from '../src/model-cache.mjs'; const base = { id: 'acme', @@ -83,3 +87,19 @@ test('the router binds to loopback only', async () => { await router.close(); } }); + +test('offline mode uses remembered lists without contacting providers', async () => { + const dir = await mkdtemp(join(tmpdir(), 'copilot-byok-offline-')); + const env = { COPILOT_BYOK_MODEL_CACHE: join(dir, 'models.json') }; + await rememberModels('acme', [{ model: 'yours-1' }], env); + + const resolved = await resolveOfflineModels([ + { id: 'acme', name: 'Acme', models: [] }, + { id: 'pinned', name: 'Pinned', models: [{ model: 'fixed-one' }] }, + { id: 'unknown', name: 'Unknown', models: [] }, + ], { env }); + + assert.deepEqual(resolved.find((entry) => entry.id === 'acme').models.map((model) => model.model), ['yours-1']); + assert.deepEqual(resolved.find((entry) => entry.id === 'pinned').models.map((model) => model.model), ['fixed-one']); + assert.deepEqual(resolved.find((entry) => entry.id === 'unknown').models, []); +}); diff --git a/test/model-cache.test.mjs b/test/model-cache.test.mjs index 46750e4..44d617b 100644 --- a/test/model-cache.test.mjs +++ b/test/model-cache.test.mjs @@ -18,7 +18,7 @@ const provider = { baseUrl: 'https://api.acme.test/v1', modelsUrl: 'https://api.acme.test/v1/models', apiKey: 'secret', - // The shipped list is generic and may name models this key cannot use. + // The pinned list is generic and may name models this key cannot use. models: [{ model: 'generic-model', label: 'Generic' }], }; @@ -52,7 +52,7 @@ test('a stale entry is ignored rather than offered as current', async () => { assert.ok(recallModels(cache, 'acme', () => 1000)); }); -test('a failed discovery offers what the key really returned, not the shipped list', async () => { +test('a failed discovery offers what the key really returned, not the pinned list', async () => { const env = await scratchEnv(); await rememberModels('acme', [{ model: 'yours-1' }, { model: 'yours-2' }], env); const diskCache = await readCache(env); @@ -70,7 +70,7 @@ test('a failed discovery offers what the key really returned, not the shipped li 'a generic entry could name a model this plan does not include'); }); -test('without a remembered list the shipped one is still used', async () => { +test('without a remembered list the configured one is still used', async () => { const env = await scratchEnv(); const result = await discoverModels(provider, { diff --git a/test/model-ranking.test.mjs b/test/model-ranking.test.mjs index bd54f15..08d0dd5 100644 --- a/test/model-ranking.test.mjs +++ b/test/model-ranking.test.mjs @@ -65,7 +65,20 @@ test('uses model name version as a tie-break when provider timestamps are equal' }, }); - assert.deepEqual(models.slice(0, 3), ['minimax-m2.7', 'minimax-m2.5', 'kimi-k2.6']); + assert.deepEqual(models.slice(0, 3), ['minimax-m2.7', 'kimi-k2.6', 'minimax-m2.5']); +}); + +test('no model family is preferred over another', () => { + const models = rankModels({ + payload: { + data: [ + { id: 'kimi-k2.5', created: 100 }, + { id: 'some-new-family-9.0', created: 100 }, + ], + }, + }); + + assert.deepEqual(models, ['some-new-family-9.0', 'kimi-k2.5']); }); test('does not reject minimal model catalogs when tool metadata is absent', () => { diff --git a/test/process-env.test.mjs b/test/process-env.test.mjs index ea7fb13..ff4267b 100644 --- a/test/process-env.test.mjs +++ b/test/process-env.test.mjs @@ -14,12 +14,17 @@ test('catalog provider keys never reach the Copilot child process', () => { QIANFAN_API_KEY: 'secret', SILICONFLOW_API_KEY: 'secret', OPENCODE_ZEN_API_KEY: 'secret', + MODEL_API_KEY: 'secret', + META_API_KEY: 'secret', + NVIDIA_API_KEY: 'secret', + NGC_API_KEY: 'secret', }); assert.equal(env.PATH, '/usr/bin'); for (const name of [ 'OPENAI_API_KEY', 'GEMINI_API_KEY', 'DASHSCOPE_API_KEY', 'ZHIPU_CN_API_KEY', 'ARK_API_KEY', 'QIANFAN_API_KEY', 'SILICONFLOW_API_KEY', 'OPENCODE_ZEN_API_KEY', + 'MODEL_API_KEY', 'META_API_KEY', 'NVIDIA_API_KEY', 'NGC_API_KEY', ]) { assert.equal(env[name], undefined, `${name} must be stripped`); } From 1db8ea01d23dfc966ef6a805f0ab93565707d981 Mon Sep 17 00:00:00 2001 From: thestreamcode Date: Wed, 14 Oct 2026 21:05:20 +0200 Subject: [PATCH 3/5] docs: drop rationale for excluded providers --- docs/provider-verification.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/provider-verification.md b/docs/provider-verification.md index 45f7f43..ae75c13 100644 --- a/docs/provider-verification.md +++ b/docs/provider-verification.md @@ -64,12 +64,6 @@ before the credential, and the probe deliberately sends a non-existent model: | Fireworks AI | `https://api.fireworks.ai/inference/v1` | 404 on an unknown model | | Nvidia NIM | `https://integrate.api.nvidia.com/v1` | 404 on an unknown model | -Perplexity, Baichuan and Baseten were probed but are not shipped: Perplexity is -search-focused rather than coding-oriented, and Baichuan and Baseten are -unverified for coding use. StepFun, Doubao, Hunyuan and Qianfan were removed -from the catalog for the same reason — the catalog keeps only coding-oriented -frontier providers. - ## Model-list endpoints Discovery reads each provider's `GET /models`, so that route was probed for every From 12f131264373e77ce6e1a68db50eb5f14e34410a Mon Sep 17 00:00:00 2001 From: thestreamcode Date: Wed, 14 Oct 2026 21:05:58 +0200 Subject: [PATCH 4/5] fix: /byok extension lists providers without shipped models configurable() required provider.models.length, which is always empty now that the catalog is discovery-only. Any provider needing a credential is worth offering. --- extensions/byok/extension.mjs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extensions/byok/extension.mjs b/extensions/byok/extension.mjs index 562c19b..c728181 100644 --- a/extensions/byok/extension.mjs +++ b/extensions/byok/extension.mjs @@ -62,15 +62,15 @@ async function currentState() { return { config, secrets }; } -/** Providers worth offering: they publish models and need a credential. */ +/** Providers worth offering: they need a credential. Models load at startup. */ function configurable(config) { - return config.providers.filter((provider) => provider.authRequired !== false && provider.models?.length); + return config.providers.filter((provider) => provider.authRequired !== false); } function describe(provider, secrets) { if (ENV_PROVIDERS.has(provider.id)) return `${provider.name} — set via environment`; if (secrets[provider.id]) return `${provider.name} — key stored`; - return `${provider.name} — not configured (${provider.models.length} models)`; + return `${provider.name} — not configured`; } async function addProvider() { @@ -106,7 +106,7 @@ async function addProvider() { await saveKey(provider.id, key.trim(), process.env); await session.log( `Saved the ${provider.name} key in ${keystorePath(process.env)}.\n` + - `Its ${provider.models.length} models will be in /model the next time you start ` + + `Its models will be in /model the next time you start ` + 'copilot-byok: Copilot caches the model list for the lifetime of a session, so ' + 'neither reopening the picker nor /restart picks them up.' ); From c32976a9c5099d93dd83a0acc73631e5c11ac1d8 Mon Sep 17 00:00:00 2001 From: thestreamcode Date: Wed, 14 Oct 2026 21:08:40 +0200 Subject: [PATCH 5/5] docs: sync /byok mock and changelog with discovery-only behavior --- CHANGELOG.md | 6 ++++++ README.md | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06a2376..f488d40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,12 @@ All notable changes to this project are documented in this file. referencing them need updating; account-specific endpoint ids still work through a custom provider entry. +### Fixed + +- The `/byok` in-session command lists every provider needing a credential + again instead of an empty list, and no longer reports model counts that only + exist after discovery. + ## [1.4.2] - 2026-08-12 ### Changed diff --git a/README.md b/README.md index c9f4661..324a7f8 100644 --- a/README.md +++ b/README.md @@ -190,9 +190,9 @@ Then `/byok` opens a provider list and asks for the key: ``` Choose a provider to configure: -❯ Anthropic — not configured (4 models) - Google Gemini — not configured (3 models) - DeepSeek — not configured (4 models) +❯ Anthropic — not configured + Google Gemini — not configured + DeepSeek — not configured OpenAI — set via environment Chutes — set via environment