diff --git a/docs/configuration.md b/docs/configuration.md index de428d5b..13e06e59 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -363,7 +363,7 @@ The plugin currently supports ten model info formats: | `"aiproxy"` | AIProxy's discovered `/models` entries plus models.dev | No | Maps inline AIProxy limits, pricing, and declared effort tiers; models.dev supplies general metadata | | `"bifrost"` | Fields in Bifrost's `/v1/models` response | No | Reads Bifrost inline limits, modalities, and base pricing when present | | `"litellm"` | Provider-specific model info endpoint | No | Uses `/v1/model/info` by default; set `modelInfoEndpoint` to override it | -| `"models.dev"` | `https://models.dev/models.json` | No | Uses the public models.dev metadata index | +| `"models.dev"` | `https://models.dev/models.json` | No | Uses the public models.dev metadata index; limits advertised inline by the provider take precedence | | `"vllm"` | Fields in the provider's `/v1/models` response | No | Reads vLLM-style `max_model_len` when present | | `"lmstudio"` | LM Studio 0.4.0+ `/api/v1/models` inventory | No | Uses `/api/v1/models` by default; set `modelInfoEndpoint` for another path | | `"llama-swap"` | Fields in llama-swap's `/v1/models` response | No | Reads inline context, modalities, and function-calling metadata when present | @@ -375,7 +375,7 @@ The plugin currently supports ten model info formats: ### AIProxy Metadata -Use `modelInfoFormat: "aiproxy"` when the configured model-list endpoint returns AIProxy's inline `limits` and `pricing` fields. The plugin makes no second request to AIProxy. It also fetches models.dev for general metadata and display names, then overlays valid AIProxy values. Explicit model configuration remains highest priority. +Use `modelInfoFormat: "aiproxy"` when the configured model-list endpoint returns AIProxy's inline `limits` and `pricing` fields. The plugin makes no second request to AIProxy. It also fetches models.dev for general metadata and display names, then overlays valid AIProxy values. Limits advertised inline in the model-list entry are kept as the base, so they win over models.dev before the AIProxy `limits` block is applied. Explicit model configuration remains highest priority. ```jsonc { @@ -690,6 +690,8 @@ When a discovered model can be matched to models.dev metadata, the plugin may po The current models.dev data uses a flat `provider/model`-keyed object. Its `limit` object is singular, and its `context`, `input`, and `output` fields are independently optional. `structured_output` and `temperature` may also be omitted; the plugin leaves omitted fields unset rather than inferring `false`. The current models.dev dataset does not provide `variants` metadata. +For gateways that expose inline limits on their `/v1/models` entries, those values take precedence over matching models.dev limits. The plugin reads `context_length` or `max_context_length` for the context window and `max_completion_tokens` or `max_output_tokens` for the output budget, then fills any remaining gap from models.dev. This keeps gateways that front several upstreams (for example a LiteLLM pool or a custom router) authoritative for what their own routes accept, while models.dev still supplies names and capabilities. A models.dev input limit is bounded by the effective context window. + Matching is intentionally conservative: - Exact model ids are preferred. diff --git a/src/core/model-types.ts b/src/core/model-types.ts index cfddfef8..f994e3a3 100644 --- a/src/core/model-types.ts +++ b/src/core/model-types.ts @@ -45,7 +45,7 @@ export interface NormalizedModelLimit { /** * Creates a normalized limit object. * - requires a positive context window - * - preserves valid explicit output limits, bounded by the context limit + * - preserves valid explicit output and input limits, bounded by the context limit * - falls back to Math.min(context, DEFAULT_OUTPUT_TOKEN_LIMIT) when output is missing or non-positive */ export function createModelLimits( @@ -58,11 +58,14 @@ export function createModelLimits( const output = typeof explicitOutput === 'number' && Number.isFinite(explicitOutput) && explicitOutput > 0 ? Math.min(explicitOutput, context) : Math.min(context, DEFAULT_OUTPUT_TOKEN_LIMIT) + const normalizedInput = typeof input === 'number' && Number.isFinite(input) && input > 0 + ? Math.min(input, context) + : undefined return { context, output, - ...(typeof input === 'number' && Number.isFinite(input) && input > 0 ? { input } : {}), + ...(normalizedInput !== undefined ? { input: normalizedInput } : {}), } } diff --git a/src/utils/model-info/aiproxy.ts b/src/utils/model-info/aiproxy.ts index a2fcd92e..b4ee03b1 100644 --- a/src/utils/model-info/aiproxy.ts +++ b/src/utils/model-info/aiproxy.ts @@ -1,4 +1,5 @@ import type { ModelEnricher, ModelEnrichmentResult } from '../../core/model-enrichment' +import { createModelLimits } from '../../core/model-types' import { createModelsDevEnricher } from './models-dev' const REASONING_EFFORTS = new Set(['none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max']) @@ -46,11 +47,11 @@ export function createAIProxyEnricher(data: unknown): ModelEnricher { const outputLimit = limits?.max_output_tokens const existingLimit = result.limit if (existingLimit && positiveNumber(existingLimit.context) && (positiveNumber(inputLimit) || positiveNumber(outputLimit))) { - result.limit = { - ...existingLimit, - ...(positiveNumber(inputLimit) ? { input: inputLimit } : {}), - ...(positiveNumber(outputLimit) ? { output: outputLimit } : {}), - } + result.limit = createModelLimits( + existingLimit.context, + positiveNumber(outputLimit) ? outputLimit : existingLimit.output, + positiveNumber(inputLimit) ? inputLimit : existingLimit.input, + ) } const pricing = object(model.pricing) diff --git a/src/utils/model-info/models-dev.ts b/src/utils/model-info/models-dev.ts index 2a1a1475..4e7b505f 100644 --- a/src/utils/model-info/models-dev.ts +++ b/src/utils/model-info/models-dev.ts @@ -1,19 +1,43 @@ import { lookupModelsDevData, type ModelsDevModel } from '../models-dev-fetcher' import type { ModelEnricher } from '../../core/model-enrichment' -import { createModelLimits } from '../../core/model-types' +import { createModelLimits, type DiscoveredRawModel } from '../../core/model-types' function hasUsableNumber(value: unknown): value is number { return typeof value === 'number' && Number.isFinite(value) && value > 0 } +function firstUsableNumber(...values: readonly unknown[]): number | undefined { + return values.find(hasUsableNumber) +} + +/** + * Reads token limits advertised inline by an OpenAI-compatible /v1/models entry. + * + * Gateways that front several upstreams (CLIProxyAPI, LiteLLM poolers, custom + * routers) advertise the limits they actually accept. Those values describe the + * live endpoint, so they take precedence over the static models.dev catalog, + * which can lag behind or overstate what a specific route enforces. + */ +function advertisedProviderLimits(model: DiscoveredRawModel): { context?: number; output?: number } | undefined { + const context = firstUsableNumber(model.context_length, model.max_context_length) + const output = firstUsableNumber(model.max_completion_tokens, model.max_output_tokens) + if (context === undefined && output === undefined) return undefined + return { + ...(context !== undefined ? { context } : {}), + ...(output !== undefined ? { output } : {}), + } +} + export function createModelsDevEnricher(data: unknown): ModelEnricher { const cache = data instanceof Map ? data as Map : new Map() return { enrich(model) { const info = lookupModelsDevData(model.id, cache) - const contextLimit = hasUsableNumber(info?.limit?.context) ? info?.limit?.context : info?.limit?.input + const advertised = advertisedProviderLimits(model) + const infoContext = hasUsableNumber(info?.limit?.context) ? info?.limit?.context : info?.limit?.input + const contextLimit = advertised?.context ?? infoContext const limits = hasUsableNumber(contextLimit) - ? createModelLimits(contextLimit, info?.limit?.output, info?.limit?.input) + ? createModelLimits(contextLimit, advertised?.output ?? info?.limit?.output, info?.limit?.input) : undefined return { metadataName: info?.name, diff --git a/test/aiproxy-model-info.test.ts b/test/aiproxy-model-info.test.ts index b98b3de6..2e27d030 100644 --- a/test/aiproxy-model-info.test.ts +++ b/test/aiproxy-model-info.test.ts @@ -60,4 +60,20 @@ describe('native AIProxy model enricher', () => { capabilities: { effort_tiers: ['unsupported'] }, }, { filterNonChat: true })).toEqual({ variants: {} }) }) + + it('keeps provider-advertised limits as the base that inline limits overlay', () => { + const enricher = createAIProxyEnricher(new Map([['gpt-6-luna', { + id: 'gpt-6-luna', + limit: { context: 1_050_000, output: 32_000 }, + }]])) + + const result = enricher.enrich({ + id: 'gpt-6-luna', + context_length: 272_000, + max_completion_tokens: 128_000, + limits: { max_input_tokens: 500_000, max_output_tokens: 64_000 }, + }, { filterNonChat: true }) + + expect(result.limit).toEqual({ context: 272_000, input: 272_000, output: 64_000 }) + }) }) diff --git a/test/core/model-types.test.ts b/test/core/model-types.test.ts index 585829ae..9f53d9b0 100644 --- a/test/core/model-types.test.ts +++ b/test/core/model-types.test.ts @@ -115,4 +115,9 @@ describe('Model limit normalization (createModelLimits)', () => { const limits = createModelLimits(32768, 4096, 28000) expect(limits).toEqual({ context: 32768, output: 4096, input: 28000 }) }) + + it('bounds explicit input limits so they do not exceed context', () => { + const limits = createModelLimits(32768, 4096, 64000) + expect(limits).toEqual({ context: 32768, output: 4096, input: 32768 }) + }) }) diff --git a/test/models-dev-model-info.test.ts b/test/models-dev-model-info.test.ts new file mode 100644 index 00000000..fa374705 --- /dev/null +++ b/test/models-dev-model-info.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from 'vitest' +import { createModelsDevEnricher } from '../src/utils/model-info/models-dev' + +function catalog(id: string, limit: { context?: number; input?: number; output?: number }, extra: Record = {}) { + return new Map([[id, { id, limit, ...extra }]]) +} + +describe('models.dev model enricher', () => { + it('prefers limits advertised by the provider over the models.dev catalog', () => { + const enricher = createModelsDevEnricher(catalog('deepseek/deepseek-v4.1-flash', { + context: 1_048_576, + output: 393_216, + })) + + const result = enricher.enrich({ + id: 'deepseek/deepseek-v4.1-flash', + context_length: 1_000_000, + max_completion_tokens: 384_000, + }, { filterNonChat: true }) + + expect(result.limit).toEqual({ context: 1_000_000, output: 384_000 }) + }) + + it('accepts max_context_length and max_output_tokens aliases', () => { + const enricher = createModelsDevEnricher(catalog('custom-model', { + context: 1_000_000, + output: 384_000, + })) + + const result = enricher.enrich({ + id: 'custom-model', + max_context_length: 262_144, + max_output_tokens: 65_536, + }, { filterNonChat: true }) + + expect(result.limit).toEqual({ context: 262_144, output: 65_536 }) + }) + + it('fills missing provider fields and bounds catalog input to provider context', () => { + const enricher = createModelsDevEnricher(catalog('custom-model', { + context: 1_000_000, + input: 900_000, + output: 32_000, + })) + + const result = enricher.enrich({ + id: 'custom-model', + context_length: 200_000, + }, { filterNonChat: true }) + + expect(result.limit).toEqual({ context: 200_000, input: 200_000, output: 32_000 }) + }) + + it('keeps the models.dev context when the provider advertises output only', () => { + const enricher = createModelsDevEnricher(catalog('custom-model', { + context: 1_000_000, + output: 393_216, + })) + + const result = enricher.enrich({ + id: 'custom-model', + max_completion_tokens: 128_000, + }, { filterNonChat: true }) + + expect(result.limit).toEqual({ context: 1_000_000, output: 128_000 }) + }) + + it('ignores malformed provider limits', () => { + const enricher = createModelsDevEnricher(catalog('custom-model', { + context: 128_000, + output: 16_384, + })) + + const result = enricher.enrich({ + id: 'custom-model', + context_length: '128000', + max_context_length: 0, + max_completion_tokens: Number.POSITIVE_INFINITY, + max_output_tokens: -1, + }, { filterNonChat: true }) + + expect(result.limit).toEqual({ context: 128_000, output: 16_384 }) + }) + + it('uses advertised context for models missing from the models.dev catalog', () => { + const enricher = createModelsDevEnricher(new Map()) + + const result = enricher.enrich({ + id: 'gateway-only-model', + context_length: 64_000, + }, { filterNonChat: true }) + + expect(result.limit).toEqual({ context: 64_000, output: 32_000 }) + }) + + it('clamps advertised output to the advertised context', () => { + const enricher = createModelsDevEnricher(new Map()) + + const result = enricher.enrich({ + id: 'gateway-only-model', + context_length: 64_000, + max_completion_tokens: 128_000, + }, { filterNonChat: true }) + + expect(result.limit).toEqual({ context: 64_000, output: 64_000 }) + }) +})