Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -363,7 +363,7 @@ The plugin currently supports ten model info formats:
| `"aiproxy"` | AIProxy's discovered `/models` entries plus models.dev | No | Maps inline AIProxy limits, pricing, and declared effort tiers; models.dev supplies general metadata |
| `"bifrost"` | Fields in Bifrost's `/v1/models` response | No | Reads Bifrost inline limits, modalities, and base pricing when present |
| `"litellm"` | Provider-specific model info endpoint | No | Uses `/v1/model/info` by default; set `modelInfoEndpoint` to override it |
| `"models.dev"` | `https://models.dev/models.json` | No | Uses the public models.dev metadata index |
| `"models.dev"` | `https://models.dev/models.json` | No | Uses the public models.dev metadata index; limits advertised inline by the provider take precedence |
| `"vllm"` | Fields in the provider's `/v1/models` response | No | Reads vLLM-style `max_model_len` when present |
| `"lmstudio"` | LM Studio 0.4.0+ `/api/v1/models` inventory | No | Uses `/api/v1/models` by default; set `modelInfoEndpoint` for another path |
| `"llama-swap"` | Fields in llama-swap's `/v1/models` response | No | Reads inline context, modalities, and function-calling metadata when present |
Expand All @@ -375,7 +375,7 @@ The plugin currently supports ten model info formats:

### AIProxy Metadata

Use `modelInfoFormat: "aiproxy"` when the configured model-list endpoint returns AIProxy's inline `limits` and `pricing` fields. The plugin makes no second request to AIProxy. It also fetches models.dev for general metadata and display names, then overlays valid AIProxy values. Explicit model configuration remains highest priority.
Use `modelInfoFormat: "aiproxy"` when the configured model-list endpoint returns AIProxy's inline `limits` and `pricing` fields. The plugin makes no second request to AIProxy. It also fetches models.dev for general metadata and display names, then overlays valid AIProxy values. Limits advertised inline in the model-list entry are kept as the base, so they win over models.dev before the AIProxy `limits` block is applied. Explicit model configuration remains highest priority.

```jsonc
{
Expand Down Expand Up @@ -690,6 +690,8 @@ When a discovered model can be matched to models.dev metadata, the plugin may po

The current models.dev data uses a flat `provider/model`-keyed object. Its `limit` object is singular, and its `context`, `input`, and `output` fields are independently optional. `structured_output` and `temperature` may also be omitted; the plugin leaves omitted fields unset rather than inferring `false`. The current models.dev dataset does not provide `variants` metadata.

For gateways that expose inline limits on their `/v1/models` entries, those values take precedence over matching models.dev limits. The plugin reads `context_length` or `max_context_length` for the context window and `max_completion_tokens` or `max_output_tokens` for the output budget, then fills any remaining gap from models.dev. This keeps gateways that front several upstreams (for example a LiteLLM pool or a custom router) authoritative for what their own routes accept, while models.dev still supplies names and capabilities. A models.dev input limit is bounded by the effective context window.

Matching is intentionally conservative:

- Exact model ids are preferred.
Expand Down
7 changes: 5 additions & 2 deletions src/core/model-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ export interface NormalizedModelLimit {
/**
* Creates a normalized limit object.
* - requires a positive context window
* - preserves valid explicit output limits, bounded by the context limit
* - preserves valid explicit output and input limits, bounded by the context limit
* - falls back to Math.min(context, DEFAULT_OUTPUT_TOKEN_LIMIT) when output is missing or non-positive
*/
export function createModelLimits(
Expand All @@ -58,11 +58,14 @@ export function createModelLimits(
const output = typeof explicitOutput === 'number' && Number.isFinite(explicitOutput) && explicitOutput > 0
? Math.min(explicitOutput, context)
: Math.min(context, DEFAULT_OUTPUT_TOKEN_LIMIT)
const normalizedInput = typeof input === 'number' && Number.isFinite(input) && input > 0
? Math.min(input, context)
: undefined

return {
context,
output,
...(typeof input === 'number' && Number.isFinite(input) && input > 0 ? { input } : {}),
...(normalizedInput !== undefined ? { input: normalizedInput } : {}),
}
}

Expand Down
11 changes: 6 additions & 5 deletions src/utils/model-info/aiproxy.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import type { ModelEnricher, ModelEnrichmentResult } from '../../core/model-enrichment'
import { createModelLimits } from '../../core/model-types'
import { createModelsDevEnricher } from './models-dev'

const REASONING_EFFORTS = new Set(['none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max'])
Expand Down Expand Up @@ -46,11 +47,11 @@ export function createAIProxyEnricher(data: unknown): ModelEnricher {
const outputLimit = limits?.max_output_tokens
const existingLimit = result.limit
if (existingLimit && positiveNumber(existingLimit.context) && (positiveNumber(inputLimit) || positiveNumber(outputLimit))) {
result.limit = {
...existingLimit,
...(positiveNumber(inputLimit) ? { input: inputLimit } : {}),
...(positiveNumber(outputLimit) ? { output: outputLimit } : {}),
}
result.limit = createModelLimits(
existingLimit.context,
positiveNumber(outputLimit) ? outputLimit : existingLimit.output,
positiveNumber(inputLimit) ? inputLimit : existingLimit.input,
)
}

const pricing = object(model.pricing)
Expand Down
30 changes: 27 additions & 3 deletions src/utils/model-info/models-dev.ts
Original file line number Diff line number Diff line change
@@ -1,19 +1,43 @@
import { lookupModelsDevData, type ModelsDevModel } from '../models-dev-fetcher'
import type { ModelEnricher } from '../../core/model-enrichment'
import { createModelLimits } from '../../core/model-types'
import { createModelLimits, type DiscoveredRawModel } from '../../core/model-types'

function hasUsableNumber(value: unknown): value is number {
return typeof value === 'number' && Number.isFinite(value) && value > 0
}

function firstUsableNumber(...values: readonly unknown[]): number | undefined {
return values.find(hasUsableNumber)
}

/**
* Reads token limits advertised inline by an OpenAI-compatible /v1/models entry.
*
* Gateways that front several upstreams (CLIProxyAPI, LiteLLM poolers, custom
* routers) advertise the limits they actually accept. Those values describe the
* live endpoint, so they take precedence over the static models.dev catalog,
* which can lag behind or overstate what a specific route enforces.
*/
function advertisedProviderLimits(model: DiscoveredRawModel): { context?: number; output?: number } | undefined {
const context = firstUsableNumber(model.context_length, model.max_context_length)
const output = firstUsableNumber(model.max_completion_tokens, model.max_output_tokens)
if (context === undefined && output === undefined) return undefined
return {
...(context !== undefined ? { context } : {}),
...(output !== undefined ? { output } : {}),
}
}

export function createModelsDevEnricher(data: unknown): ModelEnricher {
const cache = data instanceof Map ? data as Map<string, ModelsDevModel> : new Map<string, ModelsDevModel>()
return {
enrich(model) {
const info = lookupModelsDevData(model.id, cache)
const contextLimit = hasUsableNumber(info?.limit?.context) ? info?.limit?.context : info?.limit?.input
const advertised = advertisedProviderLimits(model)
const infoContext = hasUsableNumber(info?.limit?.context) ? info?.limit?.context : info?.limit?.input
const contextLimit = advertised?.context ?? infoContext
const limits = hasUsableNumber(contextLimit)
? createModelLimits(contextLimit, info?.limit?.output, info?.limit?.input)
? createModelLimits(contextLimit, advertised?.output ?? info?.limit?.output, info?.limit?.input)
: undefined
return {
metadataName: info?.name,
Expand Down
16 changes: 16 additions & 0 deletions test/aiproxy-model-info.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,4 +60,20 @@ describe('native AIProxy model enricher', () => {
capabilities: { effort_tiers: ['unsupported'] },
}, { filterNonChat: true })).toEqual({ variants: {} })
})

it('keeps provider-advertised limits as the base that inline limits overlay', () => {
const enricher = createAIProxyEnricher(new Map([['gpt-6-luna', {
id: 'gpt-6-luna',
limit: { context: 1_050_000, output: 32_000 },
}]]))

const result = enricher.enrich({
id: 'gpt-6-luna',
context_length: 272_000,
max_completion_tokens: 128_000,
limits: { max_input_tokens: 500_000, max_output_tokens: 64_000 },
}, { filterNonChat: true })

expect(result.limit).toEqual({ context: 272_000, input: 272_000, output: 64_000 })
})
})
5 changes: 5 additions & 0 deletions test/core/model-types.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -115,4 +115,9 @@ describe('Model limit normalization (createModelLimits)', () => {
const limits = createModelLimits(32768, 4096, 28000)
expect(limits).toEqual({ context: 32768, output: 4096, input: 28000 })
})

it('bounds explicit input limits so they do not exceed context', () => {
const limits = createModelLimits(32768, 4096, 64000)
expect(limits).toEqual({ context: 32768, output: 4096, input: 32768 })
})
})
107 changes: 107 additions & 0 deletions test/models-dev-model-info.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
import { describe, expect, it } from 'vitest'
import { createModelsDevEnricher } from '../src/utils/model-info/models-dev'

function catalog(id: string, limit: { context?: number; input?: number; output?: number }, extra: Record<string, unknown> = {}) {
return new Map([[id, { id, limit, ...extra }]])
}

describe('models.dev model enricher', () => {
it('prefers limits advertised by the provider over the models.dev catalog', () => {
const enricher = createModelsDevEnricher(catalog('deepseek/deepseek-v4.1-flash', {
context: 1_048_576,
output: 393_216,
}))

const result = enricher.enrich({
id: 'deepseek/deepseek-v4.1-flash',
context_length: 1_000_000,
max_completion_tokens: 384_000,
}, { filterNonChat: true })

expect(result.limit).toEqual({ context: 1_000_000, output: 384_000 })
})

it('accepts max_context_length and max_output_tokens aliases', () => {
const enricher = createModelsDevEnricher(catalog('custom-model', {
context: 1_000_000,
output: 384_000,
}))

const result = enricher.enrich({
id: 'custom-model',
max_context_length: 262_144,
max_output_tokens: 65_536,
}, { filterNonChat: true })

expect(result.limit).toEqual({ context: 262_144, output: 65_536 })
})

it('fills missing provider fields and bounds catalog input to provider context', () => {
const enricher = createModelsDevEnricher(catalog('custom-model', {
context: 1_000_000,
input: 900_000,
output: 32_000,
}))

const result = enricher.enrich({
id: 'custom-model',
context_length: 200_000,
}, { filterNonChat: true })

expect(result.limit).toEqual({ context: 200_000, input: 200_000, output: 32_000 })
})

it('keeps the models.dev context when the provider advertises output only', () => {
const enricher = createModelsDevEnricher(catalog('custom-model', {
context: 1_000_000,
output: 393_216,
}))

const result = enricher.enrich({
id: 'custom-model',
max_completion_tokens: 128_000,
}, { filterNonChat: true })

expect(result.limit).toEqual({ context: 1_000_000, output: 128_000 })
})

it('ignores malformed provider limits', () => {
const enricher = createModelsDevEnricher(catalog('custom-model', {
context: 128_000,
output: 16_384,
}))

const result = enricher.enrich({
id: 'custom-model',
context_length: '128000',
max_context_length: 0,
max_completion_tokens: Number.POSITIVE_INFINITY,
max_output_tokens: -1,
}, { filterNonChat: true })

expect(result.limit).toEqual({ context: 128_000, output: 16_384 })
})

it('uses advertised context for models missing from the models.dev catalog', () => {
const enricher = createModelsDevEnricher(new Map())

const result = enricher.enrich({
id: 'gateway-only-model',
context_length: 64_000,
}, { filterNonChat: true })

expect(result.limit).toEqual({ context: 64_000, output: 32_000 })
})

it('clamps advertised output to the advertised context', () => {
const enricher = createModelsDevEnricher(new Map())

const result = enricher.enrich({
id: 'gateway-only-model',
context_length: 64_000,
max_completion_tokens: 128_000,
}, { filterNonChat: true })

expect(result.limit).toEqual({ context: 64_000, output: 64_000 })
})
})
Loading