From 5f9be7a17e69f1195288cb060e7b34b45d8672af Mon Sep 17 00:00:00 2001 From: Ravi Tharuma Date: Mon, 20 Jul 2026 14:25:12 +0200 Subject: [PATCH 1/2] fix: isolate combo cache identities --- src/omniroute-combos.ts | 41 +++++++++++----- test/omniroute-combos.test.mjs | 87 ++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 13 deletions(-) create mode 100644 test/omniroute-combos.test.mjs diff --git a/src/omniroute-combos.ts b/src/omniroute-combos.ts index 95e5bab..1269d27 100644 --- a/src/omniroute-combos.ts +++ b/src/omniroute-combos.ts @@ -1,3 +1,5 @@ +import { createHash } from 'node:crypto'; + import type { OmniRouteConfig, OmniRouteModel, OmniRouteModelMetadata } from './types.js'; import type { ModelsDevIndex, ModelsDevModel } from './models-dev.js'; import { @@ -46,10 +48,16 @@ interface ComboCache { timestamp: number; } -// In-memory cache for combo data -let comboCache: ComboCache | null = null; +// Cache entries are isolated by endpoint and a non-reversible credential digest. +const comboCaches = new Map(); const COMBO_CACHE_TTL = 5 * 60 * 1000; // 5 minutes +function getComboCacheKey(baseUrl: string, apiKey: string): string { + const endpoint = `${baseUrl.replace(/\/v1\/?$/, '').replace(/\/$/, '')}/api/combos`; + const credentialDigest = createHash('sha256').update(apiKey).digest('hex'); + return `${endpoint}\0${credentialDigest}`; +} + /** * Fetch combo data from OmniRoute /api/combos endpoint */ @@ -59,10 +67,18 @@ export async function fetchComboData( const baseUrl = config.baseUrl; const apiKey = config.apiKey; - // Check cache first - if (comboCache && Date.now() - comboCache.timestamp < COMBO_CACHE_TTL) { + if (!baseUrl || !apiKey) { + warn('Cannot fetch combo data without baseUrl and apiKey'); + return null; + } + + const cacheKey = getComboCacheKey(baseUrl, apiKey); + + // Check the cache for this endpoint and credential identity. + const cached = comboCaches.get(cacheKey); + if (cached && Date.now() - cached.timestamp < COMBO_CACHE_TTL) { debug('Using cached combo data'); - return comboCache.combos; + return cached.combos; } const combosUrl = `${baseUrl.replace(/\/v1\/?$/, '').replace(/\/$/, '')}/api/combos`; @@ -102,11 +118,11 @@ export async function fetchComboData( } } - // Update cache - comboCache = { + // Update only this endpoint/credential cache entry. + comboCaches.set(cacheKey, { combos: comboMap, timestamp: Date.now(), - }; + }); debug(`Successfully fetched ${comboMap.size} combos`); return comboMap; @@ -122,7 +138,7 @@ export async function fetchComboData( * Clear the combo cache */ export function clearComboCache(): void { - comboCache = null; + comboCaches.clear(); debug('Combo cache cleared'); } @@ -327,11 +343,10 @@ export function isComboModel(model: OmniRouteModel): boolean { return true; } - // Fallback: check if it's in our combo cache - if (comboCache?.combos?.has(model.id)) { - return true; + // Fallback: check all endpoint/credential-specific combo caches. + for (const cached of comboCaches.values()) { + if (cached.combos.has(model.id)) return true; } - return false; } diff --git a/test/omniroute-combos.test.mjs b/test/omniroute-combos.test.mjs new file mode 100644 index 0000000..314b5a8 --- /dev/null +++ b/test/omniroute-combos.test.mjs @@ -0,0 +1,87 @@ +import { afterEach, test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { clearComboCache, fetchComboData } from '../dist/runtime.js'; + +const ORIGINAL_FETCH = global.fetch; + +function config(baseUrl, apiKey) { + return { + baseUrl, + apiKey, + apiMode: 'chat', + modelCacheTtl: 60_000, + }; +} + +afterEach(() => { + clearComboCache(); + global.fetch = ORIGINAL_FETCH; +}); + +test('combo cache is isolated by endpoint and API key', async () => { + const calls = []; + global.fetch = async (input, init = {}) => { + const url = input instanceof Request ? input.url : input.toString(); + const authorization = new Headers(init.headers).get('authorization'); + calls.push({ url, authorization }); + const suffix = url.includes('a.example') && authorization === 'Bearer key-a' ? 'a' : 'b'; + return new Response(JSON.stringify({ + combos: [{ + id: `combo-${suffix}`, + name: `combo-${suffix}`, + models: [`provider/model-${suffix}`], + strategy: 'priority', + config: {}, + createdAt: '2026-01-01T00:00:00Z', + updatedAt: '2026-01-01T00:00:00Z', + }], + }), { status: 200, headers: { 'content-type': 'application/json' } }); + }; + + const a = await fetchComboData(config('https://a.example/v1', 'key-a')); + const b = await fetchComboData(config('https://b.example/v1', 'key-b')); + const aAgain = await fetchComboData(config('https://a.example/v1', 'key-a')); + + assert.deepEqual([...a.keys()], ['combo-a']); + assert.deepEqual([...b.keys()], ['combo-b']); + assert.deepEqual([...aAgain.keys()], ['combo-a']); + assert.equal(calls.length, 2, 'each cache identity should fetch once'); +}); + +test('combo cache is isolated when credentials change for the same endpoint', async () => { + let calls = 0; + global.fetch = async (_input, init = {}) => { + calls += 1; + const authorization = new Headers(init.headers).get('authorization'); + const suffix = authorization === 'Bearer key-a' ? 'a' : 'b'; + return new Response(JSON.stringify({ + combos: [{ + id: `combo-${suffix}`, + name: `combo-${suffix}`, + models: [], + strategy: 'priority', + config: {}, + createdAt: '2026-01-01T00:00:00Z', + updatedAt: '2026-01-01T00:00:00Z', + }], + }), { status: 200, headers: { 'content-type': 'application/json' } }); + }; + + const first = await fetchComboData(config('https://same.example/v1', 'key-a')); + const second = await fetchComboData(config('https://same.example/v1', 'key-b')); + + assert.deepEqual([...first.keys()], ['combo-a']); + assert.deepEqual([...second.keys()], ['combo-b']); + assert.equal(calls, 2); +}); + + +test('fetchComboData returns null when endpoint or credential is missing', async () => { + global.fetch = async () => { + throw new Error('fetch must not be called'); + }; + + assert.equal(await fetchComboData(config('', 'key-a')), null); + assert.equal(await fetchComboData(config('https://a.example/v1', '')), null); +}); From ac93d5187aea8d664ed019e1df970e6e00a6b493 Mon Sep 17 00:00:00 2001 From: Ravi Tharuma Date: Tue, 21 Jul 2026 18:51:53 +0200 Subject: [PATCH 2/2] fix: prune stale combo caches and clear selectively Address review feedback on the multi-entry combo cache: - prune expired endpoint/credential entries before cache writes - allow clearComboCache(config) to target one identity - keep full clear when no config is provided - route clearModelCache through the selective combo clear - add regression coverage for selective clear and TTL prune --- src/models.ts | 4 +- src/omniroute-combos.ts | 24 ++++++++-- test/omniroute-combos.test.mjs | 83 ++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 6 deletions(-) diff --git a/src/models.ts b/src/models.ts index 71f6fac..a56870c 100644 --- a/src/models.ts +++ b/src/models.ts @@ -382,8 +382,8 @@ export function clearModelCache(config?: OmniRouteConfig, apiKey?: string): void modelCache.clear(); debug('All model caches cleared'); } - // Also clear combo cache - clearComboCache(); + // Also clear matching combo cache entry (or all when no config provided) + clearComboCache(config && apiKey ? { baseUrl: config.baseUrl, apiKey } : undefined); } /** diff --git a/src/omniroute-combos.ts b/src/omniroute-combos.ts index 1269d27..c7af6d7 100644 --- a/src/omniroute-combos.ts +++ b/src/omniroute-combos.ts @@ -52,6 +52,14 @@ interface ComboCache { const comboCaches = new Map(); const COMBO_CACHE_TTL = 5 * 60 * 1000; // 5 minutes +function pruneExpiredComboCaches(now = Date.now()): void { + for (const [key, cached] of comboCaches) { + if (now - cached.timestamp >= COMBO_CACHE_TTL) { + comboCaches.delete(key); + } + } +} + function getComboCacheKey(baseUrl: string, apiKey: string): string { const endpoint = `${baseUrl.replace(/\/v1\/?$/, '').replace(/\/$/, '')}/api/combos`; const credentialDigest = createHash('sha256').update(apiKey).digest('hex'); @@ -119,6 +127,7 @@ export async function fetchComboData( } // Update only this endpoint/credential cache entry. + pruneExpiredComboCaches(); comboCaches.set(cacheKey, { combos: comboMap, timestamp: Date.now(), @@ -135,11 +144,18 @@ export async function fetchComboData( } /** - * Clear the combo cache + * Clear combo cache entries. + * When config is provided, only that endpoint/credential identity is cleared. + * Without config, the entire map is cleared (legacy behavior). */ -export function clearComboCache(): void { - comboCaches.clear(); - debug('Combo cache cleared'); +export function clearComboCache(config?: Pick): void { + if (!config?.baseUrl || !config.apiKey) { + comboCaches.clear(); + debug('All combo caches cleared'); + return; + } + comboCaches.delete(getComboCacheKey(config.baseUrl, config.apiKey)); + debug('Combo cache cleared for provided configuration'); } /** diff --git a/test/omniroute-combos.test.mjs b/test/omniroute-combos.test.mjs index 314b5a8..4565ab9 100644 --- a/test/omniroute-combos.test.mjs +++ b/test/omniroute-combos.test.mjs @@ -85,3 +85,86 @@ test('fetchComboData returns null when endpoint or credential is missing', async assert.equal(await fetchComboData(config('', 'key-a')), null); assert.equal(await fetchComboData(config('https://a.example/v1', '')), null); }); + +test('clearComboCache can target one endpoint/credential pair', async () => { + clearComboCache(); + const first = config('https://a.example/v1', 'key-a'); + const second = config('https://b.example/v1', 'key-b'); + let calls = 0; + global.fetch = async (url) => { + calls += 1; + const host = String(url); + return { + ok: true, + async json() { + return { + combos: [ + { + id: host.includes('a.example') ? 'combo-a' : 'combo-b', + name: host.includes('a.example') ? 'combo-a' : 'combo-b', + models: ['model-1'], + strategy: 'priority', + config: {}, + createdAt: '2026-01-01T00:00:00Z', + updatedAt: '2026-01-01T00:00:00Z', + }, + ], + }; + }, + }; + }; + + assert.ok(await fetchComboData(first)); + assert.ok(await fetchComboData(second)); + assert.equal(calls, 2); + + clearComboCache(first); + assert.ok(await fetchComboData(first)); + assert.equal(calls, 3, 'first identity should refetch after selective clear'); + assert.ok(await fetchComboData(second)); + assert.equal(calls, 3, 'second identity should remain cached'); +}); + +test('expired combo cache entries are pruned on write', async () => { + clearComboCache(); + const first = config('https://a.example/v1', 'key-a'); + const second = config('https://b.example/v1', 'key-b'); + let now = 1_000_000; + const realNow = Date.now; + Date.now = () => now; + let calls = 0; + global.fetch = async (url) => { + calls += 1; + const host = String(url); + return { + ok: true, + async json() { + return { + combos: [ + { + id: host.includes('a.example') ? 'combo-a' : 'combo-b', + name: host.includes('a.example') ? 'combo-a' : 'combo-b', + models: ['model-1'], + strategy: 'priority', + config: {}, + createdAt: '2026-01-01T00:00:00Z', + updatedAt: '2026-01-01T00:00:00Z', + }, + ], + }; + }, + }; + }; + + try { + assert.ok(await fetchComboData(first)); + now += 6 * 60 * 1000; // beyond TTL + assert.ok(await fetchComboData(second)); + // first is expired and should have been pruned on the second write; refetch first + assert.ok(await fetchComboData(first)); + assert.equal(calls, 3); + } finally { + Date.now = realNow; + } +}); +