From 7cdb9538dc6e8d19559644a57410f7f65edafeca Mon Sep 17 00:00:00 2001 From: Vibhanshu Rana <68025665+vrana055@users.noreply.github.com> Date: Sun, 21 Jun 2026 18:31:07 +0530 Subject: [PATCH 1/3] feat: expand OpenRouter free model fallback chain to 15 models Add 5 new free models (gpt-oss-120b, qwen3-coder, nex-n2-pro, gpt-oss-20b, nemotron-omni-30b-reasoning) verified against the OpenRouter API. Inserted at high-priority slots so larger/stronger models are tried before smaller fallbacks. Co-Authored-By: Claude Sonnet 4.6 --- apps/api/app/config.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/api/app/config.py b/apps/api/app/config.py index f1a229d..e0c2f6b 100644 --- a/apps/api/app/config.py +++ b/apps/api/app/config.py @@ -34,12 +34,17 @@ class Settings(BaseSettings): # or unavailable. The primary generation_model is prepended automatically. generation_models: str = ( "qwen/qwen3-next-80b-a3b-instruct:free," + "openai/gpt-oss-120b:free," + "qwen/qwen3-coder:free," + "nex-agi/nex-n2-pro:free," "google/gemma-4-31b-it:free," "google/gemma-4-26b-a4b-it:free," "meta-llama/llama-3.3-70b-instruct:free," "nvidia/nemotron-3-super-120b-a12b:free," "nvidia/nemotron-3-ultra-550b-a55b:free," "nousresearch/hermes-3-llama-3.1-405b:free," + "openai/gpt-oss-20b:free," + "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free," "nvidia/nemotron-3-nano-30b-a3b:free," "nvidia/nemotron-nano-9b-v2:free," "meta-llama/llama-3.2-3b-instruct:free" From 0abba29ea70fcdef2a1d8c19445fafe85c0d49db Mon Sep 17 00:00:00 2001 From: Vibhanshu Rana <68025665+vrana055@users.noreply.github.com> Date: Sun, 21 Jun 2026 18:38:04 +0530 Subject: [PATCH 2/3] feat: multi-provider LLM support with Groq free-tier Refactor llm.py to route each model to its own OpenAI-compatible client via "provider>model_id" format in the chain. No prefix = openrouter (backward compat). Add Groq as second provider with 3 free models (llama-3.3-70b-versatile, qwen3.6-27b, llama-3.1-8b-instant). Adding a future provider requires: one dict entry in _provider_registry(), two env vars in config.py, and prefixed model IDs in GENERATION_MODELS. Co-Authored-By: Claude Sonnet 4.6 --- .env.example | 3 + apps/api/app/config.py | 17 +++- apps/api/app/services/generation.py | 2 +- apps/api/app/services/llm.py | 116 +++++++++++++++++++--------- 4 files changed, 99 insertions(+), 39 deletions(-) diff --git a/.env.example b/.env.example index edaf54e..f17e170 100644 --- a/.env.example +++ b/.env.example @@ -20,6 +20,9 @@ OPENAI_API_KEY=your-openai-api-key # OpenRouter (free-tier generation + vision) — get key at https://openrouter.ai/keys OPENROUTER_API_KEY=your-openrouter-api-key OPENROUTER_BASE_URL=https://openrouter.ai/api/v1 + +# Groq (free-tier, fast inference) — get key at https://console.groq.com/keys +GROQ_API_KEY=your-groq-api-key GENERATION_MODEL=qwen/qwen3-next-80b-a3b-instruct:free VISION_MODEL=meta-llama/llama-3.2-11b-vision-instruct:free diff --git a/apps/api/app/config.py b/apps/api/app/config.py index e0c2f6b..e2d1cbb 100644 --- a/apps/api/app/config.py +++ b/apps/api/app/config.py @@ -29,14 +29,24 @@ class Settings(BaseSettings): # OpenRouter (free-tier generation + vision) openrouter_api_key: str = "" openrouter_base_url: str = "https://openrouter.ai/api/v1" + + # Groq (free-tier, fast inference) — get key at https://console.groq.com/keys + groq_api_key: str = "" + groq_base_url: str = "https://api.groq.com/openai/v1" + generation_model: str = "qwen/qwen3-next-80b-a3b-instruct:free" - # Fallback chain (comma-separated): tried in order when a model is rate-limited - # or unavailable. The primary generation_model is prepended automatically. + # Fallback chain (comma-separated, "provider>model_id" format). + # No prefix = openrouter. Tried in order on 429/404/502/503. generation_models: str = ( + # OpenRouter — large free models "qwen/qwen3-next-80b-a3b-instruct:free," "openai/gpt-oss-120b:free," "qwen/qwen3-coder:free," "nex-agi/nex-n2-pro:free," + # Groq — fast free inference + "groq>llama-3.3-70b-versatile," + "groq>qwen/qwen3.6-27b," + # OpenRouter — mid-size free models "google/gemma-4-31b-it:free," "google/gemma-4-26b-a4b-it:free," "meta-llama/llama-3.3-70b-instruct:free," @@ -46,6 +56,9 @@ class Settings(BaseSettings): "openai/gpt-oss-20b:free," "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free," "nvidia/nemotron-3-nano-30b-a3b:free," + # Groq — small fast fallback + "groq>llama-3.1-8b-instant," + # OpenRouter — small fallbacks "nvidia/nemotron-nano-9b-v2:free," "meta-llama/llama-3.2-3b-instruct:free" ) diff --git a/apps/api/app/services/generation.py b/apps/api/app/services/generation.py index 6fe885d..e20ed39 100644 --- a/apps/api/app/services/generation.py +++ b/apps/api/app/services/generation.py @@ -39,7 +39,7 @@ def citations_from(chunks: list[dict]) -> list[dict]: def generation_available() -> bool: - return bool(settings.openrouter_api_key or settings.openai_api_key) + return bool(settings.openrouter_api_key or settings.openai_api_key or settings.groq_api_key) def generate_answer(question: str, chunks: list[dict], history: list[dict] | None = None) -> str: diff --git a/apps/api/app/services/llm.py b/apps/api/app/services/llm.py index 63c3aae..76ca403 100644 --- a/apps/api/app/services/llm.py +++ b/apps/api/app/services/llm.py @@ -1,44 +1,84 @@ -"""LLM client — OpenRouter (OpenAI-compatible) with a free-model fallback chain. +"""LLM client — multi-provider (OpenAI-compatible) with a free-model fallback chain. -OpenRouter free (":free") models are frequently rate-limited upstream (429) or -temporarily unavailable (404). We try a list of free models in order and move to -the next on those errors, so a grounded answer still comes back. +Model IDs use the format "provider>model_id" e.g. "groq>llama-3.3-70b-versatile". +No prefix defaults to "openrouter" for backward compatibility. + +Adding a new provider: + 1. Add its api_key / base_url fields to config.py. + 2. Add one entry to _PROVIDER_REGISTRY in this file. + 3. Prefix models in GENERATION_MODELS with "newprovider>". + +Free ":free" OpenRouter models and Groq free-tier models are frequently +rate-limited (429) or temporarily unavailable (404/502/503). We try each +(provider, model) pair in order and fall back on those errors. """ import structlog -from functools import lru_cache - from openai import APIStatusError, OpenAI from app.config import settings log = structlog.get_logger() -# Errors that mean "this model is busy/unavailable — try the next one". _FALLBACK_STATUS = {404, 429, 502, 503} +# -- Provider registry -------------------------------------------------------- +# One entry per provider. To add a new one: extend this dict + add env vars. + +def _provider_registry() -> dict[str, dict]: + return { + "openrouter": { + "api_key": settings.openrouter_api_key or settings.openai_api_key, + "base_url": settings.openrouter_base_url, + }, + "groq": { + "api_key": settings.groq_api_key, + "base_url": settings.groq_base_url, + }, + } + -@lru_cache(maxsize=1) -def get_client() -> OpenAI: - # max_retries=0: our models_chain() handles rate limits by switching models. - # The SDK's default 2 retries with backoff would stall ~50s on a 429 before - # we ever reach the next model. - return OpenAI( - api_key=settings.openrouter_api_key or settings.openai_api_key, - base_url=settings.openrouter_base_url or None, - max_retries=0, - ) +_clients: dict[str, OpenAI] = {} -def models_chain() -> list[str]: - """Ordered, de-duplicated model list: primary first, then configured fallbacks.""" - chain = [settings.generation_model] + [ +def _client_for(provider: str) -> OpenAI: + if provider not in _clients: + registry = _provider_registry() + if provider not in registry: + raise ValueError( + f"Unknown LLM provider {provider!r}. Add it to _provider_registry() in llm.py." + ) + cfg = registry[provider] + # max_retries=0: fallback chain switches providers on 429 instead of waiting. + _clients[provider] = OpenAI( + api_key=cfg["api_key"] or "sk-no-key", + base_url=cfg["base_url"] or None, + max_retries=0, + ) + return _clients[provider] + + +# -- Model chain -------------------------------------------------------------- + +def _parse_model(s: str) -> tuple[str, str]: + """'groq>llama-3.3-70b-versatile' → ('groq', 'llama-3.3-70b-versatile'). + No prefix → ('openrouter', s) for backward compatibility.""" + if ">" in s: + provider, model_id = s.split(">", 1) + return provider.strip(), model_id.strip() + return "openrouter", s + + +def models_chain() -> list[tuple[str, str]]: + """Ordered, de-duplicated list of (provider, model_id) pairs.""" + raw = [settings.generation_model] + [ m.strip() for m in settings.generation_models.split(",") if m.strip() ] - seen, out = set(), [] - for m in chain: + seen: set[str] = set() + out: list[tuple[str, str]] = [] + for m in raw: if m and m not in seen: seen.add(m) - out.append(m) + out.append(_parse_model(m)) return out @@ -46,20 +86,24 @@ def _should_fallback(err: Exception) -> bool: return isinstance(err, APIStatusError) and err.status_code in _FALLBACK_STATUS +# -- Public API --------------------------------------------------------------- + def chat(messages: list[dict], **kwargs) -> str: - """Non-streaming completion. Tries each model in the chain; raises last error if all fail.""" + """Non-streaming completion. Tries each (provider, model) in chain; raises last error if all fail.""" last: Exception | None = None - for model in models_chain(): + for provider, model in models_chain(): try: - log.info("llm.try", model=model, stream=False) - resp = get_client().chat.completions.create(model=model, messages=messages, **kwargs) + log.info("llm.try", provider=provider, model=model, stream=False) + resp = _client_for(provider).chat.completions.create( + model=model, messages=messages, **kwargs + ) usage = getattr(resp, "usage", None) - log.info("llm.ok", model=model, tokens=getattr(usage, "total_tokens", None)) + log.info("llm.ok", provider=provider, model=model, tokens=getattr(usage, "total_tokens", None)) return resp.choices[0].message.content or "" - except Exception as e: # noqa: BLE001 — decide fallback by status + except Exception as e: # noqa: BLE001 last = e if _should_fallback(e): - log.warning("llm.fallback", model=model, error=type(e).__name__) + log.warning("llm.fallback", provider=provider, model=model, error=type(e).__name__) continue raise raise last if last else RuntimeError("no generation model available") @@ -72,26 +116,26 @@ def stream_chat(messages: list[dict], **kwargs): model, an error there ends the stream (can't restart mid-answer). """ last: Exception | None = None - for model in models_chain(): + for provider, model in models_chain(): started = False try: - log.info("llm.try", model=model, stream=True) - stream = get_client().chat.completions.create( + log.info("llm.try", provider=provider, model=model, stream=True) + stream = _client_for(provider).chat.completions.create( model=model, messages=messages, stream=True, **kwargs ) for event in stream: delta = event.choices[0].delta.content if delta: if not started: - log.info("llm.stream_start", model=model) + log.info("llm.stream_start", provider=provider, model=model) started = True yield delta - log.info("llm.stream_done", model=model) + log.info("llm.stream_done", provider=provider, model=model) return except Exception as e: # noqa: BLE001 last = e if not started and _should_fallback(e): - log.warning("llm.stream_fallback", model=model, error=type(e).__name__) + log.warning("llm.stream_fallback", provider=provider, model=model, error=type(e).__name__) continue raise raise last if last else RuntimeError("no generation model available") From 8905fae2948444fdd7c2946c95e8f98a68827529 Mon Sep 17 00:00:00 2001 From: Vibhanshu Rana <68025665+vrana055@users.noreply.github.com> Date: Mon, 22 Jun 2026 09:00:33 +0530 Subject: [PATCH 3/3] feat: add Cerebras and Mistral as LLM providers Extend multi-provider registry with Cerebras (wafer-scale ~2000 tok/s) and Mistral AI (free-tier). Add 5 new models to fallback chain: cerebras>llama-3.3-70b, cerebras>qwen-3-32b, cerebras>llama-3.1-8b, mistral>mistral-small-latest, mistral>open-mistral-nemo. Co-Authored-By: Claude Sonnet 4.6 --- .env.example | 6 ++++++ apps/api/app/config.py | 16 ++++++++++++++++ apps/api/app/services/generation.py | 8 +++++++- apps/api/app/services/llm.py | 8 ++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index f17e170..6ff43e4 100644 --- a/.env.example +++ b/.env.example @@ -23,6 +23,12 @@ OPENROUTER_BASE_URL=https://openrouter.ai/api/v1 # Groq (free-tier, fast inference) — get key at https://console.groq.com/keys GROQ_API_KEY=your-groq-api-key + +# Cerebras (free-tier, ~2000 tok/s) — get key at https://cloud.cerebras.ai +CEREBRAS_API_KEY=your-cerebras-api-key + +# Mistral AI (free-tier, rate limited) — get key at https://console.mistral.ai/api-keys +MISTRAL_API_KEY=your-mistral-api-key GENERATION_MODEL=qwen/qwen3-next-80b-a3b-instruct:free VISION_MODEL=meta-llama/llama-3.2-11b-vision-instruct:free diff --git a/apps/api/app/config.py b/apps/api/app/config.py index e2d1cbb..6e4f0a9 100644 --- a/apps/api/app/config.py +++ b/apps/api/app/config.py @@ -34,6 +34,14 @@ class Settings(BaseSettings): groq_api_key: str = "" groq_base_url: str = "https://api.groq.com/openai/v1" + # Cerebras (free-tier, wafer-scale ~2000 tok/s) — get key at https://cloud.cerebras.ai + cerebras_api_key: str = "" + cerebras_base_url: str = "https://api.cerebras.ai/v1" + + # Mistral AI (free-tier, rate limited) — get key at https://console.mistral.ai/api-keys + mistral_api_key: str = "" + mistral_base_url: str = "https://api.mistral.ai/v1" + generation_model: str = "qwen/qwen3-next-80b-a3b-instruct:free" # Fallback chain (comma-separated, "provider>model_id" format). # No prefix = openrouter. Tried in order on 429/404/502/503. @@ -46,6 +54,9 @@ class Settings(BaseSettings): # Groq — fast free inference "groq>llama-3.3-70b-versatile," "groq>qwen/qwen3.6-27b," + # Cerebras — wafer-scale, extremely fast free inference + "cerebras>llama-3.3-70b," + "cerebras>qwen-3-32b," # OpenRouter — mid-size free models "google/gemma-4-31b-it:free," "google/gemma-4-26b-a4b-it:free," @@ -58,6 +69,11 @@ class Settings(BaseSettings): "nvidia/nemotron-3-nano-30b-a3b:free," # Groq — small fast fallback "groq>llama-3.1-8b-instant," + # Mistral — free tier (rate limited) + "mistral>mistral-small-latest," + "mistral>open-mistral-nemo," + # Cerebras — small fast fallback + "cerebras>llama-3.1-8b," # OpenRouter — small fallbacks "nvidia/nemotron-nano-9b-v2:free," "meta-llama/llama-3.2-3b-instruct:free" diff --git a/apps/api/app/services/generation.py b/apps/api/app/services/generation.py index e20ed39..7c78445 100644 --- a/apps/api/app/services/generation.py +++ b/apps/api/app/services/generation.py @@ -39,7 +39,13 @@ def citations_from(chunks: list[dict]) -> list[dict]: def generation_available() -> bool: - return bool(settings.openrouter_api_key or settings.openai_api_key or settings.groq_api_key) + return bool( + settings.openrouter_api_key + or settings.openai_api_key + or settings.groq_api_key + or settings.cerebras_api_key + or settings.mistral_api_key + ) def generate_answer(question: str, chunks: list[dict], history: list[dict] | None = None) -> str: diff --git a/apps/api/app/services/llm.py b/apps/api/app/services/llm.py index 76ca403..497ad38 100644 --- a/apps/api/app/services/llm.py +++ b/apps/api/app/services/llm.py @@ -34,6 +34,14 @@ def _provider_registry() -> dict[str, dict]: "api_key": settings.groq_api_key, "base_url": settings.groq_base_url, }, + "cerebras": { + "api_key": settings.cerebras_api_key, + "base_url": settings.cerebras_base_url, + }, + "mistral": { + "api_key": settings.mistral_api_key, + "base_url": settings.mistral_base_url, + }, }