Feat/more openrouter models

.env.example

@@ -20,6 +20,15 @@ OPENAI_API_KEY=your-openai-api-key
 # OpenRouter (free-tier generation + vision) — get key at https://openrouter.ai/keys
 OPENROUTER_API_KEY=your-openrouter-api-key
 OPENROUTER_BASE_URL=https://openrouter.ai/api/v1
+
+# Groq (free-tier, fast inference) — get key at https://console.groq.com/keys
+GROQ_API_KEY=your-groq-api-key
+
+# Cerebras (free-tier, ~2000 tok/s) — get key at https://cloud.cerebras.ai
+CEREBRAS_API_KEY=your-cerebras-api-key
+
+# Mistral AI (free-tier, rate limited) — get key at https://console.mistral.ai/api-keys
+MISTRAL_API_KEY=your-mistral-api-key
 GENERATION_MODEL=qwen/qwen3-next-80b-a3b-instruct:free
 VISION_MODEL=meta-llama/llama-3.2-11b-vision-instruct:free
 

apps/api/app/config.py

@@ -29,18 +29,52 @@ class Settings(BaseSettings):
     # OpenRouter (free-tier generation + vision)
     openrouter_api_key: str = ""
     openrouter_base_url: str = "https://openrouter.ai/api/v1"
+
+    # Groq (free-tier, fast inference) — get key at https://console.groq.com/keys
+    groq_api_key: str = ""
+    groq_base_url: str = "https://api.groq.com/openai/v1"
+
+    # Cerebras (free-tier, wafer-scale ~2000 tok/s) — get key at https://cloud.cerebras.ai
+    cerebras_api_key: str = ""
+    cerebras_base_url: str = "https://api.cerebras.ai/v1"
+
+    # Mistral AI (free-tier, rate limited) — get key at https://console.mistral.ai/api-keys
+    mistral_api_key: str = ""
+    mistral_base_url: str = "https://api.mistral.ai/v1"
+
     generation_model: str = "qwen/qwen3-next-80b-a3b-instruct:free"
-    # Fallback chain (comma-separated): tried in order when a model is rate-limited
-    # or unavailable. The primary generation_model is prepended automatically.
+    # Fallback chain (comma-separated, "provider>model_id" format).
+    # No prefix = openrouter. Tried in order on 429/404/502/503.
     generation_models: str = (
+        # OpenRouter — large free models
         "qwen/qwen3-next-80b-a3b-instruct:free,"
+        "openai/gpt-oss-120b:free,"
+        "qwen/qwen3-coder:free,"
+        "nex-agi/nex-n2-pro:free,"
+        # Groq — fast free inference
+        "groq>llama-3.3-70b-versatile,"
+        "groq>qwen/qwen3.6-27b,"
+        # Cerebras — wafer-scale, extremely fast free inference
+        "cerebras>llama-3.3-70b,"
+        "cerebras>qwen-3-32b,"
+        # OpenRouter — mid-size free models
         "google/gemma-4-31b-it:free,"
         "google/gemma-4-26b-a4b-it:free,"
         "meta-llama/llama-3.3-70b-instruct:free,"
         "nvidia/nemotron-3-super-120b-a12b:free,"
         "nvidia/nemotron-3-ultra-550b-a55b:free,"
         "nousresearch/hermes-3-llama-3.1-405b:free,"
+        "openai/gpt-oss-20b:free,"
+        "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free,"
         "nvidia/nemotron-3-nano-30b-a3b:free,"
+        # Groq — small fast fallback
+        "groq>llama-3.1-8b-instant,"
+        # Mistral — free tier (rate limited)
+        "mistral>mistral-small-latest,"
+        "mistral>open-mistral-nemo,"
+        # Cerebras — small fast fallback
+        "cerebras>llama-3.1-8b,"
+        # OpenRouter — small fallbacks
         "nvidia/nemotron-nano-9b-v2:free,"
         "meta-llama/llama-3.2-3b-instruct:free"
     )

apps/api/app/services/generation.py

@@ -39,7 +39,13 @@ def citations_from(chunks: list[dict]) -> list[dict]:
 
 
 def generation_available() -> bool:
-    return bool(settings.openrouter_api_key or settings.openai_api_key)
+    return bool(
+        settings.openrouter_api_key
+        or settings.openai_api_key
+        or settings.groq_api_key
+        or settings.cerebras_api_key
+        or settings.mistral_api_key
+    )
 
 
 def generate_answer(question: str, chunks: list[dict], history: list[dict] | None = None) -> str:

apps/api/app/services/llm.py

@@ -1,65 +1,117 @@
-"""LLM client — OpenRouter (OpenAI-compatible) with a free-model fallback chain.
+"""LLM client — multi-provider (OpenAI-compatible) with a free-model fallback chain.
 
-OpenRouter free (":free") models are frequently rate-limited upstream (429) or
-temporarily unavailable (404). We try a list of free models in order and move to
-the next on those errors, so a grounded answer still comes back.
+Model IDs use the format  "provider>model_id"  e.g. "groq>llama-3.3-70b-versatile".
+No prefix defaults to "openrouter" for backward compatibility.
+
+Adding a new provider:
+  1. Add its api_key / base_url fields to config.py.
+  2. Add one entry to _PROVIDER_REGISTRY in this file.
+  3. Prefix models in GENERATION_MODELS with "newprovider>".
+
+Free ":free" OpenRouter models and Groq free-tier models are frequently
+rate-limited (429) or temporarily unavailable (404/502/503). We try each
+(provider, model) pair in order and fall back on those errors.
 """
 import structlog
-from functools import lru_cache
-
 from openai import APIStatusError, OpenAI
 
 from app.config import settings
 
 log = structlog.get_logger()
 
-# Errors that mean "this model is busy/unavailable — try the next one".
 _FALLBACK_STATUS = {404, 429, 502, 503}
 
-
-@lru_cache(maxsize=1)
-def get_client() -> OpenAI:
-    # max_retries=0: our models_chain() handles rate limits by switching models.
-    # The SDK's default 2 retries with backoff would stall ~50s on a 429 before
-    # we ever reach the next model.
-    return OpenAI(
-        api_key=settings.openrouter_api_key or settings.openai_api_key,
-        base_url=settings.openrouter_base_url or None,
-        max_retries=0,
-    )
-
-
-def models_chain() -> list[str]:
-    """Ordered, de-duplicated model list: primary first, then configured fallbacks."""
-    chain = [settings.generation_model] + [
+# -- Provider registry --------------------------------------------------------
+# One entry per provider. To add a new one: extend this dict + add env vars.
+
+def _provider_registry() -> dict[str, dict]:
+    return {
+        "openrouter": {
+            "api_key": settings.openrouter_api_key or settings.openai_api_key,
+            "base_url": settings.openrouter_base_url,
+        },
+        "groq": {
+            "api_key": settings.groq_api_key,
+            "base_url": settings.groq_base_url,
+        },
+        "cerebras": {
+            "api_key": settings.cerebras_api_key,
+            "base_url": settings.cerebras_base_url,
+        },
+        "mistral": {
+            "api_key": settings.mistral_api_key,
+            "base_url": settings.mistral_base_url,
+        },
+    }
+
+
+_clients: dict[str, OpenAI] = {}
+
+
+def _client_for(provider: str) -> OpenAI:
+    if provider not in _clients:
+        registry = _provider_registry()
+        if provider not in registry:
+            raise ValueError(
+                f"Unknown LLM provider {provider!r}. Add it to _provider_registry() in llm.py."
+            )
+        cfg = registry[provider]
+        # max_retries=0: fallback chain switches providers on 429 instead of waiting.
+        _clients[provider] = OpenAI(
+            api_key=cfg["api_key"] or "sk-no-key",
+            base_url=cfg["base_url"] or None,
+            max_retries=0,
+        )
+    return _clients[provider]
+
+
+# -- Model chain --------------------------------------------------------------
+
+def _parse_model(s: str) -> tuple[str, str]:
+    """'groq>llama-3.3-70b-versatile' → ('groq', 'llama-3.3-70b-versatile').
+    No prefix → ('openrouter', s) for backward compatibility."""
+    if ">" in s:
+        provider, model_id = s.split(">", 1)
+        return provider.strip(), model_id.strip()
+    return "openrouter", s
+
+
+def models_chain() -> list[tuple[str, str]]:
+    """Ordered, de-duplicated list of (provider, model_id) pairs."""
+    raw = [settings.generation_model] + [
         m.strip() for m in settings.generation_models.split(",") if m.strip()
     ]
-    seen, out = set(), []
-    for m in chain:
+    seen: set[str] = set()
+    out: list[tuple[str, str]] = []
+    for m in raw:
         if m and m not in seen:
             seen.add(m)
-            out.append(m)
+            out.append(_parse_model(m))
     return out
 
 
 def _should_fallback(err: Exception) -> bool:
     return isinstance(err, APIStatusError) and err.status_code in _FALLBACK_STATUS
 
 
+# -- Public API ---------------------------------------------------------------
+
 def chat(messages: list[dict], **kwargs) -> str:
-    """Non-streaming completion. Tries each model in the chain; raises last error if all fail."""
+    """Non-streaming completion. Tries each (provider, model) in chain; raises last error if all fail."""
     last: Exception | None = None
-    for model in models_chain():
+    for provider, model in models_chain():
         try:
-            log.info("llm.try", model=model, stream=False)
-            resp = get_client().chat.completions.create(model=model, messages=messages, **kwargs)
+            log.info("llm.try", provider=provider, model=model, stream=False)
+            resp = _client_for(provider).chat.completions.create(
+                model=model, messages=messages, **kwargs
+            )
             usage = getattr(resp, "usage", None)
-            log.info("llm.ok", model=model, tokens=getattr(usage, "total_tokens", None))
+            log.info("llm.ok", provider=provider, model=model, tokens=getattr(usage, "total_tokens", None))
             return resp.choices[0].message.content or ""
-        except Exception as e:  # noqa: BLE001 — decide fallback by status
+        except Exception as e:  # noqa: BLE001
             last = e
             if _should_fallback(e):
-                log.warning("llm.fallback", model=model, error=type(e).__name__)
+                log.warning("llm.fallback", provider=provider, model=model, error=type(e).__name__)
                 continue
             raise
     raise last if last else RuntimeError("no generation model available")
@@ -72,26 +124,26 @@ def stream_chat(messages: list[dict], **kwargs):
     model, an error there ends the stream (can't restart mid-answer).
     """
     last: Exception | None = None
-    for model in models_chain():
+    for provider, model in models_chain():
         started = False
         try:
-            log.info("llm.try", model=model, stream=True)
-            stream = get_client().chat.completions.create(
+            log.info("llm.try", provider=provider, model=model, stream=True)
+            stream = _client_for(provider).chat.completions.create(
                 model=model, messages=messages, stream=True, **kwargs
             )
             for event in stream:
                 delta = event.choices[0].delta.content
                 if delta:
                     if not started:
-                        log.info("llm.stream_start", model=model)
+                        log.info("llm.stream_start", provider=provider, model=model)
                     started = True
                     yield delta
-            log.info("llm.stream_done", model=model)
+            log.info("llm.stream_done", provider=provider, model=model)
             return
         except Exception as e:  # noqa: BLE001
             last = e
             if not started and _should_fallback(e):
-                log.warning("llm.stream_fallback", model=model, error=type(e).__name__)
+                log.warning("llm.stream_fallback", provider=provider, model=model, error=type(e).__name__)
                 continue
             raise
     raise last if last else RuntimeError("no generation model available")