feat: establish policy-aware routing core

.github/workflows/ci.yml

@@ -0,0 +1,33 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  unit:
+    name: Unit and static analysis
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+          cache: pip
+      - run: python -m pip install -e ".[dev]"
+      - run: ruff format --check .
+      - run: ruff check .
+      - run: mypy
+      - run: pytest tests/unit --cov=llm_router --cov-report=term-missing --cov-report=xml
+      - uses: actions/upload-artifact@v4
+        with:
+          name: coverage
+          path: coverage.xml

.gitignore

@@ -0,0 +1,15 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.coverage
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+.venv/
+coverage.xml
+htmlcov/
+node_modules/
+playwright-report/
+test-results/
+.env
+

README.md

@@ -0,0 +1,22 @@
+# Production Local-LLM Inference & Routing Platform
+
+Policy-aware routing components for a production local-model inference platform.
+
+The complete architecture and design targets are documented in
+[`02-production-local-llm-inference-routing-platform.md`](02-production-local-llm-inference-routing-platform.md).
+
+## Development
+
+Requires Python 3.11 or newer.
+
+```bash
+python -m venv .venv
+python -m pip install -e ".[dev]"
+ruff format --check .
+ruff check .
+mypy
+pytest tests/unit --cov=llm_router --cov-report=term-missing
+```
+
+This first delivery slice contains deterministic routing, privacy restrictions, quotas,
+and bounded admission. API ingress and model-serving adapters are delivered separately.

pyproject.toml

@@ -0,0 +1,56 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "local-llm-router"
+version = "0.1.0"
+description = "OpenAI-compatible gateway and policy router for local LLM inference."
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+  "fastapi>=0.116,<1",
+  "pydantic-settings>=2.10,<3",
+  "uvicorn[standard]>=0.35,<1",
+]
+
+[project.optional-dependencies]
+dev = [
+  "httpx>=0.28,<1",
+  "mypy>=1.17,<2",
+  "pytest>=8.4,<9",
+  "pytest-asyncio>=1.1,<2",
+  "pytest-cov>=6.2,<7",
+  "ruff>=0.12,<1",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/llm_router"]
+
+[tool.pytest.ini_options]
+addopts = "-ra --strict-config --strict-markers"
+testpaths = ["tests"]
+markers = [
+  "integration: tests that exercise multiple application components",
+]
+
+[tool.coverage.run]
+branch = true
+source = ["llm_router"]
+
+[tool.coverage.report]
+fail_under = 90
+show_missing = true
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "ASYNC", "RUF"]
+
+[tool.mypy]
+python_version = "3.11"
+strict = true
+packages = ["llm_router"]
+

src/llm_router/__init__.py

@@ -0,0 +1,3 @@
+"""Local LLM inference gateway and routing policy package."""
+
+__version__ = "0.1.0"

src/llm_router/admission.py

@@ -0,0 +1,48 @@
+import asyncio
+import time
+from collections import defaultdict, deque
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+
+
+class AdmissionRejectedError(RuntimeError):
+    """Raised when the bounded request queue cannot admit work in time."""
+
+
+class QuotaExceededError(RuntimeError):
+    """Raised when a caller exceeds its configured sliding-window quota."""
+
+
+class AdmissionController:
+    def __init__(self, max_concurrency: int, timeout_seconds: float) -> None:
+        self._semaphore = asyncio.Semaphore(max_concurrency)
+        self._timeout_seconds = timeout_seconds
+
+    @asynccontextmanager
+    async def slot(self) -> AsyncIterator[None]:
+        try:
+            await asyncio.wait_for(self._semaphore.acquire(), timeout=self._timeout_seconds)
+        except TimeoutError as error:
+            raise AdmissionRejectedError("inference capacity is saturated") from error
+        try:
+            yield
+        finally:
+            self._semaphore.release()
+
+
+class SlidingWindowQuota:
+    def __init__(self, requests_per_minute: int) -> None:
+        self._limit = requests_per_minute
+        self._events: dict[str, deque[float]] = defaultdict(deque)
+        self._lock = asyncio.Lock()
+
+    async def consume(self, subject: str, *, now: float | None = None) -> None:
+        timestamp = time.monotonic() if now is None else now
+        cutoff = timestamp - 60
+        async with self._lock:
+            events = self._events[subject]
+            while events and events[0] <= cutoff:
+                events.popleft()
+            if len(events) >= self._limit:
+                raise QuotaExceededError("request quota exceeded")
+            events.append(timestamp)

src/llm_router/config.py

@@ -0,0 +1,32 @@
+from functools import lru_cache
+
+from pydantic import Field, model_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    """Runtime settings loaded from environment variables prefixed with ROUTER_."""
+
+    model_config = SettingsConfigDict(env_prefix="ROUTER_", extra="ignore")
+
+    environment: str = "development"
+    api_keys: str = "dev-key"
+    max_concurrency: int = Field(default=32, ge=1)
+    admission_timeout_seconds: float = Field(default=0.25, gt=0)
+    quota_requests_per_minute: int = Field(default=120, ge=1)
+    external_fallback_enabled: bool = False
+
+    @model_validator(mode="after")
+    def reject_development_key_in_shared_environments(self) -> "Settings":
+        if self.environment not in {"development", "test"} and "dev-key" in self.accepted_api_keys:
+            raise ValueError("ROUTER_API_KEYS must be set outside development and test")
+        return self
+
+    @property
+    def accepted_api_keys(self) -> frozenset[str]:
+        return frozenset(key.strip() for key in self.api_keys.split(",") if key.strip())
+
+
+@lru_cache
+def get_settings() -> Settings:
+    return Settings()

src/llm_router/models.py

@@ -0,0 +1,94 @@
+from enum import StrEnum
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field, model_validator
+
+
+class TaskClass(StrEnum):
+    EXTRACTION = "extraction"
+    CLASSIFICATION = "classification"
+    RAG = "rag"
+    SUMMARIZATION = "summarization"
+    REASONING = "reasoning"
+    CRITIQUE = "critique"
+    GENERAL = "general"
+
+
+class PrivacyClass(StrEnum):
+    PUBLIC = "public"
+    PRIVATE = "private"
+    RESTRICTED = "restricted"
+
+
+class ChatMessage(BaseModel):
+    role: Literal["system", "user", "assistant", "tool"]
+    content: str
+
+
+class RoutingOptions(BaseModel):
+    task: TaskClass | None = None
+    privacy: PrivacyClass = PrivacyClass.PRIVATE
+    latency_tier: Literal["interactive", "standard", "batch"] = "standard"
+    quality_floor: float = Field(default=0.0, ge=0.0, le=1.0)
+    allow_external_fallback: bool = False
+
+
+class ChatCompletionRequest(BaseModel):
+    model: str = "auto"
+    messages: list[ChatMessage] = Field(min_length=1)
+    max_tokens: int = Field(default=256, ge=1, le=8192)
+    temperature: float = Field(default=0.0, ge=0.0, le=2.0)
+    stream: bool = False
+    routing: RoutingOptions = Field(default_factory=RoutingOptions)
+
+    @model_validator(mode="after")
+    def reject_streaming_for_initial_slice(self) -> "ChatCompletionRequest":
+        if self.stream:
+            raise ValueError("streaming is not available in the initial control-plane slice")
+        return self
+
+    @property
+    def prompt(self) -> str:
+        return "\n".join(message.content for message in self.messages)
+
+
+class ModelProfile(BaseModel):
+    id: str
+    revision: str
+    local: bool
+    healthy: bool = True
+    context_limit: int
+    supported_tasks: frozenset[TaskClass]
+    quality: float = Field(ge=0.0, le=1.0)
+    estimated_queue_ms: int = Field(default=0, ge=0)
+    cost_weight: float = Field(default=0.0, ge=0.0)
+
+
+class RouteDecision(BaseModel):
+    profile: ModelProfile
+    task: TaskClass
+    reason: str
+    score: float
+    candidate_count: int
+
+
+class ChatCompletionChoice(BaseModel):
+    index: int = 0
+    message: ChatMessage
+    finish_reason: Literal["stop", "length"] = "stop"
+
+
+class Usage(BaseModel):
+    prompt_tokens: int
+    completion_tokens: int
+    total_tokens: int
+
+
+class ChatCompletionResponse(BaseModel):
+    id: str
+    object: Literal["chat.completion"] = "chat.completion"
+    created: int
+    model: str
+    choices: list[ChatCompletionChoice]
+    usage: Usage
+    routing: dict[str, Any]

src/llm_router/py.typed

@@ -0,0 +1 @@
+