diff --git a/.cursor/automation/improvement-loop.md b/.cursor/automation/improvement-loop.md new file mode 100644 index 0000000..6d6cbcc --- /dev/null +++ b/.cursor/automation/improvement-loop.md @@ -0,0 +1,85 @@ +# Cursor Cloud Automation — Improvement Loop + +Use this file when creating an Automation in **Cursor Dashboard → Automations → New Automation**. + +## Recommended settings + +| Field | Value | +|-------|-------| +| **Name** | Advanced-RAG Improvement Loop | +| **Repository** | `dtype2100/Advanced-RAG` | +| **Branch** | `master` (or `cursor/auto-improvement-loop-3e09` while testing) | +| **Schedule** | Daily, or after merge to master | +| **Model** | composer-2.5-fast (or higher for judge tuning) | + +## Automation prompt (copy-paste) + +``` +You are the Advanced-RAG continuous improvement agent. Follow the canonical loop +defined in app/core/improvement_loop.py in this exact order: + +1. ANALYSIS — review failing metrics / user feedback / open issues +2. VERIFICATION — confirm the problem scope before changing code +3. SEARCH — inspect relevant modules (graph, policies, evals, tests) +4. TEST — reproduce with pytest; add a failing test if missing +5. EVALUATION — run offline evals for the affected phase +6. VERIFICATION — re-run make lint && make test && make evals-ci +7. FEEDBACK — if evals fail, iterate (max 3 cycles); else open a PR + +Commands (in order each cycle): + make lint + make test + IMPROVEMENT_LOOP_CI=1 make evals-ci + +If OPENAI_API_KEY is available and judge tuning is needed: + make evals + +Rules: +- Minimal diff; match existing conventions in app/ +- Never skip tests before opening a PR +- Update evals/datasets/*.jsonl or evals/regression/golden_set.yaml when fixing routing/policy bugs +- Commit message format: "fix(improvement-loop): " +- Create a draft PR when a measurable improvement is verified +- Stop after 3 failed improvement cycles and report blockers + +Phase → code map: + analysis → app/rag/query/query_analyzer.py + verification → app/rag/policies/clarification_policy.py, rewrite_policy.py + search → app/rag/retrievers/, app/graphs/crag/nodes.py (hybrid_retrieve) + test → app/graphs/crag/nodes.py (test_retrieval), tests/unit/ + evaluation → app/graphs/crag/nodes.py (generate_answer, run_judge) + verification → app/rag/evaluators/, evaluate_grounding node + feedback → app/rag/policies/routing_policy.py (route_after_feedback) +``` + +## Trigger options + +### Option A — Scheduled (recommended) +- **Trigger:** Cron / daily +- Runs regression + proposes fixes when evals fail + +### Option B — On PR merge +- **Trigger:** SCM / merge to master +- Validates master after each merge + +### Option C — Manual +- **Trigger:** Manual / workflow_dispatch equivalent in Automations UI +- Use when tuning a specific phase (e.g. feedback routing) + +## What runs automatically without this Automation + +GitHub Actions workflow `.github/workflows/improvement-loop.yml` already runs on: +- Daily schedule (06:00 UTC) +- Push to `master` (app/evals/tests changes) +- Manual dispatch (`full_loop` input for LLM judge eval) + +CI command: `make evals-ci` (skips LLM judge phase). + +## Verify setup + +```bash +make evals-ci # local CI-safe loop +make evals # full loop (needs vLLM or OPENAI_API_KEY) +``` + +Expected CI phases: analysis → search → test → evaluation → feedback (5 phases). diff --git a/.env.example b/.env.example index cd296ff..b068ecf 100644 --- a/.env.example +++ b/.env.example @@ -25,10 +25,21 @@ COLLECTION_NAME=advanced_rag MAX_RETRIEVAL_DOCS=5 MAX_RETRIES=3 +# Server +LOG_LEVEL=INFO + +# Security (leave API_KEY empty to disable auth in development) +API_KEY= +CORS_ORIGINS=* +RATE_LIMIT_PER_MINUTE=120 + +# Observability +ENABLE_METRICS=true +HEALTH_CHECK_LLM=false + # Redis (ARQ async ingest; leave empty for sync-only) REDIS_URL= # Set true to enqueue POST /documents instead of blocking INGEST_QUEUE_ASYNC=false # ARQ queue name (must match worker) ARQ_QUEUE_NAME=arq:queue - diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..5669f44 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,28 @@ +name: CI + +on: + push: + branches: [master, "cursor/**"] + pull_request: + branches: [master] + +jobs: + lint-and-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Install dependencies + run: pip install -e ".[dev]" + + - name: Lint + run: make lint + + - name: Test + run: make test diff --git a/.github/workflows/improvement-loop.yml b/.github/workflows/improvement-loop.yml new file mode 100644 index 0000000..a98a597 --- /dev/null +++ b/.github/workflows/improvement-loop.yml @@ -0,0 +1,68 @@ +name: Improvement Loop + +on: + schedule: + # Daily at 06:00 UTC — automated quality regression check + - cron: "0 6 * * *" + workflow_dispatch: + inputs: + full_loop: + description: "Run full loop including LLM judge (requires secrets)" + required: false + default: "false" + type: choice + options: + - "false" + - "true" + push: + branches: [master] + paths: + - "app/**" + - "evals/**" + - "tests/**" + - "scripts/run_evals.py" + - "app/core/improvement_loop.py" + +jobs: + improvement-loop: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Install dependencies + run: pip install -e ".[dev]" + + - name: Lint + run: make lint + + - name: Test (integration + unit) + run: make test + + - name: Improvement loop (CI-safe) + if: github.event.inputs.full_loop != 'true' + run: make evals-ci + env: + IMPROVEMENT_LOOP_CI: "1" + + - name: Improvement loop (full, with LLM judge) + if: github.event.inputs.full_loop == 'true' + run: make evals + env: + LLM_BACKEND: openai + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + + notify-on-failure: + needs: improvement-loop + if: failure() && github.event_name == 'schedule' + runs-on: ubuntu-latest + steps: + - name: Log failure + run: | + echo "::error::Scheduled improvement loop failed. Check the Actions tab." + echo "Phases: analysis → search → test → evaluation → feedback (CI mode)" diff --git a/AGENTS.md b/AGENTS.md index 2080966..0e6a28f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,6 +38,7 @@ - **FastEmbed first-run download:** `BAAI/bge-small-en-v1.5` (~130MB) downloads on first use to `~/.cache/fastembed/`. - **Model download:** Run `huggingface-cli download Qwen/Qwen2.5-0.5B-Instruct --local-dir models/Qwen2.5-0.5B-Instruct` to get the model (~950MB). - **Ruff binary:** After pip install, ruff is at `~/.local/bin/ruff`. Ensure `PATH` includes `$HOME/.local/bin`. +- **Production ops:** Set `API_KEY`, use `/api/v1/health/ready` for readiness, scrape `/metrics` with Prometheus. CI runs via `.github/workflows/ci.yml`. ### Repository Overview - **Project:** Advanced-RAG (Retrieval-Augmented Generation) - **Status:** Newly initialized repository with only a `README.md`. No source code, dependencies, or services exist yet. diff --git a/Makefile b/Makefile index 91b8c71..a8390d5 100644 --- a/Makefile +++ b/Makefile @@ -1,39 +1,42 @@ .PHONY: install dev lint format test run vllm-serve worker evals clean +PYTHON ?= python3 +export PATH := $(HOME)/.local/bin:$(PATH) + install: - pip install -e . + $(PYTHON) -m pip install -e . dev: - pip install -e ".[dev]" + $(PYTHON) -m pip install -e ".[dev]" lint: - ruff check app/ tests/ evals/ scripts/ - ruff format --check app/ tests/ evals/ scripts/ + $(PYTHON) -m ruff check app/ tests/ evals/ scripts/ + $(PYTHON) -m ruff format --check app/ tests/ evals/ scripts/ format: - ruff check --fix app/ tests/ evals/ scripts/ - ruff format app/ tests/ evals/ scripts/ + $(PYTHON) -m ruff check --fix app/ tests/ evals/ scripts/ + $(PYTHON) -m ruff format app/ tests/ evals/ scripts/ test: - pytest -v + $(PYTHON) -m pytest -v test-unit: - pytest -v tests/unit/ + $(PYTHON) -m pytest -v tests/unit/ test-integration: - pytest -v tests/integration/ + $(PYTHON) -m pytest -v tests/integration/ run: - uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 + $(PYTHON) -m uvicorn app.main:app --reload --host 0.0.0.0 --port 8000 worker: - arq app.workers.settings.WorkerSettings + $(PYTHON) -m arq app.workers.settings.WorkerSettings vllm-serve: @echo "Starting vLLM server on port 8001..." VLLM_CPU_KVCACHE_SPACE=4 \ VLLM_CPU_OMP_THREADS_BIND=0-3 \ - python3 -m vllm.entrypoints.openai.api_server \ + $(PYTHON) -m vllm.entrypoints.openai.api_server \ --model $(or $(VLLM_MODEL_PATH),/workspace/models/Qwen2.5-0.5B-Instruct) \ --served-model-name Qwen/Qwen2.5-0.5B-Instruct \ --host 0.0.0.0 \ @@ -42,7 +45,10 @@ vllm-serve: --dtype bfloat16 evals: - python scripts/run_evals.py + $(PYTHON) scripts/run_evals.py + +evals-ci: + IMPROVEMENT_LOOP_CI=1 $(PYTHON) scripts/run_evals.py --ci clean: find . -type d -name __pycache__ -exec rm -rf {} + diff --git a/README.md b/README.md index 1fb540c..c5cf0a3 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Question → Retrieve (Qdrant) → Grade Documents (LLM) →┐ **Key features:** - Self-corrective retrieval: automatically rewrites queries when documents are irrelevant +- **Improvement loop**: analysis → verification → search → test → evaluation → verification → feedback - **vLLM**: local HuggingFace model serving via OpenAI-compatible API (CPU/GPU) - Qdrant vector store with FastEmbed (local embeddings, no API calls for embedding) - LangGraph `StateGraph` with conditional edges for the RAG loop @@ -71,10 +72,73 @@ curl -X POST http://localhost:8000/api/v1/query \ | Method | Path | Description | |--------|------|-------------| | GET | `/` | Service info | -| GET | `/api/v1/health` | Health check (Qdrant + LLM status) | +| GET | `/api/v1/health` | Health summary (dependencies + status) | +| GET | `/api/v1/health/live` | Liveness probe (process alive) | +| GET | `/api/v1/health/ready` | Readiness probe (503 if not ready) | +| GET | `/metrics` | Prometheus metrics | | POST | `/api/v1/documents` | Ingest documents into vector store | +| POST | `/api/v1/documents/async` | Async ingest (requires Redis + worker) | +| GET | `/api/v1/jobs/{job_id}` | Async job status | | POST | `/api/v1/search` | Semantic search (no LLM required) | -| POST | `/api/v1/query` | Full self-corrective RAG pipeline | +| POST | `/api/v1/query` | Full CRAG RAG pipeline | + +## Production / Operations + +### Docker Compose (recommended) + +```bash +cp .env.example .env +# Set API_KEY, QDRANT_URL is auto-configured in compose +docker compose up -d +``` + +Services: `api` (8000), `qdrant` (6333), `redis` (6379), `worker`. + +The API container exposes a readiness healthcheck on `/api/v1/health/ready`. + +### Security + +Set `API_KEY` in production. All data endpoints require the header: + +```bash +curl -H "X-API-Key: your-secret-key" \ + -X POST http://localhost:8000/api/v1/search \ + -H "Content-Type: application/json" \ + -d '{"query": "test", "top_k": 3}' +``` + +When `API_KEY` is empty, auth is disabled (development only). + +### Observability + +| Endpoint | Purpose | +|----------|---------| +| `/metrics` | Prometheus (HTTP latency, RAG queries, ingest counts) | +| `/api/v1/health/live` | Kubernetes liveness | +| `/api/v1/health/ready` | Kubernetes readiness (vector store + optional Redis/LLM) | + +Logs include `request_id` via the `X-Request-ID` header (auto-generated if omitted). + +### CI + +GitHub Actions runs `make lint` and `make test` on push/PR. + +### Automatic improvement loop + +| Trigger | Workflow | Command | +|---------|----------|---------| +| Daily 06:00 UTC | `.github/workflows/improvement-loop.yml` | `make evals-ci` | +| Push to `master` | same | `make evals-ci` | +| Manual dispatch | same | `full_loop=true` → `make evals` (needs `OPENAI_API_KEY` secret) | + +CI-safe loop skips the LLM judge phase. Full loop requires vLLM or OpenAI. + +**Cursor Cloud Automation:** copy the prompt from `.cursor/automation/improvement-loop.md` into Dashboard → Automations. + +```bash +make evals-ci # CI-safe (no LLM) +make evals # full loop +``` ## LLM Backend Configuration @@ -103,9 +167,26 @@ make lint # Run linter make format # Auto-format make test # Run tests make run # Start FastAPI dev server +make evals # Run full improvement loop (analysis → feedback) make vllm-serve # Start vLLM on port 8001 ``` +## Improvement Loop + +Runtime (CRAG graph) and offline evals (`make evals`) follow the same phase order: + +| Phase | Graph node(s) | Offline eval | +|-------|---------------|--------------| +| Analysis | `analyze_query` | `run_clarification_eval.py` | +| Verification | `decide_rewrite`, `rewrite_query` | (graph integration tests) | +| Search | `hybrid_retrieve` | `run_retrieval_eval.py` | +| Test | `test_retrieval` | `pytest tests/unit` | +| Evaluation | `generate_answer`, `run_judge` | `run_answer_eval.py` | +| Verification | `evaluate_grounding` | `run_judge_eval.py` | +| Feedback | `retry_*`, `finalize_*` | `run_feedback_eval.py` | + +Canonical definition: `app/core/improvement_loop.py` + ## Project Structure ``` @@ -143,3 +224,10 @@ tests/ | `COLLECTION_NAME` | `advanced_rag` | Qdrant collection name | | `MAX_RETRIEVAL_DOCS` | `5` | Top-K retrieval count | | `MAX_RETRIES` | `3` | Max query rewrite retries | +| `API_KEY` | (empty) | Enable API key auth when set | +| `CORS_ORIGINS` | `*` | Comma-separated allowed origins | +| `RATE_LIMIT_PER_MINUTE` | `120` | Per-IP rate limit (0 = disabled) | +| `ENABLE_METRICS` | `true` | Expose `/metrics` endpoint | +| `HEALTH_CHECK_LLM` | `false` | Include LLM ping in readiness | +| `LOG_LEVEL` | `INFO` | Application log level | +| `REDIS_URL` | (empty) | Redis for async ingest | diff --git a/app/api/dependencies.py b/app/api/dependencies.py index d5398e3..ea031d5 100644 --- a/app/api/dependencies.py +++ b/app/api/dependencies.py @@ -6,6 +6,7 @@ from __future__ import annotations +from app.core.security import verify_api_key from app.providers.vectorstore_provider import get_vectorstore from app.storage.vectorstores.base import VectorStorePort @@ -13,3 +14,7 @@ def get_store() -> VectorStorePort: """Dependency: return the active vector store instance.""" return get_vectorstore() + + +# Protect mutating / expensive endpoints when API_KEY is set. +RequireAPIKey = verify_api_key diff --git a/app/api/v1/chat.py b/app/api/v1/chat.py index 3581390..312567b 100644 --- a/app/api/v1/chat.py +++ b/app/api/v1/chat.py @@ -4,8 +4,9 @@ import logging -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, Depends, HTTPException +from app.api.dependencies import RequireAPIKey from app.core.config import settings from app.schemas.request import ChatRequest from app.schemas.response import ChatResponse @@ -17,7 +18,7 @@ @router.post("/query", response_model=ChatResponse, tags=["rag"]) -async def rag_query(req: ChatRequest) -> ChatResponse: +async def rag_query(req: ChatRequest, _: None = Depends(RequireAPIKey)) -> ChatResponse: """Run the full CRAG pipeline (analyse → retrieve → rerank → generate → evaluate).""" if not settings.using_vllm and not settings.openai_api_key: raise HTTPException( diff --git a/app/api/v1/health.py b/app/api/v1/health.py index abba927..f061d67 100644 --- a/app/api/v1/health.py +++ b/app/api/v1/health.py @@ -1,14 +1,14 @@ -"""Health / readiness check endpoint.""" +"""Health / readiness check endpoints.""" from __future__ import annotations import logging -from fastapi import APIRouter +from fastapi import APIRouter, Response from app.core.config import settings -from app.providers.vectorstore_provider import get_vectorstore -from app.schemas.response import HealthResponse +from app.core.health_checks import check_llm, check_redis, check_vectorstore, is_ready +from app.schemas.response import HealthResponse, LivenessResponse, ReadinessResponse logger = logging.getLogger(__name__) @@ -17,28 +17,48 @@ @router.get("/health", response_model=HealthResponse, tags=["system"]) async def health() -> HealthResponse: - """Return service health including vector store connection status.""" - try: - store = get_vectorstore() - # QdrantStore exposes get_client(); fall back gracefully for other adapters. - if hasattr(store, "get_client"): - client = store.get_client() - collections = [c.name for c in client.get_collections().collections] - qdrant_status = "connected" - collection_status = ( - "exists" if settings.collection_name in collections else "not_created" - ) - else: - qdrant_status = "connected" - collection_status = "unknown" - except Exception as exc: - qdrant_status = f"error: {exc}" - collection_status = "unknown" + """Return service health including dependency status (backward compatible).""" + vector = await check_vectorstore() + redis = await check_redis() + llm = await check_llm() + ready = is_ready({"vectorstore": vector, "redis": redis, "llm": llm}) return HealthResponse( - status="ok", + status="ok" if ready else "degraded", + version="0.2.0", llm_backend=settings.llm_backend, llm_model=settings.llm_model, - qdrant=qdrant_status, - collection=collection_status, + qdrant=vector["status"], + collection=vector["collection"], + redis=redis["status"], + llm=llm["status"], + auth_enabled=settings.auth_enabled, + ) + + +@router.get("/health/live", response_model=LivenessResponse, tags=["system"]) +async def liveness() -> LivenessResponse: + """Kubernetes liveness probe — process is running.""" + return LivenessResponse(status="alive") + + +@router.get("/health/ready", response_model=ReadinessResponse, tags=["system"]) +async def readiness(response: Response) -> ReadinessResponse: + """Kubernetes readiness probe — dependencies are available for traffic.""" + vector = await check_vectorstore() + redis = await check_redis() + llm = await check_llm() + checks = {"vectorstore": vector, "redis": redis, "llm": llm} + ready = is_ready(checks) + + if not ready: + response.status_code = 503 + + return ReadinessResponse( + status="ready" if ready else "not_ready", + checks={ + "vectorstore": vector, + "redis": redis, + "llm": llm, + }, ) diff --git a/app/api/v1/ingest.py b/app/api/v1/ingest.py index 6432cad..41e10b7 100644 --- a/app/api/v1/ingest.py +++ b/app/api/v1/ingest.py @@ -4,10 +4,12 @@ import logging -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import JSONResponse +from app.api.dependencies import RequireAPIKey from app.core.config import settings +from app.core.metrics import record_ingest from app.queue.pool import get_arq_pool from app.schemas.document import IngestJobResponse, IngestRequest, IngestResponse from app.schemas.request import SearchRequest @@ -41,19 +43,22 @@ async def _enqueue_ingest(raw_docs: list[dict]) -> IngestJobResponse: }, tags=["documents"], ) -async def ingest(req: IngestRequest): +async def ingest(req: IngestRequest, _: None = Depends(RequireAPIKey)): """Ingest documents (sync) or enqueue when ``INGEST_QUEUE_ASYNC`` and ``REDIS_URL`` are set.""" raw_docs = [{"text": d.text, "metadata": d.metadata} for d in req.documents] if settings.ingest_queue_async and settings.redis_url: accepted = await _enqueue_ingest(raw_docs) + record_ingest(mode="async", status="queued") return JSONResponse(status_code=202, content=accepted.model_dump()) try: count = ingest_documents(raw_docs) except Exception as exc: logger.exception("Ingestion failed") + record_ingest(mode="sync", status="error") raise HTTPException(status_code=500, detail=str(exc)) from exc + record_ingest(mode="sync", status="success") return IngestResponse(message="Documents ingested successfully", count=count) @@ -63,7 +68,7 @@ async def ingest(req: IngestRequest): status_code=202, tags=["documents"], ) -async def ingest_async(req: IngestRequest) -> IngestJobResponse: +async def ingest_async(req: IngestRequest, _: None = Depends(RequireAPIKey)) -> IngestJobResponse: """Always enqueue ingest (requires ``REDIS_URL``).""" if not settings.redis_url: raise HTTPException( @@ -71,11 +76,13 @@ async def ingest_async(req: IngestRequest) -> IngestJobResponse: detail="Async ingest requires REDIS_URL and a running ARQ worker.", ) raw_docs = [{"text": d.text, "metadata": d.metadata} for d in req.documents] - return await _enqueue_ingest(raw_docs) + result = await _enqueue_ingest(raw_docs) + record_ingest(mode="async", status="queued") + return result @router.post("/search", response_model=SearchResponse, tags=["search"]) -async def semantic_search(req: SearchRequest) -> SearchResponse: +async def semantic_search(req: SearchRequest, _: None = Depends(RequireAPIKey)) -> SearchResponse: """Perform a semantic similarity search without RAG generation.""" try: results = search_documents(req.query, top_k=req.top_k) diff --git a/app/api/v1/jobs.py b/app/api/v1/jobs.py index d99fa6c..0bd6fb4 100644 --- a/app/api/v1/jobs.py +++ b/app/api/v1/jobs.py @@ -5,8 +5,9 @@ import logging from arq.jobs import Job, JobStatus -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, Depends, HTTPException +from app.api.dependencies import RequireAPIKey from app.core.config import settings from app.queue.pool import get_arq_pool from app.schemas.document import JobStatusResponse @@ -17,7 +18,7 @@ @router.get("/jobs/{job_id}", response_model=JobStatusResponse, tags=["jobs"]) -async def get_job_status(job_id: str) -> JobStatusResponse: +async def get_job_status(job_id: str, _: None = Depends(RequireAPIKey)) -> JobStatusResponse: """Return the status and result of an async ingest job.""" if not settings.redis_url: raise HTTPException(status_code=503, detail="Job status requires REDIS_URL.") diff --git a/app/core/config.py b/app/core/config.py index 81106a8..34925ea 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -37,6 +37,16 @@ class Settings(BaseSettings): # ── Server ─────────────────────────────────────────────────────────────── host: str = "0.0.0.0" port: int = 8000 + log_level: str = "INFO" + + # ── Security ───────────────────────────────────────────────────────────── + api_key: str = "" # empty = auth disabled (dev mode) + cors_origins: str = "*" # comma-separated origins, or "*" + rate_limit_per_minute: int = 120 # 0 = disabled + + # ── Observability ──────────────────────────────────────────────────────── + enable_metrics: bool = True + health_check_llm: bool = False # ping LLM on /health/ready (adds latency) # ── Queue (ARQ + Redis) ─────────────────────────────────────────────────── redis_url: str = "" @@ -45,6 +55,11 @@ class Settings(BaseSettings): model_config = {"env_file": ".env", "env_file_encoding": "utf-8", "extra": "ignore"} + @property + def auth_enabled(self) -> bool: + """True when API key authentication is required.""" + return bool(self.api_key) + @property def qdrant_in_memory(self) -> bool: """True when no external Qdrant URL is configured (uses :memory: mode).""" diff --git a/app/core/exceptions.py b/app/core/exceptions.py new file mode 100644 index 0000000..6516bca --- /dev/null +++ b/app/core/exceptions.py @@ -0,0 +1,53 @@ +"""Global exception handlers for consistent API error responses.""" + +from __future__ import annotations + +import logging + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from starlette.exceptions import HTTPException as StarletteHTTPException + +logger = logging.getLogger(__name__) + + +def register_exception_handlers(app: FastAPI) -> None: + """Register application-wide exception handlers.""" + + @app.exception_handler(StarletteHTTPException) + async def http_exception_handler(request: Request, exc: StarletteHTTPException) -> JSONResponse: + request_id = getattr(request.state, "request_id", None) + return JSONResponse( + status_code=exc.status_code, + content={ + "detail": exc.detail, + "request_id": request_id, + }, + ) + + @app.exception_handler(RequestValidationError) + async def validation_exception_handler( + request: Request, + exc: RequestValidationError, + ) -> JSONResponse: + request_id = getattr(request.state, "request_id", None) + return JSONResponse( + status_code=422, + content={ + "detail": exc.errors(), + "request_id": request_id, + }, + ) + + @app.exception_handler(Exception) + async def unhandled_exception_handler(request: Request, exc: Exception) -> JSONResponse: + request_id = getattr(request.state, "request_id", None) + logger.exception("Unhandled exception request_id=%s path=%s", request_id, request.url.path) + return JSONResponse( + status_code=500, + content={ + "detail": "Internal server error", + "request_id": request_id, + }, + ) diff --git a/app/core/health_checks.py b/app/core/health_checks.py new file mode 100644 index 0000000..9468ed5 --- /dev/null +++ b/app/core/health_checks.py @@ -0,0 +1,85 @@ +"""Health check helpers for liveness and readiness probes.""" + +from __future__ import annotations + +import logging +from typing import Any + +import httpx + +from app.core.config import settings +from app.providers.vectorstore_provider import get_vectorstore + +logger = logging.getLogger(__name__) + + +async def check_vectorstore() -> dict[str, str]: + """Verify vector store connectivity and collection status.""" + try: + store = get_vectorstore() + if hasattr(store, "get_client"): + client = store.get_client() + collections = [c.name for c in client.get_collections().collections] + collection_status = ( + "exists" if settings.collection_name in collections else "not_created" + ) + return {"status": "connected", "collection": collection_status} + return {"status": "connected", "collection": "unknown"} + except Exception as exc: + logger.warning("Vector store health check failed: %s", exc) + return {"status": f"error: {exc}", "collection": "unknown"} + + +async def check_redis() -> dict[str, str]: + """Ping Redis when configured for async ingest.""" + if not settings.redis_url: + return {"status": "not_configured"} + + try: + from app.queue.pool import get_arq_pool + + pool = await get_arq_pool() + await pool.ping() + return {"status": "connected"} + except Exception as exc: + logger.warning("Redis health check failed: %s", exc) + return {"status": f"error: {exc}"} + + +async def check_llm() -> dict[str, str]: + """Optionally verify LLM backend reachability.""" + if not settings.health_check_llm: + return {"status": "skipped"} + + try: + if settings.using_vllm: + url = f"{settings.vllm_base_url.rstrip('/')}/models" + async with httpx.AsyncClient(timeout=5.0) as client: + resp = await client.get(url) + resp.raise_for_status() + return {"status": "connected"} + if settings.openai_api_key: + return {"status": "configured"} + return {"status": "not_configured"} + except Exception as exc: + logger.warning("LLM health check failed: %s", exc) + return {"status": f"error: {exc}"} + + +def is_ready(checks: dict[str, Any]) -> bool: + """Return True when all critical dependencies are healthy.""" + vector = checks.get("vectorstore", {}) + if not str(vector.get("status", "")).startswith("connected"): + return False + + collection = vector.get("collection") + if collection == "not_created": + return False + + redis = checks.get("redis", {}) + if settings.redis_url and not str(redis.get("status", "")).startswith("connected"): + return False + + llm = checks.get("llm", {}) + llm_status = str(llm.get("status", "")) + return not (settings.health_check_llm and llm_status.startswith("error")) diff --git a/app/core/improvement_loop.py b/app/core/improvement_loop.py new file mode 100644 index 0000000..a23b8b5 --- /dev/null +++ b/app/core/improvement_loop.py @@ -0,0 +1,110 @@ +"""Canonical improvement loop phase order for this RAG project. + +Both the CRAG LangGraph pipeline and offline eval runner follow this sequence: + + analysis → verification → search → test → evaluation → verification → feedback + +Each phase maps to concrete nodes (runtime) or scripts (offline). +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Literal + +PhaseName = Literal[ + "analysis", + "verification", + "search", + "test", + "evaluation", + "verification_post", + "feedback", +] + + +@dataclass(frozen=True) +class PipelinePhase: + """One step in the analysis-to-feedback improvement loop.""" + + name: PhaseName + label: str + graph_nodes: tuple[str, ...] + eval_target: str | None = None + requires_llm: bool = False + + +# Ordered loop — do not reorder without updating graph.py and run_evals.py together. +IMPROVEMENT_LOOP: tuple[PipelinePhase, ...] = ( + PipelinePhase( + name="analysis", + label="Query analysis (intent, slots, ambiguity)", + graph_nodes=("analyze_query",), + eval_target="evals/offline/run_clarification_eval.py", + ), + PipelinePhase( + name="verification", + label="Input verification (clarification / rewrite decision)", + graph_nodes=("decide_rewrite", "rewrite_query"), + eval_target=None, + ), + PipelinePhase( + name="search", + label="Hybrid retrieval (vector + BM25)", + graph_nodes=("hybrid_retrieve",), + eval_target="evals/offline/run_retrieval_eval.py", + ), + PipelinePhase( + name="test", + label="Retrieval quality test (expand, rerank, relevance filter)", + graph_nodes=("test_retrieval",), + eval_target="tests/unit", + ), + PipelinePhase( + name="evaluation", + label="Answer generation + LLM-as-judge", + graph_nodes=("generate_answer", "run_judge"), + eval_target="evals/offline/run_answer_eval.py", + ), + PipelinePhase( + name="verification_post", + label="Grounding verification (LLM-as-judge)", + graph_nodes=("evaluate_grounding",), + eval_target="evals/offline/run_judge_eval.py", + requires_llm=True, + ), + PipelinePhase( + name="feedback", + label="Feedback loop (retry retrieval / regeneration / reject)", + graph_nodes=( + "retry_retrieval", + "retry_generation", + "retry_with_policy", + "finalize_ok", + "finalize_rejected", + ), + eval_target="evals/offline/run_feedback_eval.py", + ), +) + + +def is_ci_mode() -> bool: + """True when running the CI-safe subset (no vLLM/OpenAI required).""" + return os.getenv("IMPROVEMENT_LOOP_CI", "").lower() in {"1", "true", "yes"} + + +def iter_eval_phases(*, ci: bool | None = None) -> tuple[PipelinePhase, ...]: + """Return phases to execute, optionally skipping LLM-dependent steps.""" + use_ci = is_ci_mode() if ci is None else ci + if not use_ci: + return IMPROVEMENT_LOOP + return tuple(p for p in IMPROVEMENT_LOOP if not p.requires_llm) + + +def graph_node_order() -> list[str]: + """Flat list of CRAG graph nodes in pipeline phase order.""" + nodes: list[str] = [] + for phase in IMPROVEMENT_LOOP: + nodes.extend(phase.graph_nodes) + return nodes diff --git a/app/core/logging.py b/app/core/logging.py index c6e40c5..1788f27 100644 --- a/app/core/logging.py +++ b/app/core/logging.py @@ -6,16 +6,23 @@ import sys -def configure_logging(level: str = "INFO") -> None: +def configure_logging(level: str | None = None) -> None: """Configure root logger with a structured format. Args: level: Log level string, e.g. "DEBUG", "INFO", "WARNING". + Defaults to ``settings.log_level`` when omitted. """ + if level is None: + from app.core.config import settings + + level = settings.log_level + logging.basicConfig( level=getattr(logging, level.upper(), logging.INFO), format="%(asctime)s %(levelname)s [%(name)s] %(message)s", stream=sys.stdout, + force=True, ) diff --git a/app/core/metrics.py b/app/core/metrics.py new file mode 100644 index 0000000..1b256c7 --- /dev/null +++ b/app/core/metrics.py @@ -0,0 +1,90 @@ +"""Prometheus metrics for HTTP and RAG pipeline observability.""" + +from __future__ import annotations + +import time +from collections.abc import Awaitable, Callable + +from prometheus_client import CONTENT_TYPE_LATEST, Counter, Histogram, generate_latest +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request +from starlette.responses import Response + +HTTP_REQUESTS_TOTAL = Counter( + "http_requests_total", + "Total HTTP requests", + ["method", "path", "status"], +) + +HTTP_REQUEST_DURATION_SECONDS = Histogram( + "http_request_duration_seconds", + "HTTP request latency in seconds", + ["method", "path"], + buckets=(0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0), +) + +RAG_QUERIES_TOTAL = Counter( + "rag_queries_total", + "Total RAG query invocations", + ["status"], +) + +RAG_QUERY_DURATION_SECONDS = Histogram( + "rag_query_duration_seconds", + "RAG query latency in seconds", + buckets=(0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0), +) + +INGEST_DOCUMENTS_TOTAL = Counter( + "ingest_documents_total", + "Total document ingest operations", + ["mode", "status"], +) + + +def metrics_response() -> Response: + """Return the Prometheus metrics exposition payload.""" + return Response(content=generate_latest(), media_type=CONTENT_TYPE_LATEST) + + +class MetricsMiddleware(BaseHTTPMiddleware): + """Record HTTP request count and latency for Prometheus.""" + + async def dispatch( + self, + request: Request, + call_next: Callable[[Request], Awaitable[Response]], + ) -> Response: + if request.url.path == "/metrics": + return await call_next(request) + + start = time.perf_counter() + response = await call_next(request) + duration = time.perf_counter() - start + + path = _normalize_path(request.url.path) + HTTP_REQUESTS_TOTAL.labels( + method=request.method, + path=path, + status=str(response.status_code), + ).inc() + HTTP_REQUEST_DURATION_SECONDS.labels(method=request.method, path=path).observe(duration) + return response + + +def _normalize_path(path: str) -> str: + """Collapse dynamic path segments to reduce metric cardinality.""" + if path.startswith("/api/v1/jobs/"): + return "/api/v1/jobs/{job_id}" + return path + + +def record_rag_query(*, status: str, duration_seconds: float) -> None: + """Increment RAG query counters after a pipeline run.""" + RAG_QUERIES_TOTAL.labels(status=status).inc() + RAG_QUERY_DURATION_SECONDS.observe(duration_seconds) + + +def record_ingest(*, mode: str, status: str) -> None: + """Increment ingest counters after a document ingest operation.""" + INGEST_DOCUMENTS_TOTAL.labels(mode=mode, status=status).inc() diff --git a/app/core/middleware.py b/app/core/middleware.py new file mode 100644 index 0000000..aec2bca --- /dev/null +++ b/app/core/middleware.py @@ -0,0 +1,120 @@ +"""HTTP middleware for request tracing, rate limiting, and security headers.""" + +from __future__ import annotations + +import logging +import time +import uuid +from collections import defaultdict, deque +from collections.abc import Awaitable, Callable + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request +from starlette.responses import JSONResponse, Response + +from app.core.config import settings + +logger = logging.getLogger(__name__) + +RequestHandler = Callable[[Request], Awaitable[Response]] + + +class RequestIDMiddleware(BaseHTTPMiddleware): + """Attach a unique request ID to every request and response.""" + + async def dispatch(self, request: Request, call_next: RequestHandler) -> Response: + request_id = request.headers.get("X-Request-ID") or str(uuid.uuid4()) + request.state.request_id = request_id + response = await call_next(request) + response.headers["X-Request-ID"] = request_id + return response + + +class RequestLoggingMiddleware(BaseHTTPMiddleware): + """Log request method, path, status, and latency.""" + + async def dispatch(self, request: Request, call_next: RequestHandler) -> Response: + start = time.perf_counter() + response = await call_next(request) + duration_ms = (time.perf_counter() - start) * 1000 + request_id = getattr(request.state, "request_id", "-") + logger.info( + "request completed method=%s path=%s status=%s duration_ms=%.1f request_id=%s", + request.method, + request.url.path, + response.status_code, + duration_ms, + request_id, + ) + return response + + +class RateLimitMiddleware(BaseHTTPMiddleware): + """Simple in-memory sliding-window rate limiter per client IP.""" + + def __init__(self, app, limit_per_minute: int) -> None: + super().__init__(app) + self._limit = max(limit_per_minute, 0) + self._windows: dict[str, deque[float]] = defaultdict(deque) + + async def dispatch(self, request: Request, call_next: RequestHandler) -> Response: + if self._limit <= 0 or request.url.path in _EXEMPT_PATHS: + return await call_next(request) + + client_ip = request.client.host if request.client else "unknown" + now = time.monotonic() + window = self._windows[client_ip] + + while window and now - window[0] > 60: + window.popleft() + + if len(window) >= self._limit: + request_id = getattr(request.state, "request_id", "-") + logger.warning( + "rate limit exceeded client=%s path=%s request_id=%s", + client_ip, + request.url.path, + request_id, + ) + return JSONResponse( + status_code=429, + content={"detail": "Rate limit exceeded. Try again later."}, + headers={"Retry-After": "60"}, + ) + + window.append(now) + return await call_next(request) + + +class SecurityHeadersMiddleware(BaseHTTPMiddleware): + """Add standard security headers to every response.""" + + async def dispatch(self, request: Request, call_next: RequestHandler) -> Response: + response = await call_next(request) + response.headers.setdefault("X-Content-Type-Options", "nosniff") + response.headers.setdefault("X-Frame-Options", "DENY") + response.headers.setdefault("Referrer-Policy", "strict-origin-when-cross-origin") + response.headers.setdefault("X-XSS-Protection", "1; mode=block") + return response + + +_EXEMPT_PATHS = frozenset( + { + "/", + "/docs", + "/openapi.json", + "/redoc", + "/api/v1/health", + "/api/v1/health/live", + "/api/v1/health/ready", + "/metrics", + } +) + + +def get_cors_origins() -> list[str]: + """Parse CORS origins from settings (comma-separated string).""" + raw = settings.cors_origins.strip() + if not raw or raw == "*": + return ["*"] + return [origin.strip() for origin in raw.split(",") if origin.strip()] diff --git a/app/core/security.py b/app/core/security.py index 63952a4..f429bd0 100644 --- a/app/core/security.py +++ b/app/core/security.py @@ -12,12 +12,14 @@ async def verify_api_key(api_key: str | None = Security(_API_KEY_HEADER)) -> None: - """Validate the X-API-Key header if a key is configured. + """Validate the X-API-Key header when ``API_KEY`` is configured. - Currently a no-op stub: configure ``API_KEY`` env var to enable. + When ``API_KEY`` is empty, authentication is disabled (development mode). """ from app.core.config import settings - configured_key = getattr(settings, "api_key", "") - if configured_key and api_key != configured_key: + if not settings.auth_enabled: + return + + if api_key != settings.api_key: raise HTTPException(status_code=401, detail="Invalid or missing API key") diff --git a/app/graphs/crag/graph.py b/app/graphs/crag/graph.py index 18c27d5..4dc7a60 100644 --- a/app/graphs/crag/graph.py +++ b/app/graphs/crag/graph.py @@ -1,27 +1,17 @@ """CRAG (Corrective RAG) LangGraph graph construction and compilation. -Graph flow: - analyze_query - ↓ (needs_clarification?) - [ask_clarification] → END (clarification_needed) - ↓ - decide_rewrite - ↓ (needs_rewrite?) - [rewrite_query] - ↓ - hybrid_retrieve - ↓ (should_expand?) - [expand_context] - ↓ - rerank_context - ↓ - generate_answer - ↓ - evaluate_grounding - ↓ (grounding_score < threshold AND retries < max?) - [retry_with_policy] → decide_rewrite (loop, max 3×) - ↓ - END +Improvement loop order (matches ``app.core.improvement_loop``): + + analyze_query → analysis + decide_rewrite → verification + rewrite_query → verification (conditional) + hybrid_retrieve → search + test_retrieval → test + generate_answer → evaluation + run_judge → evaluation + evaluate_grounding → verification (post) + [feedback routers] → retry_retrieval | retry_generation | retry_with_policy + | finalize_ok | finalize_rejected """ from __future__ import annotations @@ -33,17 +23,20 @@ ask_clarification, decide_rewrite, evaluate_grounding, - expand_context, + finalize_ok, + finalize_rejected, generate_answer, hybrid_retrieve, - rerank_context, + retry_generation, + retry_retrieval, retry_with_policy, rewrite_query, + run_judge, + test_retrieval, ) from app.graphs.crag.routes import ( route_after_analyze, - route_after_grounding, - route_after_retrieve, + route_after_feedback_eval, route_after_rewrite_decision, ) from app.graphs.crag.state import CRAGState @@ -58,22 +51,24 @@ def build_crag_graph() -> StateGraph: """ graph = StateGraph(CRAGState) - # ── Register nodes ──────────────────────────────────────────────────────── + # ── Register nodes (improvement-loop order) ─────────────────────────────── graph.add_node("analyze_query", analyze_query) graph.add_node("ask_clarification", ask_clarification) graph.add_node("decide_rewrite", decide_rewrite) graph.add_node("rewrite_query", rewrite_query) graph.add_node("hybrid_retrieve", hybrid_retrieve) - graph.add_node("expand_context", expand_context) - graph.add_node("rerank_context", rerank_context) + graph.add_node("test_retrieval", test_retrieval) graph.add_node("generate_answer", generate_answer) + graph.add_node("run_judge", run_judge) graph.add_node("evaluate_grounding", evaluate_grounding) + graph.add_node("retry_retrieval", retry_retrieval) + graph.add_node("retry_generation", retry_generation) graph.add_node("retry_with_policy", retry_with_policy) + graph.add_node("finalize_ok", finalize_ok) + graph.add_node("finalize_rejected", finalize_rejected) - # ── Entry point ─────────────────────────────────────────────────────────── + # ── Phase 1–2: analysis → verification ─────────────────────────────────── graph.set_entry_point("analyze_query") - - # ── Conditional: clarification ──────────────────────────────────────────── graph.add_conditional_edges( "analyze_query", route_after_analyze, @@ -81,7 +76,6 @@ def build_crag_graph() -> StateGraph: ) graph.add_edge("ask_clarification", END) - # ── Conditional: rewrite ────────────────────────────────────────────────── graph.add_conditional_edges( "decide_rewrite", route_after_rewrite_decision, @@ -89,22 +83,30 @@ def build_crag_graph() -> StateGraph: ) graph.add_edge("rewrite_query", "hybrid_retrieve") - # ── Conditional: context expansion ─────────────────────────────────────── - graph.add_conditional_edges( - "hybrid_retrieve", - route_after_retrieve, - {"expand_context": "expand_context", "rerank_context": "rerank_context"}, - ) - graph.add_edge("expand_context", "rerank_context") - graph.add_edge("rerank_context", "generate_answer") - graph.add_edge("generate_answer", "evaluate_grounding") + # ── Phase 3–4: search → test ────────────────────────────────────────────── + graph.add_edge("hybrid_retrieve", "test_retrieval") + + # ── Phase 5–6: evaluation → verification ─────────────────────────────────── + graph.add_edge("test_retrieval", "generate_answer") + graph.add_edge("generate_answer", "run_judge") + graph.add_edge("run_judge", "evaluate_grounding") - # ── Conditional: hallucination feedback loop (max 3×) ──────────────────── + # ── Phase 7: feedback loop ──────────────────────────────────────────────── graph.add_conditional_edges( "evaluate_grounding", - route_after_grounding, - {"retry_with_policy": "retry_with_policy", "end": END}, + route_after_feedback_eval, + { + "end": "finalize_ok", + "reject": "finalize_rejected", + "retry_retrieval": "retry_retrieval", + "retry_generation": "retry_generation", + "retry_with_policy": "retry_with_policy", + }, ) + graph.add_edge("finalize_ok", END) + graph.add_edge("finalize_rejected", END) + graph.add_edge("retry_retrieval", "decide_rewrite") + graph.add_edge("retry_generation", "generate_answer") graph.add_edge("retry_with_policy", "decide_rewrite") return graph.compile() diff --git a/app/graphs/crag/nodes.py b/app/graphs/crag/nodes.py index b36e91a..218fcee 100644 --- a/app/graphs/crag/nodes.py +++ b/app/graphs/crag/nodes.py @@ -109,6 +109,40 @@ def hybrid_retrieve(state: CRAGState) -> dict: return {"retrieved_children": children, "retrieval_attempt": attempt} +# ── test_retrieval ──────────────────────────────────────────────────────────── + + +def test_retrieval(state: CRAGState) -> dict: + """Test retrieved context quality: expand, rerank, and relevance filter. + + Phase 4 of the improvement loop (search → **test** → evaluation). + """ + from app.providers.reranker_provider import get_reranker + from app.rag.guards.relevance_guard import filter_relevant + from app.rag.policies.expansion_policy import should_expand + + children = state.get("retrieved_children", []) + query = _active_query(state) + + if should_expand(state): + contexts = expand_context(state).get("expanded_contexts", children) + else: + contexts = children + logger.info("Test phase: evaluating %d candidate chunks", len(contexts)) + + reranker = get_reranker() + if reranker is not None: + contexts = reranker.rerank(query, contexts) + + filtered = filter_relevant(contexts) + logger.info( + "Test phase: %d → %d contexts passed relevance filter", + len(contexts), + len(filtered), + ) + return {"expanded_contexts": filtered} + + # ── expand_context ──────────────────────────────────────────────────────────── @@ -211,3 +245,40 @@ def retry_with_policy(state: CRAGState) -> dict: "hallucination_attempt": attempt, "needs_rewrite": True, } + + +def retry_retrieval(state: CRAGState) -> dict: + """Feedback: low faithfulness — loop back to search with a rewritten query.""" + attempt = state.get("hallucination_attempt", 0) + 1 + retrieval = state.get("retrieval_attempt", 0) + 1 + logger.info( + "Feedback retry_retrieval: hallucination=%d retrieval=%d", + attempt, + retrieval, + ) + return { + "hallucination_attempt": attempt, + "retrieval_attempt": retrieval, + "needs_rewrite": True, + } + + +def retry_generation(state: CRAGState) -> dict: + """Feedback: low overall score — regenerate answer without re-retrieval.""" + attempt = state.get("hallucination_attempt", 0) + 1 + logger.info("Feedback retry_generation: attempt %d/%d", attempt, settings.max_retries) + return {"hallucination_attempt": attempt} + + +def finalize_ok(state: CRAGState) -> dict: + """Mark the pipeline as successfully completed.""" + return {"final_status": "ok"} + + +def finalize_rejected(state: CRAGState) -> dict: + """Mark the pipeline as rejected after exhausting feedback retries.""" + answer = state.get("answer", "") + suffix = " [Answer did not pass quality verification]" + if suffix not in answer: + answer = f"{answer}{suffix}".strip() + return {"final_status": "rejected", "answer": answer} diff --git a/app/graphs/crag/routes.py b/app/graphs/crag/routes.py index 6b736cb..1b3c549 100644 --- a/app/graphs/crag/routes.py +++ b/app/graphs/crag/routes.py @@ -15,6 +15,7 @@ from app.graphs.crag.state import CRAGState from app.rag.policies.routing_policy import ( route_after_clarification_check, + route_after_feedback, route_after_grounding, route_after_judge, route_after_retrieval, @@ -73,3 +74,16 @@ def route_after_grounding_eval(state: CRAGState) -> str: settings.max_retries, ) return result + + +def route_after_feedback_eval(state: CRAGState) -> str: + """Route after post-verification — feedback loop or end.""" + result = route_after_feedback(state) + logger.info( + "Route after feedback → %s (grounding=%.2f, attempt %d/%d)", + result, + state.get("grounding_score", 0.0), + state.get("hallucination_attempt", 0), + settings.max_retries, + ) + return result diff --git a/app/main.py b/app/main.py index 09a9bcd..d15eab3 100644 --- a/app/main.py +++ b/app/main.py @@ -4,14 +4,24 @@ from contextlib import asynccontextmanager -from fastapi import FastAPI +from fastapi import FastAPI, Response +from fastapi.middleware.cors import CORSMiddleware from app.api.v1.chat import router as chat_router from app.api.v1.health import router as health_router from app.api.v1.ingest import router as ingest_router from app.api.v1.jobs import router as jobs_router from app.core.config import settings +from app.core.exceptions import register_exception_handlers from app.core.logging import configure_logging, get_logger +from app.core.metrics import MetricsMiddleware, metrics_response +from app.core.middleware import ( + RateLimitMiddleware, + RequestIDMiddleware, + RequestLoggingMiddleware, + SecurityHeadersMiddleware, + get_cors_origins, +) from app.queue.pool import close_arq_pool from app.services.index_service import ensure_index @@ -29,6 +39,7 @@ async def lifespan(app: FastAPI): ) logger.info("Embedding model: %s", settings.embedding_model) logger.info("LLM backend: %s model: %s", settings.llm_backend, settings.llm_model) + logger.info("API auth: %s", "enabled" if settings.auth_enabled else "disabled") if settings.using_vllm: logger.info("vLLM endpoint: %s", settings.vllm_base_url) @@ -52,6 +63,23 @@ async def lifespan(app: FastAPI): lifespan=lifespan, ) +register_exception_handlers(app) + +app.add_middleware(SecurityHeadersMiddleware) +app.add_middleware(RateLimitMiddleware, limit_per_minute=settings.rate_limit_per_minute) +if settings.enable_metrics: + app.add_middleware(MetricsMiddleware) +app.add_middleware(RequestLoggingMiddleware) +app.add_middleware(RequestIDMiddleware) + +app.add_middleware( + CORSMiddleware, + allow_origins=get_cors_origins(), + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + app.include_router(health_router, prefix="/api/v1") app.include_router(ingest_router, prefix="/api/v1") app.include_router(chat_router, prefix="/api/v1") @@ -66,3 +94,11 @@ async def root(): "version": "0.2.0", "docs": "/docs", } + + +@app.get("/metrics", tags=["system"], include_in_schema=settings.enable_metrics) +async def metrics(): + """Prometheus metrics exposition endpoint.""" + if not settings.enable_metrics: + return Response(status_code=404, content="Metrics disabled") + return metrics_response() diff --git a/app/rag/policies/routing_policy.py b/app/rag/policies/routing_policy.py index bcce622..1efa862 100644 --- a/app/rag/policies/routing_policy.py +++ b/app/rag/policies/routing_policy.py @@ -89,3 +89,45 @@ def route_after_grounding(state: dict[str, Any]) -> str: from app.rag.policies.retry_policy import should_retry return "retry_with_policy" if should_retry(state) else "end" + + +def route_after_feedback(state: dict[str, Any]) -> str: + """Post-verification feedback router — judge verdict first, then grounding. + + Implements the final **feedback** phase of the improvement loop: + analysis → verification → search → test → evaluation → verification → **feedback**. + + Priority: + 1. Judge says reject → ``"reject"`` + 2. Judge says retry_retrieval → ``"retry_retrieval"`` + 3. Judge says retry_generation → ``"retry_generation"`` + 4. Grounding below threshold → ``"retry_with_policy"`` + 5. Otherwise → ``"end"`` + + Returns: + Routing key for the feedback conditional edges in the CRAG graph. + """ + from app.core.config import settings + from app.rag.policies.judge_policy import decide_next_action + from app.rag.policies.retry_policy import should_retry + + verdict = state.get("judge_verdict") + attempt = state.get("hallucination_attempt", 0) + + if verdict is not None: + action = decide_next_action( + verdict, + hallucination_attempt=attempt, + max_retries=settings.max_retries, + ) + if action == "reject": + return "reject" + if action == "retry_retrieval": + return "retry_retrieval" + if action == "retry_generation": + return "retry_generation" + + if should_retry(state): + return "retry_with_policy" + + return "end" diff --git a/app/schemas/response.py b/app/schemas/response.py index 86f8fcf..b0009b8 100644 --- a/app/schemas/response.py +++ b/app/schemas/response.py @@ -34,7 +34,24 @@ class HealthResponse(BaseModel): """Health / readiness check response.""" status: str + version: str = "0.2.0" llm_backend: str llm_model: str qdrant: str collection: str + redis: str = "not_configured" + llm: str = "skipped" + auth_enabled: bool = False + + +class LivenessResponse(BaseModel): + """Liveness probe response.""" + + status: str + + +class ReadinessResponse(BaseModel): + """Readiness probe response with per-dependency checks.""" + + status: str + checks: dict[str, dict[str, str]] diff --git a/app/services/chat_service.py b/app/services/chat_service.py index b7e6567..6ee66ae 100644 --- a/app/services/chat_service.py +++ b/app/services/chat_service.py @@ -3,8 +3,10 @@ from __future__ import annotations import logging +import time from typing import Any +from app.core.metrics import record_rag_query from app.graphs.crag.graph import crag_chain from app.rag.guards.policy_guard import is_allowed @@ -25,6 +27,7 @@ def run_chat( Result dict with ``answer``, ``final_status``, and graph state fields. """ if not is_allowed(question): + record_rag_query(status="blocked", duration_seconds=0.0) return { "answer": "I'm sorry, but I can't help with that request.", "final_status": "blocked", @@ -36,10 +39,14 @@ def run_chat( "hallucination_attempt": 0, } + start = time.perf_counter() try: result = crag_chain.invoke(initial_state) - logger.info("Chat completed, status=%s", result.get("final_status", "ok")) + status = str(result.get("final_status", "ok")) + record_rag_query(status=status, duration_seconds=time.perf_counter() - start) + logger.info("Chat completed, status=%s", status) return result except Exception: + record_rag_query(status="error", duration_seconds=time.perf_counter() - start) logger.exception("Chat service error for question: %s", question[:80]) raise diff --git a/docker-compose.yml b/docker-compose.yml index d5899b6..615d0f7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,12 @@ services: condition: service_started qdrant: condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/health/ready"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 30s volumes: - ./data:/app/data restart: unless-stopped diff --git a/docker/Dockerfile b/docker/Dockerfile index a37c560..036e3a7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,6 +2,9 @@ FROM python:3.12-slim WORKDIR /app +RUN apt-get update && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* + COPY pyproject.toml . RUN pip install --no-cache-dir -e . diff --git a/evals/offline/run_feedback_eval.py b/evals/offline/run_feedback_eval.py new file mode 100644 index 0000000..6e5a82d --- /dev/null +++ b/evals/offline/run_feedback_eval.py @@ -0,0 +1,90 @@ +"""Offline feedback-loop evaluation against the golden regression set. + +Validates routing policy decisions for the feedback phase of the improvement loop. + +Usage: + python evals/offline/run_feedback_eval.py +""" + +from __future__ import annotations + +import logging +from pathlib import Path + +import yaml + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +GOLDEN_SET = Path(__file__).parent.parent / "regression" / "golden_set.yaml" + +_BAD_VERDICT = dict(correctness=0.2, faithfulness=0.1, completeness=0.2, conciseness=0.2) +_GOOD_VERDICT = dict(correctness=0.9, faithfulness=0.9, completeness=0.9, conciseness=0.9) + + +def run() -> None: + """Evaluate feedback routing against golden_set.yaml expectations.""" + from app.rag.evaluators.llm_judge_evaluator import JudgeVerdict + from app.rag.policies.clarification_policy import needs_clarification + from app.rag.policies.rewrite_policy import needs_rewrite + from app.rag.policies.routing_policy import route_after_feedback + from app.rag.query.query_analyzer import analyze + + with GOLDEN_SET.open() as f: + data = yaml.safe_load(f) + + failed = [] + for item in data.get("golden_set", []): + item_id = item["id"] + query = item.get("query", "") + + if "expected_clarification" in item: + analysis = analyze(query) + predicted = needs_clarification(analysis) + expected = item["expected_clarification"] + if predicted != expected: + failed.append(f"{item_id}: clarification expected={expected} got={predicted}") + continue + + if item.get("expected_rewrite") is not None: + predicted_rewrite = needs_rewrite(query, {}) + expected = item["expected_rewrite"] + if predicted_rewrite != expected: + failed.append(f"{item_id}: rewrite expected={expected} got={predicted_rewrite}") + continue + + if "answer_hallucinated" in item: + verdict = JudgeVerdict(**_BAD_VERDICT) + state = { + "judge_verdict": verdict, + "grounding_score": 0.2, + "hallucination_attempt": 0, + } + action = route_after_feedback(state) + expected = item.get("expected_judge_action", "retry_retrieval") + if action not in {expected, "retry_with_policy", "retry_retrieval"}: + failed.append(f"{item_id}: feedback action expected~={expected} got={action}") + + if "answer_correct" in item: + verdict = JudgeVerdict(**_GOOD_VERDICT) + state = { + "judge_verdict": verdict, + "grounding_score": 0.9, + "hallucination_attempt": 0, + } + action = route_after_feedback(state) + if action != "end": + failed.append(f"{item_id}: feedback action expected=end got={action}") + + logger.info("✓ %s", item_id) + + if failed: + for msg in failed: + logger.error("✗ %s", msg) + raise SystemExit(1) + + logger.info("Feedback golden set: all %d cases passed", len(data.get("golden_set", []))) + + +if __name__ == "__main__": + run() diff --git a/pyproject.toml b/pyproject.toml index 68b58c0..e180922 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,9 @@ dependencies = [ "python-multipart>=0.0.12", "python-dotenv>=1.0.0", "arq>=0.26.0", + "prometheus-client>=0.21.0", + "httpx>=0.27.0", + "pyyaml>=6.0.0", ] [project.optional-dependencies] diff --git a/scripts/run_evals.py b/scripts/run_evals.py index ed6c16f..ebc19af 100644 --- a/scripts/run_evals.py +++ b/scripts/run_evals.py @@ -1,39 +1,76 @@ -"""CLI script — run all offline evaluations. +"""CLI script — run the full improvement loop (analysis → feedback). Usage: - python scripts/run_evals.py + python scripts/run_evals.py # full loop (needs LLM for judge phase) + python scripts/run_evals.py --ci # CI-safe subset (no LLM) + IMPROVEMENT_LOOP_CI=1 make evals-ci """ from __future__ import annotations +import argparse import logging import subprocess import sys +from app.core.improvement_loop import iter_eval_phases + logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -EVAL_SCRIPTS = [ - "evals/offline/run_clarification_eval.py", - "evals/offline/run_retrieval_eval.py", - "evals/offline/run_answer_eval.py", - "evals/offline/run_judge_eval.py", -] + +def _run_pytest(target: str) -> int: + """Run pytest against a test directory or file.""" + return subprocess.run( + [sys.executable, "-m", "pytest", target, "-q", "--tb=short"], + capture_output=False, + ).returncode + + +def _run_script(script: str) -> int: + """Run a Python eval script.""" + return subprocess.run([sys.executable, script], capture_output=False).returncode def main() -> None: - """Run each eval script in sequence and report pass/fail.""" - failed = [] - for script in EVAL_SCRIPTS: - logger.info("Running %s ...", script) - result = subprocess.run([sys.executable, script], capture_output=False) - if result.returncode != 0: - failed.append(script) + """Run each improvement-loop phase in canonical order.""" + parser = argparse.ArgumentParser(description="Run the Advanced-RAG improvement loop") + parser.add_argument( + "--ci", + action="store_true", + help="Skip LLM-dependent phases (safe for GitHub Actions without vLLM)", + ) + args = parser.parse_args() + + phases = iter_eval_phases(ci=args.ci) + if args.ci: + logger.info("CI mode: skipping LLM-dependent phases") + + failed: list[str] = [] + + for phase in phases: + logger.info("=== Phase [%s] %s ===", phase.name, phase.label) + + if phase.name == "verification" and phase.eval_target is None: + logger.info( + "Skipping runtime-only verification phase (covered by graph integration tests)" + ) + continue + + target = phase.eval_target + if target is None: + continue + + code = _run_pytest(target) if target.startswith("tests/") else _run_script(target) + + if code != 0: + failed.append(f"{phase.name}:{target}") if failed: - logger.error("Failed evals: %s", failed) + logger.error("Failed phases: %s", failed) sys.exit(1) - logger.info("All evals passed.") + + logger.info("Improvement loop complete — all phases passed.") if __name__ == "__main__": diff --git a/tests/integration/test_ops.py b/tests/integration/test_ops.py new file mode 100644 index 0000000..b6a902a --- /dev/null +++ b/tests/integration/test_ops.py @@ -0,0 +1,86 @@ +"""Integration tests for operational endpoints (health, metrics, auth, rate limit).""" + +from __future__ import annotations + +import pytest +from fastapi.testclient import TestClient + +from app.core.config import settings +from app.main import app + + +@pytest.fixture +def authed_client(monkeypatch): + """Test client with API key auth enabled.""" + monkeypatch.setattr(settings, "api_key", "test-secret-key") + with TestClient(app) as client: + yield client + + +def test_liveness(client): + resp = client.get("/api/v1/health/live") + assert resp.status_code == 200 + assert resp.json()["status"] == "alive" + + +def test_readiness(client): + resp = client.get("/api/v1/health/ready") + assert resp.status_code == 200 + data = resp.json() + assert data["status"] == "ready" + assert data["checks"]["vectorstore"]["status"] == "connected" + + +def test_health_includes_ops_fields(client): + resp = client.get("/api/v1/health") + assert resp.status_code == 200 + data = resp.json() + assert data["status"] in {"ok", "degraded"} + assert data["version"] == "0.2.0" + assert "redis" in data + assert "auth_enabled" in data + + +def test_metrics_endpoint(client): + resp = client.get("/metrics") + assert resp.status_code == 200 + assert "http_requests_total" in resp.text + + +def test_request_id_header(client): + resp = client.get("/") + assert resp.status_code == 200 + assert "X-Request-ID" in resp.headers + + +def test_security_headers(client): + resp = client.get("/") + assert resp.headers.get("X-Content-Type-Options") == "nosniff" + assert resp.headers.get("X-Frame-Options") == "DENY" + + +def test_api_key_required_when_configured(authed_client): + resp = authed_client.post("/api/v1/search", json={"query": "test", "top_k": 1}) + assert resp.status_code == 401 + + resp = authed_client.post( + "/api/v1/search", + json={"query": "test", "top_k": 1}, + headers={"X-API-Key": "test-secret-key"}, + ) + assert resp.status_code == 200 + + +def test_health_endpoints_skip_auth_when_configured(authed_client): + resp = authed_client.get("/api/v1/health/live") + assert resp.status_code == 200 + + resp = authed_client.get("/metrics") + assert resp.status_code == 200 + + +def test_error_response_includes_request_id(client): + resp = client.post("/api/v1/documents", json={"documents": []}) + assert resp.status_code == 422 + body = resp.json() + assert "request_id" in body diff --git a/tests/unit/test_feedback_routing.py b/tests/unit/test_feedback_routing.py new file mode 100644 index 0000000..c04ba7d --- /dev/null +++ b/tests/unit/test_feedback_routing.py @@ -0,0 +1,36 @@ +"""Tests for the post-verification feedback router.""" + +from __future__ import annotations + +from app.rag.evaluators.llm_judge_evaluator import JudgeVerdict +from app.rag.policies.routing_policy import route_after_feedback + + +def test_feedback_accepts_good_answer(): + verdict = JudgeVerdict(correctness=0.9, faithfulness=0.9, completeness=0.9, conciseness=0.9) + state = {"judge_verdict": verdict, "grounding_score": 0.9, "hallucination_attempt": 0} + assert route_after_feedback(state) == "end" + + +def test_feedback_retry_retrieval_low_faithfulness(): + verdict = JudgeVerdict(correctness=0.9, faithfulness=0.2, completeness=0.9, conciseness=0.9) + state = {"judge_verdict": verdict, "grounding_score": 0.9, "hallucination_attempt": 0} + assert route_after_feedback(state) == "retry_retrieval" + + +def test_feedback_retry_generation_low_overall(): + verdict = JudgeVerdict(correctness=0.2, faithfulness=0.6, completeness=0.2, conciseness=0.2) + state = {"judge_verdict": verdict, "grounding_score": 0.9, "hallucination_attempt": 0} + assert route_after_feedback(state) == "retry_generation" + + +def test_feedback_grounding_retry_when_judge_passes(): + verdict = JudgeVerdict(correctness=0.9, faithfulness=0.9, completeness=0.9, conciseness=0.9) + state = {"judge_verdict": verdict, "grounding_score": 0.2, "hallucination_attempt": 0} + assert route_after_feedback(state) == "retry_with_policy" + + +def test_feedback_reject_at_max_retries(): + verdict = JudgeVerdict(correctness=0.1, faithfulness=0.1, completeness=0.1, conciseness=0.1) + state = {"judge_verdict": verdict, "grounding_score": 0.1, "hallucination_attempt": 3} + assert route_after_feedback(state) == "reject"