feat(rag): add pdf-inspector fallback, optimize batch embeddings, and fix migration

.gitignore

@@ -75,6 +75,9 @@ evaluation/results/
 evaluation/evaluation/
 evaluation/benchmark_dataset.json
 evaluation/benchmark_corpus.json
+evaluation/*_queries.json
+scripts/ingest_*.py
+scripts/run_*_benchmark.py
 evaluation/*.log
 evaluation/*.sh
 uvicorn.log

api/main.py

@@ -22,7 +22,11 @@ async def lifespan(_app: FastAPI):
 
 settings = get_settings()
 app = FastAPI(title="DepthAPI", version="0.1.0", lifespan=lifespan)
-limiter = Limiter(key_func=get_remote_address, default_limits=[f"{settings.slowapi_default_limit_per_minute}/minute"])
+limiter = Limiter(
+    key_func=get_remote_address,
+    default_limits=[f"{settings.slowapi_default_limit_per_minute}/minute"],
+    enabled=settings.slowapi_enabled,
+)
 app.state.limiter = limiter
 app.add_exception_handler(RateLimitExceeded, lambda _request, _exc: JSONResponse(status_code=429, content={"error": "rate limit exceeded"}))
 app.add_middleware(SlowAPIMiddleware)

api/routers/ingest.py

@@ -436,7 +436,7 @@ async def ingest(
     )
 
 
-MAX_FILE_BYTES = 50 * 1024 * 1024  # 50 MB limit
+MAX_FILE_BYTES = 75 * 1024 * 1024  # 75 MB limit
 
 
 @router.post("/ingest/file", response_model=IngestResponse)
@@ -490,6 +490,19 @@ async def ingest_file(
         except Exception as exc:
             log.warning("depth_engine.to_markdown failed on %s: %s", filename, exc)
 
+    # High-accuracy fallback for PDFs using pdf_inspector
+    if filename.lower().endswith(".pdf") and (not raw_text or user_metadata.get("parser_confidence", 1.0) < 0.8):
+        try:
+            import pdf_inspector
+            pi_text = await asyncio.to_thread(pdf_inspector.extract_text_bytes, file_bytes)
+            if pi_text and len(pi_text.strip()) > 30:
+                log.info("pdf_inspector extracted %d chars for %s", len(pi_text), filename)
+                raw_text = pi_text
+                user_metadata["parser_engine"] = "pdf_inspector"
+                user_metadata["parser_confidence"] = 0.95
+        except Exception as pi_exc:
+            log.debug("pdf_inspector fallback skipped or failed: %s", pi_exc)
+
     if not raw_text:
         # Fallback for plain text, markdown, or csv
         try:

api/services/rag/embeddings.py

@@ -1,6 +1,7 @@
 """Embedding generation with local SentenceTransformer and deterministic fallback."""
 from __future__ import annotations
 
+import asyncio
 import functools
 import hashlib
 import logging
@@ -70,7 +71,13 @@ async def embed_texts(texts: Sequence[str]) -> list[str]:
     model = get_local_transformer(DEFAULT_LOCAL_MODEL)
     if model is not None:
         try:
-            embeddings = model.encode(list(texts), normalize_embeddings=True)
+            embeddings = await asyncio.to_thread(
+                model.encode,
+                list(texts),
+                batch_size=64,
+                normalize_embeddings=True,
+                show_progress_bar=False,
+            )
             return [_vector_literal(emb.round(8).tolist()) for emb in embeddings]
         except Exception as exc:
             log.warning("Local neural embedding failed, falling back to hash: %s", exc)

db/migrations/005_rank_by_attribute.sql

@@ -24,6 +24,12 @@
 --   * quality_score is already guaranteed [0, 1] by the ingestion pipeline, so
 --     no normalisation is needed.
 
+-- Add quality_score column to knowledge_chunks if not yet present.
+ALTER TABLE knowledge_chunks ADD COLUMN IF NOT EXISTS quality_score float;
+CREATE INDEX IF NOT EXISTS knowledge_chunks_quality_idx
+    ON knowledge_chunks (quality_score)
+    WHERE quality_score IS NOT NULL;
+
 CREATE OR REPLACE FUNCTION hybrid_search_v6(
     query_text           text,
     query_embedding      vector(768),
@@ -128,12 +134,3 @@ CREATE OR REPLACE FUNCTION hybrid_search_trusted_v6(
         recency_weight, quality_weight
     )
 $$;
-
--- Add quality_score column to knowledge_chunks if not yet present.
--- (The ingestion pipeline already writes this; migration is idempotent.)
-ALTER TABLE knowledge_chunks ADD COLUMN IF NOT EXISTS quality_score float;
-
--- Index on quality_score for future filter-by-quality queries.
-CREATE INDEX IF NOT EXISTS knowledge_chunks_quality_idx
-    ON knowledge_chunks (quality_score)
-    WHERE quality_score IS NOT NULL;

scripts/validate_rag_local.sh

@@ -31,7 +31,7 @@ else
 fi
 
 for _ in {1..30}; do
-  if docker exec depthapi-postgres pg_isready -U depthapi -d depthapi >/dev/null 2>&1; then break; fi
+  if docker compose exec -T postgres pg_isready -U depthapi -d depthapi >/dev/null 2>&1 || docker exec depthapi-postgres pg_isready -U depthapi -d depthapi >/dev/null 2>&1; then break; fi
   sleep 1
 done