From 218c688a7677769060e68a0433db32785c04e389 Mon Sep 17 00:00:00 2001 From: sanjeevafk <230678117+sanjeevafk@users.noreply.github.com> Date: Tue, 6 Oct 2026 01:20:10 +0530 Subject: [PATCH 1/2] feat(rag): add pdf-inspector fallback, optimize batch embeddings, and fix migration - Add pdf-inspector fallback in ingest router for PDFs with low confidence or failed streams - Bump MAX_FILE_BYTES to 75 MB to support large monograph and book uploads - Optimize local neural embeddings with asyncio.to_thread and batch_size 64 - Fix slowapi rate limiter initialization to honor settings.slowapi_enabled - Fix migration 005_rank_by_attribute.sql to add quality_score column before hybrid_search_v6 - Improve Docker Compose v2 compatibility in validate_rag_local.sh --- api/main.py | 6 +++++- api/routers/ingest.py | 15 ++++++++++++++- api/services/rag/embeddings.py | 9 ++++++++- db/migrations/005_rank_by_attribute.sql | 15 ++++++--------- scripts/validate_rag_local.sh | 2 +- 5 files changed, 34 insertions(+), 13 deletions(-) diff --git a/api/main.py b/api/main.py index d2cf487..6223779 100644 --- a/api/main.py +++ b/api/main.py @@ -22,7 +22,11 @@ async def lifespan(_app: FastAPI): settings = get_settings() app = FastAPI(title="DepthAPI", version="0.1.0", lifespan=lifespan) -limiter = Limiter(key_func=get_remote_address, default_limits=[f"{settings.slowapi_default_limit_per_minute}/minute"]) +limiter = Limiter( + key_func=get_remote_address, + default_limits=[f"{settings.slowapi_default_limit_per_minute}/minute"], + enabled=settings.slowapi_enabled, +) app.state.limiter = limiter app.add_exception_handler(RateLimitExceeded, lambda _request, _exc: JSONResponse(status_code=429, content={"error": "rate limit exceeded"})) app.add_middleware(SlowAPIMiddleware) diff --git a/api/routers/ingest.py b/api/routers/ingest.py index 5a252c5..04c47d7 100644 --- a/api/routers/ingest.py +++ b/api/routers/ingest.py @@ -436,7 +436,7 @@ async def ingest( ) -MAX_FILE_BYTES = 50 * 1024 * 1024 # 50 MB limit +MAX_FILE_BYTES = 75 * 1024 * 1024 # 75 MB limit @router.post("/ingest/file", response_model=IngestResponse) @@ -490,6 +490,19 @@ async def ingest_file( except Exception as exc: log.warning("depth_engine.to_markdown failed on %s: %s", filename, exc) + # High-accuracy fallback for PDFs using pdf_inspector + if filename.lower().endswith(".pdf") and (not raw_text or user_metadata.get("parser_confidence", 1.0) < 0.8): + try: + import pdf_inspector + pi_text = await asyncio.to_thread(pdf_inspector.extract_text_bytes, file_bytes) + if pi_text and len(pi_text.strip()) > 30: + log.info("pdf_inspector extracted %d chars for %s", len(pi_text), filename) + raw_text = pi_text + user_metadata["parser_engine"] = "pdf_inspector" + user_metadata["parser_confidence"] = 0.95 + except Exception as pi_exc: + log.debug("pdf_inspector fallback skipped or failed: %s", pi_exc) + if not raw_text: # Fallback for plain text, markdown, or csv try: diff --git a/api/services/rag/embeddings.py b/api/services/rag/embeddings.py index 4497237..8701d7e 100644 --- a/api/services/rag/embeddings.py +++ b/api/services/rag/embeddings.py @@ -1,6 +1,7 @@ """Embedding generation with local SentenceTransformer and deterministic fallback.""" from __future__ import annotations +import asyncio import functools import hashlib import logging @@ -70,7 +71,13 @@ async def embed_texts(texts: Sequence[str]) -> list[str]: model = get_local_transformer(DEFAULT_LOCAL_MODEL) if model is not None: try: - embeddings = model.encode(list(texts), normalize_embeddings=True) + embeddings = await asyncio.to_thread( + model.encode, + list(texts), + batch_size=64, + normalize_embeddings=True, + show_progress_bar=False, + ) return [_vector_literal(emb.round(8).tolist()) for emb in embeddings] except Exception as exc: log.warning("Local neural embedding failed, falling back to hash: %s", exc) diff --git a/db/migrations/005_rank_by_attribute.sql b/db/migrations/005_rank_by_attribute.sql index 2b0ed27..f2bf4ae 100644 --- a/db/migrations/005_rank_by_attribute.sql +++ b/db/migrations/005_rank_by_attribute.sql @@ -24,6 +24,12 @@ -- * quality_score is already guaranteed [0, 1] by the ingestion pipeline, so -- no normalisation is needed. +-- Add quality_score column to knowledge_chunks if not yet present. +ALTER TABLE knowledge_chunks ADD COLUMN IF NOT EXISTS quality_score float; +CREATE INDEX IF NOT EXISTS knowledge_chunks_quality_idx + ON knowledge_chunks (quality_score) + WHERE quality_score IS NOT NULL; + CREATE OR REPLACE FUNCTION hybrid_search_v6( query_text text, query_embedding vector(768), @@ -128,12 +134,3 @@ CREATE OR REPLACE FUNCTION hybrid_search_trusted_v6( recency_weight, quality_weight ) $$; - --- Add quality_score column to knowledge_chunks if not yet present. --- (The ingestion pipeline already writes this; migration is idempotent.) -ALTER TABLE knowledge_chunks ADD COLUMN IF NOT EXISTS quality_score float; - --- Index on quality_score for future filter-by-quality queries. -CREATE INDEX IF NOT EXISTS knowledge_chunks_quality_idx - ON knowledge_chunks (quality_score) - WHERE quality_score IS NOT NULL; diff --git a/scripts/validate_rag_local.sh b/scripts/validate_rag_local.sh index f899fed..b8be7c3 100755 --- a/scripts/validate_rag_local.sh +++ b/scripts/validate_rag_local.sh @@ -31,7 +31,7 @@ else fi for _ in {1..30}; do - if docker exec depthapi-postgres pg_isready -U depthapi -d depthapi >/dev/null 2>&1; then break; fi + if docker compose exec -T postgres pg_isready -U depthapi -d depthapi >/dev/null 2>&1 || docker exec depthapi-postgres pg_isready -U depthapi -d depthapi >/dev/null 2>&1; then break; fi sleep 1 done From f4afd0eae4091b0ff99dde4cfae417051e949341 Mon Sep 17 00:00:00 2001 From: sanjeevafk <230678117+sanjeevafk@users.noreply.github.com> Date: Tue, 6 Oct 2026 01:24:36 +0530 Subject: [PATCH 2/2] chore: ignore custom ad-hoc dataset scripts and benchmark query files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 0a80fa0..ba2d4e6 100755 --- a/.gitignore +++ b/.gitignore @@ -75,6 +75,9 @@ evaluation/results/ evaluation/evaluation/ evaluation/benchmark_dataset.json evaluation/benchmark_corpus.json +evaluation/*_queries.json +scripts/ingest_*.py +scripts/run_*_benchmark.py evaluation/*.log evaluation/*.sh uvicorn.log