feat: add full/sliced file reading and large file summarization

ARCHITECTURE.md

@@ -1,4 +1,4 @@
-# Architecture: ContextCortex (v2.14.0)
+# Architecture: ContextCortex (v2.15.0)
 
 ContextCortex provides fast, local, syntax-aware semantic and hybrid search over codebases, git repositories, markdown notes, architecture documents, and system documentation. It is built natively on the **Model Context Protocol (MCP) SDK 2.0.0+** using `FastMCP`, with an integrated FastAPI web engine, real-time diagnostic logging, pluggable relational and vector store backends (PostgreSQL 16 with pgvector, Qdrant, ChromaDB, and SQLite), automatic polling daemons, multi-provider webhooks, interactive dependency topology graph explorer, RFC 9728 OAuth 2.1 Protected Resource Server, 3-tier API key RBAC, and a React 19 administrative dashboard.
 

DEVELOPER_DOCS.md

@@ -1,4 +1,4 @@
-# Developer Documentation: ContextCortex (v2.14.0)
+# Developer Documentation: ContextCortex (v2.15.0)
 
 This document provides instructions for developing, testing, configuring, and running ContextCortex locally.
 

README.md

@@ -1,4 +1,4 @@
-# ContextCortex (v2.14.0)
+# ContextCortex (v2.15.0)
 
 
 [![Build and Publish Docker Image](https://github.com/spelech/contextcortex/actions/workflows/docker-publish.yml/badge.svg)](https://github.com/spelech/contextcortex/actions/workflows/docker-publish.yml)

REQUIREMENTS.md

@@ -1,8 +1,8 @@
-# Software Requirements Specification: ContextCortex (v2.14.0)
+# Software Requirements Specification: ContextCortex (v2.15.0)
 
 > **Note:** This document is automatically generated and verified against the live test suite by `scripts/generate_requirements.py` and `tests/backend/test_requirements_sync.py`.
 
-**Test Verification Baseline:** **934 Automated Tests** (619 Pytest Backend + 269 Vitest Frontend + 46 Playwright E2E).
+**Test Verification Baseline:** **968 Automated Tests** (651 Pytest Backend + 271 Vitest Frontend + 46 Playwright E2E).
 
 ---
 
@@ -934,6 +934,29 @@ classDiagram
 - `test_embedding_cache_invalidation`
 - `test_embedding_cache_empty_inputs`
 
+#### `tests/test_file_mcp_and_api.py` (5 tests)
+- `test_env`
+- `test_mcp_handle_read_file`
+- `test_mcp_handle_summarize_file`
+- `test_api_read_file`
+- `test_api_summarize_file`
+
+#### `tests/test_file_reader.py` (8 tests)
+- `test_safe_path_resolution_local_storage`
+- `test_safe_path_resolution_indexed_paths`
+- `test_path_traversal_and_invalid_paths_rejected`
+- `test_read_text_file_full_and_line_slicing`
+- `test_read_file_capping_max_lines_and_truncation`
+- `test_binary_file_detection_and_rejection`
+- `test_read_nonexistent_file_raises_not_found`
+- `test_get_file_reader_service_singleton`
+
+#### `tests/test_file_settings.py` (4 tests)
+- `test_file_summaries_table_has_summary_text_column`
+- `test_file_settings_defaults`
+- `test_set_file_settings_persists`
+- `test_file_settings_api_endpoints`
+
 #### `tests/test_git_incremental.py` (6 tests)
 - `test_compute_git_repo_delta`
 - `test_compute_git_repo_delta_custom_extensions`
@@ -1100,12 +1123,31 @@ and leaves the prior indexed state intact without data loss._
 - `test_process_file_content_doc_caching`
 - `test_process_file_content_file_size_guard`
 
+#### `tests/test_processor_summarization.py` (3 tests)
+- `test_env`
+- `test_large_file_auto_summarization`
+- `test_large_file_disabled_summarization`
+
 #### `tests/test_storage_api_routes.py` (4 tests)
 - `test_storage_upload_and_get_file`
 - `test_storage_upload_multipart_and_put`
 - `test_storage_validation_and_errors`
 - `test_ingestion_catalog_endpoint`
 
+#### `tests/test_summarizer.py` (12 tests)
+- `test_db` - _Sets up an isolated SQLite database for summarizer testing._
+- `test_generate_file_summary_success`
+- `test_generate_file_summary_handles_litellm_exception`
+- `test_get_or_create_summary_returns_cached_summary`
+- `test_get_or_create_summary_cache_miss_reads_disk_and_persists`
+- `test_get_or_create_summary_force_refresh_updates_existing_cache`
+- `test_get_or_create_summary_litellm_failure_does_not_crash`
+- `test_get_summarizer_service_uses_settings_model`
+- `test_get_or_create_summary_nonexistent_file_raises_not_found`
+- `test_get_or_create_summary_vector_store_failure_resilience`
+- `test_get_or_create_summary_reads_via_file_reader`
+- `test_generate_file_summary_truncates_huge_content`
+
 #### `tests/test_webhooks.py` (12 tests)
 - `test_github_webhook_no_secret`
 - `test_github_webhook_with_secret_valid_and_invalid`
@@ -1176,6 +1218,10 @@ and leaves the prior indexed state intact without data loss._
 - switches to manual input when Custom is selected from dropdown or link clicked
 - disables discover button and shows spinner while isDiscovering is true
 
+#### `FileSettings.test.tsx` (2 tests)
+- renders settings fields and loads data from api
+- submits updated settings when Save button is clicked
+
 #### `GitRepoManager.test.tsx` (13 tests)
 - renders repository list with status badges, auto-sync buttons, and details
 - shows empty state when no repositories are registered

app/api/routers/files.py

@@ -0,0 +1,77 @@
+import logging
+from typing import Optional
+from pydantic import BaseModel, Field
+from fastapi import APIRouter, Query
+from fastapi.responses import JSONResponse
+
+from app.services.file_reader import get_file_reader_service
+from app.services.summarizer import get_summarizer_service
+from app.services.auth import ForbiddenError
+
+logger = logging.getLogger("contextcortex.api.files")
+router = APIRouter()
+
+
+class FileSummarizePayload(BaseModel):
+    path: str = Field(..., description="Path to the file to summarize")
+    repo: Optional[str] = Field(None, description="Optional repository or storage namespace")
+    force_refresh: bool = Field(False, description="Whether to bypass cache and regenerate summary")
+
+
+@router.get("/admin/api/files/read")
+async def api_read_file(
+    path: str = Query(..., description="Target file path (relative to repo/storage or absolute within indexed root)"),
+    repo: Optional[str] = Query(None, description="Optional repo or storage namespace"),
+    start_line: Optional[int] = Query(None, description="1-based starting line number"),
+    end_line: Optional[int] = Query(None, description="1-based ending line number"),
+):
+    try:
+        reader = get_file_reader_service()
+        res = reader.read_file(
+            path=path,
+            repo=repo,
+            start_line=start_line,
+            end_line=end_line
+        )
+        return res
+    except ForbiddenError as fe:
+        logger.warning(f"Forbidden access reading file '{path}': {fe}")
+        return JSONResponse(status_code=403, content={"error": "Access denied. Path is outside allowed repositories."})
+    except FileNotFoundError as fne:
+        logger.warning(f"File not found reading '{path}': {fne}")
+        return JSONResponse(status_code=404, content={"error": "File not found."})
+    except ValueError as ve:
+        logger.warning(f"Validation error reading file '{path}': {ve}")
+        return JSONResponse(status_code=400, content={"error": "Invalid file parameters."})
+    except Exception as e:
+        logger.error(f"Error reading file '{path}': {e}")
+        return JSONResponse(status_code=500, content={"error": "Internal server error reading file."})
+
+
+@router.post("/admin/api/files/summarize")
+async def api_summarize_file(payload: FileSummarizePayload):
+    try:
+        summarizer = get_summarizer_service()
+        summary_text = summarizer.get_or_create_summary(
+            filepath=payload.path,
+            repo=payload.repo,
+            force_refresh=payload.force_refresh
+        )
+        return {
+            "path": payload.path,
+            "repo": payload.repo,
+            "summary": summary_text,
+            "status": "success"
+        }
+    except ForbiddenError as fe:
+        logger.warning(f"Forbidden access summarizing file '{payload.path}': {fe}")
+        return JSONResponse(status_code=403, content={"error": "Access denied. Path is outside allowed repositories."})
+    except FileNotFoundError as fne:
+        logger.warning(f"File not found summarizing '{payload.path}': {fne}")
+        return JSONResponse(status_code=404, content={"error": "File not found."})
+    except ValueError as ve:
+        logger.warning(f"Validation error summarizing file '{payload.path}': {ve}")
+        return JSONResponse(status_code=400, content={"error": "Invalid file parameters."})
+    except Exception as e:
+        logger.error(f"Error summarizing file '{payload.path}': {e}")
+        return JSONResponse(status_code=500, content={"error": "Internal server error summarizing file."})

app/api/routers/settings.py

@@ -10,7 +10,8 @@
 from app.models.schemas import (
     TokenRequest, HostCredentialRequest,
     VectorStoreTestRequest, VectorStoreSwitchRequest,
-    AutoSyncSettingsRequest, EmbeddingSettingsRequest
+    AutoSyncSettingsRequest, EmbeddingSettingsRequest,
+    FileSettingsRequest
 )
 import app.services.database as db_service
 import app.services.git_manager as gm_service
@@ -363,3 +364,23 @@ async def api_save_embedding_settings(payload: EmbeddingSettingsRequest):
         logger.error(f"Error updating embedding settings: {e}")
         return JSONResponse(status_code=500, content={"status": "error", "error": str(e), "message": str(e)})
 
+@router.get("/admin/api/settings/files")
+async def api_get_file_settings():
+    try:
+        cfg = db_service.get_file_settings()
+        return cfg
+    except Exception as e:
+        logger.error(f"Error reading file settings: {e}")
+        return JSONResponse(status_code=500, content={"error": "Failed to read file settings."})
+
+@router.post("/admin/api/settings/files")
+async def api_save_file_settings(payload: FileSettingsRequest):
+    try:
+        data = payload.model_dump(exclude_unset=True)
+        updated = db_service.set_file_settings(data)
+        return updated
+    except Exception as e:
+        logger.error(f"Error saving file settings: {e}")
+        return JSONResponse(status_code=500, content={"error": "Failed to save file settings."})
+
+

app/api/routes.py

@@ -42,6 +42,7 @@
 from app.api.routers.storage import router as storage_router
 from app.api.routers.ingestion import router as ingestion_router
 from app.api.routers.navigator import router as navigator_router
+from app.api.routers.files import router as files_router
 
 logger = logging.getLogger("contextcortex.api")
 
@@ -54,6 +55,7 @@
 router.include_router(storage_router)
 router.include_router(ingestion_router)
 router.include_router(navigator_router)
+router.include_router(files_router)
 
 __all__ = [
     "router",

app/mcp/handlers/__init__.py

@@ -27,6 +27,10 @@
     handle_manage_local_file,
     handle_what_is_ingested,
 )
+from app.mcp.handlers.file_handlers import (
+    handle_read_file,
+    handle_summarize_file,
+)
 
 __all__ = [
     "handle_search_code",
@@ -46,4 +50,6 @@
     "handle_manage_adr",
     "handle_manage_local_file",
     "handle_what_is_ingested",
+    "handle_read_file",
+    "handle_summarize_file",
 ]

app/mcp/handlers/file_handlers.py

@@ -0,0 +1,88 @@
+import logging
+import sys
+from typing import Optional, Annotated
+from pydantic import Field
+
+from app.services.auth import enforce_tool_permission, Role, ForbiddenError
+from app.services.file_reader import get_file_reader_service, FileReaderService
+from app.services.summarizer import get_summarizer_service, SummarizerService
+
+logger = logging.getLogger("contextcortex.mcp.files")
+
+
+def _get_tools_attr(name, default):
+    t_mod = sys.modules.get("app.mcp.tools")
+    return getattr(t_mod, name, default) if t_mod else default
+
+
+async def handle_read_file(
+    path: Annotated[str, Field(description="Relative or absolute file path within a watched local directory or uploaded local storage.")],
+    repo: Annotated[Optional[str], Field(description="Optional repository identifier or storage namespace to target.")] = None,
+    start_line: Annotated[Optional[int], Field(description="1-based starting line number (inclusive).")] = None,
+    end_line: Annotated[Optional[int], Field(description="1-based ending line number (inclusive).")] = None
+) -> str:
+    """Read entire file content or bounded line ranges from monitored local directories or local storage with safety limits."""
+    try:
+        enforce_tool_permission(Role.VIEWER)
+        reader_fn = _get_tools_attr("get_file_reader_service", get_file_reader_service)
+        reader = reader_fn()
+
+        result = reader.read_file(
+            path=path,
+            repo=repo,
+            start_line=start_line,
+            end_line=end_line
+        )
+
+        trunc_msg = f" (truncated at line limit)" if result.get("truncated") else ""
+        header = (
+            f"### File: `{result['filepath']}` ({result['size_bytes']} bytes, "
+            f"lines {result['start_line']}-{result['end_line']} of {result['total_lines']}{trunc_msg})\n"
+            f"- **Source:** `{result['source']}`\n\n"
+        )
+        return f"{header}```\n{result['content']}\n```"
+
+    except ForbiddenError as fe:
+        logger.warning(f"Forbidden error in handle_read_file ({path}): {fe}")
+        return f"Forbidden: {str(fe)}"
+    except FileNotFoundError as fne:
+        return f"Error: File not found: {str(fne)}"
+    except ValueError as ve:
+        return f"Error: {str(ve)}"
+    except Exception as e:
+        logger.error(f"Error reading file '{path}': {e}")
+        return f"Error reading file: {str(e)}"
+
+
+async def handle_summarize_file(
+    path: Annotated[str, Field(description="Path to the file to summarize (monitored local path or uploaded local storage).")],
+    repo: Annotated[Optional[str], Field(description="Optional repository or storage namespace filter.")] = None,
+    force_refresh: Annotated[bool, Field(description="If True, bypasses SQLite cache and regenerates a fresh LLM summary.")] = False
+) -> str:
+    """Retrieve an existing summary or generate a structured LLM executive summary for a large file."""
+    try:
+        enforce_tool_permission(Role.VIEWER)
+        summarizer_fn = _get_tools_attr("get_summarizer_service", get_summarizer_service)
+        summarizer = summarizer_fn()
+
+        summary_text = summarizer.get_or_create_summary(
+            filepath=path,
+            repo=repo,
+            force_refresh=force_refresh
+        )
+
+        if not summary_text or not summary_text.strip():
+            return f"Notice: Could not generate summary for `{path}` (file may be empty or summarization failed)."
+
+        return f"### Summary: `{path}`\n\n{summary_text.strip()}"
+
+    except ForbiddenError as fe:
+        logger.warning(f"Forbidden error in handle_summarize_file ({path}): {fe}")
+        return f"Forbidden: {str(fe)}"
+    except FileNotFoundError as fne:
+        return f"Error: File not found: {str(fne)}"
+    except ValueError as ve:
+        return f"Error: {str(ve)}"
+    except Exception as e:
+        logger.error(f"Error summarizing file '{path}': {e}")
+        return f"Error summarizing file: {str(e)}"

app/mcp/tools.py

@@ -28,6 +28,8 @@
     handle_manage_adr,
     handle_manage_local_file,
     handle_what_is_ingested,
+    handle_read_file,
+    handle_summarize_file,
 )
 
 logger = logging.getLogger("contextcortex.mcp")
@@ -124,6 +126,19 @@ def register_mcp_tools_and_resources(server=None):
             description="Inspect all ingested Git repositories, monitored local paths, and uploaded local storage files with optional filtering and detailed file trees."
         )(handle_what_is_ingested)
 
+    if "read_file" not in existing_tools:
+        server.tool(
+            name="read_file",
+            description="Read entire file content or bounded line ranges from monitored local directories or local storage with safety limits."
+        )(handle_read_file)
+
+    if "summarize_file" not in existing_tools:
+        server.tool(
+            name="summarize_file",
+            description="Retrieve an existing summary or generate a structured LLM executive summary for a large file."
+        )(handle_summarize_file)
+
+
     existing_resources = {str(r.uri) for r in server._resource_manager.list_resources()}
     if "knowledge://catalog/summary" not in existing_resources:
         server.resource(

app/models/schemas.py

@@ -51,6 +51,14 @@ class ApiClientCallRecord(BaseModel):
     line_number: int
     created_at: Optional[str] = None
 
+class FileSettingsRequest(BaseModel):
+    summary_enabled: Optional[bool] = None
+    summary_threshold_kb: Optional[int] = None
+    summary_max_file_size_mb: Optional[int] = None
+    read_file_max_lines: Optional[int] = None
+    summary_chat_model: Optional[str] = None
+
+
 class CodeSymbol(BaseModel):
     name: str
     full_symbol: str